pymongo-3.2/0000755000175000017500000000000012631423130014747 5ustar behackettbehackett00000000000000pymongo-3.2/MANIFEST.in0000644000175000017500000000055012630145074016514 0ustar behackettbehackett00000000000000include README.rst include LICENSE include ez_setup.py recursive-include doc *.rst recursive-include doc *.py recursive-include doc *.conf recursive-include doc *.css recursive-include doc *.js recursive-include doc *.png recursive-include tools *.py include tools/README.rst recursive-include test *.pem recursive-include test *.py recursive-include bson *.h pymongo-3.2/pymongo.egg-info/0000755000175000017500000000000012631423130020131 5ustar behackettbehackett00000000000000pymongo-3.2/pymongo.egg-info/SOURCES.txt0000644000175000017500000001147312631423130022023 0ustar behackettbehackett00000000000000LICENSE MANIFEST.in README.rst ez_setup.py setup.py bson/__init__.py bson/_cbsonmodule.c bson/_cbsonmodule.h bson/binary.py bson/buffer.c bson/buffer.h bson/code.py bson/codec_options.py bson/dbref.py bson/encoding_helpers.c bson/encoding_helpers.h bson/errors.py bson/int64.py bson/json_util.py bson/max_key.py bson/min_key.py bson/objectid.py bson/py3compat.py bson/raw_bson.py bson/regex.py bson/son.py bson/time64.c bson/time64.h bson/time64_config.h bson/time64_limits.h bson/timestamp.py bson/tz_util.py doc/__init__.py doc/changelog.rst doc/conf.py doc/contributors.rst doc/faq.rst doc/index.rst doc/installation.rst doc/migrate-to-pymongo3.rst doc/mongo_extensions.py doc/python3.rst doc/tools.rst doc/tutorial.rst doc/api/index.rst doc/api/bson/binary.rst doc/api/bson/code.rst doc/api/bson/codec_options.rst doc/api/bson/dbref.rst doc/api/bson/errors.rst doc/api/bson/index.rst doc/api/bson/int64.rst doc/api/bson/json_util.rst doc/api/bson/max_key.rst doc/api/bson/min_key.rst doc/api/bson/objectid.rst doc/api/bson/raw_bson.rst doc/api/bson/regex.rst doc/api/bson/son.rst doc/api/bson/timestamp.rst doc/api/bson/tz_util.rst doc/api/gridfs/errors.rst doc/api/gridfs/grid_file.rst doc/api/gridfs/index.rst doc/api/pymongo/bulk.rst doc/api/pymongo/collection.rst doc/api/pymongo/command_cursor.rst doc/api/pymongo/cursor.rst doc/api/pymongo/cursor_manager.rst doc/api/pymongo/database.rst doc/api/pymongo/errors.rst doc/api/pymongo/index.rst doc/api/pymongo/message.rst doc/api/pymongo/mongo_client.rst doc/api/pymongo/mongo_replica_set_client.rst doc/api/pymongo/monitoring.rst doc/api/pymongo/operations.rst doc/api/pymongo/pool.rst doc/api/pymongo/read_concern.rst doc/api/pymongo/read_preferences.rst doc/api/pymongo/results.rst doc/api/pymongo/son_manipulator.rst doc/api/pymongo/uri_parser.rst doc/api/pymongo/write_concern.rst doc/developer/index.rst doc/developer/periodic_executor.rst doc/examples/aggregation.rst doc/examples/authentication.rst doc/examples/bulk.rst doc/examples/copydb.rst doc/examples/custom_type.rst doc/examples/datetimes.rst doc/examples/geo.rst doc/examples/gevent.rst doc/examples/gridfs.rst doc/examples/high_availability.rst doc/examples/index.rst doc/examples/mod_wsgi.rst doc/examples/tls.rst doc/pydoctheme/theme.conf doc/pydoctheme/static/pydoctheme.css doc/static/periodic-executor-refs.png doc/static/sidebar.js gridfs/__init__.py gridfs/errors.py gridfs/grid_file.py pymongo/__init__.py pymongo/_cmessagemodule.c pymongo/auth.py pymongo/bulk.py pymongo/client_options.py pymongo/collection.py pymongo/command_cursor.py pymongo/common.py pymongo/cursor.py pymongo/cursor_manager.py pymongo/database.py pymongo/errors.py pymongo/helpers.py pymongo/ismaster.py pymongo/message.py pymongo/mongo_client.py pymongo/mongo_replica_set_client.py pymongo/monitor.py pymongo/monitoring.py pymongo/monotonic.py pymongo/network.py pymongo/operations.py pymongo/periodic_executor.py pymongo/pool.py pymongo/read_concern.py pymongo/read_preferences.py pymongo/response.py pymongo/results.py pymongo/server.py pymongo/server_description.py pymongo/server_selectors.py pymongo/server_type.py pymongo/settings.py pymongo/son_manipulator.py pymongo/ssl_context.py pymongo/ssl_match_hostname.py pymongo/ssl_support.py pymongo/thread_util.py pymongo/topology.py pymongo/topology_description.py pymongo/uri_parser.py pymongo/write_concern.py pymongo.egg-info/PKG-INFO pymongo.egg-info/SOURCES.txt pymongo.egg-info/dependency_links.txt pymongo.egg-info/top_level.txt test/__init__.py test/pymongo_mocks.py test/qcheck.py test/test_auth.py test/test_binary.py test/test_bson.py test/test_bulk.py test/test_client.py test/test_code.py test/test_collection.py test/test_command_monitoring_spec.py test/test_common.py test/test_crud.py test/test_cursor.py test/test_cursor_manager.py test/test_database.py test/test_dbref.py test/test_discovery_and_monitoring.py test/test_grid_file.py test/test_gridfs.py test/test_gridfs_bucket.py test/test_gridfs_spec.py test/test_json_util.py test/test_legacy_api.py test/test_mongos_load_balancing.py test/test_monitor.py test/test_monitoring.py test/test_objectid.py test/test_pooling.py test/test_pymongo.py test/test_raw_bson.py test/test_read_concern.py test/test_read_preferences.py test/test_replica_set_client.py test/test_replica_set_reconfig.py test/test_server.py test/test_server_description.py test/test_server_selection.py test/test_server_selection_rtt.py test/test_son.py test/test_son_manipulator.py test/test_ssl.py test/test_threads.py test/test_timestamp.py test/test_topology.py test/test_uri_parser.py test/test_uri_spec.py test/utils.py test/version.py test/certificates/ca.pem test/certificates/client.pem test/certificates/crl.pem test/certificates/server.pem test/high_availability/ha_tools.py test/high_availability/test_ha.py test/mod_wsgi_test/test_client.py tools/README.rst tools/benchmark.py tools/clean.py tools/fail_if_no_c.pypymongo-3.2/pymongo.egg-info/PKG-INFO0000644000175000017500000002141612631423130021232 0ustar behackettbehackett00000000000000Metadata-Version: 1.1 Name: pymongo Version: 3.2 Summary: Python driver for MongoDB Home-page: http://github.com/mongodb/mongo-python-driver Author: Bernie Hackett Author-email: bernie@mongodb.com License: Apache License, Version 2.0 Description: ======= PyMongo ======= :Info: See `the mongo site `_ for more information. See `github `_ for the latest source. :Author: Mike Dirolf :Maintainer: Bernie Hackett About ===== The PyMongo distribution contains tools for interacting with MongoDB database from Python. The ``bson`` package is an implementation of the `BSON format `_ for Python. The ``pymongo`` package is a native Python driver for MongoDB. The ``gridfs`` package is a `gridfs `_ implementation on top of ``pymongo``. Support / Feedback ================== For issues with, questions about, or feedback for PyMongo, please look into our `support channels `_. Please do not email any of the PyMongo developers directly with issues or questions - you're more likely to get an answer on the `mongodb-user `_ list on Google Groups. Bugs / Feature Requests ======================= Think you’ve found a bug? Want to see a new feature in PyMongo? Please open a case in our issue management tool, JIRA: - `Create an account and login `_. - Navigate to `the PYTHON project `_. - Click **Create Issue** - Please provide as much information as possible about the issue type and how to reproduce it. Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the Core Server (i.e. SERVER) project are **public**. How To Ask For Help ------------------- Please include all of the following information when opening an issue: - Detailed steps to reproduce the problem, including full traceback, if possible. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used, with patch level:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The operating system and version (e.g. Windows 7, OSX 10.8, ...) - Web framework or asynchronous network library used, if any, with version (e.g. Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...) Security Vulnerabilities ------------------------ If you’ve identified a security vulnerability in a driver or any other MongoDB project, please report it according to the `instructions here `_. Installation ============ If you have `setuptools `_ installed you should be able to do **easy_install pymongo** to install PyMongo. Otherwise you can download the project source and do **python setup.py install** to install. Do **not** install the "bson" package. PyMongo comes with its own bson package; doing "easy_install bson" installs a third-party package that is incompatible with PyMongo. Dependencies ============ The PyMongo distribution is supported and tested on Python 2.x (where x >= 6) and Python 3.x (where x >= 2). PyMongo versions before 3.0 also support Python 2.4, 2.5, and 3.1. Optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python older than 2.7.8, or on Python 3 before Python 3.4. - `pykerberos `_ is required for the GSSAPI authentication mechanism. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3.3+. - `wincertstore `_ adds support for verifying server SSL certificates using Windows provided CA certificates on older versions of python. Not needed or used with versions of Python 2 beginning with 2.7.9, or versions of Python 3 beginning with 3.4.0. - `certifi `_ adds support for using the Mozilla CA bundle with SSL to verify server certificates. Not needed or used with versions of Python 2 beginning with 2.7.9 on any OS, versions of Python 3 beginning with Python 3.4.0 on Windows, or versions of Python 3 beginning with Python 3.2.0 on operating systems other than Windows. Additional dependencies are: - (to generate documentation) sphinx_ - (to run the tests under Python 2.6) unittest2_ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: pycon >>> import pymongo >>> client = pymongo.MongoClient("localhost", 27017) >>> db = client.test >>> db.name u'test' >>> db.my_collection Collection(Database(MongoClient('localhost', 27017), u'test'), u'my_collection') >>> db.my_collection.insert_one({"x": 10}).inserted_id ObjectId('4aba15ebe23f6b53b0000000') >>> db.my_collection.insert_one({"x": 8}).inserted_id ObjectId('4aba160ee23f6b543e000000') >>> db.my_collection.insert_one({"x": 11}).inserted_id ObjectId('4aba160ee23f6b543e000002') >>> db.my_collection.find_one() {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')} >>> for item in db.my_collection.find(): ... print(item["x"]) ... 10 8 11 >>> db.my_collection.create_index("x") u'x_1' >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING): ... print(item["x"]) ... 8 10 11 >>> [item["x"] for item in db.my_collection.find().limit(2).skip(1)] [8, 11] Documentation ============= You will need sphinx_ installed to generate the documentation. Documentation can be generated by running **python setup.py doc**. Generated documentation can be found in the *doc/build/html/* directory. Testing ======= The easiest way to run the tests is to run **python setup.py test** in the root of the distribution. Note that you will need unittest2_ to run the tests under Python 2.6. To verify that PyMongo works with Gevent's monkey-patching:: $ python green_framework_test.py gevent Or with Eventlet's:: $ python green_framework_test.py eventlet .. _sphinx: http://sphinx.pocoo.org/ .. _unittest2: https://pypi.python.org/pypi/unittest2 Keywords: mongo,mongodb,pymongo,gridfs,bson Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Database pymongo-3.2/pymongo.egg-info/top_level.txt0000644000175000017500000000002412631423130022657 0ustar behackettbehackett00000000000000bson gridfs pymongo pymongo-3.2/pymongo.egg-info/dependency_links.txt0000644000175000017500000000000112631423130024177 0ustar behackettbehackett00000000000000 pymongo-3.2/PKG-INFO0000644000175000017500000002141612631423130016050 0ustar behackettbehackett00000000000000Metadata-Version: 1.1 Name: pymongo Version: 3.2 Summary: Python driver for MongoDB Home-page: http://github.com/mongodb/mongo-python-driver Author: Bernie Hackett Author-email: bernie@mongodb.com License: Apache License, Version 2.0 Description: ======= PyMongo ======= :Info: See `the mongo site `_ for more information. See `github `_ for the latest source. :Author: Mike Dirolf :Maintainer: Bernie Hackett About ===== The PyMongo distribution contains tools for interacting with MongoDB database from Python. The ``bson`` package is an implementation of the `BSON format `_ for Python. The ``pymongo`` package is a native Python driver for MongoDB. The ``gridfs`` package is a `gridfs `_ implementation on top of ``pymongo``. Support / Feedback ================== For issues with, questions about, or feedback for PyMongo, please look into our `support channels `_. Please do not email any of the PyMongo developers directly with issues or questions - you're more likely to get an answer on the `mongodb-user `_ list on Google Groups. Bugs / Feature Requests ======================= Think you’ve found a bug? Want to see a new feature in PyMongo? Please open a case in our issue management tool, JIRA: - `Create an account and login `_. - Navigate to `the PYTHON project `_. - Click **Create Issue** - Please provide as much information as possible about the issue type and how to reproduce it. Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the Core Server (i.e. SERVER) project are **public**. How To Ask For Help ------------------- Please include all of the following information when opening an issue: - Detailed steps to reproduce the problem, including full traceback, if possible. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used, with patch level:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The operating system and version (e.g. Windows 7, OSX 10.8, ...) - Web framework or asynchronous network library used, if any, with version (e.g. Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...) Security Vulnerabilities ------------------------ If you’ve identified a security vulnerability in a driver or any other MongoDB project, please report it according to the `instructions here `_. Installation ============ If you have `setuptools `_ installed you should be able to do **easy_install pymongo** to install PyMongo. Otherwise you can download the project source and do **python setup.py install** to install. Do **not** install the "bson" package. PyMongo comes with its own bson package; doing "easy_install bson" installs a third-party package that is incompatible with PyMongo. Dependencies ============ The PyMongo distribution is supported and tested on Python 2.x (where x >= 6) and Python 3.x (where x >= 2). PyMongo versions before 3.0 also support Python 2.4, 2.5, and 3.1. Optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python older than 2.7.8, or on Python 3 before Python 3.4. - `pykerberos `_ is required for the GSSAPI authentication mechanism. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3.3+. - `wincertstore `_ adds support for verifying server SSL certificates using Windows provided CA certificates on older versions of python. Not needed or used with versions of Python 2 beginning with 2.7.9, or versions of Python 3 beginning with 3.4.0. - `certifi `_ adds support for using the Mozilla CA bundle with SSL to verify server certificates. Not needed or used with versions of Python 2 beginning with 2.7.9 on any OS, versions of Python 3 beginning with Python 3.4.0 on Windows, or versions of Python 3 beginning with Python 3.2.0 on operating systems other than Windows. Additional dependencies are: - (to generate documentation) sphinx_ - (to run the tests under Python 2.6) unittest2_ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: pycon >>> import pymongo >>> client = pymongo.MongoClient("localhost", 27017) >>> db = client.test >>> db.name u'test' >>> db.my_collection Collection(Database(MongoClient('localhost', 27017), u'test'), u'my_collection') >>> db.my_collection.insert_one({"x": 10}).inserted_id ObjectId('4aba15ebe23f6b53b0000000') >>> db.my_collection.insert_one({"x": 8}).inserted_id ObjectId('4aba160ee23f6b543e000000') >>> db.my_collection.insert_one({"x": 11}).inserted_id ObjectId('4aba160ee23f6b543e000002') >>> db.my_collection.find_one() {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')} >>> for item in db.my_collection.find(): ... print(item["x"]) ... 10 8 11 >>> db.my_collection.create_index("x") u'x_1' >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING): ... print(item["x"]) ... 8 10 11 >>> [item["x"] for item in db.my_collection.find().limit(2).skip(1)] [8, 11] Documentation ============= You will need sphinx_ installed to generate the documentation. Documentation can be generated by running **python setup.py doc**. Generated documentation can be found in the *doc/build/html/* directory. Testing ======= The easiest way to run the tests is to run **python setup.py test** in the root of the distribution. Note that you will need unittest2_ to run the tests under Python 2.6. To verify that PyMongo works with Gevent's monkey-patching:: $ python green_framework_test.py gevent Or with Eventlet's:: $ python green_framework_test.py eventlet .. _sphinx: http://sphinx.pocoo.org/ .. _unittest2: https://pypi.python.org/pypi/unittest2 Keywords: mongo,mongodb,pymongo,gridfs,bson Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Database pymongo-3.2/tools/0000755000175000017500000000000012631423130016107 5ustar behackettbehackett00000000000000pymongo-3.2/tools/clean.py0000644000175000017500000000213012560753776017566 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Clean up script for build artifacts. Only really intended to be used by internal build scripts. """ import os import sys try: os.remove("pymongo/_cmessage.so") os.remove("bson/_cbson.so") except: pass try: os.remove("pymongo/_cmessage.pyd") os.remove("bson/_cbson.pyd") except: pass try: from pymongo import _cmessage sys.exit("could still import _cmessage") except ImportError: pass try: from bson import _cbson sys.exit("could still import _cbson") except ImportError: pass pymongo-3.2/tools/benchmark.py0000644000175000017500000001346012630145074020426 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """MongoDB benchmarking suite.""" import time import sys sys.path[0:0] = [""] import datetime from pymongo import mongo_client from pymongo import ASCENDING trials = 2 per_trial = 5000 batch_size = 100 small = {} medium = {"integer": 5, "number": 5.05, "boolean": False, "array": ["test", "benchmark"] } # this is similar to the benchmark data posted to the user list large = {"base_url": "http://www.example.com/test-me", "total_word_count": 6743, "access_time": datetime.datetime.utcnow(), "meta_tags": {"description": "i am a long description string", "author": "Holly Man", "dynamically_created_meta_tag": "who know\n what" }, "page_structure": {"counted_tags": 3450, "no_of_js_attached": 10, "no_of_images": 6 }, "harvested_words": ["10gen", "web", "open", "source", "application", "paas", "platform-as-a-service", "technology", "helps", "developers", "focus", "building", "mongodb", "mongo"] * 20 } def setup_insert(db, collection, object): db.drop_collection(collection) def insert(db, collection, object): for i in range(per_trial): to_insert = object.copy() to_insert["x"] = i db[collection].insert(to_insert) def insert_batch(db, collection, object): for i in range(per_trial / batch_size): db[collection].insert([object] * batch_size) def find_one(db, collection, x): for _ in range(per_trial): db[collection].find_one({"x": x}) def find(db, collection, x): for _ in range(per_trial): for _ in db[collection].find({"x": x}): pass def timed(name, function, args=[], setup=None): times = [] for _ in range(trials): if setup: setup(*args) start = time.time() function(*args) times.append(time.time() - start) best_time = min(times) print "%s%d" % (name + (60 - len(name)) * ".", per_trial / best_time) return best_time def main(): c = mongo_client.MongoClient(connectTimeoutMS=60*1000) # jack up timeout c.drop_database("benchmark") db = c.benchmark timed("insert (small, no index)", insert, [db, 'small_none', small], setup_insert) timed("insert (medium, no index)", insert, [db, 'medium_none', medium], setup_insert) timed("insert (large, no index)", insert, [db, 'large_none', large], setup_insert) db.small_index.create_index("x", ASCENDING) timed("insert (small, indexed)", insert, [db, 'small_index', small]) db.medium_index.create_index("x", ASCENDING) timed("insert (medium, indexed)", insert, [db, 'medium_index', medium]) db.large_index.create_index("x", ASCENDING) timed("insert (large, indexed)", insert, [db, 'large_index', large]) timed("batch insert (small, no index)", insert_batch, [db, 'small_bulk', small], setup_insert) timed("batch insert (medium, no index)", insert_batch, [db, 'medium_bulk', medium], setup_insert) timed("batch insert (large, no index)", insert_batch, [db, 'large_bulk', large], setup_insert) timed("find_one (small, no index)", find_one, [db, 'small_none', per_trial / 2]) timed("find_one (medium, no index)", find_one, [db, 'medium_none', per_trial / 2]) timed("find_one (large, no index)", find_one, [db, 'large_none', per_trial / 2]) timed("find_one (small, indexed)", find_one, [db, 'small_index', per_trial / 2]) timed("find_one (medium, indexed)", find_one, [db, 'medium_index', per_trial / 2]) timed("find_one (large, indexed)", find_one, [db, 'large_index', per_trial / 2]) timed("find (small, no index)", find, [db, 'small_none', per_trial / 2]) timed("find (medium, no index)", find, [db, 'medium_none', per_trial / 2]) timed("find (large, no index)", find, [db, 'large_none', per_trial / 2]) timed("find (small, indexed)", find, [db, 'small_index', per_trial / 2]) timed("find (medium, indexed)", find, [db, 'medium_index', per_trial / 2]) timed("find (large, indexed)", find, [db, 'large_index', per_trial / 2]) # timed("find range (small, no index)", find, # [db, 'small_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) # timed("find range (medium, no index)", find, # [db, 'medium_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) # timed("find range (large, no index)", find, # [db, 'large_none', # {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}]) timed("find range (small, indexed)", find, [db, 'small_index', {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) timed("find range (medium, indexed)", find, [db, 'medium_index', {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) timed("find range (large, indexed)", find, [db, 'large_index', {"$gt": per_trial / 2, "$lt": per_trial / 2 + batch_size}]) if __name__ == "__main__": # cProfile.run("main()") main() pymongo-3.2/tools/fail_if_no_c.py0000644000175000017500000000151212560753776021076 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Fail if the C extension module doesn't exist. Only really intended to be used by internal build scripts. """ import sys sys.path[0:0] = [""] import bson import pymongo if not pymongo.has_c() or not bson.has_c(): sys.exit("could not load C extensions") pymongo-3.2/tools/README.rst0000644000175000017500000000011712507072032017600 0ustar behackettbehackett00000000000000Tools ===== This directory contains tools for use with the ``pymongo`` module. pymongo-3.2/setup.py0000755000175000017500000002577512631420450016506 0ustar behackettbehackett00000000000000import os import platform import re import subprocess import sys import warnings # Hack to silence atexit traceback in some Python versions try: import multiprocessing except ImportError: pass # Don't force people to install setuptools unless # we have to. try: from setuptools import setup except ImportError: from ez_setup import use_setuptools use_setuptools() from setuptools import setup from distutils.cmd import Command from distutils.command.build_ext import build_ext from distutils.errors import CCompilerError, DistutilsOptionError from distutils.errors import DistutilsPlatformError, DistutilsExecError from distutils.core import Extension version = "3.2" f = open("README.rst") try: try: readme_content = f.read() except: readme_content = "" finally: f.close() # PYTHON-654 - Clang doesn't support -mno-fused-madd but the pythons Apple # ships are built with it. This is a problem starting with Xcode 5.1 # since clang 3.4 errors out when it encounters unrecognized compiler # flags. This hack removes -mno-fused-madd from the CFLAGS automatically # generated by distutils for Apple provided pythons, allowing C extension # builds to complete without error. The inspiration comes from older # versions of distutils.sysconfig.get_config_vars. if sys.platform == 'darwin' and 'clang' in platform.python_compiler().lower(): from distutils.sysconfig import get_config_vars res = get_config_vars() for key in ('CFLAGS', 'PY_CFLAGS'): if key in res: flags = res[key] flags = re.sub('-mno-fused-madd', '', flags) res[key] = flags class test(Command): description = "run the tests" user_options = [ ("test-module=", "m", "Discover tests in specified module"), ("test-suite=", "s", "Test suite to run (e.g. 'some_module.test_suite')"), ("failfast", "f", "Stop running tests on first failure or error") ] def initialize_options(self): self.test_module = None self.test_suite = None self.failfast = False def finalize_options(self): if self.test_suite is None and self.test_module is None: self.test_module = 'test' elif self.test_module is not None and self.test_suite is not None: raise DistutilsOptionError( "You may specify a module or suite, but not both" ) def run(self): # Installing required packages, running egg_info and build_ext are # part of normal operation for setuptools.command.test.test if self.distribution.install_requires: self.distribution.fetch_build_eggs( self.distribution.install_requires) if self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) self.run_command('egg_info') build_ext_cmd = self.reinitialize_command('build_ext') build_ext_cmd.inplace = 1 self.run_command('build_ext') # Construct a TextTestRunner directly from the unittest imported from # test (this will be unittest2 under Python 2.6), which creates a # TestResult that supports the 'addSkip' method. setuptools will by # default create a TextTestRunner that uses the old TestResult class, # resulting in DeprecationWarnings instead of skipping tests under 2.6. from test import unittest, PymongoTestRunner, test_cases if self.test_suite is None: all_tests = unittest.defaultTestLoader.discover(self.test_module) suite = unittest.TestSuite() suite.addTests(sorted(test_cases(all_tests), key=lambda x: x.__module__)) else: suite = unittest.defaultTestLoader.loadTestsFromName( self.test_suite) result = PymongoTestRunner(verbosity=2, failfast=self.failfast).run(suite) sys.exit(not result.wasSuccessful()) class doc(Command): description = "generate or test documentation" user_options = [("test", "t", "run doctests instead of generating documentation")] boolean_options = ["test"] def initialize_options(self): self.test = False def finalize_options(self): pass def run(self): if self.test: path = "doc/_build/doctest" mode = "doctest" else: path = "doc/_build/%s" % version mode = "html" try: os.makedirs(path) except: pass status = subprocess.call(["sphinx-build", "-E", "-b", mode, "doc", path]) if status: raise RuntimeError("documentation step '%s' failed" % (mode,)) sys.stdout.write("\nDocumentation step '%s' performed, results here:\n" " %s/\n" % (mode, path)) if sys.platform == 'win32' and sys.version_info > (2, 6): # 2.6's distutils.msvc9compiler can raise an IOError when failing to # find the compiler build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError) else: build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) class custom_build_ext(build_ext): """Allow C extension building to fail. The C extension speeds up BSON encoding, but is not essential. """ warning_message = """ ******************************************************************** WARNING: %s could not be compiled. No C extensions are essential for PyMongo to run, although they do result in significant speed improvements. %s Please see the installation docs for solutions to build issues: http://api.mongodb.org/python/current/installation.html Here are some hints for popular operating systems: If you are seeing this message on Linux you probably need to install GCC and/or the Python development package for your version of Python. Debian and Ubuntu users should issue the following command: $ sudo apt-get install build-essential python-dev Users of Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, Fedora, etc.) should issue the following command: $ sudo yum install gcc python-devel If you are seeing this message on Microsoft Windows please install PyMongo using the MS Windows installer for your version of Python, available on pypi here: http://pypi.python.org/pypi/pymongo/#downloads If you are seeing this message on OSX please read the documentation here: http://api.mongodb.org/python/current/installation.html#osx ******************************************************************** """ def run(self): try: build_ext.run(self) except DistutilsPlatformError: e = sys.exc_info()[1] sys.stdout.write('%s\n' % str(e)) warnings.warn(self.warning_message % ("Extension modules", "There was an issue with " "your platform configuration" " - see above.")) def build_extension(self, ext): name = ext.name if sys.version_info[:3] >= (2, 6, 0): try: build_ext.build_extension(self, ext) except build_errors: e = sys.exc_info()[1] sys.stdout.write('%s\n' % str(e)) warnings.warn(self.warning_message % ("The %s extension " "module" % (name,), "The output above " "this warning shows how " "the compilation " "failed.")) else: warnings.warn(self.warning_message % ("The %s extension " "module" % (name,), "Please use Python >= 2.6 " "to take advantage of the " "extension.")) ext_modules = [Extension('bson._cbson', include_dirs=['bson'], sources=['bson/_cbsonmodule.c', 'bson/time64.c', 'bson/buffer.c', 'bson/encoding_helpers.c']), Extension('pymongo._cmessage', include_dirs=['bson'], sources=['pymongo/_cmessagemodule.c', 'bson/buffer.c'])] extra_opts = { "packages": ["bson", "pymongo", "gridfs"] } if sys.version_info[:2] == (2, 6): extra_opts['tests_require'] = "unittest2" if "--no_ext" in sys.argv: sys.argv.remove("--no_ext") elif (sys.platform.startswith("java") or sys.platform == "cli" or "PyPy" in sys.version): sys.stdout.write(""" *****************************************************\n The optional C extensions are currently not supported\n by this python implementation.\n *****************************************************\n """) elif sys.byteorder == "big": sys.stdout.write(""" *****************************************************\n The optional C extensions are currently not supported\n on big endian platforms and will not be built.\n Performance may be degraded.\n *****************************************************\n """) else: extra_opts['ext_modules'] = ext_modules setup( name="pymongo", version=version, description="Python driver for MongoDB ", long_description=readme_content, author="Mike Dirolf", author_email="mongodb-user@googlegroups.com", maintainer="Bernie Hackett", maintainer_email="bernie@mongodb.com", url="http://github.com/mongodb/mongo-python-driver", keywords=["mongo", "mongodb", "pymongo", "gridfs", "bson"], install_requires=[], license="Apache License, Version 2.0", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Database"], cmdclass={"build_ext": custom_build_ext, "doc": doc, "test": test}, **extra_opts ) pymongo-3.2/gridfs/0000755000175000017500000000000012631423130016225 5ustar behackettbehackett00000000000000pymongo-3.2/gridfs/errors.py0000644000175000017500000000204012630145074020116 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by the :mod:`gridfs` package""" from pymongo.errors import PyMongoError class GridFSError(PyMongoError): """Base class for all GridFS exceptions.""" class CorruptGridFile(GridFSError): """Raised when a file in :class:`~gridfs.GridFS` is malformed.""" class NoFile(GridFSError): """Raised when trying to read from a non-existent file.""" class FileExists(GridFSError): """Raised when trying to create a file that already exists.""" pymongo-3.2/gridfs/__init__.py0000644000175000017500000006542112630145074020355 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """GridFS is a specification for storing large objects in Mongo. The :mod:`gridfs` package is an implementation of GridFS on top of :mod:`pymongo`, exposing a file-like interface. .. mongodoc:: gridfs """ from collections import Mapping from gridfs.errors import NoFile from gridfs.grid_file import (GridIn, GridOut, GridOutCursor, DEFAULT_CHUNK_SIZE) from pymongo import (ASCENDING, DESCENDING) from pymongo.common import UNAUTHORIZED_CODES, validate_string from pymongo.database import Database from pymongo.errors import ConfigurationError, OperationFailure class GridFS(object): """An instance of GridFS on top of a single Database. """ def __init__(self, database, collection="fs"): """Create a new instance of :class:`GridFS`. Raises :class:`TypeError` if `database` is not an instance of :class:`~pymongo.database.Database`. :Parameters: - `database`: database to use - `collection` (optional): root collection to use .. versionchanged:: 3.1 Indexes are only ensured on the first write to the DB. .. versionchanged:: 3.0 `database` must use an acknowledged :attr:`~pymongo.database.Database.write_concern` .. mongodoc:: gridfs """ if not isinstance(database, Database): raise TypeError("database must be an instance of Database") if not database.write_concern.acknowledged: raise ConfigurationError('database must use ' 'acknowledged write_concern') self.__database = database self.__collection = database[collection] self.__files = self.__collection.files self.__chunks = self.__collection.chunks def new_file(self, **kwargs): """Create a new file in GridFS. Returns a new :class:`~gridfs.grid_file.GridIn` instance to which data can be written. Any keyword arguments will be passed through to :meth:`~gridfs.grid_file.GridIn`. If the ``"_id"`` of the file is manually specified, it must not already exist in GridFS. Otherwise :class:`~gridfs.errors.FileExists` is raised. :Parameters: - `**kwargs` (optional): keyword arguments for file creation """ # No need for __ensure_index_files_id() here; GridIn ensures # the (files_id, n) index when needed. return GridIn(self.__collection, **kwargs) def put(self, data, **kwargs): """Put data in GridFS as a new file. Equivalent to doing:: try: f = new_file(**kwargs) f.write(data) finally: f.close() `data` can be either an instance of :class:`str` (:class:`bytes` in python 3) or a file-like object providing a :meth:`read` method. If an `encoding` keyword argument is passed, `data` can also be a :class:`unicode` (:class:`str` in python 3) instance, which will be encoded as `encoding` before being written. Any keyword arguments will be passed through to the created file - see :meth:`~gridfs.grid_file.GridIn` for possible arguments. Returns the ``"_id"`` of the created file. If the ``"_id"`` of the file is manually specified, it must not already exist in GridFS. Otherwise :class:`~gridfs.errors.FileExists` is raised. :Parameters: - `data`: data to be written as a file. - `**kwargs` (optional): keyword arguments for file creation .. versionchanged:: 3.0 w=0 writes to GridFS are now prohibited. """ grid_file = GridIn(self.__collection, **kwargs) try: grid_file.write(data) finally: grid_file.close() return grid_file._id def get(self, file_id): """Get a file from GridFS by ``"_id"``. Returns an instance of :class:`~gridfs.grid_file.GridOut`, which provides a file-like interface for reading. :Parameters: - `file_id`: ``"_id"`` of the file to get """ gout = GridOut(self.__collection, file_id) # Raise NoFile now, instead of on first attribute access. gout._ensure_file() return gout def get_version(self, filename=None, version=-1, **kwargs): """Get a file from GridFS by ``"filename"`` or metadata fields. Returns a version of the file in GridFS whose filename matches `filename` and whose metadata fields match the supplied keyword arguments, as an instance of :class:`~gridfs.grid_file.GridOut`. Version numbering is a convenience atop the GridFS API provided by MongoDB. If more than one file matches the query (either by `filename` alone, by metadata fields, or by a combination of both), then version ``-1`` will be the most recently uploaded matching file, ``-2`` the second most recently uploaded, etc. Version ``0`` will be the first version uploaded, ``1`` the second version, etc. So if three versions have been uploaded, then version ``0`` is the same as version ``-3``, version ``1`` is the same as version ``-2``, and version ``2`` is the same as version ``-1``. Raises :class:`~gridfs.errors.NoFile` if no such version of that file exists. :Parameters: - `filename`: ``"filename"`` of the file to get, or `None` - `version` (optional): version of the file to get (defaults to -1, the most recent version uploaded) - `**kwargs` (optional): find files by custom metadata. .. versionchanged:: 3.1 ``get_version`` no longer ensures indexes. """ query = kwargs if filename is not None: query["filename"] = filename cursor = self.__files.find(query) if version < 0: skip = abs(version) - 1 cursor.limit(-1).skip(skip).sort("uploadDate", DESCENDING) else: cursor.limit(-1).skip(version).sort("uploadDate", ASCENDING) try: grid_file = next(cursor) return GridOut(self.__collection, file_document=grid_file) except StopIteration: raise NoFile("no version %d for filename %r" % (version, filename)) def get_last_version(self, filename=None, **kwargs): """Get the most recent version of a file in GridFS by ``"filename"`` or metadata fields. Equivalent to calling :meth:`get_version` with the default `version` (``-1``). :Parameters: - `filename`: ``"filename"`` of the file to get, or `None` - `**kwargs` (optional): find files by custom metadata. """ return self.get_version(filename=filename, **kwargs) # TODO add optional safe mode for chunk removal? def delete(self, file_id): """Delete a file from GridFS by ``"_id"``. Deletes all data belonging to the file with ``"_id"``: `file_id`. .. warning:: Any processes/threads reading from the file while this method is executing will likely see an invalid/corrupt file. Care should be taken to avoid concurrent reads to a file while it is being deleted. .. note:: Deletes of non-existent files are considered successful since the end result is the same: no file with that _id remains. :Parameters: - `file_id`: ``"_id"`` of the file to delete .. versionchanged:: 3.1 ``delete`` no longer ensures indexes. """ self.__files.delete_one({"_id": file_id}) self.__chunks.delete_many({"files_id": file_id}) def list(self): """List the names of all files stored in this instance of :class:`GridFS`. .. versionchanged:: 3.1 ``list`` no longer ensures indexes. """ # With an index, distinct includes documents with no filename # as None. return [ name for name in self.__files.distinct("filename") if name is not None] def find_one(self, filter=None, *args, **kwargs): """Get a single file from gridfs. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single :class:`~gridfs.grid_file.GridOut`, or ``None`` if no matching file is found. For example:: file = fs.find_one({"filename": "lisa.txt"}) :Parameters: - `filter` (optional): a dictionary specifying the query to be performing OR any other type to be used as the value for a query for ``"_id"`` in the file collection. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. """ if filter is not None and not isinstance(filter, Mapping): filter = {"_id": filter} for f in self.find(filter, *args, **kwargs): return f return None def find(self, *args, **kwargs): """Query GridFS for files. Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: for grid_out in fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True): data = grid_out.read() would iterate through all versions of "lisa.txt" stored in GridFS. Note that setting no_cursor_timeout to True may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) would return a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~pymongo.collection.Collection.find` in :class:`~pymongo.collection.Collection`. :Parameters: - `filter` (optional): a SON object specifying elements which must be present for a document to be included in the result set - `skip` (optional): the number of files to omit (from the start of the result set) when returning the results - `limit` (optional): the maximum number of results to return - `no_cursor_timeout` (optional): if False (the default), any returned cursor is closed by the server after 10 minutes of inactivity. If set to True, the returned cursor will never time out on the server. Care should be taken to ensure that cursors with no_cursor_timeout turned on are properly closed. - `sort` (optional): a list of (key, direction) pairs specifying the sort order for this query. See :meth:`~pymongo.cursor.Cursor.sort` for details. Raises :class:`TypeError` if any of the arguments are of improper type. Returns an instance of :class:`~gridfs.grid_file.GridOutCursor` corresponding to this query. .. versionchanged:: 3.0 Removed the read_preference, tag_sets, and secondary_acceptable_latency_ms options. .. versionadded:: 2.7 .. mongodoc:: find """ return GridOutCursor(self.__collection, *args, **kwargs) def exists(self, document_or_id=None, **kwargs): """Check if a file exists in this instance of :class:`GridFS`. The file to check for can be specified by the value of its ``_id`` key, or by passing in a query document. A query document can be passed in as dictionary, or by using keyword arguments. Thus, the following three calls are equivalent: >>> fs.exists(file_id) >>> fs.exists({"_id": file_id}) >>> fs.exists(_id=file_id) As are the following two calls: >>> fs.exists({"filename": "mike.txt"}) >>> fs.exists(filename="mike.txt") And the following two: >>> fs.exists({"foo": {"$gt": 12}}) >>> fs.exists(foo={"$gt": 12}) Returns ``True`` if a matching file exists, ``False`` otherwise. Calls to :meth:`exists` will not automatically create appropriate indexes; application developers should be sure to create indexes if needed and as appropriate. :Parameters: - `document_or_id` (optional): query document, or _id of the document to check for - `**kwargs` (optional): keyword arguments are used as a query document, if they're present. """ if kwargs: return self.__files.find_one(kwargs, ["_id"]) is not None return self.__files.find_one(document_or_id, ["_id"]) is not None class GridFSBucket(object): """An instance of GridFS on top of a single Database.""" def __init__(self, db, bucket_name="fs", chunk_size_bytes=DEFAULT_CHUNK_SIZE, write_concern=None, read_preference=None): """Create a new instance of :class:`GridFSBucket`. Raises :exc:`TypeError` if `database` is not an instance of :class:`~pymongo.database.Database`. Raises :exc:`~pymongo.errors.ConfigurationError` if `write_concern` is not acknowledged. :Parameters: - `database`: database to use. - `bucket_name` (optional): The name of the bucket. Defaults to 'fs'. - `chunk_size_bytes` (optional): The chunk size in bytes. Defaults to 255KB. - `write_concern` (optional): The :class:`~pymongo.write_concern.WriteConcern` to use. If ``None`` (the default) db.write_concern is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) db.read_preference is used. .. versionadded:: 3.1 .. mongodoc:: gridfs """ if not isinstance(db, Database): raise TypeError("database must be an instance of Database") wtc = write_concern if write_concern is not None else db.write_concern if not wtc.acknowledged: raise ConfigurationError('write concern must be acknowledged') self._db = db self._bucket_name = bucket_name self._collection = db[bucket_name] self._chunks = self._collection.chunks.with_options( write_concern=write_concern, read_preference=read_preference) self._files = self._collection.files.with_options( write_concern=write_concern, read_preference=read_preference) self._chunk_size_bytes = chunk_size_bytes def open_upload_stream(self, filename, chunk_size_bytes=None, metadata=None): """Opens a Stream that the application can write the contents of the file to. The user must specify the filename, and can choose to add any additional information in the metadata field of the file document or modify the chunk size. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream( "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) grid_in.write("data I want to store!") grid_in.close() # uploaded on close Returns an instance of :class:`~gridfs.grid_file.GridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`GridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. """ validate_string("filename", filename) opts = {"filename": filename, "chunk_size": (chunk_size_bytes if chunk_size_bytes is not None else self._chunk_size_bytes)} if metadata is not None: opts["metadata"] = metadata return GridIn(self._collection, **opts) def upload_from_stream(self, filename, source, chunk_size_bytes=None, metadata=None): """Uploads a user file to a GridFS bucket. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) file_id = fs.upload_from_stream( "test_file", "data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Returns the _id of the uploaded file. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`GridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. """ with self.open_upload_stream( filename, chunk_size_bytes, metadata) as gin: gin.write(source) return gin._id def open_download_stream(self, file_id): """Opens a Stream from which the application can read the contents of the stored file specified by file_id. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # get _id of file to read. file_id = fs.upload_from_stream("test_file", "data I want to store!") grid_out = fs.open_download_stream(file_id) contents = grid_out.read() Returns an instance of :class:`~gridfs.grid_file.GridOut`. Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: -`file_id`: The _id of the file to be downloaded. """ gout = GridOut(self._collection, file_id) # Raise NoFile now, instead of on first attribute access. gout._ensure_file() return gout def download_to_stream(self, file_id, destination): """Downloads the contents of the stored file specified by file_id and writes the contents to `destination`. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get _id of file to read file_id = fs.upload_from_stream("test_file", "data I want to store!") # Get file to write to file = open('myfile','rw') fs.download_to_stream(file_id, file) contents = file.read() Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: -`file_id`: The _id of the file to be downloaded. -`destination`: a file-like object implementing :meth:`write`. """ gout = self.open_download_stream(file_id) destination.write(gout) def delete(self, file_id): """Given an file_id, delete this stored file's files collection document and associated chunks from a GridFS bucket. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get _id of file to delete file_id = fs.upload_from_stream("test_file", "data I want to store!") fs.delete(file_id) Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: -`file_id`: The _id of the file to be deleted. """ res = self._files.delete_one({"_id": file_id}) self._chunks.delete_many({"files_id": file_id}) if not res.deleted_count: raise NoFile( "no file could be deleted because none matched %s" % file_id) def find(self, *args, **kwargs): """Find and return the files collection documents that match ``filter`` Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: for grid_data in fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True): data = grid_data.read() would iterate through all versions of "lisa.txt" stored in GridFS. Note that setting no_cursor_timeout to True may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) would return a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~pymongo.collection.Collection.find` in :class:`~pymongo.collection.Collection`. :Parameters: - `filter`: Search query. - `batch_size` (optional): The number of documents to return per batch. - `limit` (optional): The maximum number of documents to return. - `no_cursor_timeout` (optional): The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to True prevent that. - `skip` (optional): The number of documents to skip before returning. - `sort` (optional): The order by which to sort results. Defaults to None. """ return GridOutCursor(self._collection, *args, **kwargs) def open_download_stream_by_name(self, filename, revision=-1): """Opens a Stream from which the application can read the contents of `filename` and optional `revision`. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) grid_out = fs.open_download_stream_by_name("test_file") contents = grid_out.read() Returns an instance of :class:`~gridfs.grid_file.GridOut`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` filename is not a string. :Parameters: - `filename`: The name of the file to read from. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). :Note: Revision numbers are defined as follows: 0 = the original stored file 1 = the first revision 2 = the second revision etc... -2 = the second most recent revision -1 = the most recent revision """ validate_string("filename", filename) query = {"filename": filename} cursor = self._files.find(query) if revision < 0: skip = abs(revision) - 1 cursor.limit(-1).skip(skip).sort("uploadDate", DESCENDING) else: cursor.limit(-1).skip(revision).sort("uploadDate", ASCENDING) try: grid_file = next(cursor) return GridOut(self._collection, file_document=grid_file) except StopIteration: raise NoFile( "no version %d for filename %r" % (revision, filename)) def download_to_stream_by_name(self, filename, destination, revision=-1): """Write the contents of `filename` (with optional `revision`) to `destination`. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get file to write to file = open('myfile','w') fs.download_to_stream_by_name("test_file", file) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to read from. - `destination`: A file-like object that implements :meth:`write`. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). :Note: Revision numbers are defined as follows: 0 = the original stored file 1 = the first revision 2 = the second revision etc... -2 = the second most recent revision -1 = the most recent revision """ gout = self.open_download_stream_by_name(filename, revision) destination.write(gout) def rename(self, file_id, new_filename): """Renames the stored file with the specified file_id. For example:: my_db = MongoClient().test fs = GridFSBucket(my_db) # Get _id of file to rename file_id = fs.upload_from_stream("test_file", "data I want to store!") fs.rename(file_id, "new_test_name") Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be renamed. - `new_filename`: The new name of the file. """ result = self._files.update_one({"_id": file_id}, {"$set": {"filename": new_filename}}) if not result.matched_count: raise NoFile("no files could be renamed %r because none " "matched file_id %i" % (new_filename, file_id)) pymongo-3.2/gridfs/grid_file.py0000644000175000017500000006001412630427733020540 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing files stored in GridFS.""" import datetime import math import os from hashlib import md5 from bson.binary import Binary from bson.objectid import ObjectId from bson.py3compat import text_type, StringIO from gridfs.errors import CorruptGridFile, FileExists, NoFile from pymongo import ASCENDING from pymongo.collection import Collection from pymongo.cursor import Cursor from pymongo.errors import (ConfigurationError, DuplicateKeyError, OperationFailure) from pymongo.read_preferences import ReadPreference try: _SEEK_SET = os.SEEK_SET _SEEK_CUR = os.SEEK_CUR _SEEK_END = os.SEEK_END # before 2.5 except AttributeError: _SEEK_SET = 0 _SEEK_CUR = 1 _SEEK_END = 2 EMPTY = b"" NEWLN = b"\n" """Default chunk size, in bytes.""" # Slightly under a power of 2, to work well with server's record allocations. DEFAULT_CHUNK_SIZE = 255 * 1024 _C_INDEX = [("files_id", ASCENDING), ("n", ASCENDING)] _F_INDEX = [("filename", ASCENDING), ("uploadDate", ASCENDING)] def _grid_in_property(field_name, docstring, read_only=False, closed_only=False): """Create a GridIn property.""" def getter(self): if closed_only and not self._closed: raise AttributeError("can only get %r on a closed file" % field_name) # Protect against PHP-237 if field_name == 'length': return self._file.get(field_name, 0) return self._file.get(field_name, None) def setter(self, value): if self._closed: self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {field_name: value}}) self._file[field_name] = value if read_only: docstring += "\n\nThis attribute is read-only." elif closed_only: docstring = "%s\n\n%s" % (docstring, "This attribute is read-only and " "can only be read after :meth:`close` " "has been called.") if not read_only and not closed_only: return property(getter, setter, doc=docstring) return property(getter, doc=docstring) def _grid_out_property(field_name, docstring): """Create a GridOut property.""" def getter(self): self._ensure_file() # Protect against PHP-237 if field_name == 'length': return self._file.get(field_name, 0) return self._file.get(field_name, None) docstring += "\n\nThis attribute is read-only." return property(getter, doc=docstring) class GridIn(object): """Class to write data to GridFS. """ def __init__(self, root_collection, **kwargs): """Write a file to GridFS Application developers should generally not need to instantiate this class directly - instead see the methods provided by :class:`~gridfs.GridFS`. Raises :class:`TypeError` if `root_collection` is not an instance of :class:`~pymongo.collection.Collection`. Any of the file level options specified in the `GridFS Spec `_ may be passed as keyword arguments. Any additional keyword arguments will be set as additional fields on the file document. Valid keyword arguments include: - ``"_id"``: unique ID for this file (default: :class:`~bson.objectid.ObjectId`) - this ``"_id"`` must not have already been used for another file - ``"filename"``: human name for the file - ``"contentType"`` or ``"content_type"``: valid mime-type for the file - ``"chunkSize"`` or ``"chunk_size"``: size of each of the chunks, in bytes (default: 255 kb) - ``"encoding"``: encoding used for this file. In Python 2, any :class:`unicode` that is written to the file will be converted to a :class:`str`. In Python 3, any :class:`str` that is written to the file will be converted to :class:`bytes`. :Parameters: - `root_collection`: root collection to write to - `**kwargs` (optional): file level options (see above) .. versionchanged:: 3.0 `root_collection` must use an acknowledged :attr:`~pymongo.collection.Collection.write_concern` """ if not isinstance(root_collection, Collection): raise TypeError("root_collection must be an " "instance of Collection") # With w=0, 'filemd5' might run before the final chunks are written. if not root_collection.write_concern.acknowledged: raise ConfigurationError('root_collection must use ' 'acknowledged write_concern') # Handle alternative naming if "content_type" in kwargs: kwargs["contentType"] = kwargs.pop("content_type") if "chunk_size" in kwargs: kwargs["chunkSize"] = kwargs.pop("chunk_size") coll = root_collection.with_options( read_preference=ReadPreference.PRIMARY) kwargs['md5'] = md5() # Defaults kwargs["_id"] = kwargs.get("_id", ObjectId()) kwargs["chunkSize"] = kwargs.get("chunkSize", DEFAULT_CHUNK_SIZE) object.__setattr__(self, "_coll", coll) object.__setattr__(self, "_chunks", coll.chunks) object.__setattr__(self, "_file", kwargs) object.__setattr__(self, "_buffer", StringIO()) object.__setattr__(self, "_position", 0) object.__setattr__(self, "_chunk_number", 0) object.__setattr__(self, "_closed", False) object.__setattr__(self, "_ensured_index", False) def __create_index(self, collection, index, unique): doc = collection.find_one(projection={"_id": 1}) if doc is None: try: indexes = list(collection.list_indexes()) except OperationFailure: indexes = [] if index not in indexes: collection.create_index(index, unique=unique) def __ensure_indexes(self): if not object.__getattribute__(self, "_ensured_index"): self.__create_index(self._coll.files, _F_INDEX, False) self.__create_index(self._coll.chunks, _C_INDEX, True) object.__setattr__(self, "_ensured_index", True) def abort(self): """Remove all chunks/files that may have been uploaded and close. """ self._coll.chunks.delete_many({"files_id": self._file['_id']}) self._coll.files.delete_one({"_id": self._file['_id']}) object.__setattr__(self, "_closed", True) @property def closed(self): """Is this file closed? """ return self._closed _id = _grid_in_property("_id", "The ``'_id'`` value for this file.", read_only=True) filename = _grid_in_property("filename", "Name of this file.") name = _grid_in_property("filename", "Alias for `filename`.") content_type = _grid_in_property("contentType", "Mime-type for this file.") length = _grid_in_property("length", "Length (in bytes) of this file.", closed_only=True) chunk_size = _grid_in_property("chunkSize", "Chunk size for this file.", read_only=True) upload_date = _grid_in_property("uploadDate", "Date that this file was uploaded.", closed_only=True) md5 = _grid_in_property("md5", "MD5 of the contents of this file " "(generated on the server).", closed_only=True) def __getattr__(self, name): if name in self._file: return self._file[name] raise AttributeError("GridIn object has no attribute '%s'" % name) def __setattr__(self, name, value): # For properties of this instance like _buffer, or descriptors set on # the class like filename, use regular __setattr__ if name in self.__dict__ or name in self.__class__.__dict__: object.__setattr__(self, name, value) else: # All other attributes are part of the document in db.fs.files. # Store them to be sent to server on close() or if closed, send # them now. self._file[name] = value if self._closed: self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {name: value}}) def __flush_data(self, data): """Flush `data` to a chunk. """ # Ensure the index, even if there's nothing to write, so # the filemd5 command always succeeds. self.__ensure_indexes() self._file['md5'].update(data) if not data: return assert(len(data) <= self.chunk_size) chunk = {"files_id": self._file["_id"], "n": self._chunk_number, "data": Binary(data)} try: self._chunks.insert_one(chunk) except DuplicateKeyError: self._raise_file_exists(self._file['_id']) self._chunk_number += 1 self._position += len(data) def __flush_buffer(self): """Flush the buffer contents out to a chunk. """ self.__flush_data(self._buffer.getvalue()) self._buffer.close() self._buffer = StringIO() def __flush(self): """Flush the file to the database. """ try: self.__flush_buffer() self._file['md5'] = self._file["md5"].hexdigest() self._file["length"] = self._position self._file["uploadDate"] = datetime.datetime.utcnow() return self._coll.files.insert_one(self._file) except DuplicateKeyError: self._raise_file_exists(self._id) def _raise_file_exists(self, file_id): """Raise a FileExists exception for the given file_id.""" raise FileExists("file with _id %r already exists" % file_id) def close(self): """Flush the file and close it. A closed file cannot be written any more. Calling :meth:`close` more than once is allowed. """ if not self._closed: self.__flush() object.__setattr__(self, "_closed", True) def write(self, data): """Write data to the file. There is no return value. `data` can be either a string of bytes or a file-like object (implementing :meth:`read`). If the file has an :attr:`encoding` attribute, `data` can also be a :class:`unicode` (:class:`str` in python 3) instance, which will be encoded as :attr:`encoding` before being written. Due to buffering, the data may not actually be written to the database until the :meth:`close` method is called. Raises :class:`ValueError` if this file is already closed. Raises :class:`TypeError` if `data` is not an instance of :class:`str` (:class:`bytes` in python 3), a file-like object, or an instance of :class:`unicode` (:class:`str` in python 3). Unicode data is only allowed if the file has an :attr:`encoding` attribute. :Parameters: - `data`: string of bytes or file-like object to be written to the file """ if self._closed: raise ValueError("cannot write to a closed file") try: # file-like read = data.read except AttributeError: # string if not isinstance(data, (text_type, bytes)): raise TypeError("can only write strings or file-like objects") if isinstance(data, text_type): try: data = data.encode(self.encoding) except AttributeError: raise TypeError("must specify an encoding for file in " "order to write %s" % (text_type.__name__,)) read = StringIO(data).read if self._buffer.tell() > 0: # Make sure to flush only when _buffer is complete space = self.chunk_size - self._buffer.tell() if space: try: to_write = read(space) except: self.abort() raise self._buffer.write(to_write) if len(to_write) < space: return # EOF or incomplete self.__flush_buffer() to_write = read(self.chunk_size) while to_write and len(to_write) == self.chunk_size: self.__flush_data(to_write) to_write = read(self.chunk_size) self._buffer.write(to_write) def writelines(self, sequence): """Write a sequence of strings to the file. Does not add seperators. """ for line in sequence: self.write(line) def __enter__(self): """Support for the context manager protocol. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """Support for the context manager protocol. Close the file and allow exceptions to propagate. """ self.close() # propagate exceptions return False class GridOut(object): """Class to read data out of GridFS. """ def __init__(self, root_collection, file_id=None, file_document=None): """Read a file from GridFS Application developers should generally not need to instantiate this class directly - instead see the methods provided by :class:`~gridfs.GridFS`. Either `file_id` or `file_document` must be specified, `file_document` will be given priority if present. Raises :class:`TypeError` if `root_collection` is not an instance of :class:`~pymongo.collection.Collection`. :Parameters: - `root_collection`: root collection to read from - `file_id` (optional): value of ``"_id"`` for the file to read - `file_document` (optional): file document from `root_collection.files` .. versionchanged:: 3.0 Creating a GridOut does not immediately retrieve the file metadata from the server. Metadata is fetched when first needed. """ if not isinstance(root_collection, Collection): raise TypeError("root_collection must be an " "instance of Collection") self.__chunks = root_collection.chunks self.__files = root_collection.files self.__file_id = file_id self.__buffer = EMPTY self.__position = 0 self._file = file_document _id = _grid_out_property("_id", "The ``'_id'`` value for this file.") filename = _grid_out_property("filename", "Name of this file.") name = _grid_out_property("filename", "Alias for `filename`.") content_type = _grid_out_property("contentType", "Mime-type for this file.") length = _grid_out_property("length", "Length (in bytes) of this file.") chunk_size = _grid_out_property("chunkSize", "Chunk size for this file.") upload_date = _grid_out_property("uploadDate", "Date that this file was first uploaded.") aliases = _grid_out_property("aliases", "List of aliases for this file.") metadata = _grid_out_property("metadata", "Metadata attached to this file.") md5 = _grid_out_property("md5", "MD5 of the contents of this file " "(generated on the server).") def _ensure_file(self): if not self._file: self._file = self.__files.find_one({"_id": self.__file_id}) if not self._file: raise NoFile("no file in gridfs collection %r with _id %r" % (self.__files, self.__file_id)) def __getattr__(self, name): self._ensure_file() if name in self._file: return self._file[name] raise AttributeError("GridOut object has no attribute '%s'" % name) def readchunk(self): """Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. """ received = len(self.__buffer) chunk_data = EMPTY chunk_size = int(self.chunk_size) if received > 0: chunk_data = self.__buffer elif self.__position < int(self.length): chunk_number = int((received + self.__position) / chunk_size) chunk = self.__chunks.find_one({"files_id": self._id, "n": chunk_number}) if not chunk: raise CorruptGridFile("no chunk #%d" % chunk_number) chunk_data = chunk["data"][self.__position % chunk_size:] if not chunk_data: raise CorruptGridFile("truncated chunk") self.__position += len(chunk_data) self.__buffer = EMPTY return chunk_data def read(self, size=-1): """Read at most `size` bytes from the file (less if there isn't enough data). The bytes are returned as an instance of :class:`str` (:class:`bytes` in python 3). If `size` is negative or omitted all data is read. :Parameters: - `size` (optional): the number of bytes to read """ self._ensure_file() if size == 0: return EMPTY remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder received = 0 data = StringIO() while received < size: chunk_data = self.readchunk() received += len(chunk_data) data.write(chunk_data) # Detect extra chunks. max_chunk_n = math.ceil(self.length / float(self.chunk_size)) chunk = self.__chunks.find_one({"files_id": self._id, "n": {"$gte": max_chunk_n}}) # According to spec, ignore extra chunks if they are empty. if chunk is not None and len(chunk['data']): raise CorruptGridFile( "Extra chunk found: expected %i chunks but found " "chunk with n=%i" % (max_chunk_n, chunk['n'])) self.__position -= received - size # Return 'size' bytes and store the rest. data.seek(size) self.__buffer = data.read() data.seek(0) return data.read(size) def readline(self, size=-1): """Read one line or up to `size` bytes from the file. :Parameters: - `size` (optional): the maximum number of bytes to read """ if size == 0: return b'' remainder = int(self.length) - self.__position if size < 0 or size > remainder: size = remainder received = 0 data = StringIO() while received < size: chunk_data = self.readchunk() pos = chunk_data.find(NEWLN, 0, size) if pos != -1: size = received + pos + 1 received += len(chunk_data) data.write(chunk_data) if pos != -1: break self.__position -= received - size # Return 'size' bytes and store the rest. data.seek(size) self.__buffer = data.read() data.seek(0) return data.read(size) def tell(self): """Return the current position of this file. """ return self.__position def seek(self, pos, whence=_SEEK_SET): """Set the current position of this file. :Parameters: - `pos`: the position (or offset if using relative positioning) to seek to - `whence` (optional): where to seek from. :attr:`os.SEEK_SET` (``0``) for absolute file positioning, :attr:`os.SEEK_CUR` (``1``) to seek relative to the current position, :attr:`os.SEEK_END` (``2``) to seek relative to the file's end. """ if whence == _SEEK_SET: new_pos = pos elif whence == _SEEK_CUR: new_pos = self.__position + pos elif whence == _SEEK_END: new_pos = int(self.length) + pos else: raise IOError(22, "Invalid value for `whence`") if new_pos < 0: raise IOError(22, "Invalid value for `pos` - must be positive") self.__position = new_pos self.__buffer = EMPTY def __iter__(self): """Return an iterator over all of this file's data. The iterator will return chunk-sized instances of :class:`str` (:class:`bytes` in python 3). This can be useful when serving files using a webserver that handles such an iterator efficiently. """ return GridOutIterator(self, self.__chunks) def close(self): """Make GridOut more generically file-like.""" pass def __enter__(self): """Makes it possible to use :class:`GridOut` files with the context manager protocol. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """Makes it possible to use :class:`GridOut` files with the context manager protocol. """ return False class GridOutIterator(object): def __init__(self, grid_out, chunks): self.__id = grid_out._id self.__chunks = chunks self.__current_chunk = 0 self.__max_chunk = math.ceil(float(grid_out.length) / grid_out.chunk_size) def __iter__(self): return self def next(self): if self.__current_chunk >= self.__max_chunk: raise StopIteration chunk = self.__chunks.find_one({"files_id": self.__id, "n": self.__current_chunk}) if not chunk: raise CorruptGridFile("no chunk #%d" % self.__current_chunk) self.__current_chunk += 1 return bytes(chunk["data"]) __next__ = next class GridOutCursor(Cursor): """A cursor / iterator for returning GridOut objects as the result of an arbitrary query against the GridFS files collection. """ def __init__(self, collection, filter=None, skip=0, limit=0, no_cursor_timeout=False, sort=None, batch_size=0): """Create a new cursor, similar to the normal :class:`~pymongo.cursor.Cursor`. Should not be called directly by application developers - see the :class:`~gridfs.GridFS` method :meth:`~gridfs.GridFS.find` instead. .. versionadded 2.7 .. mongodoc:: cursors """ # Hold on to the base "fs" collection to create GridOut objects later. self.__root_collection = collection super(GridOutCursor, self).__init__( collection.files, filter, skip=skip, limit=limit, no_cursor_timeout=no_cursor_timeout, sort=sort, batch_size=batch_size) def next(self): """Get next GridOut object from cursor. """ # Work around "super is not iterable" issue in Python 3.x next_file = super(GridOutCursor, self).next() return GridOut(self.__root_collection, file_document=next_file) __next__ = next def add_option(self, *args, **kwargs): raise NotImplementedError("Method does not exist for GridOutCursor") def remove_option(self, *args, **kwargs): raise NotImplementedError("Method does not exist for GridOutCursor") def _clone_base(self): """Creates an empty GridOutCursor for information to be copied into. """ return GridOutCursor(self.__root_collection) pymongo-3.2/doc/0000755000175000017500000000000012631423130015514 5ustar behackettbehackett00000000000000pymongo-3.2/doc/conf.py0000644000175000017500000001211612630145074017023 0ustar behackettbehackett00000000000000# -*- coding: utf-8 -*- # # PyMongo documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. import sys, os sys.path[0:0] = [os.path.abspath('..')] import pymongo # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.todo', 'doc.mongo_extensions', 'sphinx.ext.intersphinx'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = u'PyMongo' copyright = u'2008 - 2015, MongoDB, Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pymongo.version # The full version, including alpha/beta/rc tags. release = pymongo.version # List of documents that shouldn't be included in the build. unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for extensions ---------------------------------------------------- autoclass_content = 'init' doctest_path = os.path.abspath('..') doctest_test_doctest_blocks = False doctest_global_setup = """ from pymongo.mongo_client import MongoClient client = MongoClient() client.drop_database("doctest_test") db = client.doctest_test """ # -- Options for HTML output --------------------------------------------------- # Theme gratefully vendored from CPython source. html_theme = "pydoctheme" html_theme_path = ["."] html_theme_options = {'collapsiblesidebar': True} # Additional static files. html_static_path = ['static'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'PyMongo' + release.replace('.', '_') # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'PyMongo.tex', u'PyMongo Documentation', u'Michael Dirolf', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_use_modindex = True intersphinx_mapping = { 'gevent': ('http://www.gevent.org/', None), } pymongo-3.2/doc/installation.rst0000644000175000017500000002072712630157526020773 0ustar behackettbehackett00000000000000Installing / Upgrading ====================== .. highlight:: bash **PyMongo** is in the `Python Package Index `_. .. warning:: **Do not install the "bson" package.** PyMongo comes with its own bson package; doing "pip install bson" or "easy_install bson" installs a third-party package that is incompatible with PyMongo. Optional packages ----------------- PyMongo has no required dependencies, but it can take advantage of these packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python older than 2.7.8, or on Python 3 before Python 3.4. - `pykerberos `_ is required for the GSSAPI authentication mechanism. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3.3+. - `wincertstore `_ adds support for verifying server SSL certificates using Windows provided CA certificates on older versions of python. Not needed or used with versions of Python 2 beginning with 2.7.9, or versions of Python 3 beginning with 3.4.0. - `certifi `_ adds support for using the Mozilla CA bundle with SSL to verify server certificates. Not needed or used with versions of Python 2 beginning with 2.7.9 on any OS, versions of Python 3 beginning with Python 3.4.0 on Windows, or versions of Python 3 beginning with Python 3.2.0 on operating systems other than Windows. Installing with pip ------------------- We recommend using `pip `_ to install pymongo on all platforms:: $ python -m pip install pymongo To get a specific version of pymongo:: $ python -m pip install pymongo==3.1.1 To upgrade using pip:: $ python -m pip install --upgrade pymongo .. note:: pip does not support installing python packages in .egg format. If you would like to install PyMongo from a .egg provided on pypi use easy_install instead. Installing with easy_install ---------------------------- To use ``easy_install`` from `setuptools `_ do:: $ python -m easy_install pymongo To upgrade do:: $ python -m easy_install -U pymongo Dependencies for installing C Extensions on Unix ------------------------------------------------ MongoDB, Inc. does not provide statically linked binary packages for Unix flavors other than OSX. To build the optional C extensions you must have the GNU C compiler (gcc) installed. Depending on your flavor of Unix (or Linux distribution) you may also need a python development package that provides the necessary header files for your version of Python. The package name may vary from distro to distro. Debian and Ubuntu users should issue the following command:: $ sudo apt-get install build-essential python-dev Users of Red Hat based distributions (RHEL, CentOS, Amazon Linux, Oracle Linux, Fedora, etc.) should issue the following command:: $ sudo yum install gcc python-devel Installing from source ---------------------- If you'd rather install directly from the source (i.e. to stay on the bleeding edge), install the C extension dependencies then check out the latest source from github and install the driver from the resulting tree:: $ git clone git://github.com/mongodb/mongo-python-driver.git pymongo $ cd pymongo/ $ python setup.py install Installing from source on OSX ............................. If you want to install PyMongo from source on OSX you will have to install the following to build the C extensions: **Snow Leopard (10.6)** - Xcode 3 with 'UNIX Development Support'. **Snow Leopard Xcode 4**: The Python versions shipped with OSX 10.6.x are universal binaries. They support i386, PPC, and (in the case of python2.6) x86_64. Xcode 4 removed support for PPC, causing the distutils version shipped with Apple's builds of Python to fail to build the C extensions if you have Xcode 4 installed. There is a workaround:: # For Apple-supplied Python2.6 (installed at /usr/bin/python2.6) and # some builds from python.org $ env ARCHFLAGS='-arch i386 -arch x86_64' python -m easy_install pymongo See `http://bugs.python.org/issue11623 `_ for a more detailed explanation. **Lion (10.7) and newer** - PyMongo's C extensions can be built against versions of Python 2.7 >= 2.7.4 or Python 3.x >= 3.2.4 downloaded from python.org. In all cases Xcode must be installed with 'UNIX Development Support'. **Xcode 5.1**: Starting with version 5.1 the version of clang that ships with Xcode throws an error when it encounters compiler flags it doesn't recognize. This may cause C extension builds to fail with an error similar to:: clang: error: unknown argument: '-mno-fused-madd' [-Wunused-command-line-argument-hard-error-in-future] There are workarounds:: # Apple specified workaround for Xcode 5.1 # easy_install $ ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future easy_install pymongo # or pip $ ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future pip install pymongo # Alternative workaround using CFLAGS # easy_install $ CFLAGS=-Qunused-arguments easy_install pymongo # or pip $ CFLAGS=-Qunused-arguments pip install pymongo Installing from source on Windows ................................. If you want to install PyMongo with C extensions from source the following requirements apply to both CPython and ActiveState's ActivePython: 64-bit Windows ~~~~~~~~~~~~~~ For Python 3.5 and newer install Visual Studio 2015. For Python 3.3 and 3.4 install Visual Studio 2010. For Python 3.2 and older install Visual Studio 2008, or the Microsoft Visual C++ Compiler for Python 2.7. You must use the full version of Visual Studio 2010 or 2008 as Visual C++ Express does not provide 64-bit compilers. Make sure that you check the "x64 Compilers and Tools" option under Visual C++. 32-bit Windows ~~~~~~~~~~~~~~ For Python 3.5 and newer install Visual Studio 2015. For Python 3.3 and 3.4 install Visual C++ 2010 Express. For Python 2.6 through 3.2 install Visual C++ 2008 Express SP1. .. _install-no-c: Installing Without C Extensions ------------------------------- By default, the driver attempts to build and install optional C extensions (used for increasing performance) when it is installed. If any extension fails to build the driver will be installed anyway but a warning will be printed. If you wish to install PyMongo without the C extensions, even if the extensions build properly, it can be done using a command line option to *setup.py*:: $ python setup.py --no_ext install Building PyMongo egg Packages ----------------------------- Some organizations do not allow compilers and other build tools on production systems. To install PyMongo on these systems with C extensions you may need to build custom egg packages. Make sure that you have installed the dependencies listed above for your operating system then run the following command in the PyMongo source directory:: $ python setup.py bdist_egg The egg package can be found in the dist/ subdirectory. The file name will resemble “pymongo-3.1-py2.7-linux-x86_64.egg” but may have a different name depending on your platform and the version of python you use to compile. .. warning:: These “binary distributions,” will only work on systems that resemble the environment on which you built the package. In other words, ensure that operating systems and versions of Python and architecture (i.e. “32” or “64” bit) match. Copy this file to the target system and issue the following command to install the package:: $ sudo python -m easy_install pymongo-3.1-py2.7-linux-x86_64.egg Installing a beta or release candidate -------------------------------------- MongoDB, Inc. may occasionally tag a beta or release candidate for testing by the community before final release. These releases will not be uploaded to pypi but can be found on the `github tags page `_. They can be installed by passing the full URL for the tag to pip:: $ python -m pip install https://github.com/mongodb/mongo-python-driver/archive/3.2rc0.tar.gz or easy_install:: $ python -m easy_install https://github.com/mongodb/mongo-python-driver/archive/3.2rc0.tar.gz pymongo-3.2/doc/changelog.rst0000644000175000017500000023425512630145074020217 0ustar behackettbehackett00000000000000Changelog ========= Changes in Version 3.2 ---------------------- Version 3.2 implements the new server features introduced in MongoDB 3.2. Highlights include: - Full support for MongoDB 3.2 including: - Support for :class:`~pymongo.read_concern.ReadConcern` - :class:`~pymongo.write_concern.WriteConcern` is now applied to :meth:`~pymongo.collection.Collection.find_one_and_replace`, :meth:`~pymongo.collection.Collection.find_one_and_update`, and :meth:`~pymongo.collection.Collection.find_one_and_delete`. - Support for the new `bypassDocumentValidation` option in write helpers. - Support for reading and writing raw BSON with :class:`~bson.raw_bson.RawBSONDocument` .. note:: Certain :class:`~pymongo.mongo_client.MongoClient` properties now block until a connection is established or raise :exc:`~pymongo.errors.ServerSelectionTimeoutError` if no server is available. See :class:`~pymongo.mongo_client.MongoClient` for details. Changes in Version 3.1.1 ------------------------ Version 3.1.1 fixes a few issues reported since the release of 3.1, including a regression in error handling for oversize command documents and interrupt handling issues in the C extensions. Issues Resolved ............... See the `PyMongo 3.1.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.1.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/16211 Changes in Version 3.1 ---------------------- Version 3.1 implements a few new features and fixes bugs reported since the release of 3.0.3. Highlights include: - Command monitoring support. See :mod:`~pymongo.monitoring` for details. - Configurable error handling for :exc:`UnicodeDecodeError`. See the `unicode_decode_error_handler` option of :class:`~bson.codec_options.CodecOptions`. - Optional automatic timezone conversion when decoding BSON datetime. See the `tzinfo` option of :class:`~bson.codec_options.CodecOptions`. - An implementation of :class:`~gridfs.GridFSBucket` from the new GridFS spec. - Compliance with the new Connection String spec. - Reduced idle CPU usage in Python 2. Changes in internal classes ........................... The private ``PeriodicExecutor`` class no longer takes a ``condition_class`` option, and the private ``thread_util.Event`` class is removed. Issues Resolved ............... See the `PyMongo 3.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14796 Changes in Version 3.0.3 ------------------------ Version 3.0.3 fixes issues reported since the release of 3.0.2, including a feature breaking bug in the GSSAPI implementation. Issues Resolved ............... See the `PyMongo 3.0.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0.3 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/15528 Changes in Version 3.0.2 ------------------------ Version 3.0.2 fixes issues reported since the release of 3.0.1, most importantly a bug that could route operations to replica set members that are not in primary or secondary state when using :class:`~pymongo.read_preferences.PrimaryPreferred` or :class:`~pymongo.read_preferences.Nearest`. It is a recommended upgrade for all users of PyMongo 3.0.x. Issues Resolved ............... See the `PyMongo 3.0.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/15430 Changes in Version 3.0.1 ------------------------ Version 3.0.1 fixes issues reported since the release of 3.0, most importantly a bug in GridFS.delete that could prevent file chunks from actually being deleted. Issues Resolved ............... See the `PyMongo 3.0.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/15322 Changes in Version 3.0 ---------------------- PyMongo 3.0 is a partial rewrite of PyMongo bringing a large number of improvements: - A unified client class. MongoClient is the one and only client class for connecting to a standalone mongod, replica set, or sharded cluster. Migrating from a standalone, to a replica set, to a sharded cluster can be accomplished with only a simple URI change. - MongoClient is much more responsive to configuration changes in your MongoDB deployment. All connected servers are monitored in a non-blocking manner. Slow to respond or down servers no longer block server discovery, reducing application startup time and time to respond to new or reconfigured servers and replica set failovers. - A unified CRUD API. All official MongoDB drivers now implement a standard CRUD API allowing polyglot developers to move from language to language with ease. - Single source support for Python 2.x and 3.x. PyMongo no longer relies on 2to3 to support Python 3. - A rewritten pure Python BSON implementation, improving performance with pypy and cpython deployments without support for C extensions. - Better support for greenlet based async frameworks including eventlet. - Immutable client, database, and collection classes, avoiding a host of thread safety issues in client applications. PyMongo 3.0 brings a large number of API changes. Be sure to read the changes listed below before upgrading from PyMongo 2.x. .. warning:: PyMongo no longer supports Python 2.4, 2.5, or 3.1. If you must use PyMongo with these versions of Python the 2.x branch of PyMongo will be minimally supported for some time. SONManipulator changes ...................... The :class:`~pymongo.son_manipulator.SONManipulator` API has limitations as a technique for transforming your data. Instead, it is more flexible and straightforward to transform outgoing documents in your own code before passing them to PyMongo, and transform incoming documents after receiving them from PyMongo. Thus the :meth:`~pymongo.database.Database.add_son_manipulator` method is deprecated. PyMongo 3's new CRUD API does **not** apply SON manipulators to documents passed to :meth:`~pymongo.collection.Collection.bulk_write`, :meth:`~pymongo.collection.Collection.insert_one`, :meth:`~pymongo.collection.Collection.insert_many`, :meth:`~pymongo.collection.Collection.update_one`, or :meth:`~pymongo.collection.Collection.update_many`. SON manipulators are **not** applied to documents returned by the new methods :meth:`~pymongo.collection.Collection.find_one_and_delete`, :meth:`~pymongo.collection.Collection.find_one_and_replace`, and :meth:`~pymongo.collection.Collection.find_one_and_update`. SSL/TLS changes ............... When `ssl` is ``True`` the `ssl_cert_reqs` option now defaults to :attr:`ssl.CERT_REQUIRED` if not provided. PyMongo will attempt to load OS provided CA certificates to verify the server, raising :exc:`~pymongo.errors.ConfigurationError` if it cannot. Gevent Support .............. In previous versions, PyMongo supported Gevent in two modes: you could call ``gevent.monkey.patch_socket()`` and pass ``use_greenlets=True`` to :class:`~pymongo.mongo_client.MongoClient`, or you could simply call ``gevent.monkey.patch_all()`` and omit the ``use_greenlets`` argument. In PyMongo 3.0, the ``use_greenlets`` option is gone. To use PyMongo with Gevent simply call ``gevent.monkey.patch_all()``. For more information, see :doc:`PyMongo's Gevent documentation `. :class:`~pymongo.mongo_client.MongoClient` changes .................................................. :class:`~pymongo.mongo_client.MongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into ``MongoReplicaSetClient``: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. :class:`~pymongo.mongo_client.MongoClient` now also supports the full :class:`~pymongo.read_preferences.ReadPreference` API. The obsolete classes ``MasterSlaveConnection``, ``Connection``, and ``ReplicaSetConnection`` are removed. The :class:`~pymongo.mongo_client.MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :class:`~pymongo.errors.ConfigurationError` if the user's credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. The ``connect`` option is added to control whether these threads are started immediately, or when the client is first used. Therefore the ``alive`` method is removed since it no longer provides meaningful information; even if the client is disconnected, it may discover a server in time to fulfill the next operation. In PyMongo 2.x, :class:`~pymongo.mongo_client.MongoClient` accepted a list of standalone MongoDB servers and used the first it could connect to:: MongoClient(['host1.com:27017', 'host2.com:27017']) A list of multiple standalones is no longer supported; if multiple servers are listed they must be members of the same replica set, or mongoses in the same sharded cluster. The behavior for a list of mongoses is changed from "high availability" to "load balancing". Before, the client connected to the lowest-latency mongos in the list, and used it until a network error prompted it to re-evaluate all mongoses' latencies and reconnect to one of them. In PyMongo 3, the client monitors its network latency to all the mongoses continuously, and distributes operations evenly among those with the lowest latency. See :ref:`mongos-load-balancing` for more information. The client methods ``start_request``, ``in_request``, and ``end_request`` are removed, and so is the ``auto_start_request`` option. Requests were designed to make read-your-writes consistency more likely with the ``w=0`` write concern. Additionally, a thread in a request used the same member for all secondary reads in a replica set. To ensure read-your-writes consistency in PyMongo 3.0, do not override the default write concern with ``w=0``, and do not override the default :ref:`read preference ` of PRIMARY. Support for the ``slaveOk`` (or ``slave_okay``), ``safe``, and ``network_timeout`` options has been removed. Use :attr:`~pymongo.read_preferences.ReadPreference.SECONDARY_PREFERRED` instead of slave_okay. Accept the default write concern, acknowledged writes, instead of setting safe=True. Use socketTimeoutMS in place of network_timeout (note that network_timeout was in seconds, where as socketTimeoutMS is milliseconds). The ``max_pool_size`` option has been removed. It is replaced by the ``maxPoolSize`` MongoDB URI option. ``maxPoolSize`` is now a supported URI option in PyMongo and can be passed as a keyword argument. The ``copy_database`` method is removed, see the :doc:`copy_database examples ` for alternatives. The ``disconnect`` method is removed. Use :meth:`~pymongo.mongo_client.MongoClient.close` instead. The ``get_document_class`` method is removed. Use :attr:`~pymongo.mongo_client.MongoClient.codec_options` instead. The ``get_lasterror_options``, ``set_lasterror_options``, and ``unset_lasterror_options`` methods are removed. Write concern options can be passed to :class:`~pymongo.mongo_client.MongoClient` as keyword arguments or MongoDB URI options. The :meth:`~pymongo.mongo_client.MongoClient.get_database` method is added for getting a Database instance with its options configured differently than the MongoClient's. The following read-only attributes have been added: - :attr:`~pymongo.mongo_client.MongoClient.codec_options` The following attributes are now read-only: - :attr:`~pymongo.mongo_client.MongoClient.read_preference` - :attr:`~pymongo.mongo_client.MongoClient.write_concern` The following attributes have been removed: - :attr:`~pymongo.mongo_client.MongoClient.document_class` (use :attr:`~pymongo.mongo_client.MongoClient.codec_options` instead) - :attr:`~pymongo.mongo_client.MongoClient.host` (use :attr:`~pymongo.mongo_client.MongoClient.address` instead) - :attr:`~pymongo.mongo_client.MongoClient.min_wire_version` - :attr:`~pymongo.mongo_client.MongoClient.max_wire_version` - :attr:`~pymongo.mongo_client.MongoClient.port` (use :attr:`~pymongo.mongo_client.MongoClient.address` instead) - :attr:`~pymongo.mongo_client.MongoClient.safe` (use :attr:`~pymongo.mongo_client.MongoClient.write_concern` instead) - :attr:`~pymongo.mongo_client.MongoClient.slave_okay` (use :attr:`~pymongo.mongo_client.MongoClient.read_preference` instead) - :attr:`~pymongo.mongo_client.MongoClient.tag_sets` (use :attr:`~pymongo.mongo_client.MongoClient.read_preference` instead) - :attr:`~pymongo.mongo_client.MongoClient.tz_aware` (use :attr:`~pymongo.mongo_client.MongoClient.codec_options` instead) The following attributes have been renamed: - :attr:`~pymongo.mongo_client.MongoClient.secondary_acceptable_latency_ms` is now :attr:`~pymongo.mongo_client.MongoClient.local_threshold_ms` and is now read-only. :class:`~pymongo.cursor.Cursor` changes ....................................... The ``conn_id`` property is renamed to :attr:`~pymongo.cursor.Cursor.address`. Cursor management changes ......................... :class:`~pymongo.cursor_manager.CursorManager` and :meth:`~pymongo.mongo_client.MongoClient.set_cursor_manager` are no longer deprecated. If you subclass :class:`~pymongo.cursor_manager.CursorManager` your implementation of :meth:`~pymongo.cursor_manager.CursorManager.close` must now take a second parameter, `address`. The ``BatchCursorManager`` class is removed. The second parameter to :meth:`~pymongo.mongo_client.MongoClient.close_cursor` is renamed from ``_conn_id`` to ``address``. :meth:`~pymongo.mongo_client.MongoClient.kill_cursors` now accepts an `address` parameter. :class:`~pymongo.database.Database` changes ........................................... The ``connection`` property is renamed to :attr:`~pymongo.database.Database.client`. The following read-only attributes have been added: - :attr:`~pymongo.database.Database.codec_options` The following attributes are now read-only: - :attr:`~pymongo.database.Database.read_preference` - :attr:`~pymongo.database.Database.write_concern` Use :meth:`~pymongo.mongo_client.MongoClient.get_database` for getting a Database instance with its options configured differently than the MongoClient's. The following attributes have been removed: - :attr:`~pymongo.database.Database.safe` - :attr:`~pymongo.database.Database.secondary_acceptable_latency_ms` - :attr:`~pymongo.database.Database.slave_okay` - :attr:`~pymongo.database.Database.tag_sets` The following methods have been added: - :meth:`~pymongo.database.Database.get_collection` The following methods have been changed: - :meth:`~pymongo.database.Database.command`. Support for `as_class`, `uuid_subtype`, `tag_sets`, and `secondary_acceptable_latency_ms` have been removed. You can instead pass an instance of :class:`~bson.codec_options.CodecOptions` as `codec_options` and an instance of a read preference class from :mod:`~pymongo.read_preferences` as `read_preference`. The `fields` and `compile_re` options are also removed. The `fields` options was undocumented and never really worked. Regular expressions are always decoded to :class:`~bson.regex.Regex`. The following methods have been deprecated: - :meth:`~pymongo.database.Database.add_son_manipulator` The following methods have been removed: The ``get_lasterror_options``, ``set_lasterror_options``, and ``unset_lasterror_options`` methods have been removed. Use :class:`~pymongo.write_concern.WriteConcern` with :meth:`~pymongo.mongo_client.MongoClient.get_database` instead. :class:`~pymongo.collection.Collection` changes ............................................... The following read-only attributes have been added: - :attr:`~pymongo.collection.Collection.codec_options` The following attributes are now read-only: - :attr:`~pymongo.collection.Collection.read_preference` - :attr:`~pymongo.collection.Collection.write_concern` Use :meth:`~pymongo.database.Database.get_collection` or :meth:`~pymongo.collection.Collection.with_options` for getting a Collection instance with its options configured differently than the Database's. The following attributes have been removed: - :attr:`~pymongo.collection.Collection.safe` - :attr:`~pymongo.collection.Collection.secondary_acceptable_latency_ms` - :attr:`~pymongo.collection.Collection.slave_okay` - :attr:`~pymongo.collection.Collection.tag_sets` The following methods have been added: - :meth:`~pymongo.collection.Collection.bulk_write` - :meth:`~pymongo.collection.Collection.insert_one` - :meth:`~pymongo.collection.Collection.insert_many` - :meth:`~pymongo.collection.Collection.update_one` - :meth:`~pymongo.collection.Collection.update_many` - :meth:`~pymongo.collection.Collection.replace_one` - :meth:`~pymongo.collection.Collection.delete_one` - :meth:`~pymongo.collection.Collection.delete_many` - :meth:`~pymongo.collection.Collection.find_one_and_delete` - :meth:`~pymongo.collection.Collection.find_one_and_replace` - :meth:`~pymongo.collection.Collection.find_one_and_update` - :meth:`~pymongo.collection.Collection.with_options` - :meth:`~pymongo.collection.Collection.create_indexes` - :meth:`~pymongo.collection.Collection.list_indexes` The following methods have changed: - :meth:`~pymongo.collection.Collection.aggregate` now **always** returns an instance of :class:`~pymongo.command_cursor.CommandCursor`. See the documentation for all options. - :meth:`~pymongo.collection.Collection.count` now optionally takes a filter argument, as well as other options supported by the count command. - :meth:`~pymongo.collection.Collection.distinct` now optionally takes a filter argument. - :meth:`~pymongo.collection.Collection.create_index` no longer caches indexes, therefore the `cache_for` parameter has been removed. It also no longer supports the `bucket_size` and `drop_dups` aliases for `bucketSize` and `dropDups`. The following methods are deprecated: - :meth:`~pymongo.collection.Collection.save` - :meth:`~pymongo.collection.Collection.insert` - :meth:`~pymongo.collection.Collection.update` - :meth:`~pymongo.collection.Collection.remove` - :meth:`~pymongo.collection.Collection.find_and_modify` - :meth:`~pymongo.collection.Collection.ensure_index` The following methods have been removed: The ``get_lasterror_options``, ``set_lasterror_options``, and ``unset_lasterror_options`` methods have been removed. Use :class:`~pymongo.write_concern.WriteConcern` with :meth:`~pymongo.collection.Collection.with_options` instead. Changes to :meth:`~pymongo.collection.Collection.find` and :meth:`~pymongo.collection.Collection.find_one` `````````````````````````````````````````````````````````````````````````````````````````````````````````` The following find/find_one options have been renamed: These renames only affect your code if you passed these as keyword arguments, like find(fields=['fieldname']). If you passed only positional parameters these changes are not significant for your application. - spec -> filter - fields -> projection - partial -> allow_partial_results The following find/find_one options have been added: - cursor_type (see :class:`~pymongo.cursor.CursorType` for values) - oplog_replay - modifiers The following find/find_one options have been removed: - network_timeout (use :meth:`~pymongo.cursor.Cursor.max_time_ms` instead) - slave_okay (use one of the read preference classes from :mod:`~pymongo.read_preferences` and :meth:`~pymongo.collection.Collection.with_options` instead) - read_preference (use :meth:`~pymongo.collection.Collection.with_options` instead) - tag_sets (use one of the read preference classes from :mod:`~pymongo.read_preferences` and :meth:`~pymongo.collection.Collection.with_options` instead) - secondary_acceptable_latency_ms (use the `localThresholdMS` URI option instead) - max_scan (use the new `modifiers` option instead) - snapshot (use the new `modifiers` option instead) - tailable (use the new `cursor_type` option instead) - await_data (use the new `cursor_type` option instead) - exhaust (use the new `cursor_type` option instead) - as_class (use :meth:`~pymongo.collection.Collection.with_options` with :class:`~bson.codec_options.CodecOptions` instead) - compile_re (BSON regular expressions are always decoded to :class:`~bson.regex.Regex`) The following find/find_one options are deprecated: - manipulate The following renames need special handling. - timeout -> no_cursor_timeout - The default for `timeout` was True. The default for `no_cursor_timeout` is False. If you were previously passing False for `timeout` you must pass **True** for `no_cursor_timeout` to keep the previous behavior. :mod:`~pymongo.errors` changes .............................. The exception classes ``UnsupportedOption`` and ``TimeoutError`` are deleted. :mod:`~gridfs` changes ...................... Since PyMongo 1.6, methods ``open`` and ``close`` of :class:`~gridfs.GridFS` raised an ``UnsupportedAPI`` exception, as did the entire ``GridFile`` class. The unsupported methods, the class, and the exception are all deleted. :mod:`~bson` changes .................... The `compile_re` option is removed from all methods that accepted it in :mod:`~bson` and :mod:`~bson.json_util`. Additionally, it is removed from :meth:`~pymongo.collection.Collection.find`, :meth:`~pymongo.collection.Collection.find_one`, :meth:`~pymongo.collection.Collection.aggregate`, :meth:`~pymongo.database.Database.command`, and so on. PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. This prevents errors for incompatible patterns, see `PYTHON-500`_. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. PyMongo now decodes the int64 BSON type to :class:`~bson.int64.Int64`, a trivial wrapper around long (in python 2.x) or int (in python 3.x). This allows BSON int64 to be round tripped without losing type information in python 3. Note that if you store a python long (or a python int larger than 4 bytes) it will be returned from PyMongo as :class:`~bson.int64.Int64`. The `as_class`, `tz_aware`, and `uuid_subtype` options are removed from all BSON encoding and decoding methods. Use :class:`~bson.codec_options.CodecOptions` to configure these options. The APIs affected are: - :func:`~bson.decode_all` - :func:`~bson.decode_iter` - :func:`~bson.decode_file_iter` - :meth:`~bson.BSON.encode` - :meth:`~bson.BSON.decode` This is a breaking change for any application that uses the BSON API directly and changes any of the named parameter defaults. No changes are required for applications that use the default values for these options. The behavior remains the same. .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 Issues Resolved ............... See the `PyMongo 3.0 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 3.0 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12501 Changes in Version 2.9 ---------------------- Version 2.9 provides an upgrade path to PyMongo 3.x. Most of the API changes from PyMongo 3.0 have been backported in a backward compatible way, allowing applications to be written against PyMongo >= 2.9, rather then PyMongo 2.x or PyMongo 3.x. See the :doc:`/migrate-to-pymongo3` for detailed examples. .. note:: There are a number of new deprecations in this release for features that were removed in PyMongo 3.0. :class:`~pymongo.mongo_client.MongoClient`: - :attr:`~pymongo.mongo_client.MongoClient.host` - :attr:`~pymongo.mongo_client.MongoClient.port` - :attr:`~pymongo.mongo_client.MongoClient.use_greenlets` - :attr:`~pymongo.mongo_client.MongoClient.document_class` - :attr:`~pymongo.mongo_client.MongoClient.tz_aware` - :attr:`~pymongo.mongo_client.MongoClient.secondary_acceptable_latency_ms` - :attr:`~pymongo.mongo_client.MongoClient.tag_sets` - :attr:`~pymongo.mongo_client.MongoClient.uuid_subtype` - :meth:`~pymongo.mongo_client.MongoClient.disconnect` - :meth:`~pymongo.mongo_client.MongoClient.alive` :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`: - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.use_greenlets` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.document_class` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.tz_aware` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.secondary_acceptable_latency_ms` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.tag_sets` - :attr:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.uuid_subtype` - :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.alive` :class:`~pymongo.database.Database`: - :attr:`~pymongo.database.Database.secondary_acceptable_latency_ms` - :attr:`~pymongo.database.Database.tag_sets` - :attr:`~pymongo.database.Database.uuid_subtype` :class:`~pymongo.collection.Collection`: - :attr:`~pymongo.collection.Collection.secondary_acceptable_latency_ms` - :attr:`~pymongo.collection.Collection.tag_sets` - :attr:`~pymongo.collection.Collection.uuid_subtype` .. warning:: In previous versions of PyMongo, changing the value of :attr:`~pymongo.mongo_client.MongoClient.document_class` changed the behavior of all existing instances of :class:`~pymongo.collection.Collection`:: >>> coll = client.test.test >>> coll.find_one() {u'_id': ObjectId('5579dc7cfba5220cc14d9a18')} >>> from bson.son import SON >>> client.document_class = SON >>> coll.find_one() SON([(u'_id', ObjectId('5579dc7cfba5220cc14d9a18'))]) The document_class setting is now configurable at the client, database, collection, and per-operation level. This required breaking the existing behavior. To change the document class per operation in a forward compatible way use :meth:`~pymongo.collection.Collection.with_options`:: >>> coll.find_one() {u'_id': ObjectId('5579dc7cfba5220cc14d9a18')} >>> from bson.codec_options import CodecOptions >>> coll.with_options(CodecOptions(SON)).find_one() SON([(u'_id', ObjectId('5579dc7cfba5220cc14d9a18'))]) Issues Resolved ............... See the `PyMongo 2.9 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.9 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14795 Changes in Version 2.8.1 ------------------------ Version 2.8.1 fixes a number of issues reported since the release of PyMongo 2.8. It is a recommended upgrade for all users of PyMongo 2.x. Issues Resolved ............... See the `PyMongo 2.8.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.8.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/15324 Changes in Version 2.8 ---------------------- Version 2.8 is a major release that provides full support for MongoDB 3.0 and fixes a number of bugs. Special thanks to Don Mitchell, Ximing, Can Zhang, Sergey Azovskov, and Heewa Barfchin for their contributions to this release. Highlights include: - Support for the SCRAM-SHA-1 authentication mechanism (new in MongoDB 3.0). - JSON decoder support for the new $numberLong and $undefined types. - JSON decoder support for the $date type as an ISO-8601 string. - Support passing an index name to :meth:`~pymongo.cursor.Cursor.hint`. - The :meth:`~pymongo.cursor.Cursor.count` method will use a hint if one has been provided through :meth:`~pymongo.cursor.Cursor.hint`. - A new socketKeepAlive option for the connection pool. - New generator based BSON decode functions, :func:`~bson.decode_iter` and :func:`~bson.decode_file_iter`. - Internal changes to support alternative storage engines like wiredtiger. .. note:: There are a number of deprecations in this release for features that will be removed in PyMongo 3.0. These include: - :meth:`~pymongo.mongo_client.MongoClient.start_request` - :meth:`~pymongo.mongo_client.MongoClient.in_request` - :meth:`~pymongo.mongo_client.MongoClient.end_request` - :meth:`~pymongo.mongo_client.MongoClient.copy_database` - :meth:`~pymongo.database.Database.error` - :meth:`~pymongo.database.Database.last_status` - :meth:`~pymongo.database.Database.previous_error` - :meth:`~pymongo.database.Database.reset_error_history` - :class:`~pymongo.master_slave_connection.MasterSlaveConnection` The JSON format for :class:`~bson.timestamp.Timestamp` has changed from '{"t": , "i": }' to '{"$timestamp": {"t": , "i": }}'. This new format will be decoded to an instance of :class:`~bson.timestamp.Timestamp`. The old format will continue to be decoded to a python dict as before. Encoding to the old format is no longer supported as it was never correct and loses type information. Issues Resolved ............... See the `PyMongo 2.8 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.8 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14223 Changes in Version 2.7.2 ------------------------ Version 2.7.2 includes fixes for upsert reporting in the bulk API for MongoDB versions previous to 2.6, a regression in how son manipulators are applied in :meth:`~pymongo.collection.Collection.insert`, a few obscure connection pool semaphore leaks, and a few other minor issues. See the list of issues resolved for full details. Issues Resolved ............... See the `PyMongo 2.7.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.7.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14005 Changes in Version 2.7.1 ------------------------ Version 2.7.1 fixes a number of issues reported since the release of 2.7, most importantly a fix for creating indexes and manipulating users through mongos versions older than 2.4.0. Issues Resolved ............... See the `PyMongo 2.7.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.7.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/13823 Changes in Version 2.7 ---------------------- PyMongo 2.7 is a major release with a large number of new features and bug fixes. Highlights include: - Full support for MongoDB 2.6. - A new :doc:`bulk write operations API `. - Support for server side query timeouts using :meth:`~pymongo.cursor.Cursor.max_time_ms`. - Support for writing :meth:`~pymongo.collection.Collection.aggregate` output to a collection. - A new :meth:`~pymongo.collection.Collection.parallel_scan` helper. - :class:`~pymongo.errors.OperationFailure` and its subclasses now include a :attr:`~pymongo.errors.OperationFailure.details` attribute with complete error details from the server. - A new GridFS :meth:`~gridfs.GridFS.find` method that returns a :class:`~gridfs.grid_file.GridOutCursor`. - Greatly improved :doc:`support for mod_wsgi ` when using PyMongo's C extensions. Read `Jesse's blog post `_ for details. - Improved C extension support for ARM little endian. Breaking changes ................ Version 2.7 drops support for replica sets running MongoDB versions older than 1.6.2. Issues Resolved ............... See the `PyMongo 2.7 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.7 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12892 Changes in Version 2.6.3 ------------------------ Version 2.6.3 fixes issues reported since the release of 2.6.2, most importantly a semaphore leak when a connection to the server fails. Issues Resolved ............... See the `PyMongo 2.6.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6.3 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/13098 Changes in Version 2.6.2 ------------------------ Version 2.6.2 fixes a :exc:`TypeError` problem when max_pool_size=None is used in Python 3. Issues Resolved ............... See the `PyMongo 2.6.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12910 Changes in Version 2.6.1 ------------------------ Version 2.6.1 fixes a reference leak in the :meth:`~pymongo.collection.Collection.insert` method. Issues Resolved ............... See the `PyMongo 2.6.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12905 Changes in Version 2.6 ---------------------- Version 2.6 includes some frequently requested improvements and adds support for some early MongoDB 2.6 features. Special thanks go to Justin Patrin for his work on the connection pool in this release. Important new features: - The ``max_pool_size`` option for :class:`~pymongo.mongo_client.MongoClient` and :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` now actually caps the number of sockets the pool will open concurrently. Once the pool has reached :attr:`~pymongo.mongo_client.MongoClient.max_pool_size` operations will block waiting for a socket to become available. If ``waitQueueTimeoutMS`` is set, an operation that blocks waiting for a socket will raise :exc:`~pymongo.errors.ConnectionFailure` after the timeout. By default ``waitQueueTimeoutMS`` is not set. See :ref:`connection-pooling` for more information. - The :meth:`~pymongo.collection.Collection.insert` method automatically splits large batches of documents into multiple insert messages based on :attr:`~pymongo.mongo_client.MongoClient.max_message_size` - Support for the exhaust cursor flag. See :meth:`~pymongo.collection.Collection.find` for details and caveats. - Support for the PLAIN and MONGODB-X509 authentication mechanisms. See :doc:`the authentication docs ` for more information. - Support aggregation output as a :class:`~pymongo.cursor.Cursor`. See :meth:`~pymongo.collection.Collection.aggregate` for details. .. warning:: SIGNIFICANT BEHAVIOR CHANGE in 2.6. Previously, `max_pool_size` would limit only the idle sockets the pool would hold onto, not the number of open sockets. The default has also changed, from 10 to 100. If you pass a value for ``max_pool_size`` make sure it is large enough for the expected load. (Sockets are only opened when needed, so there is no cost to having a ``max_pool_size`` larger than necessary. Err towards a larger value.) If your application accepts the default, continue to do so. See :ref:`connection-pooling` for more information. Issues Resolved ............... See the `PyMongo 2.6 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.6 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12380 Changes in Version 2.5.2 ------------------------ Version 2.5.2 fixes a NULL pointer dereference issue when decoding an invalid :class:`~bson.dbref.DBRef`. Issues Resolved ............... See the `PyMongo 2.5.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.5.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12581 Changes in Version 2.5.1 ------------------------ Version 2.5.1 is a minor release that fixes issues discovered after the release of 2.5. Most importantly, this release addresses some race conditions in replica set monitoring. Issues Resolved ............... See the `PyMongo 2.5.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.5.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12484 Changes in Version 2.5 ---------------------- Version 2.5 includes changes to support new features in MongoDB 2.4. Important new features: - Support for :ref:`GSSAPI (Kerberos) authentication `. - Support for SSL certificate validation with hostname matching. - Support for delegated and role based authentication. - New GEOSPHERE (2dsphere) and HASHED index constants. .. note:: :meth:`~pymongo.database.Database.authenticate` now raises a subclass of :class:`~pymongo.errors.PyMongoError` if authentication fails due to invalid credentials or configuration issues. Issues Resolved ............... See the `PyMongo 2.5 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.5 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/11981 Changes in Version 2.4.2 ------------------------ Version 2.4.2 is a minor release that fixes issues discovered after the release of 2.4.1. Most importantly, PyMongo will no longer select a replica set member for read operations that is not in primary or secondary state. Issues Resolved ............... See the `PyMongo 2.4.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.4.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12299 Changes in Version 2.4.1 ------------------------ Version 2.4.1 is a minor release that fixes issues discovered after the release of 2.4. Most importantly, this release fixes a regression using :meth:`~pymongo.collection.Collection.aggregate`, and possibly other commands, with mongos. Issues Resolved ............... See the `PyMongo 2.4.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.4.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/12286 Changes in Version 2.4 ---------------------- Version 2.4 includes a few important new features and a large number of bug fixes. Important new features: - New :class:`~pymongo.mongo_client.MongoClient` and :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` classes - these connection classes do acknowledged write operations (previously referred to as 'safe' writes) by default. :class:`~pymongo.connection.Connection` and :class:`~pymongo.replica_set_connection.ReplicaSetConnection` are deprecated but still support the old default fire-and-forget behavior. - A new write concern API implemented as a :attr:`~pymongo.collection.Collection.write_concern` attribute on the connection, :class:`~pymongo.database.Database`, or :class:`~pymongo.collection.Collection` classes. - :class:`~pymongo.mongo_client.MongoClient` (and :class:`~pymongo.connection.Connection`) now support Unix Domain Sockets. - :class:`~pymongo.cursor.Cursor` can be copied with functions from the :mod:`copy` module. - The :meth:`~pymongo.database.Database.set_profiling_level` method now supports a `slow_ms` option. - The replica set monitor task (used by :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` and :class:`~pymongo.replica_set_connection.ReplicaSetConnection`) is a daemon thread once again, meaning you won't have to call :meth:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient.close` before exiting the python interactive shell. .. warning:: The constructors for :class:`~pymongo.mongo_client.MongoClient`, :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`, :class:`~pymongo.connection.Connection`, and :class:`~pymongo.replica_set_connection.ReplicaSetConnection` now raise :exc:`~pymongo.errors.ConnectionFailure` instead of its subclass :exc:`~pymongo.errors.AutoReconnect` if the server is unavailable. Applications that expect to catch :exc:`~pymongo.errors.AutoReconnect` should now catch :exc:`~pymongo.errors.ConnectionFailure` while creating a new connection. Issues Resolved ............... See the `PyMongo 2.4 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.4 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/11485 Changes in Version 2.3 ---------------------- Version 2.3 adds support for new features and behavior changes in MongoDB 2.2. Important New Features: - Support for expanded read preferences including directing reads to tagged servers - See :ref:`secondary-reads` for more information. - Support for mongos failover. - A new :meth:`~pymongo.collection.Collection.aggregate` method to support MongoDB's new `aggregation framework `_. - Support for legacy Java and C# byte order when encoding and decoding UUIDs. - Support for connecting directly to an arbiter. .. warning:: Starting with MongoDB 2.2 the getLastError command requires authentication when the server's `authentication features `_ are enabled. Changes to PyMongo were required to support this behavior change. Users of authentication must upgrade to PyMongo 2.3 (or newer) for "safe" write operations to function correctly. Issues Resolved ............... See the `PyMongo 2.3 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.3 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/11146 Changes in Version 2.2.1 ------------------------ Version 2.2.1 is a minor release that fixes issues discovered after the release of 2.2. Most importantly, this release fixes an incompatibility with mod_wsgi 2.x that could cause connections to leak. Users of mod_wsgi 2.x are strongly encouraged to upgrade from PyMongo 2.2. Issues Resolved ............... See the `PyMongo 2.2.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.2.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/11185 Changes in Version 2.2 ---------------------- Version 2.2 adds a few more frequently requested features and fixes a number of bugs. Special thanks go to Alex Grönholm for his contributions to Python 3 support and maintaining the original pymongo3 port. Christoph Simon, Wouter Bolsterlee, Mike O'Brien, and Chris Tompkinson also contributed to this release. Important New Features: - Support for Python 3 - See the :doc:`python3` for more information. - Support for Gevent - See :doc:`examples/gevent` for more information. - Improved connection pooling. See `PYTHON-287 `_. .. warning:: A number of methods and method parameters that were deprecated in PyMongo 1.9 or older versions have been removed in this release. The full list of changes can be found in the following JIRA ticket: https://jira.mongodb.org/browse/PYTHON-305 BSON module aliases from the pymongo package that were deprecated in PyMongo 1.9 have also been removed in this release. See the following JIRA ticket for details: https://jira.mongodb.org/browse/PYTHON-304 As a result of this cleanup some minor code changes may be required to use this release. Issues Resolved ............... See the `PyMongo 2.2 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/10584 Changes in Version 2.1.1 ------------------------ Version 2.1.1 is a minor release that fixes a few issues discovered after the release of 2.1. You can now use :class:`~pymongo.replica_set_connection.ReplicaSetConnection` to run inline map reduce commands on secondaries. See :meth:`~pymongo.collection.Collection.inline_map_reduce` for details. Special thanks go to Samuel Clay and Ross Lawley for their contributions to this release. Issues Resolved ............... See the `PyMongo 2.1.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.1.1 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/11081 Changes in Version 2.1 ---------------------- Version 2.1 adds a few frequently requested features and includes the usual round of bug fixes and improvements. Special thanks go to Alexey Borzenkov, Dan Crosta, Kostya Rybnikov, Flavio Percoco Premoli, Jonas Haag, and Jesse Davis for their contributions to this release. Important New Features: - ReplicaSetConnection - :class:`~pymongo.replica_set_connection.ReplicaSetConnection` can be used to distribute reads to secondaries in a replica set. It supports automatic failover handling and periodically checks the state of the replica set to handle issues like primary stepdown or secondaries being removed for backup operations. Read preferences are defined through :class:`~pymongo.read_preferences.ReadPreference`. - PyMongo supports the new BSON binary subtype 4 for UUIDs. The default subtype to use can be set through :attr:`~pymongo.collection.Collection.uuid_subtype` The current default remains :attr:`~bson.binary.OLD_UUID_SUBTYPE` but will be changed to :attr:`~bson.binary.UUID_SUBTYPE` in a future release. - The getLastError option 'w' can be set to a string, allowing for options like "majority" available in newer version of MongoDB. - Added support for the MongoDB URI options socketTimeoutMS and connectTimeoutMS. - Added support for the ContinueOnError insert flag. - Added basic SSL support. - Added basic support for Jython. - Secondaries can be used for :meth:`~pymongo.cursor.Cursor.count`, :meth:`~pymongo.cursor.Cursor.distinct`, :meth:`~pymongo.collection.Collection.group`, and querying :class:`~gridfs.GridFS`. - Added document_class and tz_aware options to :class:`~pymongo.master_slave_connection.MasterSlaveConnection` Issues Resolved ............... See the `PyMongo 2.1 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=10583 Changes in Version 2.0.1 ------------------------ Version 2.0.1 fixes a regression in :class:`~gridfs.grid_file.GridIn` when writing pre-chunked strings. Thanks go to Alexey Borzenkov for reporting the issue and submitting a patch. Issues Resolved ............... - `PYTHON-271 `_: Regression in GridFS leads to serious loss of data. Changes in Version 2.0 ---------------------- Version 2.0 adds a large number of features and fixes a number of issues. Special thanks go to James Murty, Abhay Vardhan, David Pisoni, Ryan Smith-Roberts, Andrew Pendleton, Mher Movsisyan, Reed O'Brien, Michael Schurter, Josip Delic and Jonas Haag for their contributions to this release. Important New Features: - PyMongo now performs automatic per-socket database authentication. You no longer have to re-authenticate for each new thread or after a replica set failover. Authentication credentials are cached by the driver until the application calls :meth:`~pymongo.database.Database.logout`. - slave_okay can be set independently at the connection, database, collection or query level. Each level will inherit the slave_okay setting from the previous level and each level can override the previous level's setting. - safe and getLastError options (e.g. w, wtimeout, etc.) can be set independently at the connection, database, collection or query level. Each level will inherit settings from the previous level and each level can override the previous level's setting. - PyMongo now supports the `await_data` and `partial` cursor flags. If the `await_data` flag is set on a `tailable` cursor the server will block for some extra time waiting for more data to return. The `partial` flag tells a mongos to return partial data for a query if not all shards are available. - :meth:`~pymongo.collection.Collection.map_reduce` will accept a `dict` or instance of :class:`~bson.son.SON` as the `out` parameter. - The URI parser has been moved into its own module and can be used directly by application code. - AutoReconnect exception now provides information about the error that actually occured instead of a generic failure message. - A number of new helper methods have been added with options for setting and unsetting cursor flags, re-indexing a collection, fsync and locking a server, and getting the server's current operations. API changes: - If only one host:port pair is specified :class:`~pymongo.connection.Connection` will make a direct connection to only that host. Please note that `slave_okay` must be `True` in order to query from a secondary. - If more than one host:port pair is specified or the `replicaset` option is used PyMongo will treat the specified host:port pair(s) as a seed list and connect using replica set behavior. .. warning:: The default subtype for :class:`~bson.binary.Binary` has changed from :const:`~bson.binary.OLD_BINARY_SUBTYPE` (2) to :const:`~bson.binary.BINARY_SUBTYPE` (0). Issues Resolved ............... See the `PyMongo 2.0 release notes in JIRA`_ for the list of resolved issues in this release. .. _PyMongo 2.0 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=10274 Changes in Version 1.11 ----------------------- Version 1.11 adds a few new features and fixes a few more bugs. New Features: - Basic IPv6 support: pymongo prefers IPv4 but will try IPv6. You can also specify an IPv6 address literal in the `host` parameter or a MongoDB URI provided it is enclosed in '[' and ']'. - max_pool_size option: previously pymongo had a hard coded pool size of 10 connections. With this change you can specify a different pool size as a parameter to :class:`~pymongo.connection.Connection` (max_pool_size=) or in the MongoDB URI (maxPoolSize=). - Find by metadata in GridFS: You can know specify query fields as keyword parameters for :meth:`~gridfs.GridFS.get_version` and :meth:`~gridfs.GridFS.get_last_version`. - Per-query slave_okay option: slave_okay=True is now a valid keyword argument for :meth:`~pymongo.collection.Collection.find` and :meth:`~pymongo.collection.Collection.find_one`. API changes: - :meth:`~pymongo.database.Database.validate_collection` now returns a dict instead of a string. This change was required to deal with an API change on the server. This method also now takes the optional `scandata` and `full` parameters. See the documentation for more details. .. warning:: The `pool_size`, `auto_start_request`, and `timeout` parameters for :class:`~pymongo.connection.Connection` have been completely removed in this release. They were deprecated in pymongo-1.4 and have had no effect since then. Please make sure that your code doesn't currently pass these parameters when creating a Connection instance. Issues resolved ............... - `PYTHON-241 `_: Support setting slaveok at the cursor level. - `PYTHON-240 `_: Queries can sometimes permanently fail after a replica set fail over. - `PYTHON-238 `_: error after few million requests - `PYTHON-237 `_: Basic IPv6 support. - `PYTHON-236 `_: Restore option to specify pool size in Connection. - `PYTHON-212 `_: pymongo does not recover after stale config - `PYTHON-138 `_: Find method for GridFS Changes in Version 1.10.1 ------------------------- Version 1.10.1 is primarily a bugfix release. It fixes a regression in version 1.10 that broke pickling of ObjectIds. A number of other bugs have been fixed as well. There are two behavior changes to be aware of: - If a read slave raises :class:`~pymongo.errors.AutoReconnect` :class:`~pymongo.master_slave_connection.MasterSlaveConnection` will now retry the query on each slave until it is successful or all slaves have raised :class:`~pymongo.errors.AutoReconnect`. Any other exception will immediately be raised. The order that the slaves are tried is random. Previously the read would be sent to one randomly chosen slave and :class:`~pymongo.errors.AutoReconnect` was immediately raised in case of a connection failure. - A Python `long` is now always BSON encoded as an int64. Previously the encoding was based only on the value of the field and a `long` with a value less than `2147483648` or greater than `-2147483649` would always be BSON encoded as an int32. Issues resolved ............... - `PYTHON-234 `_: Fix setup.py to raise exception if any when building extensions - `PYTHON-233 `_: Add information to build and test with extensions on windows - `PYTHON-232 `_: Traceback when hashing a DBRef instance - `PYTHON-231 `_: Traceback when pickling a DBRef instance - `PYTHON-230 `_: Pickled ObjectIds are not compatible between pymongo 1.9 and 1.10 - `PYTHON-228 `_: Cannot pickle bson.ObjectId - `PYTHON-227 `_: Traceback when calling find() on system.js - `PYTHON-216 `_: MasterSlaveConnection is missing disconnect() method - `PYTHON-186 `_: When storing integers, type is selected according to value instead of type - `PYTHON-173 `_: as_class option is not propogated by Cursor.clone - `PYTHON-113 `_: Redunducy in MasterSlaveConnection Changes in Version 1.10 ----------------------- Version 1.10 includes changes to support new features in MongoDB 1.8.x. Highlights include a modified map/reduce API including an inline map/reduce helper method, a new find_and_modify helper, and the ability to query the server for the maximum BSON document size it supports. - added :meth:`~pymongo.collection.Collection.find_and_modify`. - added :meth:`~pymongo.collection.Collection.inline_map_reduce`. - changed :meth:`~pymongo.collection.Collection.map_reduce`. .. warning:: MongoDB versions greater than 1.7.4 no longer generate temporary collections for map/reduce results. An output collection name must be provided and the output will replace any existing output collection with the same name. :meth:`~pymongo.collection.Collection.map_reduce` now requires the `out` parameter. Issues resolved ............... - PYTHON-225: :class:`~pymongo.objectid.ObjectId` class definition should use __slots__. - PYTHON-223: Documentation fix. - PYTHON-220: Documentation fix. - PYTHON-219: KeyError in :meth:`~pymongo.collection.Collection.find_and_modify` - PYTHON-213: Query server for maximum BSON document size. - PYTHON-208: Fix :class:`~pymongo.connection.Connection` __repr__. - PYTHON-207: Changes to Map/Reduce API. - PYTHON-205: Accept slaveOk in the URI to match the URI docs. - PYTHON-203: When slave_okay=True and we only specify one host don't autodetect other set members. - PYTHON-194: Show size when whining about a document being too large. - PYTHON-184: Raise :class:`~pymongo.errors.DuplicateKeyError` for duplicate keys in capped collections. - PYTHON-178: Don't segfault when trying to encode a recursive data structure. - PYTHON-177: Don't segfault when decoding dicts with broken iterators. - PYTHON-172: Fix a typo. - PYTHON-170: Add :meth:`~pymongo.collection.Collection.find_and_modify`. - PYTHON-169: Support deepcopy of DBRef. - PYTHON-167: Duplicate of PYTHON-166. - PYTHON-166: Fixes a concurrency issue. - PYTHON-158: Add code and err string to `db assertion` messages. Changes in Version 1.9 ---------------------- Version 1.9 adds a new package to the PyMongo distribution, :mod:`bson`. :mod:`bson` contains all of the `BSON `_ encoding and decoding logic, and the BSON types that were formerly in the :mod:`pymongo` package. The following modules have been renamed: - :mod:`pymongo.bson` -> :mod:`bson` - :mod:`pymongo._cbson` -> :mod:`bson._cbson` and :mod:`pymongo._cmessage` - :mod:`pymongo.binary` -> :mod:`bson.binary` - :mod:`pymongo.code` -> :mod:`bson.code` - :mod:`pymongo.dbref` -> :mod:`bson.dbref` - :mod:`pymongo.json_util` -> :mod:`bson.json_util` - :mod:`pymongo.max_key` -> :mod:`bson.max_key` - :mod:`pymongo.min_key` -> :mod:`bson.min_key` - :mod:`pymongo.objectid` -> :mod:`bson.objectid` - :mod:`pymongo.son` -> :mod:`bson.son` - :mod:`pymongo.timestamp` -> :mod:`bson.timestamp` - :mod:`pymongo.tz_util` -> :mod:`bson.tz_util` In addition, the following exception classes have been renamed: - :class:`pymongo.errors.InvalidBSON` -> :class:`bson.errors.InvalidBSON` - :class:`pymongo.errors.InvalidStringData` -> :class:`bson.errors.InvalidStringData` - :class:`pymongo.errors.InvalidDocument` -> :class:`bson.errors.InvalidDocument` - :class:`pymongo.errors.InvalidId` -> :class:`bson.errors.InvalidId` The above exceptions now inherit from :class:`bson.errors.BSONError` rather than :class:`pymongo.errors.PyMongoError`. .. note:: All of the renamed modules and exceptions above have aliases created with the old names, so these changes should not break existing code. The old names will eventually be deprecated and then removed, so users should begin migrating towards the new names now. .. warning:: The change to the exception hierarchy mentioned above is possibly breaking. If your code is catching :class:`~pymongo.errors.PyMongoError`, then the exceptions raised by :mod:`bson` will not be caught, even though they would have been caught previously. Before upgrading, it is recommended that users check for any cases like this. - the C extension now shares buffer.c/h with the Ruby driver - :mod:`bson` no longer raises :class:`~pymongo.errors.InvalidName`, all occurrences have been replaced with :class:`~bson.errors.InvalidDocument`. - renamed :meth:`bson._to_dicts` to :meth:`~bson.decode_all`. - renamed :meth:`~bson.BSON.from_dict` to :meth:`~bson.BSON.encode` and :meth:`~bson.BSON.to_dict` to :meth:`~bson.BSON.decode`. - added :meth:`~pymongo.cursor.Cursor.batch_size`. - allow updating (some) file metadata after a :class:`~gridfs.grid_file.GridIn` instance has been closed. - performance improvements for reading from GridFS. - special cased slice with the same start and stop to return an empty cursor. - allow writing :class:`unicode` to GridFS if an :attr:`encoding` attribute has been specified for the file. - added :meth:`gridfs.GridFS.get_version`. - scope variables for :class:`~bson.code.Code` can now be specified as keyword arguments. - added :meth:`~gridfs.grid_file.GridOut.readline` to :class:`~gridfs.grid_file.GridOut`. - make a best effort to transparently auto-reconnect if a :class:`~pymongo.connection.Connection` has been idle for a while. - added :meth:`~pymongo.database.SystemJS.list` to :class:`~pymongo.database.SystemJS`. - added `file_document` argument to :meth:`~gridfs.grid_file.GridOut` to allow initializing from an existing file document. - raise :class:`~pymongo.errors.TimeoutError` even if the ``getLastError`` command was run manually and not through "safe" mode. - added :class:`uuid` support to :mod:`~bson.json_util`. Changes in Version 1.8.1 ------------------------ - fixed a typo in the C extension that could cause safe-mode operations to report a failure (:class:`SystemError`) even when none occurred. - added a :meth:`__ne__` implementation to any class where we define :meth:`__eq__`. Changes in Version 1.8 ---------------------- Version 1.8 adds support for connecting to replica sets, specifying per-operation values for `w` and `wtimeout`, and decoding to timezone-aware datetimes. - fixed a reference leak in the C extension when decoding a :class:`~bson.dbref.DBRef`. - added support for `w`, `wtimeout`, and `fsync` (and any other options for `getLastError`) to "safe mode" operations. - added :attr:`~pymongo.connection.Connection.nodes` property. - added a maximum pool size of 10 sockets. - added support for replica sets. - DEPRECATED :meth:`~pymongo.connection.Connection.from_uri` and :meth:`~pymongo.connection.Connection.paired`, both are supplanted by extended functionality in :meth:`~pymongo.connection.Connection`. - added tz aware support for datetimes in :class:`~bson.objectid.ObjectId`, :class:`~bson.timestamp.Timestamp` and :mod:`~bson.json_util` methods. - added :meth:`~pymongo.collection.Collection.drop` helper. - reuse the socket used for finding the master when a :class:`~pymongo.connection.Connection` is first created. - added support for :class:`~bson.min_key.MinKey`, :class:`~bson.max_key.MaxKey` and :class:`~bson.timestamp.Timestamp` to :mod:`~bson.json_util`. - added support for decoding datetimes as aware (UTC) - it is highly recommended to enable this by setting the `tz_aware` parameter to :meth:`~pymongo.connection.Connection` to ``True``. - added `network_timeout` option for individual calls to :meth:`~pymongo.collection.Collection.find` and :meth:`~pymongo.collection.Collection.find_one`. - added :meth:`~gridfs.GridFS.exists` to check if a file exists in GridFS. - added support for additional keys in :class:`~bson.dbref.DBRef` instances. - added :attr:`~pymongo.errors.OperationFailure.code` attribute to :class:`~pymongo.errors.OperationFailure` exceptions. - fixed serialization of int and float subclasses in the C extension. Changes in Version 1.7 ---------------------- Version 1.7 is a recommended upgrade for all PyMongo users. The full release notes are below, and some more in depth discussion of the highlights is `here `_. - no longer attempt to build the C extension on big-endian systems. - added :class:`~bson.min_key.MinKey` and :class:`~bson.max_key.MaxKey`. - use unsigned for :class:`~bson.timestamp.Timestamp` in BSON encoder/decoder. - support ``True`` as ``"ok"`` in command responses, in addition to ``1.0`` - necessary for server versions **>= 1.5.X** - BREAKING change to :meth:`~pymongo.collection.Collection.index_information` to add support for querying unique status and other index information. - added :attr:`~pymongo.connection.Connection.document_class`, to specify class for returned documents. - added `as_class` argument for :meth:`~pymongo.collection.Collection.find`, and in the BSON decoder. - added support for creating :class:`~bson.timestamp.Timestamp` instances using a :class:`~datetime.datetime`. - allow `dropTarget` argument for :class:`~pymongo.collection.Collection.rename`. - handle aware :class:`~datetime.datetime` instances, by converting to UTC. - added support for :class:`~pymongo.cursor.Cursor.max_scan`. - raise :class:`~gridfs.errors.FileExists` exception when creating a duplicate GridFS file. - use `y2038 `_ for time handling in the C extension - eliminates 2038 problems when extension is installed. - added `sort` parameter to :meth:`~pymongo.collection.Collection.find` - finalized deprecation of changes from versions **<= 1.4** - take any non-:class:`dict` as an ``"_id"`` query for :meth:`~pymongo.collection.Collection.find_one` or :meth:`~pymongo.collection.Collection.remove` - added ability to pass a :class:`dict` for `fields` argument to :meth:`~pymongo.collection.Collection.find` (supports ``"$slice"`` and field negation) - simplified code to find master, since paired setups don't always have a remote - fixed bug in C encoder for certain invalid types (like :class:`~pymongo.collection.Collection` instances). - don't transparently map ``"filename"`` key to :attr:`name` attribute for GridFS. Changes in Version 1.6 ---------------------- The biggest change in version 1.6 is a complete re-implementation of :mod:`gridfs` with a lot of improvements over the old implementation. There are many details and examples of using the new API in `this blog post `_. The old API has been removed in this version, so existing code will need to be modified before upgrading to 1.6. - fixed issue where connection pool was being shared across :class:`~pymongo.connection.Connection` instances. - more improvements to Python code caching in C extension - should improve behavior on mod_wsgi. - added :meth:`~bson.objectid.ObjectId.from_datetime`. - complete rewrite of :mod:`gridfs` support. - improvements to the :meth:`~pymongo.database.Database.command` API. - fixed :meth:`~pymongo.collection.Collection.drop_indexes` behavior on non-existent collections. - disallow empty bulk inserts. Changes in Version 1.5.2 ------------------------ - fixed response handling to ignore unknown response flags in queries. - handle server versions containing '-pre-'. Changes in Version 1.5.1 ------------------------ - added :data:`~gridfs.grid_file.GridFile._id` property for :class:`~gridfs.grid_file.GridFile` instances. - fix for making a :class:`~pymongo.connection.Connection` (with `slave_okay` set) directly to a slave in a replica pair. - accept kwargs for :meth:`~pymongo.collection.Collection.create_index` and :meth:`~pymongo.collection.Collection.ensure_index` to support all indexing options. - add :data:`pymongo.GEO2D` and support for geo indexing. - improvements to Python code caching in C extension - should improve behavior on mod_wsgi. Changes in Version 1.5 ---------------------- - added subtype constants to :mod:`~bson.binary` module. - DEPRECATED `options` argument to :meth:`~pymongo.collection.Collection` and :meth:`~pymongo.database.Database.create_collection` in favor of kwargs. - added :meth:`~pymongo.has_c` to check for C extension. - added :meth:`~pymongo.connection.Connection.copy_database`. - added :data:`~pymongo.cursor.Cursor.alive` to tell when a cursor might have more data to return (useful for tailable cursors). - added :class:`~bson.timestamp.Timestamp` to better support dealing with internal MongoDB timestamps. - added `name` argument for :meth:`~pymongo.collection.Collection.create_index` and :meth:`~pymongo.collection.Collection.ensure_index`. - fixed connection pooling w/ fork - :meth:`~pymongo.connection.Connection.paired` takes all kwargs that are allowed for :meth:`~pymongo.connection.Connection`. - :meth:`~pymongo.collection.Collection.insert` returns list for bulk inserts of size one. - fixed handling of :class:`datetime.datetime` instances in :mod:`~bson.json_util`. - added :meth:`~pymongo.connection.Connection.from_uri` to support MongoDB connection uri scheme. - fixed chunk number calculation when unaligned in :mod:`gridfs`. - :meth:`~pymongo.database.Database.command` takes a string for simple commands. - added :data:`~pymongo.database.Database.system_js` helper for dealing with server-side JS. - don't wrap queries containing ``"$query"`` (support manual use of ``"$min"``, etc.). - added :class:`~gridfs.errors.GridFSError` as base class for :mod:`gridfs` exceptions. Changes in Version 1.4 ---------------------- Perhaps the most important change in version 1.4 is that we have decided to **no longer support Python 2.3**. The most immediate reason for this is to allow some improvements to connection pooling. This will also allow us to use some new (as in Python 2.4 ;) idioms and will help begin the path towards supporting Python 3.0. If you need to use Python 2.3 you should consider using version 1.3 of this driver, although that will no longer be actively supported. Other changes: - move ``"_id"`` to front only for top-level documents (fixes some corner cases). - :meth:`~pymongo.collection.Collection.update` and :meth:`~pymongo.collection.Collection.remove` return the entire response to the *lastError* command when safe is ``True``. - completed removal of things that were deprecated in version 1.2 or earlier. - enforce that collection names do not contain the NULL byte. - fix to allow using UTF-8 collection names with the C extension. - added :class:`~pymongo.errors.PyMongoError` as base exception class for all :mod:`~pymongo.errors`. this changes the exception hierarchy somewhat, and is a BREAKING change if you depend on :class:`~pymongo.errors.ConnectionFailure` being a :class:`IOError` or :class:`~bson.errors.InvalidBSON` being a :class:`ValueError`, for example. - added :class:`~pymongo.errors.DuplicateKeyError` for calls to :meth:`~pymongo.collection.Collection.insert` or :meth:`~pymongo.collection.Collection.update` with `safe` set to ``True``. - removed :mod:`~pymongo.thread_util`. - added :meth:`~pymongo.database.Database.add_user` and :meth:`~pymongo.database.Database.remove_user` helpers. - fix for :meth:`~pymongo.database.Database.authenticate` when using non-UTF-8 names or passwords. - minor fixes for :class:`~pymongo.master_slave_connection.MasterSlaveConnection`. - clean up all cases where :class:`~pymongo.errors.ConnectionFailure` is raised. - simplification of connection pooling - makes driver ~2x faster for simple benchmarks. see :ref:`connection-pooling` for more information. - DEPRECATED `pool_size`, `auto_start_request` and `timeout` parameters to :class:`~pymongo.connection.Connection`. DEPRECATED :meth:`~pymongo.connection.Connection.start_request`. - use :meth:`socket.sendall`. - removed :meth:`~bson.son.SON.from_xml` as it was only being used for some internal testing - also eliminates dependency on :mod:`elementtree`. - implementation of :meth:`~pymongo.message.update` in C. - deprecate :meth:`~pymongo.database.Database._command` in favor of :meth:`~pymongo.database.Database.command`. - send all commands without wrapping as ``{"query": ...}``. - support string as `key` argument to :meth:`~pymongo.collection.Collection.group` (keyf) and run all groups as commands. - support for equality testing for :class:`~bson.code.Code` instances. - allow the NULL byte in strings and disallow it in key names or regex patterns Changes in Version 1.3 ---------------------- - DEPRECATED running :meth:`~pymongo.collection.Collection.group` as :meth:`~pymongo.database.Database.eval`, also changed default for :meth:`~pymongo.collection.Collection.group` to running as a command - remove :meth:`pymongo.cursor.Cursor.__len__`, which was deprecated in 1.1.1 - needed to do this aggressively due to it's presence breaking **Django** template *for* loops - DEPRECATED :meth:`~pymongo.connection.Connection.host`, :meth:`~pymongo.connection.Connection.port`, :meth:`~pymongo.database.Database.connection`, :meth:`~pymongo.database.Database.name`, :meth:`~pymongo.collection.Collection.database`, :meth:`~pymongo.collection.Collection.name` and :meth:`~pymongo.collection.Collection.full_name` in favor of :attr:`~pymongo.connection.Connection.host`, :attr:`~pymongo.connection.Connection.port`, :attr:`~pymongo.database.Database.connection`, :attr:`~pymongo.database.Database.name`, :attr:`~pymongo.collection.Collection.database`, :attr:`~pymongo.collection.Collection.name` and :attr:`~pymongo.collection.Collection.full_name`, respectively. The deprecation schedule for this change will probably be faster than usual, as it carries some performance implications. - added :meth:`~pymongo.connection.Connection.disconnect` Changes in Version 1.2.1 ------------------------ - added :doc:`changelog` to docs - added ``setup.py doc --test`` to run doctests for tutorial, examples - moved most examples to Sphinx docs (and remove from *examples/* directory) - raise :class:`~bson.errors.InvalidId` instead of :class:`TypeError` when passing a 24 character string to :class:`~bson.objectid.ObjectId` that contains non-hexadecimal characters - allow :class:`unicode` instances for :class:`~bson.objectid.ObjectId` init Changes in Version 1.2 ---------------------- - `spec` parameter for :meth:`~pymongo.collection.Collection.remove` is now optional to allow for deleting all documents in a :class:`~pymongo.collection.Collection` - always wrap queries with ``{query: ...}`` even when no special options - get around some issues with queries on fields named ``query`` - enforce 4MB document limit on the client side - added :meth:`~pymongo.collection.Collection.map_reduce` helper - see :doc:`example ` - added :meth:`~pymongo.cursor.Cursor.distinct` method on :class:`~pymongo.cursor.Cursor` instances to allow distinct with queries - fix for :meth:`~pymongo.cursor.Cursor.__getitem__` after :meth:`~pymongo.cursor.Cursor.skip` - allow any UTF-8 string in :class:`~bson.BSON` encoder, not just ASCII subset - added :attr:`~bson.objectid.ObjectId.generation_time` - removed support for legacy :class:`~bson.objectid.ObjectId` format - pretty sure this was never used, and is just confusing - DEPRECATED :meth:`~bson.objectid.ObjectId.url_encode` and :meth:`~bson.objectid.ObjectId.url_decode` in favor of :meth:`str` and :meth:`~bson.objectid.ObjectId`, respectively - allow *oplog.$main* as a valid collection name - some minor fixes for installation process - added support for datetime and regex in :mod:`~bson.json_util` Changes in Version 1.1.2 ------------------------ - improvements to :meth:`~pymongo.collection.Collection.insert` speed (using C for insert message creation) - use random number for request_id - fix some race conditions with :class:`~pymongo.errors.AutoReconnect` Changes in Version 1.1.1 ------------------------ - added `multi` parameter for :meth:`~pymongo.collection.Collection.update` - fix unicode regex patterns with C extension - added :meth:`~pymongo.collection.Collection.distinct` - added `database` support for :class:`~bson.dbref.DBRef` - added :mod:`~bson.json_util` with helpers for encoding / decoding special types to JSON - DEPRECATED :meth:`pymongo.cursor.Cursor.__len__` in favor of :meth:`~pymongo.cursor.Cursor.count` with `with_limit_and_skip` set to ``True`` due to performance regression - switch documentation to Sphinx Changes in Version 1.1 ---------------------- - added :meth:`__hash__` for :class:`~bson.dbref.DBRef` and :class:`~bson.objectid.ObjectId` - bulk :meth:`~pymongo.collection.Collection.insert` works with any iterable - fix :class:`~bson.objectid.ObjectId` generation when using :mod:`multiprocessing` - added :attr:`~pymongo.cursor.Cursor.collection` - added `network_timeout` parameter for :meth:`~pymongo.connection.Connection` - DEPRECATED `slave_okay` parameter for individual queries - fix for `safe` mode when multi-threaded - added `safe` parameter for :meth:`~pymongo.collection.Collection.remove` - added `tailable` parameter for :meth:`~pymongo.collection.Collection.find` Changes in Version 1.0 ---------------------- - fixes for :class:`~pymongo.master_slave_connection.MasterSlaveConnection` - added `finalize` parameter for :meth:`~pymongo.collection.Collection.group` - improvements to :meth:`~pymongo.collection.Collection.insert` speed - improvements to :mod:`gridfs` speed - added :meth:`~pymongo.cursor.Cursor.__getitem__` and :meth:`~pymongo.cursor.Cursor.__len__` for :class:`~pymongo.cursor.Cursor` instances Changes in Version 0.16 ----------------------- - support for encoding/decoding :class:`uuid.UUID` instances - fix for :meth:`~pymongo.cursor.Cursor.explain` with limits Changes in Version 0.15.2 ------------------------- - documentation changes only Changes in Version 0.15.1 ------------------------- - various performance improvements - API CHANGE no longer need to specify direction for :meth:`~pymongo.collection.Collection.create_index` and :meth:`~pymongo.collection.Collection.ensure_index` when indexing a single key - support for encoding :class:`tuple` instances as :class:`list` instances Changes in Version 0.15 ----------------------- - fix string representation of :class:`~bson.objectid.ObjectId` instances - added `timeout` parameter for :meth:`~pymongo.collection.Collection.find` - allow scope for `reduce` function in :meth:`~pymongo.collection.Collection.group` Changes in Version 0.14.2 ------------------------- - minor bugfixes Changes in Version 0.14.1 ------------------------- - :meth:`~gridfs.grid_file.GridFile.seek` and :meth:`~gridfs.grid_file.GridFile.tell` for (read mode) :class:`~gridfs.grid_file.GridFile` instances Changes in Version 0.14 ----------------------- - support for long in :class:`~bson.BSON` - added :meth:`~pymongo.collection.Collection.rename` - added `snapshot` parameter for :meth:`~pymongo.collection.Collection.find` Changes in Version 0.13 ----------------------- - better :class:`~pymongo.master_slave_connection.MasterSlaveConnection` support - API CHANGE :meth:`~pymongo.collection.Collection.insert` and :meth:`~pymongo.collection.Collection.save` both return inserted ``_id`` - DEPRECATED passing an index name to :meth:`~pymongo.cursor.Cursor.hint` Changes in Version 0.12 ----------------------- - improved :class:`~bson.objectid.ObjectId` generation - added :class:`~pymongo.errors.AutoReconnect` exception for when reconnection is possible - make :mod:`gridfs` thread-safe - fix for :mod:`gridfs` with non :class:`~bson.objectid.ObjectId` ``_id`` Changes in Version 0.11.3 ------------------------- - don't allow NULL bytes in string encoder - fixes for Python 2.3 Changes in Version 0.11.2 ------------------------- - PEP 8 - updates for :meth:`~pymongo.collection.Collection.group` - VS build Changes in Version 0.11.1 ------------------------- - fix for connection pooling under Python 2.5 Changes in Version 0.11 ----------------------- - better build failure detection - driver support for selecting fields in sub-documents - disallow insertion of invalid key names - added `timeout` parameter for :meth:`~pymongo.connection.Connection` Changes in Version 0.10.3 ------------------------- - fix bug with large :meth:`~pymongo.cursor.Cursor.limit` - better exception when modules get reloaded out from underneath the C extension - better exception messages when calling a :class:`~pymongo.collection.Collection` or :class:`~pymongo.database.Database` instance Changes in Version 0.10.2 ------------------------- - support subclasses of :class:`dict` in C encoder Changes in Version 0.10.1 ------------------------- - alias :class:`~pymongo.connection.Connection` as :attr:`pymongo.Connection` - raise an exception rather than silently overflowing in encoder Changes in Version 0.10 ----------------------- - added :meth:`~pymongo.collection.Collection.ensure_index` Changes in Version 0.9.7 ------------------------ - allow sub-collections of *$cmd* as valid :class:`~pymongo.collection.Collection` names - add version as :attr:`pymongo.version` - add ``--no_ext`` command line option to *setup.py* .. toctree:: :hidden: python3 examples/gevent pymongo-3.2/doc/pydoctheme/0000755000175000017500000000000012631423130017655 5ustar behackettbehackett00000000000000pymongo-3.2/doc/pydoctheme/theme.conf0000644000175000017500000000102312630145074021631 0ustar behackettbehackett00000000000000[theme] inherit = default stylesheet = pydoctheme.css pygments_style = sphinx [options] bodyfont = 'Lucida Grande', Arial, sans-serif headfont = 'Lucida Grande', Arial, sans-serif footerbgcolor = white footertextcolor = #555555 relbarbgcolor = white relbartextcolor = #666666 relbarlinkcolor = #444444 sidebarbgcolor = white sidebartextcolor = #444444 sidebarlinkcolor = #444444 bgcolor = white textcolor = #222222 linkcolor = #0090c0 visitedlinkcolor = #00608f headtextcolor = #1a1a1a headbgcolor = white headlinkcolor = #aaaaaa pymongo-3.2/doc/pydoctheme/static/0000755000175000017500000000000012631423130021144 5ustar behackettbehackett00000000000000pymongo-3.2/doc/pydoctheme/static/pydoctheme.css0000644000175000017500000000526712630145074024040 0ustar behackettbehackett00000000000000@import url("default.css"); body { background-color: white; margin-left: 1em; margin-right: 1em; } div.related { margin-bottom: 1.2em; padding: 0.5em 0; border-top: 1px solid #ccc; margin-top: 0.5em; } div.related a:hover { color: #0095C4; } div.related:first-child { border-top: 0; border-bottom: 1px solid #ccc; } div.sphinxsidebar { background-color: #eeeeee; border-radius: 5px; line-height: 130%; font-size: smaller; } div.sphinxsidebar h3, div.sphinxsidebar h4 { margin-top: 1.5em; } div.sphinxsidebarwrapper > h3:first-child { margin-top: 0.2em; } div.sphinxsidebarwrapper > ul > li > ul > li { margin-bottom: 0.4em; } div.sphinxsidebar a:hover { color: #0095C4; } div.sphinxsidebar input { font-family: 'Lucida Grande',Arial,sans-serif; border: 1px solid #999999; font-size: smaller; border-radius: 3px; } div.sphinxsidebar input[type=text] { max-width: 150px; } div.body { padding: 0 0 0 1.2em; } div.body p { line-height: 140%; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { margin: 0; border: 0; padding: 0.3em 0; } div.body hr { border: 0; background-color: #ccc; height: 1px; } div.body pre { border-radius: 3px; border: 1px solid #ac9; } div.body div.admonition, div.body div.impl-detail { border-radius: 3px; } div.body div.impl-detail > p { margin: 0; } div.body div.seealso { border: 1px solid #dddd66; } div.body a { color: #0072aa; } div.body a:visited { color: #6363bb; } div.body a:hover { color: #00B0E4; } tt, code, pre { font-family: monospace, sans-serif; font-size: 96.5%; } div.body tt, div.body code { border-radius: 3px; } div.body tt.descname, div.body code.descname { font-size: 120%; } div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { font-weight: normal; } .deprecated { border-radius: 3px; } table.docutils { border: 1px solid #ddd; min-width: 20%; border-radius: 3px; margin-top: 10px; margin-bottom: 10px; } table.docutils td, table.docutils th { border: 1px solid #ddd !important; border-radius: 3px; } table p, table li { text-align: left !important; } table.docutils th { background-color: #eee; padding: 0.3em 0.5em; } table.docutils td { background-color: white; padding: 0.3em 0.5em; } table.footnote, table.footnote td { border: 0 !important; } div.footer { line-height: 150%; margin-top: -2em; text-align: right; width: auto; margin-right: 10px; } div.footer a:hover { color: #0095C4; } .refcount { color: #060; } .stableabi { color: #229; } pymongo-3.2/doc/faq.rst0000644000175000017500000004354012630165620017031 0ustar behackettbehackett00000000000000Frequently Asked Questions ========================== .. contents:: Is PyMongo thread-safe? ----------------------- PyMongo is thread-safe and provides built-in connection pooling for threaded applications. .. _connection-pooling: How does connection pooling work in PyMongo? -------------------------------------------- Every :class:`~pymongo.mongo_client.MongoClient` instance has a built-in connection pool. The client opens sockets on demand to support the number of concurrent MongoDB operations your application requires. There is no thread-affinity for sockets. The client instance opens one additional socket per server in your MongoDB topology for monitoring the server's state. The size of each connection pool is capped at ``maxPoolSize``, which defaults to 100. When a thread in your application begins an operation on MongoDB, if all other sockets are in use and the pool has reached its maximum, the thread pauses, waiting for a socket to be returned to the pool by another thread. The default configuration for a :class:`~pymongo.mongo_client.MongoClient` works for most applications:: client = MongoClient(host, port) Create this client **once** when your program starts up, and reuse it for all operations. It is a common mistake to create a new client for each request, which is very inefficient. To support extremely high numbers of concurrent MongoDB operations within one process, increase ``maxPoolSize``:: client = MongoClient(host, port, maxPoolSize=200) ... or make it unbounded:: client = MongoClient(host, port, maxPoolSize=None) By default, any number of threads are allowed to wait for sockets to become available, and they can wait any length of time. Override ``waitQueueMultiple`` to cap the number of waiting threads. E.g., to keep the number of waiters less than or equal to 500:: client = MongoClient(host, port, maxPoolSize=50, waitQueueMultiple=10) When 500 threads are waiting for a socket, the 501st that needs a socket raises :exc:`~pymongo.errors.ExceededMaxWaiters`. Use this option to bound the amount of queueing in your application during a load spike, at the cost of additional exceptions. Once the pool reaches its max size, additional threads are allowed to wait indefinitely for sockets to become available, unless you set ``waitQueueTimeoutMS``:: client = MongoClient(host, port, waitQueueTimeoutMS=100) A thread that waits more than 100ms (in this example) for a socket raises :exc:`~pymongo.errors.ConnectionFailure`. Use this option if it is more important to bound the duration of operations during a load spike than it is to complete every operation. When :meth:`~pymongo.mongo_client.MongoClient.close` is called by any thread, all sockets are closed. Does PyMongo support Python 3? ------------------------------ PyMongo supports Python 3.x where x >= 2. See the :doc:`python3` for details. Does PyMongo support asynchronous frameworks like Gevent, asyncio, Tornado, or Twisted? --------------------------------------------------------------------------------------- PyMongo fully supports :doc:`Gevent `. To use MongoDB with `asyncio ` or `Tornado `_, see the `Motor `_ project. For `Twisted `_, see `TxMongo `_. Its stated mission is to keep feature parity with PyMongo. .. _writes-and-ids: Why does PyMongo add an _id field to all of my documents? --------------------------------------------------------- When a document is inserted to MongoDB using :meth:`~pymongo.collection.Collection.insert_one`, :meth:`~pymongo.collection.Collection.insert_many`, or :meth:`~pymongo.collection.Collection.bulk_write`, and that document does not include an ``_id`` field, PyMongo automatically adds one for you, set to an instance of :class:`~bson.objectid.ObjectId`. For example:: >>> my_doc = {'x': 1} >>> collection.insert_one(my_doc) >>> my_doc {'x': 1, '_id': ObjectId('560db337fba522189f171720')} Users often discover this behavior when calling :meth:`~pymongo.collection.Collection.insert_many` with a list of references to a single document raises :exc:`~pymongo.errors.BulkWriteError`. Several Python idioms lead to this pitfall:: >>> doc = {} >>> collection.insert_many(doc for _ in range(10)) Traceback (most recent call last): ... pymongo.errors.BulkWriteError: batch op errors occurred >>> doc {'_id': ObjectId('560f171cfba52279f0b0da0c')} >>> docs = [{}] >>> collection.insert_many(docs * 10) Traceback (most recent call last): ... pymongo.errors.BulkWriteError: batch op errors occurred >>> docs [{'_id': ObjectId('560f1933fba52279f0b0da0e')}] PyMongo adds an ``_id`` field in this manner for a few reasons: - All MongoDB documents are required to have an ``_id`` field. - If PyMongo were to insert a document without an ``_id`` MongoDB would add one itself, but it would not report the value back to PyMongo. - Copying the document to insert before adding the ``_id`` field would be prohibitively expensive for most high write volume applications. If you don't want PyMongo to add an ``_id`` to your documents, insert only documents that already have an ``_id`` field, added by your application. Key order in subdocuments -- why does my query work in the shell but not PyMongo? --------------------------------------------------------------------------------- .. testsetup:: key-order from bson.son import SON from pymongo.mongo_client import MongoClient collection = MongoClient().test.collection collection.drop() collection.insert_one({'_id': 1.0, 'subdocument': SON([('b', 1.0), ('a', 1.0)])}) The key-value pairs in a BSON document can have any order (except that ``_id`` is always first). The mongo shell preserves key order when reading and writing data. Observe that "b" comes before "a" when we create the document and when it is displayed: .. code-block:: javascript > // mongo shell. > db.collection.insert( { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } ) WriteResult({ "nInserted" : 1 }) > db.collection.find() { "_id" : 1, "subdocument" : { "b" : 1, "a" : 1 } } PyMongo represents BSON documents as Python dicts by default, and the order of keys in dicts is not defined. That is, a dict declared with the "a" key first is the same, to Python, as one with "b" first: .. doctest:: key-order >>> print {'a': 1.0, 'b': 1.0} {'a': 1.0, 'b': 1.0} >>> print {'b': 1.0, 'a': 1.0} {'a': 1.0, 'b': 1.0} Therefore, Python dicts are not guaranteed to show keys in the order they are stored in BSON. Here, "a" is shown before "b": .. doctest:: key-order >>> print collection.find_one() {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}} To preserve order when reading BSON, use the :class:`~bson.son.SON` class, which is a dict that remembers its key order. First, get a handle to the collection, configured to use :class:`~bson.son.SON` instead of dict: .. doctest:: key-order >>> from bson import CodecOptions, SON >>> opts = CodecOptions(document_class=SON) >>> opts # doctest: +NORMALIZE_WHITESPACE CodecOptions(document_class=, tz_aware=False, uuid_representation=PYTHON_LEGACY, unicode_decode_error_handler='strict', tzinfo=None) >>> collection_son = collection.with_options(codec_options=opts) Now, documents and subdocuments in query results are represented with :class:`~bson.son.SON` objects: .. doctest:: key-order >>> print collection_son.find_one() SON([(u'_id', 1.0), (u'subdocument', SON([(u'b', 1.0), (u'a', 1.0)]))]) The subdocument's actual storage layout is now visible: "b" is before "a". Because a dict's key order is not defined, you cannot predict how it will be serialized **to** BSON. But MongoDB considers subdocuments equal only if their keys have the same order. So if you use a dict to query on a subdocument it may not match: .. doctest:: key-order >>> collection.find_one({'subdocument': {'a': 1.0, 'b': 1.0}}) is None True Swapping the key order in your query makes no difference: .. doctest:: key-order >>> collection.find_one({'subdocument': {'b': 1.0, 'a': 1.0}}) is None True ... because, as we saw above, Python considers the two dicts the same. There are two solutions. First, you can match the subdocument field-by-field: .. doctest:: key-order >>> collection.find_one({'subdocument.a': 1.0, ... 'subdocument.b': 1.0}) {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}} The query matches any subdocument with an "a" of 1.0 and a "b" of 1.0, regardless of the order you specify them in Python or the order they are stored in BSON. Additionally, this query now matches subdocuments with additional keys besides "a" and "b", whereas the previous query required an exact match. The second solution is to use a :class:`~bson.son.SON` to specify the key order: .. doctest:: key-order >>> query = {'subdocument': SON([('b', 1.0), ('a', 1.0)])} >>> collection.find_one(query) {u'_id': 1.0, u'subdocument': {u'a': 1.0, u'b': 1.0}} The key order you use when you create a :class:`~bson.son.SON` is preserved when it is serialized to BSON and used as a query. Thus you can create a subdocument that exactly matches the subdocument in the collection. .. seealso:: `MongoDB Manual entry on subdocument matching `_. What does *CursorNotFound* cursor id not valid at server mean? -------------------------------------------------------------- Cursors in MongoDB can timeout on the server if they've been open for a long time without any operations being performed on them. This can lead to an :class:`~pymongo.errors.CursorNotFound` exception being raised when attempting to iterate the cursor. How do I change the timeout value for cursors? ---------------------------------------------- MongoDB doesn't support custom timeouts for cursors, but cursor timeouts can be turned off entirely. Pass ``no_cursor_timeout=True`` to :meth:`~pymongo.collection.Collection.find`. How can I store :mod:`decimal.Decimal` instances? ------------------------------------------------- MongoDB only supports IEEE 754 floating points - the same as the Python float type. The only way PyMongo could store Decimal instances would be to convert them to this standard, so you'd really only be storing floats anyway - we force users to do this conversion explicitly so that they are aware that it is happening. I'm saving ``9.99`` but when I query my document contains ``9.9900000000000002`` - what's going on here? -------------------------------------------------------------------------------------------------------- The database representation is ``9.99`` as an IEEE floating point (which is common to MongoDB and Python as well as most other modern languages). The problem is that ``9.99`` cannot be represented exactly with a double precision floating point - this is true in some versions of Python as well: >>> 9.99 9.9900000000000002 The result that you get when you save ``9.99`` with PyMongo is exactly the same as the result you'd get saving it with the JavaScript shell or any of the other languages (and as the data you're working with when you type ``9.99`` into a Python program). Can you add attribute style access for documents? ------------------------------------------------- This request has come up a number of times but we've decided not to implement anything like this. The relevant `jira case `_ has some information about the decision, but here is a brief summary: 1. This will pollute the attribute namespace for documents, so could lead to subtle bugs / confusing errors when using a key with the same name as a dictionary method. 2. The only reason we even use SON objects instead of regular dictionaries is to maintain key ordering, since the server requires this for certain operations. So we're hesitant to needlessly complicate SON (at some point it's hypothetically possible we might want to revert back to using dictionaries alone, without breaking backwards compatibility for everyone). 3. It's easy (and Pythonic) for new users to deal with documents, since they behave just like dictionaries. If we start changing their behavior it adds a barrier to entry for new users - another class to learn. What is the correct way to handle time zones with PyMongo? ---------------------------------------------------------- See :doc:`examples/datetimes` for examples on how to handle :class:`~datetime.datetime` objects correctly. How can I save a :mod:`datetime.date` instance? ----------------------------------------------- PyMongo doesn't support saving :mod:`datetime.date` instances, since there is no BSON type for dates without times. Rather than having the driver enforce a convention for converting :mod:`datetime.date` instances to :mod:`datetime.datetime` instances for you, any conversion should be performed in your client code. .. _web-application-querying-by-objectid: When I query for a document by ObjectId in my web application I get no result ----------------------------------------------------------------------------- It's common in web applications to encode documents' ObjectIds in URLs, like:: "/posts/50b3bda58a02fb9a84d8991e" Your web framework will pass the ObjectId portion of the URL to your request handler as a string, so it must be converted to :class:`~bson.objectid.ObjectId` before it is passed to :meth:`~pymongo.collection.Collection.find_one`. It is a common mistake to forget to do this conversion. Here's how to do it correctly in Flask_ (other web frameworks are similar):: from pymongo import MongoClient from bson.objectid import ObjectId from flask import Flask, render_template client = MongoClient() app = Flask(__name__) @app.route("/posts/<_id>") def show_post(_id): # NOTE!: converting _id from string to ObjectId before passing to find_one post = client.db.posts.find_one({'_id': ObjectId(_id)}) return render_template('post.html', post=post) if __name__ == "__main__": app.run() .. _Flask: http://flask.pocoo.org/ .. seealso:: :ref:`querying-by-objectid` How can I use PyMongo from Django? ---------------------------------- `Django `_ is a popular Python web framework. Django includes an ORM, :mod:`django.db`. Currently, there's no official MongoDB backend for Django. `django-mongodb-engine `_ is an unofficial MongoDB backend that supports Django aggregations, (atomic) updates, embedded objects, Map/Reduce and GridFS. It allows you to use most of Django's built-in features, including the ORM, admin, authentication, site and session frameworks and caching. However, it's easy to use MongoDB (and PyMongo) from Django without using a Django backend. Certain features of Django that require :mod:`django.db` (admin, authentication and sessions) will not work using just MongoDB, but most of what Django provides can still be used. One project which should make working with MongoDB and Django easier is `mango `_. Mango is a set of MongoDB backends for Django sessions and authentication (bypassing :mod:`django.db` entirely). .. _using-with-mod-wsgi: Does PyMongo work with **mod_wsgi**? ------------------------------------ Yes. See the configuration guide for :ref:`pymongo-and-mod_wsgi`. How can I use something like Python's :mod:`json` module to encode my documents to JSON? ---------------------------------------------------------------------------------------- The :mod:`json` module won't work out of the box with all documents from PyMongo as PyMongo supports some special types (like :class:`~bson.objectid.ObjectId` and :class:`~bson.dbref.DBRef`) that are not supported in JSON. We've added some utilities for working with JSON in the :mod:`~bson.json_util` module. Why do I get OverflowError decoding dates stored by another language's driver? ------------------------------------------------------------------------------ PyMongo decodes BSON datetime values to instances of Python's :class:`datetime.datetime`. Instances of :class:`datetime.datetime` are limited to years between :data:`datetime.MINYEAR` (usually 1) and :data:`datetime.MAXYEAR` (usually 9999). Some MongoDB drivers (e.g. the PHP driver) can store BSON datetimes with year values far outside those supported by :class:`datetime.datetime`. There are a few ways to work around this issue. One option is to filter out documents with values outside of the range supported by :class:`datetime.datetime`:: >>> from datetime import datetime >>> coll = client.test.dates >>> cur = coll.find({'dt': {'$gte': datetime.min, '$lte': datetime.max}}) Another option, assuming you don't need the datetime field, is to filter out just that field:: >>> cur = coll.find({}, projection={'dt': False}) .. _multiprocessing: Using PyMongo with Multiprocessing ---------------------------------- There are a few things to be aware of when using multiprocessing with PyMongo. On certain platforms (`defined here `_) :class:`~pymongo.mongo_client.MongoClient` MUST be initialized with ``connect=False`` if a :class:`~pymongo.mongo_client.MongoClient` used in a child process is initialized before forking. If ``connect`` cannot be False, then :class:`~pymongo.mongo_client.MongoClient` must be initialized AFTER forking. This is because CPython must acquire a lock before calling `getaddrinfo() `_. A deadlock will occur if the :class:`~pymongo.mongo_client.MongoClient`'s parent process forks (on the main thread) while its monitor thread is in the getaddrinfo() system call. PyMongo will issue a warning if there is a chance of this deadlock occurring. pymongo-3.2/doc/migrate-to-pymongo3.rst0000644000175000017500000004006712622440614022104 0ustar behackettbehackett00000000000000PyMongo 3 Migration Guide ========================= .. contents:: .. testsetup:: from pymongo import MongoClient, ReadPreference client = MongoClient() collection = client.my_database.my_collection PyMongo 3 is a partial rewrite bringing a large number of improvements. It also brings a number of backward breaking changes. This guide provides a roadmap for migrating an existing application from PyMongo 2.x to 3.x or writing libraries that will work with both PyMongo 2.x and 3.x. PyMongo 2.9 ----------- The first step in any successful migration involves upgrading to, or requiring, at least PyMongo 2.9. If your project has a requirements.txt file, add the line "pymongo >= 2.9, < 3.0" until you have completely migrated to PyMongo 3. Most of the key new methods and options from PyMongo 3.0 are backported in PyMongo 2.9 making migration much easier. Enable Deprecation Warnings --------------------------- Starting with PyMongo 2.9, :exc:`DeprecationWarning` is raised by most methods removed in PyMongo 3.0. Make sure you enable runtime warnings to see where deprecated functions and methods are being used in your application:: python -Wd Warnings can also be changed to errors:: python -Wd -Werror .. note:: Not all deprecated features raise :exc:`DeprecationWarning` when used. For example, the :meth:`~pymongo.collection.Collection.find` options renamed in PyMongo 3.0 do not raise :exc:`DeprecationWarning` when used in PyMongo 2.x. See also `Removed features with no migration path`_. CRUD API -------- Changes to find() and find_one() ................................ "spec" renamed "filter" ~~~~~~~~~~~~~~~~~~~~~~~ The `spec` option has been renamed to `filter`. Code like this:: >>> cursor = collection.find(spec={"a": 1}) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find(filter={"a": 1}) or this with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"a": 1}) "fields" renamed "projection" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `fields` option has been renamed to `projection`. Code like this:: >>> cursor = collection.find({"a": 1}, fields={"_id": False}) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, projection={"_id": False}) or this with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"a": 1}, {"_id": False}) "partial" renamed "allow_partial_results" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `partial` option has been renamed to `allow_partial_results`. Code like this:: >>> cursor = collection.find({"a": 1}, partial=True) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, allow_partial_results=True) "timeout" replaced by "no_cursor_timeout" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `timeout` option has been replaced by `no_cursor_timeout`. Code like this:: >>> cursor = collection.find({"a": 1}, timeout=False) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, no_cursor_timeout=True) "snapshot" and "max_scan" replaced by "modifiers" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `snapshot` and `max_scan` options have been removed. They can now be set, along with other $ query modifiers, through the `modifiers` option. Code like this:: >>> cursor = collection.find({"a": 1}, snapshot=True) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> cursor = collection.find({"a": 1}, modifiers={"$snapshot": True}) or with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"$query": {"a": 1}, "$snapshot": True}) "network_timeout" is removed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `network_timeout` option has been removed. This option was always the wrong solution for timing out long running queries and should never be used in production. Starting with **MongoDB 2.6** you can use the $maxTimeMS query modifier. Code like this:: # Set a 5 second select() timeout. >>> cursor = collection.find({"a": 1}, network_timeout=5) can be changed to this with PyMongo 2.9 or later: .. doctest:: # Set a 5 second (5000 millisecond) server side query timeout. >>> cursor = collection.find({"a": 1}, modifiers={"$maxTimeMS": 5000}) or with any version of PyMongo: .. doctest:: >>> cursor = collection.find({"$query": {"a": 1}, "$maxTimeMS": 5000}) .. seealso:: `$maxTimeMS `_ Tailable cursors ~~~~~~~~~~~~~~~~ The `tailable` and `await_data` options have been replaced by `cursor_type`. Code like this:: >>> cursor = collection.find({"a": 1}, tailable=True) >>> cursor = collection.find({"a": 1}, tailable=True, await_data=True) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo import CursorType >>> cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE) >>> cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE_AWAIT) Other removed options ~~~~~~~~~~~~~~~~~~~~~ The `slave_okay`, `read_preference`, `tag_sets`, and `secondary_acceptable_latency_ms` options have been removed. See the `Read Preferences`_ section for solutions. The aggregate method always returns a cursor ............................................ PyMongo 2.6 added an option to return an iterable cursor from :meth:`~pymongo.collection.Collection.aggregate`. In PyMongo 3 :meth:`~pymongo.collection.Collection.aggregate` always returns a cursor. Use the `cursor` option for consistent behavior with PyMongo 2.9 and later: .. doctest:: >>> for result in collection.aggregate([], cursor={}): ... pass Read Preferences ---------------- The "slave_okay" option is removed .................................. The `slave_okay` option is removed from PyMongo's API. The secondaryPreferred read preference provides the same behavior. Code like this:: >>> client = MongoClient(slave_okay=True) can be changed to this with PyMongo 2.9 or newer: .. doctest:: >>> client = MongoClient(readPreference="secondaryPreferred") The "read_preference" attribute is immutable ............................................ Code like this:: >>> from pymongo import ReadPreference >>> db = client.my_database >>> db.read_preference = ReadPreference.SECONDARY can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> db = client.get_database("my_database", ... read_preference=ReadPreference.SECONDARY) Code like this:: >>> cursor = collection.find({"a": 1}, ... read_preference=ReadPreference.SECONDARY) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> coll2 = collection.with_options(read_preference=ReadPreference.SECONDARY) >>> cursor = coll2.find({"a": 1}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` The "tag_sets" option and attribute are removed ............................................... The `tag_sets` MongoClient option is removed. The `read_preference` option can be used instead. Code like this:: >>> client = MongoClient( ... read_preference=ReadPreference.SECONDARY, ... tag_sets=[{"dc": "ny"}, {"dc": "sf"}]) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo.read_preferences import Secondary >>> client = MongoClient(read_preference=Secondary([{"dc": "ny"}])) To change the tags sets for a Database or Collection, code like this:: >>> db = client.my_database >>> db.read_preference = ReadPreference.SECONDARY >>> db.tag_sets = [{"dc": "ny"}] can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> db = client.get_database("my_database", ... read_preference=Secondary([{"dc": "ny"}])) Code like this:: >>> cursor = collection.find( ... {"a": 1}, ... read_preference=ReadPreference.SECONDARY, ... tag_sets=[{"dc": "ny"}]) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo.read_preferences import Secondary >>> coll2 = collection.with_options( ... read_preference=Secondary([{"dc": "ny"}])) >>> cursor = coll2.find({"a": 1}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` The "secondary_acceptable_latency_ms" option and attribute are removed ...................................................................... PyMongo 2.x supports `secondary_acceptable_latency_ms` as an option to methods throughout the driver, but mongos only supports a global latency option. PyMongo 3.x has changed to match the behavior of mongos, allowing migration from a single server, to a replica set, to a sharded cluster without a surprising change in server selection behavior. A new option, `localThresholdMS`, is available through MongoClient and should be used in place of `secondaryAcceptableLatencyMS`. Code like this:: >>> client = MongoClient(readPreference="nearest", ... secondaryAcceptableLatencyMS=100) can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> client = MongoClient(readPreference="nearest", ... localThresholdMS=100) Write Concern ------------- The "safe" option is removed ............................ In PyMongo 3 the `safe` option is removed from the entire API. :class:`~pymongo.mongo_client.MongoClient` has always defaulted to acknowledged write operations and continues to do so in PyMongo 3. The "write_concern" attribute is immutable .......................................... The `write_concern` attribute is immutable in PyMongo 3. Code like this:: >>> client = MongoClient() >>> client.write_concern = {"w": "majority"} can be changed to this with any version of PyMongo: .. doctest:: >>> client = MongoClient(w="majority") Code like this:: >>> db = client.my_database >>> db.write_concern = {"w": "majority"} can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo import WriteConcern >>> db = client.get_database("my_database", ... write_concern=WriteConcern(w="majority")) The new CRUD API write methods do not accept write concern options. Code like this:: >>> oid = collection.insert({"a": 2}, w="majority") can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo import WriteConcern >>> coll2 = collection.with_options( ... write_concern=WriteConcern(w="majority")) >>> oid = coll2.insert({"a": 2}) .. seealso:: :meth:`~pymongo.database.Database.get_collection` Codec Options ------------- The "document_class" attribute is removed ......................................... Code like this:: >>> from bson.son import SON >>> client = MongoClient() >>> client.document_class = SON can be replaced by this in any version of PyMongo: .. doctest:: >>> from bson.son import SON >>> client = MongoClient(document_class=SON) or to change the `document_class` for a :class:`~pymongo.database.Database` with PyMongo 2.9 or later: .. doctest:: >>> from bson.codec_options import CodecOptions >>> from bson.son import SON >>> db = client.get_database("my_database", CodecOptions(SON)) .. seealso:: :meth:`~pymongo.database.Database.get_collection` and :meth:`~pymongo.collection.Collection.with_options` The "uuid_subtype" option and attribute are removed ................................................... Code like this:: >>> from bson.binary import JAVA_LEGACY >>> db = client.my_database >>> db.uuid_subtype = JAVA_LEGACY can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> from bson.binary import JAVA_LEGACY >>> from bson.codec_options import CodecOptions >>> db = client.get_database("my_database", ... CodecOptions(uuid_representation=JAVA_LEGACY)) .. seealso:: :meth:`~pymongo.database.Database.get_collection` and :meth:`~pymongo.collection.Collection.with_options` MongoClient ----------- MongoClient connects asynchronously ................................... In PyMongo 3, the :class:`~pymongo.mongo_client.MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :exc:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :exc:`~pymongo.errors.ConfigurationError` if the user’s credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. The `connect` option is added to control whether these threads are started immediately, or when the client is first used. For consistent behavior in PyMongo 2.x and PyMongo 3.x, code like this:: >>> from pymongo.errors import ConnectionFailure >>> try: ... client = MongoClient() ... except ConnectionFailure: ... print("Server not available") >>> can be changed to this with PyMongo 2.9 or later: .. doctest:: >>> from pymongo.errors import ConnectionFailure >>> client = MongoClient(connect=False) >>> try: ... result = client.admin.command("ismaster") ... except ConnectionFailure: ... print("Server not available") >>> Any operation can be used to determine if the server is available. We choose the "ismaster" command here because it is cheap and does not require auth, so it is a simple way to check whether the server is available. The max_pool_size parameter is removed ...................................... PyMongo 3 replaced the max_pool_size parameter with support for the MongoDB URI `maxPoolSize` option. Code like this:: >>> client = MongoClient(max_pool_size=10) can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> client = MongoClient(maxPoolSize=10) >>> client = MongoClient("mongodb://localhost:27017/?maxPoolSize=10") The "disconnect" method is removed .................................. Code like this:: >>> client.disconnect() can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> client.close() The host and port attributes are removed ........................................ Code like this:: >>> host = client.host >>> port = client.port can be replaced by this with PyMongo 2.9 or later: .. doctest:: >>> address = client.address >>> host, port = address or (None, None) BSON ---- "as_class", "tz_aware", and "uuid_subtype" are removed ...................................................... The `as_class`, `tz_aware`, and `uuid_subtype` parameters have been removed from the functions provided in :mod:`bson`. Code like this:: >>> from bson import BSON >>> from bson.son import SON >>> encoded = BSON.encode({"a": 1}, as_class=SON) can be replaced by this in PyMongo 2.9 or later: .. doctest:: >>> from bson import BSON >>> from bson.codec_options import CodecOptions >>> from bson.son import SON >>> encoded = BSON.encode({"a": 1}, codec_options=CodecOptions(SON)) Removed features with no migration path --------------------------------------- MasterSlaveConnection is removed ................................ Master slave deployments are deprecated in MongoDB. Starting with MongoDB 3.0 a replica set can have up to 50 members and that limit is likely to be removed in later releases. We recommend migrating to replica sets instead. Requests are removed .................... The client methods `start_request`, `in_request`, and `end_request` are removed. Requests were designed to make read-your-writes consistency more likely with the w=0 write concern. Additionally, a thread in a request used the same member for all secondary reads in a replica set. To ensure read-your-writes consistency in PyMongo 3.0, do not override the default write concern with w=0, and do not override the default read preference of PRIMARY. The "compile_re" option is removed .................................. In PyMongo 3 regular expressions are never compiled to Python match objects. The "use_greenlets" option is removed ..................................... The `use_greenlets` option was meant to allow use of PyMongo with Gevent without the use of gevent.monkey.patch_threads(). This option caused a lot of confusion and made it difficult to support alternative asyncio libraries like Eventlet. Users of Gevent should use gevent.monkey.patch_all() instead. .. seealso:: :doc:`examples/gevent` pymongo-3.2/doc/contributors.rst0000644000175000017500000000421212630145074021011 0ustar behackettbehackett00000000000000Contributors ============ The following is a list of people who have contributed to **PyMongo**. If you belong here and are missing please let us know (or send a pull request after adding yourself to the list): - Mike Dirolf (mdirolf) - Jeff Jenkins (jeffjenkins) - Jim Jones - Eliot Horowitz (erh) - Michael Stephens (mikejs) - Joakim Sernbrant (serbaut) - Alexander Artemenko (svetlyak40wt) - Mathias Stearn (RedBeard0531) - Fajran Iman Rusadi (fajran) - Brad Clements (bkc) - Andrey Fedorov (andreyf) - Joshua Roesslein (joshthecoder) - Gregg Lind (gregglind) - Michael Schurter (schmichael) - Daniel Lundin - Michael Richardson (mtrichardson) - Dan McKinley (mcfunley) - David Wolever (wolever) - Carlos Valiente (carletes) - Jehiah Czebotar (jehiah) - Drew Perttula (drewp) - Carl Baatz (c-w-b) - Johan Bergstrom (jbergstroem) - Jonas Haag (jonashaag) - Kristina Chodorow (kchodorow) - Andrew Sibley (sibsibsib) - Flavio Percoco Premoli (FlaPer87) - Ken Kurzweil (kurzweil) - Christian Wyglendowski (dowski) - James Murty (jmurty) - Brendan W. McAdams (bwmcadams) - Bernie Hackett (behackett) - Reed O'Brien (reedobrien) - Francisco Souza (fsouza) - Alexey I. Froloff (raorn) - Steve Lacy (slacy) - Richard Shea (shearic) - Vladimir Sidorenko (gearheart) - Aaron Westendorf (awestendorf) - Dan Crosta (dcrosta) - Ryan Smith-Roberts (rmsr) - David Pisoni (gefilte) - Abhay Vardhan (abhayv) - Alexey Borzenkov (snaury) - Kostya Rybnikov (k-bx) - A Jesse Jiryu Davis (ajdavis) - Samuel Clay (samuelclay) - Ross Lawley (rozza) - Wouter Bolsterlee (wbolster) - Alex Grönholm (agronholm) - Christoph Simon (kalanzun) - Chris Tompkinson (tompko) - Mike O'Brien (mpobrien) - T Dampier (dampier) - Michael Henson (hensom) - Craig Hobbs (craigahobbs) - Emily Stolfo (estolfo) - Sam Helman (shelman) - Justin Patrin (reversefold) - Xiuming Chen (cxmcc) - Tyler Jones (thomascirca) - Amalia Hawkins (hawka) - Yuchen Ying (yegle) - Kyle Erf (3rf) - Luke Lovett (lovett89) - Jaroslav Semančík (girogiro) - Don Mitchell (dmitchell) - Ximing (armnotstrong) - Can Zhang (cannium) - Sergey Azovskov (last-g) - Heewa Barfchin (heewa) - Anna Herlihy (aherlihy) - Len Buckens (buckensl) - ultrabug pymongo-3.2/doc/static/0000755000175000017500000000000012631423130017003 5ustar behackettbehackett00000000000000pymongo-3.2/doc/static/sidebar.js0000644000175000017500000001421412630145074020763 0ustar behackettbehackett00000000000000/* * sidebar.js * ~~~~~~~~~~ * * This script makes the Sphinx sidebar collapsible and implements intelligent * scrolling. * * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to * collapse and expand the sidebar. * * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the * width of the sidebar and the margin-left of the document are decreased. * When the sidebar is expanded the opposite happens. This script saves a * per-browser/per-session cookie used to remember the position of the sidebar * among the pages. Once the browser is closed the cookie is deleted and the * position reset to the default (expanded). * * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ $(function() { // global elements used by the functions. // the 'sidebarbutton' element is defined as global after its // creation, in the add_sidebar_button function var jwindow = $(window); var jdocument = $(document); var bodywrapper = $('.bodywrapper'); var sidebar = $('.sphinxsidebar'); var sidebarwrapper = $('.sphinxsidebarwrapper'); // original margin-left of the bodywrapper and width of the sidebar // with the sidebar expanded var bw_margin_expanded = bodywrapper.css('margin-left'); var ssb_width_expanded = sidebar.width(); // margin-left of the bodywrapper and width of the sidebar // with the sidebar collapsed var bw_margin_collapsed = '.8em'; var ssb_width_collapsed = '.8em'; // colors used by the current theme var dark_color = '#AAAAAA'; var light_color = '#CCCCCC'; function get_viewport_height() { if (window.innerHeight) return window.innerHeight; else return jwindow.height(); } function sidebar_is_collapsed() { return sidebarwrapper.is(':not(:visible)'); } function toggle_sidebar() { if (sidebar_is_collapsed()) expand_sidebar(); else collapse_sidebar(); // adjust the scrolling of the sidebar scroll_sidebar(); } function collapse_sidebar() { sidebarwrapper.hide(); sidebar.css('width', ssb_width_collapsed); bodywrapper.css('margin-left', bw_margin_collapsed); sidebarbutton.css({ 'margin-left': '0', 'height': bodywrapper.height(), 'border-radius': '5px' }); sidebarbutton.find('span').text('»'); sidebarbutton.attr('title', _('Expand sidebar')); document.cookie = 'sidebar=collapsed'; } function expand_sidebar() { bodywrapper.css('margin-left', bw_margin_expanded); sidebar.css('width', ssb_width_expanded); sidebarwrapper.show(); sidebarbutton.css({ 'margin-left': ssb_width_expanded-12, 'height': bodywrapper.height(), 'border-radius': '0 5px 5px 0' }); sidebarbutton.find('span').text('«'); sidebarbutton.attr('title', _('Collapse sidebar')); //sidebarwrapper.css({'padding-top': // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); document.cookie = 'sidebar=expanded'; } function add_sidebar_button() { sidebarwrapper.css({ 'float': 'left', 'margin-right': '0', 'width': ssb_width_expanded - 28 }); // create the button sidebar.append( '
«
' ); var sidebarbutton = $('#sidebarbutton'); // find the height of the viewport to center the '<<' in the page var viewport_height = get_viewport_height(); var sidebar_offset = sidebar.offset().top; var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); sidebarbutton.find('span').css({ 'display': 'block', 'position': 'fixed', 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 }); sidebarbutton.click(toggle_sidebar); sidebarbutton.attr('title', _('Collapse sidebar')); sidebarbutton.css({ 'border-radius': '0 5px 5px 0', 'color': '#444444', 'background-color': '#CCCCCC', 'font-size': '1.2em', 'cursor': 'pointer', 'height': sidebar_height, 'padding-top': '1px', 'padding-left': '1px', 'margin-left': ssb_width_expanded - 12 }); sidebarbutton.hover( function () { $(this).css('background-color', dark_color); }, function () { $(this).css('background-color', light_color); } ); } function set_position_from_cookie() { if (!document.cookie) return; var items = document.cookie.split(';'); for(var k=0; k wintop && curbot > winbot) { sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); } else if (curtop < wintop && curbot < winbot) { sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, jdocument.height() - sidebar_height - 200])); } } } jwindow.scroll(scroll_sidebar); }); pymongo-3.2/doc/static/periodic-executor-refs.png0000644000175000017500000011524212630145074024114 0ustar behackettbehackett00000000000000PNG  IHDR?{LsRGB@IDATxUǯ-v'6"*݉-*⫂5.l9<ĝ{w̹;Ap31 Cxa2Q~AkZk>`@+"`ʯ{lΔ=!ВkV[ !믿/R/B?'L~ᇜ~r>dM9934eYrg?nmsL3MuUU'|}'?cg`i߾C!9ngQ\38*lEw=Q^'(O~+=ߔ{-n饗m:S9ܔ_ܺc,^z)֣X^^,nXc6[,QX?(d7|uuYjV_}\sy*) @S 0~xcGyD>M>jA@V]uU2˸駟!u&?{w+^~eF&x]Ν݆nF  GaÆ#F^{MJЩS'k馛.s~{9裏7knos j)m123]c[mͲ2kou]!=Gn]vqM5> s+ݨQt1gϞjzn)hiXla%o`}q]tqF,BԊ3r`/w_|&lk/׭[7u+9Q~wwwk=SnEqGy}SOJB1dK/uoA+0'z4Gqw}ݴN[GM0*G%\`w]ؙ|.r֕38GLϭS~eex' Z]R6*f'0up駻AW^]qnWR jv!sTS"8SMewVƱx yeW}2eݻw(vaA)D\ve裏V[o"+<0 XZ,S|n__|E7n8 {+QZU hLBM7T7 .8}k(/*[L DLUcd!㏫s27tS"露nuu]vunvٜ_YC`PJ*n-p7xcnfߪF.,rH!UK`l!Vcg 3ڠͽlG@ MǎqE<c 6bpnaVmӌcǎU9B.D\wuK}wm {Bc I/ZEF{꘍U Aidwv #嗆N6D7ȓqGr3"|H.W(Q(;˝:âSHl*CJL]Q%Li׮]p`f+$*BA>2d+ # 3<̣bO+qdطo_OzH|]reeZ.H#cu3[::1w⋫1~+[iԒfAÍ6HdwoEWiLBw[nI;CAqd\##a9[9GuL^P$(|z-z 7JKyPͯ:P; TXT7pCT2>'p9rxrzc\$AaCłҎI:T0WL C~aj?$qZwf @l2?ݻwfŚ ?OIUZ.SXd N:>=t|c=6q?eN/T}|U/kQXZlwaf 21J rcE)YIMIp/]xչsu.?s1dҲ2 0\s5sF o!L 6C^XقJKh,S2aF%ϼ5}S~hwCeƏSfc4.@,*dwYge2,~p|J9(>##\!1Wg! O[^e+:\4XeI7[XeU XH!IRkY\#Hɐ a7C !,t!YSNѕ_Cg~֥)rS#_}UQzZPZ SY=5/rDzS] A9%M_*-kĉ~Bvm7U~xaxJZIWP w9]H{;8Q[}wܱPUd0*B@A W [_IoJY z ق:rHpwk0pb%.x %!RY9TY(2+I7|sL\\/\}Ձ02Kl zhgΝI<9!y ?w=Ju֍Ylow5 <^ó(_la?'T %&M,.VWY"2˕9BHc1%[*KX k:-8t?cٻCB ٹHfe !PPF|˝ڐX~А{,^;oA+NV}ɮ2 -6'ԹC)&Sa K;ud5Rf Vs~廤4^OSWNFK/s`g]zWj3}kvbKF;f~rlt >ljJx[ + ,H8] Crh"ɝ4EBݡ~2o)+PUR0Vc!^Vvgh!dߢXM2Xtq)yփ +-&az0#M̵`KJ/xBvU:˪jB`6Se&LPvi bW,Xxv_㰶;!{bs/%^HY~H09ҍ=nk EY$,?ABJ@-,D}Pt uqw;1mҥK(k [|u` :wR6V{0[| 7du~wV% (;ＳfK⣭6{> "P/{KMF2ϏK)[MA'ZZz[*6fEl/-=ih9{]wyۉ!P *W؜_@i@6InS~kR/{KLӔjk vSC>vo6ɼڔ_2jDҊpڂG+9xF,t$27?2aC n5>@V@xzS Z{z͵6h{K~XoRAeo/wC e!nLQZ*4@[]tQ׾}{AŔ_{Ž(eY&\]ZcL:qroak_ҟ(!R iQ~fUv!D{K\Q~ƍs;7nms~ug#lvm,IKRoY]S";u.b7qDmk[AgyVB -(Q[x[*f^xZk)aTSMv}w7,/A(j24( CN~-q箾jMbg}ӛ6B:!r]l B\$ݺus:t!1t&#`ʯ~ C H/"gSht %!P?_~ yɠv'k@,@K (s~f83uAϮ5BDlaOt(?p*ڛԞz1&k/wie oرqV1˯bDC Z|<SL u>;Βs)nw5r@u]z뭱e[` 'Ql؛^:7>c^~mf">?ЍZiM;n9ps9kE/: 6`!(_q_=Z˛iJ+V[m5V_}QH>Ӷ"*dl (%")~aD1a V^yeK]XĊ¢ܫ^yGH{^xa׵kW3nʯ. atR+ n.]Ν;֬o' Q(DVQ;.0 iFÔ_ 4wZwPGM@#GnMS`~{dxlR*NJݐ!C[o~T\^@c[orhPX&En \IKC9DޛoK颒fh{3<%{#FBKԨkv"c9R.m馎U+BDS+Ygdí暪CMj2嗚BvmƺB;V蚤[jT b qIzª)rbO>?]Uל~]ys{Cm>0Kc f!f/d@x pgǰdlY{(D@zׇڐ/|駡٨L5 iOC N8w:&عa|O>3?P`ےcVߢ.w-k:K8_׾}{S~e'!{=n 6./F\dpuYqWW'b]l_DC\|+5N03 {/*r KIS~I1ou=ꨣܹkC6=>5ﾻ#,n,~r#XIT~FlPSP`oUWa̬nTZ+[tE5+'7ɺKv@dAF *C\^{(M3Eo9Rs0޶Og}&5a ~$ Vzj/mM5~wڻ2h2r2TX^馛z쉿KX\UvA +9dAIjF ѓ]s-;<o*{s4˯C@bqdJMz멿a*v[lM(4z)[י[k4vZ.L<#v8!8{uX0)3lذVT| 99~>loСGR@r{"R"Xȼ֪bʯU{>f {ӬnE]`laԨQ-?5`~CǼ{zz֪aV[mф܇$>GbeJcq /B_Ĕ_:̪d$яUeŧ3;2Q"Xo(̱cǺ]P$ɔWA\'ɘZ߼z5jFb85{+4A,C|PpIq,=s ^/4x펆@a|:KrNm)Fi)V!Pn!wh==vC"COV&bN2V!ﾻ^s5aiA_`l! X=ӧ8plFZ垦ZA1cȑ#S$VےkV#4v&Ӭ-A ^,[kX[Eٰ7U;ȏd:|S~+L<3 4IS~I9w 1q!0Wzv!!qv{!K.a/ڔ_  KKrYSV5_r%i+$b/ɽguo Շki4@),IHrY[w/#%[GJ:Қrː!CR5reK(^h/-qu{-`e*fs~eÆ@0zB 2@^naȠ K-l\flnРAu#C/X *n/NrDg}2,U6oX xF'9z_V^yJNS~I1! 4W_u38cyd#a@h oVr;AL -@^^{ yy`LבhUVItS~JkH!HѣGkL)V{ʬ@Ln/^ pωۑ Y~hwC axaÆ^u!L34Y#6 #@]2r|*),?ӵk׮*$!P7|]|Ūn&!?c׷r[߯CW\ѡ/Q}= @RO>2-&X~&0˯0.5wuccKMΎ@Xn܃>s}ưsG[ͭMEjD,wygQVy>c:_=47]K {g}iʔ_6H {v]tQkA^꫻`Ii/ɽgu7!=؜!0/QLWM-@Ǎ?f,>0W3v!`4 +uM1zzF1/jmjYPz~[k &ԌC^\Ғ(GD駟t Z_U~'ďBQyPqG($tM禝vZ4Lp:.~ v"= 矻^z9k;<ǿeOGڷob 7lEB -o4\[Յ7nn<|I~$[P.3nEu-XfC4!(splZldgXds9gdf!%f1N<kZkY/8￯+RJcGۈ /y xC6Dല`=y)g}/l+br٘oxJ bD/:[x =sy,@XPvcL<2V#H=t>'&H ^~eB-Қ3 +JI*d1X&c g[wuIsr=[oU+;jR/÷V[nE_L({̟gsO={GO<b7 \"/8ZKczE,6m0}x݀ZpnJL+ JpСjn"VB XXt @~0 ,:dx㍁yJ,t%X@<%ˋ]qS,e_ k*gCp 7b(6@<#QVbXʏ}恬(jd'|Q6ʎ\{;j> K,Dp12_x yQ}~6&))QI]ѩS@OhYtO?6B'E 2/EK <U(%qȢJZS2?lھm&x뭷lGrx駃 6@_{g @9'\':a,g]ӎL$}v\uU%& q/@@FvġҲFSc!I8ʏ7>g ZY c@eq?@|F'H8Yb׊J$S~鋑gCTj}OVjmVRY*Uo %$:O,!n{B +sW&"p%8Sxmʏa,2!i&}=}`זCaÆ$߫a2DÔ>'o&%5F;0'n#OLw DC5.vObG._N[:r;Վ GaGkH0xNUVVi8%n(O"%N?4$I';yi҂V"V^GI`W˴+%/wOrD,Gw9h}p dE"?E0/WaKsjudNb#*B2S;{֥ s[|mS,L=VhO!(%~^xyk00e G0G( '[yj-*Y9._t<3%f9dSIXm:[KY #s3GJ;³.N?:ԉBc|f+cp˞_=I؀|f?^hޑ,w cn11wYHHP)Vl!:@O^('*"܍YY4iQ ⋫DM*%ڙ!˦̗*-69{zW)ߑ{뎐Lw-1/ŤKe+Zm~88ꨣtQ/',pyf(4̅|uʞ+ts?~ /r2te&eEu0ioc1*r BH..֓qJ_~ńeʨDsY=eBܧsHJW"mJ/TЪui=Չ(wFM%FqhV5؝Cl (2+hm,V ٣$D6{õzaIa "A ֋?RoiXjD^QHw}kj^e#y4Cѡze o^?LUgWwVR(cqqvucJYkꪫVv<E\]tSǛH i]f4.Lvu.S&~5\aDC R=roΗЪ6`/ "F$[ $.tl~FɼW':A;@3r)V t?+Q2KNSxHɪb!nS \SB=e ϾUxc%^^6,IR˦yg%{|#g_C֯: 5$dPڮ ّ@cC)sX2_@Z-9<>e"^!F:L: 7c=V)Vs($>PDdLrV_@De %B[pU($\ԛBK :+^tNP{lQrW96ulxP򴇗g&^|@]x0|G- 󀂒}>J3>/OIhj}뮻2wi>( C"$"F'Nr+~dh27a )wϫZyF{JSw`җʏs ĄT˩ЏwR9-pA2+n$X.ME!Xkm&ȃ vD.h(X1g\!Ѩ}E@k؉?a½ixr &IC<`0Q"ftpS11@@,ePYU2o 0,HD^V) RV)/> /\U9)&FTj.s WP`b޸LH츲c驻V@׳!m.ߤ&}c|H"C&;"%܈3T\4vбKJ4| H"]~ΤؠOJEi/1eG\ |%d N %f,~nE3(7pa$ݷo_,DA &?& pj-k1ǙRGQr.C6 @e;7:atXr@IDAT51(yc6@|&fUrh:TCT C#^QߣbҚn^*B`Ű!a&6Nt#A}<(8H%qiP#u qlg$r B)yTxT/oTd;9Ox^$mV3s/Jf3Yp eyѷ l؉8C9VeOz!Qu˃ V5Jψ%}(a9zɽe _B~'/z)(k~k~4'> JnuuY:_f8 ֍O `!Vyw>Ɍ`OB}r?'P^ԍٟ&+euق敼 (#4}<.Js%5=6:CIѯyy~$yyneo\Oڃ ?[|Oɡ JQmXnW#,>Dj?VNg^sA)-|2H)=jxO&@KJ(_DO>9a+xXX$V /Y/PLLŵCwVok^u @B Ņj!, [!adxӖZZ䚚*kJ \DhbwxQf(>%_mŦĔ_L{wVW%#\LkhժpY0`.jHohī 3׈2FXH!F+)6zҷsLhժ$b }IYJ$U.~qo3٘Yd"Bs{B8l!.۟`ITj5mht3bcIh,z oLeA$ ` +- N&U\jZ`T-iT2D}L:hS~ $~P] :TGݘ*OBvATld5i~6mILzj^UrLY8=ܖ گ\*AΉ5HǓ0HV[m뺇Q9vin 6p/&ܹsEVW~꒒5X_؇~=2vXiK/./ m,|̔_)tplܸqHM@cUK`yu(VX!M]LUR OaMM_ZlǎcؔD4o?wl 2@/.pAQ|Z6C U;6lSٶ(99('\vIR~Ě[s5<̣,˦ !e*E_H<|WL2}ݎ;~mNs曻:ʝ|Gu7_9!P-5Ֆd7hm"= Isx8&3*u޽Bj<;J*B>C }(=[H&AA<6z,<#ivZ|]tќ%pa!0a}ņU>C*8.]pޔ:UrS,6۬ͩ 塟RU)dW{3'{6BDh\ΔsRX;'qT~cƌq}y)? G1L^Z-JαuYg6U)RQn^f{Ǐbb+=W[m5U`<(lEwCrDiBd@|Z!eB2˸^Z?b;%,aRE :tP:ZX)V&n4(`W^領e72;qN[o$z_.sVkxԨQp/,!+ Nq~TX(n}aQ2T&Qn6xccdiRgroRu} yP5 +b6{++oQѣGGzg=Z'S~9tMkln5,sei n'ƒyƍ6HBƷF3ר`ꫯvW]u:bCgr9lT5bwɼ {4lt0,rU(5+:3thcg] I+RyA5x'gb=U;ja$ڸN{ /3,[OL(ExCg^??VEGn&!$guk׮`ge0Wj3.sX\s{Wե Ƥ6Zk-7rHWJ+d3l'KIB?O$=L%W1gJ/pWĆndj8p#ɛo Z;jkvNsMQ?a0矯 ՗dWCx{5|dF]w]x鼕Ŕ_뭷j?BI" P{hw}wWp&Vca\day ! M*ChVft%S~=+9gA )N:餜cOc` V $nDiꖟ {|xz-;ܓ)*J:,w7Gtː1/=}yKA%6s{:,_J޺B ŧr1 C^,uLJ.E<<$0+LTU^z92UvuWMI]M bK1dIe]7ܯaMB5~vZ=p^X9l^3s]w)s $:E믿VrQuőτ;HwY !/yGg"fjh'd99 6@_Ea 8@C@C 27)$ /sfm{@^{-J`La2ȏX u $$s_ʕ#J)| D;S O>D޴S2zꩺ ~@#<2Hf7=WhQXȁ$Z dqH! N|:$aQ U+/\ 5"!02&e>K"2gVX`E "\,zle+VL0SB{ａfKr$YM b 2IH^ 4[Z;zK]Y LXkٴ |H^+%)z,?.K<l+r24)xƇ-rܣ:*S4M.#qpq>g}\9,R@[d.%3=ap m q: .p; A^@y-{XF_7_ԧ~3VeOa^9JRQp2/\`vr@Ob|SQʏ a9b.3is{ҒV{+۫eU؂"k9y'XY2szrX!YaHyeS4~$''wI#gu&m ](IZ]LU0T%l\~(D_s֡o߾S-BrXEc_'LN=-R,^ex*zJGH![0CO[6b0o)&w]F !qm EbWQ./DWW%+W B L Ɂ\b$izJ2ל}ف(<]':/=cdv Xd`e܋ _\D7~x-KH!)D J<NK,|{iiNjPϗ]Eer8.saGVző[Tpy' 9䐀">zoI-Qvhp+b-cǎkV:_G(\Ia^xဗY)j!Y4;j.\϶*0r'ݼ&"V' /[Q_@ H*Wq>I/NRIz30#6(61Rd?&[Hmw-hVsL/c۰Et a-,FlPmC]m ܋/Xv~O!iC/"eɔ4UV{kmРAQbW~JKB@lX\Ybq"Ŕ_ O.!S`m@ݕIx饗Uj_B ?Yяe#)fXP'G)ocAK/uD@X!?jCLUV޹8xG /w&7U:3 ~VVz0Bߒ6np׭ڮO>rOE jxw7+0R/ :ԍ5ʉӲКŰ2[+0à` 5mV_O.$AKׯ siwKgE㮽ZVꫯԚHt6Zhrș袋?fJԲBɇ {8T* S~5VEp Ӊ 8w{SVOIO2gr^fU9??<̙?lʯ]D=UHJAs=wӴB@ 6LY"i[neQk*w}w [WULUW'hJ3dFTda#r 6<"+D9O'0}ض[ngZ\L@J5ra +ٛpšՁ!/?c<DŰ!c "8C;ȎUVY 0 2Z)8v~l>* 'J"Ffr޲faFY|Z-DL0=ܨSN8J+mm Ji/ ~uFL B0?I5 ) ߄iYאo6|s# ߹+lPޣ7K.*d2:(=sQxbwzhlԤ mMBےul7n&?+$KsUTbS@$:''q-[l1'lϮ,ua *ٔ_:+̟\ ]ZGydG7o zEm1RwI$LMo䜬 BbbhbW\QOQvm`B#FE4:;197?C` !\W%vI zmFlTD aԨQa4 S~M17FabYL"kU Uqk?\&#!`/v]^ U víFWa+h @}+HvqG +W02jo~!08S|)\kRذn[oVof ELE_J6Y5\KL_G}Hd/\Cpt6S~)} +֬j 0'7Tz)`#a6)%M,Icoرc?4W0ȫ^&fo[o5tȻZG 5oR'Ej}71aȋ7LM"5"T~b~A!B,r [¨IcoƐߍ~#_~H`d/R$,oݳLՂZ̯!1o :T35vOCLUR5oVvJ0W J :믿V +7KHgibS~qUPeh {t7es5Wt7  `/݄fež\`1)IX'|]tT~Lg^8{W,?GVRfU_ݦ ’ܝ(EY$V x5Ɨu˄ Lo/$K=v/&l2ۤ F'ߔ_ IO?Q^{5cȋذ7|S~x$?(?vu C^Ĕ_n#;Vǯ[kޣ啟qA~%@`ܸqnyƞZ?_V"Y~>$ rwIE`wڵsv[lpWvGqvNS_M;U,U oxc+Zkv{nuu Dzƌb\jaoj9PQ蓑#G.GQg!(k߇ 8w]wۯ0\ght?~?5wGķW^y%3WbV+Fj q\#\< bHL F!`ʯQHG|8+?ݡZ>q;h)0lbYqV~ .wq68iy Nxށ(?|䦙f+gy;S ֍n杸^h/%=|_!xO:$w9:d+Q}' kq48#I<ꨣ_[mI];@cIŦRҫ~՚> 4(Sg/G(_p&Gk )$rqM7]!C ?ES~v2%nKfNΥI~-w}w7S޽{^{ :T}u 'l%SM5.p==2s93܆$PMWo%A$S~x$?,f)1uȏ_~Y?x G35jn_|o-2z\rIX`UTz?A۪l;< "Zs5].]tC!T"X}Ϥ-D_guq\ÇW~'|Rk:w6xc*7$~ꩧWnA2ݻww={t믿~Ji B`)B$pC9J1F5+=C:Ƕ[2tMc"[o=ﾱ̋Ҩ>M}joJzfXp袋taa7w7k*skK-;`(Bb*סCKs=ŚRߌEX8V{1;$GB ĿkVz}饗RԴv3La+ffɉ0`ÂJ ;+AX$S~c5j;zh&8q` r>g}w}ל^Z^^<&C_`6b{ 78p@F\s5WR)ɜ8wǺ6ȑ<$YKV-3âO>sr3Eh}'_|Qux!'EL )¸$n/a[PG!'Nt]vu$""./;vT6L{(2~i]믿j>YGJz8*=1-)A,f@(t . :ûA%73Zc̔_Jaa !so;L#pg'T])(?HL #`ʯ0.q^_As}Uhn%J#oʯ4>9wkT:q?k )}f IQvXC`?9S6p5!ml+)$(sLa)Ĵ?neD\ު1ͨ)Ҩ+Ob~Ǻa\ݵkͬ{ou7Na*|Vɉn,nkƭqO 2IBeH-*@x.T#FZh@(Kcwq'*W\0ńZV7K;l-ðp 8nVQDBT꫇nvI25׆=?S]LPsE.$ 1ci CMCltJBo/ԉW[6$7Zsy$ C$wygv&s[m2$exR<~pru^\Cl+MP^>CvErW$F=mhqĺk$ ](wZ/9,08f2Yh^lQeh@U;mYϓU'ˁ _IY:_ ^ (59QEH\%u ԉr1ۜ_qlw֕?zCa8KW",!Nj DVZi%u]9ʪ8/a3\b-i쳵밆J.dS|fS~觪j U]XdrX|2#,;:ĥ=D|1!W\qsDmtf/E雂ՃP0E2t"J)|aEaHjտDn(mZ}Ck/=X(?[':Db-tU8{ŭ-mݦ){JOX8kt\sMD€뮻*#GLyMEqoe/Cθ ɼY\+*c$ UiP$I-L`vA}!ɔ_`Ʋ(oר x( UǏXy裏,mĢK+|%II+y&]vE 801 fKY`nIk I5H7KN_TSV,QsOM"Sg6dܝ_AL%=;X%mZħdkbԊ)ZKu }%. $E-?S~E! `ʯ~ 34R/%-+?7S~þawB"9ܒ.6Mzƣעo߾.2lذ gިNwݿ-UK/Kl؛GMţR~)i$ inbhCfwI^S~˲-㏜'[P7UxEkF^Fp8M{zғOT$~lț?7Ncs0W lIP~ؑB|͗{+)Ф 'zw&Q3s_}Ƥ2LUSz/KDao^/uY~m)ǔ_1dRcgu;5PҚj޶=Mi{D_AXҿs7WJ#<2qao.3-._)tR~|XnmbZk_ۮZguLS~%IArX{n-T߹$&!`Jwc$e]ٰ71]늚uD_EYqJu=|ĆIє_(RLذ7 :E^;rtM~"h׃)?{m1i~8EYlINL%"%\Igao{'9u3嗜-;t{#_7ao.r#FpiE_YZ;̭ n=pO,on뮻ܽޛ+) SL12dE8 {s{eرniE_YZ[ѣc {s/0 IE;%X‘608Cj@*0W^-sSN~o;cnNr`pa$L*fʯRZeY}27adȋILUTw]޽{9(؜ߤn`,v7 JUHy^{ޓk(w}{5gy;j*@5j8qbfv&;vt rLSTd2aTw݊9R9H뮻6 ,ρ7~dEdAfrIaϦ"0܆M?97֭R_׽m*#zws0? Ňճ;{+0b^OsIzJ+=z_UBr!K.Ϯnn/xDbbT)jjs۵knVG}Os饗*2\sM>ܓT|2ܗ!P Au$ž-ܢNЏ?x&x񎅈{EiXo¾kNM0!G)aa-l*˯W^>C$JcSN_|Sa$n ]Ȗo\8)LT $h& j R4|%(41)``"!CE %J@Nsνwfcc;wfc?{fݽZPDH֭*bK$82dDwy~ K:QwUŦ<܏ S) U~ᖺ[m}W-%]&n|[RyIxhѢEfIJYP7,MJ@ 믿.3-jVXafWjzQaqUQ.6olQ) U~ʧ7n,t˗/`>A1=@if:4hPSsNV T>ݥK/ecC,ńao@}=zYTy}U~Kl|Msˌ,[ !ҢE ӭ[7):,;v0.{=| x\Re$n)TTT|qSλ-[ 6dM6 ?;wJL/~F#6mT&M͛mfdiF,?]pfժU¸Nѕ;c]v(.fNG/qF 8P\(1^9PrK4Q'~g7PQMm>|п:rBbvx 77T@h4C[/!筟hĴlReB 9I P:uz<H,H[MjƍR~>}ٚDq?^z+J~ Uʌ@yi!`3a„+V8V9ַͱJ3b *XBPǦtlә8q\ ; 3tP*Ofdsrwuӿdž9LԱKwquʖ{bt>/;hԱa]ʵdN߾}|26|$.{:gmU<@c| A}'ls:Or۵k'%{夥ԝ^gABey]=o~םw"%9g#*_*xYʺA!Vى+½E 98_޸mbyI'y&id1 /\0bߎٗ?*mao*( gX\!.X_.̊H-I2HngϞfn1^q&C?:A!V_ʞJAg1h)qB`РAA8V l\\lr⦂ؼr9b ?e˖~-͛;6αJO]T,\wkpx?ֈ"p[:yq{qo&'e{[WXvU֒ ϵV[ܹs* .B3&vY^2s?Wpщo-9%(֭[g[>X5_PĤuvsu%OW%O(#I|2f5ʱ{q].F姞zjrӴF۽I.?)Fw8(jϢZwj׊9AlK1轵#y{)B؛ 7$Jb€BX 1@#ۂ[Jqy=#6(c̘1r]>ԇD- k"y~<[rԫIFVZڎ;"UkpĵNh w ~ KkTC 2𣞔QQщz*%2䞵QF9_?6f|\r 3ŋ,o֬YH89#8 K9R-#_PSW",( rBJKK%yϻQU-5QޡC1;={aSU|E@pja‚4ɋ /er@+7 1LIĒ:^B@.`HN3^ؒmNqaQWT V L" Q:$Bpi۶lVf$an t 1eʱi&Wء ca5w}eN- gb= =GJf(OԆʁrM ߴ⛠*5{[F3.]4Z&avYeGK/@wFH\(=r"(^xfٱMOL 8AT V9DŽ СCͤIӛ.k7D@LoŝRQE U~ 536-*NLǴ# "`I2uGڮPGX1E8\9TP~Mnlq%K)<^vҩS847mԙ_GM\<?υzx8/OƶjiËAjyǍWce6hvZR*ӰGB&f,@DҥKj EO}oGr˅զep*UW&?@ZUѼጌ*ppZ"^M81"- !RU~©E b~nʊZ{nj֯_/QZEeiӢ0 I~N`P'`#!C$SɗիVjD@W0KG9t"3t"@a=>t旾1F%K5E/E]-[3fTg<ڷ`Gju}ޑ(%Qé)JNnG#/%3|p3uT{y+۶m3 }T_C`ȏǗo^x C%pPZKm~3ݱcGYZh"ӱcGӬYԟJUqԵ oʔ)ծq_Þ={Q9X6mjXG0ɥGaVT+`干= / 3: L%2P蛴#~N:Ɍ;6T(#S[۶mC7합K7@_?~A~ ѭl oRz%/!۷o7j*괎6&(0={_Kff=TES{VE O|nVӧOs7 ?#e9!U~ H ֭0!U(QF63k5lʯkFC5s1첋T›7ojx۷rao>pӟ$J^zuc?sG'hwO_G[͜9SXiXnҤIz&(Uz^{^ |y뭷͘1c̷-sG?0s|r䀚87# ardwc7nl7onpi>CK Cfo л Ӭ^ZUVkך> GPl6l l%cU+IlْQ泌~\*bVΝ;"OW*!3?ԒbWXW^ feE꺤0;ZB0s0QD74\'Z$:ڵ( ړ2+U`!!Y XWX!3fV(Keq#*ŒY($7њYkoN8Y>G1h* 6#(dQQFwU~ mYy淿c=̏cQzALϘ1CyL_zs1{ ( EJ6j#6wС'##GK.4h ڃN4{[88k-A!]f58 CC???3'O7fԨQ}fʔ)aԘĥ\Hi; @xwa:u$$~aK/W#=ڼ 830KǼϯ\y商,yO; @̻X[haƏo^~eK.榛nJ-*jZps/3?XMR88n38vXÌ4&QWwF,#8(_uUbrh"aC4($."\$41+s=vm78q|K+wޢ }*:Ũi/"̚5 4He|IY=4*4rx뭷_BӾŸ'n:D"GtȻ:%6D-"`[ի,wRP x !wc$N NZC)Ucǎc)E3%"rJ17N"BJ,&)iG(Qؗ.\(ssL5gjfL@=^{I({iAKtAfn0g}'~|BD Kގu`LEԱ!b:-ZN>dRK9Y~uU'AջuV*qWU͎B0Ī(Bw^c#.̫-[ԯ_߱.9׿xۃ:6_I(Sj-|[+ tw}k-kVIxbpC7O>%z 4# 81ҥK=n@r f*z)qFӧOZ!ؓ&M[N*}'$Dp@]xBm@6'LU|3^`=R(-ԩSEAAğK+ƤŬ-^ۙ<'Q`CDJZ"O?T,q^N}ݎaԎ۷owݱuK/mYdLlT\veGYd<*YۜBXg_ %_Z|;E[f`{'dFɍ.Oeܹ\qzɧx˜[?<{m$H9ڒ_C*F-:/x]weM&Gt.H. 29yK{1(P_ IXZ`]f {a&"\|*F$/"ZH MZg{ U~hX"^Qnך{ǴnZ(IDC>`;,qrNV䜦NI_a&810я~$ ,(AAyeV ;>{I8@}<ZJ ڴiԩׯ$2#tMDzUT[{ .&R#=B Q֯*¯+@P>r!*@y⯵+@PW&ZE@(/Q #*6IENDB`pymongo-3.2/doc/__init__.py0000644000175000017500000000000112507072032017617 0ustar behackettbehackett00000000000000 pymongo-3.2/doc/tutorial.rst0000644000175000017500000003240412630145074020123 0ustar behackettbehackett00000000000000Tutorial ======== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('test-database') This tutorial is intended as an introduction to working with **MongoDB** and **PyMongo**. Prerequisites ------------- Before we start, make sure that you have the **PyMongo** distribution :doc:`installed `. In the Python shell, the following should run without raising an exception: .. doctest:: >>> import pymongo This tutorial also assumes that a MongoDB instance is running on the default host and port. Assuming you have `downloaded and installed `_ MongoDB, you can start it like so: .. code-block:: bash $ mongod Making a Connection with MongoClient ------------------------------------ The first step when working with **PyMongo** is to create a :class:`~pymongo.mongo_client.MongoClient` to the running **mongod** instance. Doing so is easy: .. doctest:: >>> from pymongo import MongoClient >>> client = MongoClient() The above code will connect on the default host and port. We can also specify the host and port explicitly, as follows: .. doctest:: >>> client = MongoClient('localhost', 27017) Or use the MongoDB URI format: .. doctest:: >>> client = MongoClient('mongodb://localhost:27017/') Getting a Database ------------------ A single instance of MongoDB can support multiple independent `databases `_. When working with PyMongo you access databases using attribute style access on :class:`~pymongo.mongo_client.MongoClient` instances: .. doctest:: >>> db = client.test_database If your database name is such that using attribute style access won't work (like ``test-database``), you can use dictionary style access instead: .. doctest:: >>> db = client['test-database'] Getting a Collection -------------------- A `collection `_ is a group of documents stored in MongoDB, and can be thought of as roughly the equivalent of a table in a relational database. Getting a collection in PyMongo works the same as getting a database: .. doctest:: >>> collection = db.test_collection or (using dictionary style access): .. doctest:: >>> collection = db['test-collection'] An important note about collections (and databases) in MongoDB is that they are created lazily - none of the above commands have actually performed any operations on the MongoDB server. Collections and databases are created when the first document is inserted into them. Documents --------- Data in MongoDB is represented (and stored) using JSON-style documents. In PyMongo we use dictionaries to represent documents. As an example, the following dictionary might be used to represent a blog post: .. doctest:: >>> import datetime >>> post = {"author": "Mike", ... "text": "My first blog post!", ... "tags": ["mongodb", "python", "pymongo"], ... "date": datetime.datetime.utcnow()} Note that documents can contain native Python types (like :class:`datetime.datetime` instances) which will be automatically converted to and from the appropriate `BSON `_ types. .. todo:: link to table of Python <-> BSON types Inserting a Document -------------------- To insert a document into a collection we can use the :meth:`~pymongo.collection.Collection.insert_one` method: .. doctest:: >>> posts = db.posts >>> post_id = posts.insert_one(post).inserted_id >>> post_id ObjectId('...') When a document is inserted a special key, ``"_id"``, is automatically added if the document doesn't already contain an ``"_id"`` key. The value of ``"_id"`` must be unique across the collection. :meth:`~pymongo.collection.Collection.insert_one` returns an instance of :class:`~pymongo.results.InsertOneResult`. For more information on ``"_id"``, see the `documentation on _id `_. After inserting the first document, the *posts* collection has actually been created on the server. We can verify this by listing all of the collections in our database: .. doctest:: >>> db.collection_names(include_system_collections=False) [u'posts'] Getting a Single Document With :meth:`~pymongo.collection.Collection.find_one` ------------------------------------------------------------------------------ The most basic type of query that can be performed in MongoDB is :meth:`~pymongo.collection.Collection.find_one`. This method returns a single document matching a query (or ``None`` if there are no matches). It is useful when you know there is only one matching document, or are only interested in the first match. Here we use :meth:`~pymongo.collection.Collection.find_one` to get the first document from the posts collection: .. doctest:: >>> posts.find_one() {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']} The result is a dictionary matching the one that we inserted previously. .. note:: The returned document contains an ``"_id"``, which was automatically added on insert. :meth:`~pymongo.collection.Collection.find_one` also supports querying on specific elements that the resulting document must match. To limit our results to a document with author "Mike" we do: .. doctest:: >>> posts.find_one({"author": "Mike"}) {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']} If we try with a different author, like "Eliot", we'll get no result: .. doctest:: >>> posts.find_one({"author": "Eliot"}) >>> .. _querying-by-objectid: Querying By ObjectId -------------------- We can also find a post by its ``_id``, which in our example is an ObjectId: .. doctest:: >>> post_id ObjectId(...) >>> posts.find_one({"_id": post_id}) {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']} Note that an ObjectId is not the same as its string representation: .. doctest:: >>> post_id_as_str = str(post_id) >>> posts.find_one({"_id": post_id_as_str}) # No result >>> A common task in web applications is to get an ObjectId from the request URL and find the matching document. It's necessary in this case to **convert the ObjectId from a string** before passing it to ``find_one``:: from bson.objectid import ObjectId # The web framework gets post_id from the URL and passes it as a string def get(post_id): # Convert from string to ObjectId: document = client.db.collection.find_one({'_id': ObjectId(post_id)}) .. seealso:: :ref:`web-application-querying-by-objectid` A Note On Unicode Strings ------------------------- You probably noticed that the regular Python strings we stored earlier look different when retrieved from the server (e.g. u'Mike' instead of 'Mike'). A short explanation is in order. MongoDB stores data in `BSON format `_. BSON strings are UTF-8 encoded so PyMongo must ensure that any strings it stores contain only valid UTF-8 data. Regular strings () are validated and stored unaltered. Unicode strings () are encoded UTF-8 first. The reason our example string is represented in the Python shell as u'Mike' instead of 'Mike' is that PyMongo decodes each BSON string to a Python unicode string, not a regular str. `You can read more about Python unicode strings here `_. Bulk Inserts ------------ In order to make querying a little more interesting, let's insert a few more documents. In addition to inserting a single document, we can also perform *bulk insert* operations, by passing a list as the first argument to :meth:`~pymongo.collection.Collection.insert_many`. This will insert each document in the list, sending only a single command to the server: .. doctest:: >>> new_posts = [{"author": "Mike", ... "text": "Another post!", ... "tags": ["bulk", "insert"], ... "date": datetime.datetime(2009, 11, 12, 11, 14)}, ... {"author": "Eliot", ... "title": "MongoDB is fun", ... "text": "and pretty easy too!", ... "date": datetime.datetime(2009, 11, 10, 10, 45)}] >>> result = posts.insert_many(new_posts) >>> result.inserted_ids [ObjectId('...'), ObjectId('...')] There are a couple of interesting things to note about this example: - The result from :meth:`~pymongo.collection.Collection.insert_many` now returns two :class:`~bson.objectid.ObjectId` instances, one for each inserted document. - ``new_posts[1]`` has a different "shape" than the other posts - there is no ``"tags"`` field and we've added a new field, ``"title"``. This is what we mean when we say that MongoDB is *schema-free*. Querying for More Than One Document ----------------------------------- To get more than a single document as the result of a query we use the :meth:`~pymongo.collection.Collection.find` method. :meth:`~pymongo.collection.Collection.find` returns a :class:`~pymongo.cursor.Cursor` instance, which allows us to iterate over all matching documents. For example, we can iterate over every document in the ``posts`` collection: .. doctest:: >>> for post in posts.find(): ... post ... {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']} {u'date': datetime.datetime(2009, 11, 12, 11, 14), u'text': u'Another post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'bulk', u'insert']} {u'date': datetime.datetime(2009, 11, 10, 10, 45), u'text': u'and pretty easy too!', u'_id': ObjectId('...'), u'author': u'Eliot', u'title': u'MongoDB is fun'} Just like we did with :meth:`~pymongo.collection.Collection.find_one`, we can pass a document to :meth:`~pymongo.collection.Collection.find` to limit the returned results. Here, we get only those documents whose author is "Mike": .. doctest:: >>> for post in posts.find({"author": "Mike"}): ... post ... {u'date': datetime.datetime(...), u'text': u'My first blog post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'mongodb', u'python', u'pymongo']} {u'date': datetime.datetime(2009, 11, 12, 11, 14), u'text': u'Another post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'bulk', u'insert']} Counting -------- If we just want to know how many documents match a query we can perform a :meth:`~pymongo.cursor.Cursor.count` operation instead of a full query. We can get a count of all of the documents in a collection: .. doctest:: >>> posts.count() 3 or just of those documents that match a specific query: .. doctest:: >>> posts.find({"author": "Mike"}).count() 2 Range Queries ------------- MongoDB supports many different types of `advanced queries `_. As an example, lets perform a query where we limit results to posts older than a certain date, but also sort the results by author: .. doctest:: >>> d = datetime.datetime(2009, 11, 12, 12) >>> for post in posts.find({"date": {"$lt": d}}).sort("author"): ... print post ... {u'date': datetime.datetime(2009, 11, 10, 10, 45), u'text': u'and pretty easy too!', u'_id': ObjectId('...'), u'author': u'Eliot', u'title': u'MongoDB is fun'} {u'date': datetime.datetime(2009, 11, 12, 11, 14), u'text': u'Another post!', u'_id': ObjectId('...'), u'author': u'Mike', u'tags': [u'bulk', u'insert']} Here we use the special ``"$lt"`` operator to do a range query, and also call :meth:`~pymongo.cursor.Cursor.sort` to sort the results by author. Indexing -------- Adding indexes can help accelerate certain queries and can also add additional functionality to querying and storing documents. In this example, we'll demonstrate how to create a `unique index `_ on a key that rejects documents whose value for that key already exists in the index. First, we'll need to create the index: .. doctest:: >>> result = db.profiles.create_index([('user_id', pymongo.ASCENDING)], ... unique=True) >>> list(db.profiles.index_information()) [u'user_id_1', u'_id_'] Notice that we have two indexes now: one is the index on ``_id`` that MongoDB creates automatically, and the other is the index on ``user_id`` we just created. Now let's set up some user profiles: .. doctest:: >>> user_profiles = [ ... {'user_id': 211, 'name': 'Luke'}, ... {'user_id': 212, 'name': 'Ziltoid'}] >>> result = db.profiles.insert_many(user_profiles) The index prevents us from inserting a document whose ``user_id`` is already in the collection: .. doctest:: >>> new_profile = {'user_id': 213, 'name': 'Drew'} >>> duplicate_profile = {'user_id': 212, 'name': 'Tommy'} >>> result = db.profiles.insert_one(new_profile) # This is fine. >>> result = db.profiles.insert_one(duplicate_profile) Traceback (most recent call last): pymongo.errors.DuplicateKeyError: E11000 duplicate key error index: test_database.profiles.$user_id_1 dup key: { : 212 } .. seealso:: The MongoDB documentation on `indexes `_ pymongo-3.2/doc/api/0000755000175000017500000000000012631423130016265 5ustar behackettbehackett00000000000000pymongo-3.2/doc/api/gridfs/0000755000175000017500000000000012631423130017543 5ustar behackettbehackett00000000000000pymongo-3.2/doc/api/gridfs/index.rst0000644000175000017500000000036312507072032021411 0ustar behackettbehackett00000000000000:mod:`gridfs` -- Tools for working with GridFS ============================================== .. automodule:: gridfs :synopsis: Tools for working with GridFS :members: Sub-modules: .. toctree:: :maxdepth: 2 errors grid_file pymongo-3.2/doc/api/gridfs/grid_file.rst0000644000175000017500000000070312630145074022230 0ustar behackettbehackett00000000000000:mod:`grid_file` -- Tools for representing files stored in GridFS ================================================================= .. automodule:: gridfs.grid_file :synopsis: Tools for representing files stored in GridFS .. autoclass:: GridIn :members: .. autoattribute:: _id .. autoclass:: GridOut :members: .. autoattribute:: _id .. automethod:: __iter__ .. autoclass:: GridOutCursor :members: pymongo-3.2/doc/api/gridfs/errors.rst0000644000175000017500000000034412507072032021615 0ustar behackettbehackett00000000000000:mod:`errors` -- Exceptions raised by the :mod:`gridfs` package ================================================================= .. automodule:: gridfs.errors :synopsis: Exceptions raised by the gridfs package :members: pymongo-3.2/doc/api/index.rst0000644000175000017500000000074512507072032020137 0ustar behackettbehackett00000000000000API Documentation ================= The PyMongo distribution contains three top-level packages for interacting with MongoDB. :mod:`bson` is an implementation of the `BSON format `_, :mod:`pymongo` is a full-featured driver for MongoDB, and :mod:`gridfs` is a set of tools for working with the `GridFS `_ storage specification. .. toctree:: :maxdepth: 2 bson/index pymongo/index gridfs/index pymongo-3.2/doc/api/bson/0000755000175000017500000000000012631423130017226 5ustar behackettbehackett00000000000000pymongo-3.2/doc/api/bson/tz_util.rst0000644000175000017500000000035212507072032021455 0ustar behackettbehackett00000000000000:mod:`tz_util` -- Utilities for dealing with timezones in Python ================================================================ .. automodule:: bson.tz_util :synopsis: Utilities for dealing with timezones in Python :members: pymongo-3.2/doc/api/bson/binary.rst0000644000175000017500000000141212630145074021251 0ustar behackettbehackett00000000000000:mod:`binary` -- Tools for representing binary data to be stored in MongoDB =========================================================================== .. automodule:: bson.binary :synopsis: Tools for representing binary data to be stored in MongoDB .. autodata:: BINARY_SUBTYPE .. autodata:: FUNCTION_SUBTYPE .. autodata:: OLD_BINARY_SUBTYPE .. autodata:: OLD_UUID_SUBTYPE .. autodata:: UUID_SUBTYPE .. autodata:: STANDARD .. autodata:: PYTHON_LEGACY .. autodata:: JAVA_LEGACY .. autodata:: CSHARP_LEGACY .. autodata:: MD5_SUBTYPE .. autodata:: USER_DEFINED_SUBTYPE .. autoclass:: Binary(data, subtype=BINARY_SUBTYPE) :members: :show-inheritance: .. autoclass:: UUIDLegacy(obj) :members: :show-inheritance: pymongo-3.2/doc/api/bson/max_key.rst0000644000175000017500000000037012630145074021424 0ustar behackettbehackett00000000000000:mod:`max_key` -- Representation for the MongoDB internal MaxKey type ===================================================================== .. automodule:: bson.max_key :synopsis: Representation for the MongoDB internal MaxKey type :members: pymongo-3.2/doc/api/bson/raw_bson.rst0000644000175000017500000000034012630145074021576 0ustar behackettbehackett00000000000000:mod:`raw_bson` -- Tools for representing raw BSON documents. ============================================================= .. automodule:: bson.raw_bson :synopsis: Tools for representing raw BSON documents. :members: pymongo-3.2/doc/api/bson/timestamp.rst0000644000175000017500000000037312630145074021775 0ustar behackettbehackett00000000000000:mod:`timestamp` -- Tools for representing MongoDB internal Timestamps ====================================================================== .. automodule:: bson.timestamp :synopsis: Tools for representing MongoDB internal Timestamps :members: pymongo-3.2/doc/api/bson/int64.rst0000644000175000017500000000032312630145074020731 0ustar behackettbehackett00000000000000:mod:`int64` -- Tools for representing BSON int64 ================================================= .. versionadded:: 3.0 .. automodule:: bson.int64 :synopsis: Tools for representing BSON int64 :members: pymongo-3.2/doc/api/bson/son.rst0000644000175000017500000000033612507072032020564 0ustar behackettbehackett00000000000000:mod:`son` -- Tools for working with SON, an ordered mapping ============================================================ .. automodule:: bson.son :synopsis: Tools for working with SON, an ordered mapping :members: pymongo-3.2/doc/api/bson/regex.rst0000644000175000017500000000040612507072032021075 0ustar behackettbehackett00000000000000:mod:`regex` -- Tools for representing MongoDB regular expressions ================================================================== .. versionadded:: 2.7 .. automodule:: bson.regex :synopsis: Tools for representing MongoDB regular expressions :members: pymongo-3.2/doc/api/bson/code.rst0000644000175000017500000000043112630145074020677 0ustar behackettbehackett00000000000000:mod:`code` -- Tools for representing JavaScript code ===================================================== .. automodule:: bson.code :synopsis: Tools for representing JavaScript code .. autoclass:: Code(code, scope=None, **kwargs) :members: :show-inheritance: pymongo-3.2/doc/api/bson/codec_options.rst0000644000175000017500000000035012551555337022626 0ustar behackettbehackett00000000000000:mod:`codec_options` -- Tools for specifying BSON codec options =============================================================== .. automodule:: bson.codec_options :synopsis: Tools for specifying BSON codec options. :members: pymongo-3.2/doc/api/bson/index.rst0000644000175000017500000000063112630145074021076 0ustar behackettbehackett00000000000000:mod:`bson` -- BSON (Binary JSON) Encoding and Decoding ======================================================= .. automodule:: bson :synopsis: BSON (Binary JSON) Encoding and Decoding :members: Sub-modules: .. toctree:: :maxdepth: 2 binary code codec_options dbref errors int64 json_util max_key min_key objectid raw_bson regex son timestamp tz_util pymongo-3.2/doc/api/bson/dbref.rst0000644000175000017500000000046512507072032021052 0ustar behackettbehackett00000000000000:mod:`dbref` -- Tools for manipulating DBRefs (references to documents stored in MongoDB) ========================================================================================= .. automodule:: bson.dbref :synopsis: Tools for manipulating DBRefs (references to documents stored in MongoDB) :members: pymongo-3.2/doc/api/bson/json_util.rst0000644000175000017500000000046112630145074021776 0ustar behackettbehackett00000000000000:mod:`json_util` -- Tools for using Python's :mod:`json` module with BSON documents ====================================================================================== .. automodule:: bson.json_util :synopsis: Tools for using Python's json module with BSON documents :members: :undoc-members: pymongo-3.2/doc/api/bson/objectid.rst0000644000175000017500000000127512630145074021557 0ustar behackettbehackett00000000000000:mod:`objectid` -- Tools for working with MongoDB ObjectIds =========================================================== .. automodule:: bson.objectid :synopsis: Tools for working with MongoDB ObjectIds .. autoclass:: bson.objectid.ObjectId(oid=None) :members: .. describe:: str(o) Get a hex encoded version of :class:`ObjectId` `o`. The following property always holds: .. testsetup:: from bson.objectid import ObjectId .. doctest:: >>> o = ObjectId() >>> o == ObjectId(str(o)) True This representation is useful for urls or other places where ``o.binary`` is inappropriate. pymongo-3.2/doc/api/bson/min_key.rst0000644000175000017500000000037012630145074021422 0ustar behackettbehackett00000000000000:mod:`min_key` -- Representation for the MongoDB internal MinKey type ===================================================================== .. automodule:: bson.min_key :synopsis: Representation for the MongoDB internal MinKey type :members: pymongo-3.2/doc/api/bson/errors.rst0000644000175000017500000000033512507072032021300 0ustar behackettbehackett00000000000000:mod:`errors` -- Exceptions raised by the :mod:`bson` package ================================================================ .. automodule:: bson.errors :synopsis: Exceptions raised by the bson package :members: pymongo-3.2/doc/api/pymongo/0000755000175000017500000000000012631423130017755 5ustar behackettbehackett00000000000000pymongo-3.2/doc/api/pymongo/uri_parser.rst0000644000175000017500000000035012507072032022663 0ustar behackettbehackett00000000000000:mod:`uri_parser` -- Tools to parse and validate a MongoDB URI ============================================================== .. automodule:: pymongo.uri_parser :synopsis: Tools to parse and validate a MongoDB URI. :members: pymongo-3.2/doc/api/pymongo/write_concern.rst0000644000175000017500000000033412551555337023370 0ustar behackettbehackett00000000000000:mod:`write_concern` -- Tools for specifying write concern ========================================================== .. automodule:: pymongo.write_concern :synopsis: Tools for specifying write concern. :members: pymongo-3.2/doc/api/pymongo/read_concern.rst0000644000175000017500000000036512630145074023144 0ustar behackettbehackett00000000000000:mod:`read_concern` -- Tools for working with read concern. =========================================================== .. automodule:: pymongo.read_concern :synopsis: Tools for working with read concern. :members: :inherited-members: pymongo-3.2/doc/api/pymongo/results.rst0000644000175000017500000000030212551555337022223 0ustar behackettbehackett00000000000000:mod:`results` -- Result class definitions ========================================== .. automodule:: pymongo.results :synopsis: Result class definitions :members: :inherited-members: pymongo-3.2/doc/api/pymongo/monitoring.rst0000644000175000017500000000102412630145074022700 0ustar behackettbehackett00000000000000:mod:`monitoring` -- Tools for monitoring driver events. ======================================================== .. automodule:: pymongo.monitoring :synopsis: Tools for monitoring driver events. .. autofunction:: register(listener) .. autoclass:: CommandListener :members: .. autoclass:: CommandStartedEvent :members: :inherited-members: .. autoclass:: CommandSucceededEvent :members: :inherited-members: .. autoclass:: CommandFailedEvent :members: :inherited-members: pymongo-3.2/doc/api/pymongo/collection.rst0000644000175000017500000000605512630145074022657 0ustar behackettbehackett00000000000000:mod:`collection` -- Collection level operations ================================================ .. automodule:: pymongo.collection :synopsis: Collection level operations .. autodata:: pymongo.ASCENDING .. autodata:: pymongo.DESCENDING .. autodata:: pymongo.GEO2D .. autodata:: pymongo.GEOHAYSTACK .. autodata:: pymongo.GEOSPHERE .. autodata:: pymongo.HASHED .. autodata:: pymongo.TEXT .. autoclass:: pymongo.collection.ReturnDocument .. autoattribute:: BEFORE :annotation: .. autoattribute:: AFTER :annotation: .. autoclass:: pymongo.collection.Collection(database, name, create=False, **kwargs) .. describe:: c[name] || c.name Get the `name` sub-collection of :class:`Collection` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. .. autoattribute:: full_name .. autoattribute:: name .. autoattribute:: database .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. automethod:: with_options .. automethod:: bulk_write .. automethod:: insert_one .. automethod:: insert_many .. automethod:: replace_one .. automethod:: update_one .. automethod:: update_many .. automethod:: delete_one .. automethod:: delete_many .. automethod:: aggregate .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, manipulate=True) .. automethod:: find_one(filter_or_id=None, *args, **kwargs) .. automethod:: find_one_and_delete .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, **kwargs) .. automethod:: find_one_and_update(filter, update, projection=None, sort=None, return_document=ReturnDocument.BEFORE, **kwargs) .. automethod:: count .. automethod:: distinct .. automethod:: create_index .. automethod:: create_indexes .. automethod:: drop_index .. automethod:: drop_indexes .. automethod:: reindex .. automethod:: list_indexes .. automethod:: index_information .. automethod:: drop .. automethod:: rename .. automethod:: options .. automethod:: group .. automethod:: map_reduce .. automethod:: inline_map_reduce .. automethod:: parallel_scan .. automethod:: initialize_unordered_bulk_op .. automethod:: initialize_ordered_bulk_op .. automethod:: insert(doc_or_docs, manipulate=True, check_keys=True, continue_on_error=False, **kwargs) .. automethod:: save(to_save, manipulate=True, check_keys=True, **kwargs) .. automethod:: update(spec, document, upsert=False, manipulate=False, multi=False, check_keys=True, **kwargs) .. automethod:: remove(spec_or_id=None, multi=True, **kwargs) .. automethod:: find_and_modify .. automethod:: ensure_index pymongo-3.2/doc/api/pymongo/cursor.rst0000644000175000017500000000154712630145074022042 0ustar behackettbehackett00000000000000:mod:`cursor` -- Tools for iterating over MongoDB query results =============================================================== .. automodule:: pymongo.cursor :synopsis: Tools for iterating over MongoDB query results .. autoclass:: pymongo.cursor.CursorType .. autoattribute:: NON_TAILABLE :annotation: .. autoattribute:: TAILABLE :annotation: .. autoattribute:: TAILABLE_AWAIT :annotation: .. autoattribute:: EXHAUST :annotation: .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, manipulate=True) :members: .. describe:: c[index] See :meth:`__getitem__`. .. automethod:: __getitem__ pymongo-3.2/doc/api/pymongo/cursor_manager.rst0000644000175000017500000000045712630145074023533 0ustar behackettbehackett00000000000000:mod:`cursor_manager` -- Managers to handle when cursors are killed after being closed ====================================================================================== .. automodule:: pymongo.cursor_manager :synopsis: Managers to handle when cursors are killed after being closed :members: pymongo-3.2/doc/api/pymongo/read_preferences.rst0000644000175000017500000000166012630145074024015 0ustar behackettbehackett00000000000000:mod:`read_preferences` -- Utilities for choosing which member of a replica set to read from. ============================================================================================= .. automodule:: pymongo.read_preferences :synopsis: Utilities for choosing which member of a replica set to read from. .. autoclass:: pymongo.read_preferences.Primary :inherited-members: .. autoclass:: pymongo.read_preferences.PrimaryPreferred :inherited-members: .. autoclass:: pymongo.read_preferences.Secondary :inherited-members: .. autoclass:: pymongo.read_preferences.SecondaryPreferred :inherited-members: .. autoclass:: pymongo.read_preferences.Nearest :inherited-members: .. autoclass:: ReadPreference .. autoattribute:: PRIMARY .. autoattribute:: PRIMARY_PREFERRED .. autoattribute:: SECONDARY .. autoattribute:: SECONDARY_PREFERRED .. autoattribute:: NEAREST pymongo-3.2/doc/api/pymongo/operations.rst0000644000175000017500000000027512551555337022716 0ustar behackettbehackett00000000000000:mod:`operations` -- Operation class definitions ================================================ .. automodule:: pymongo.operations :synopsis: Operation class definitions :members: pymongo-3.2/doc/api/pymongo/message.rst0000644000175000017500000000036612507072032022143 0ustar behackettbehackett00000000000000:mod:`message` -- Tools for creating messages to be sent to MongoDB =================================================================== .. automodule:: pymongo.message :synopsis: Tools for creating messages to be sent to MongoDB :members: pymongo-3.2/doc/api/pymongo/bulk.rst0000644000175000017500000000030412507072032021444 0ustar behackettbehackett00000000000000:mod:`bulk` -- The bulk write operations interface ================================================== .. automodule:: pymongo.bulk :synopsis: The bulk write operations interface. :members: pymongo-3.2/doc/api/pymongo/pool.rst0000644000175000017500000000033512507072032021464 0ustar behackettbehackett00000000000000:mod:`pool` -- Pool module for use with a MongoDB client. ============================================================== .. automodule:: pymongo.pool :synopsis: Pool module for use with a MongoDB client. :members: pymongo-3.2/doc/api/pymongo/database.rst0000644000175000017500000000171712630145074022270 0ustar behackettbehackett00000000000000:mod:`database` -- Database level operations ============================================ .. automodule:: pymongo.database :synopsis: Database level operations .. autodata:: pymongo.auth.MECHANISMS .. autodata:: pymongo.OFF .. autodata:: pymongo.SLOW_ONLY .. autodata:: pymongo.ALL .. autoclass:: pymongo.database.Database :members: .. describe:: db[collection_name] || db.collection_name Get the `collection_name` :class:`~pymongo.collection.Collection` of :class:`Database` `db`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. .. note:: Use dictionary style access if `collection_name` is an attribute of the :class:`Database` class eg: db[`collection_name`]. .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. autoclass:: pymongo.database.SystemJS :members: pymongo-3.2/doc/api/pymongo/index.rst0000644000175000017500000000200412630145074021621 0ustar behackettbehackett00000000000000:mod:`pymongo` -- Python driver for MongoDB =========================================== .. automodule:: pymongo :synopsis: Python driver for MongoDB .. autodata:: version .. data:: MongoClient Alias for :class:`pymongo.mongo_client.MongoClient`. .. data:: MongoReplicaSetClient Alias for :class:`pymongo.mongo_replica_set_client.MongoReplicaSetClient`. .. data:: ReadPreference Alias for :class:`pymongo.read_preferences.ReadPreference`. .. autofunction:: has_c .. data:: MIN_SUPPORTED_WIRE_VERSION The minimum wire protocol version PyMongo supports. .. data:: MAX_SUPPORTED_WIRE_VERSION The maximum wire protocol version PyMongo supports. Sub-modules: .. toctree:: :maxdepth: 2 database collection command_cursor cursor bulk errors message monitoring mongo_client mongo_replica_set_client operations pool read_concern read_preferences results son_manipulator cursor_manager uri_parser write_concern pymongo-3.2/doc/api/pymongo/mongo_client.rst0000644000175000017500000000257212630145074023201 0ustar behackettbehackett00000000000000:mod:`mongo_client` -- Tools for connecting to MongoDB ====================================================== .. automodule:: pymongo.mongo_client :synopsis: Tools for connecting to MongoDB .. autoclass:: pymongo.mongo_client.MongoClient(host='localhost', port=27017, document_class=dict, tz_aware=False, connect=True, **kwargs) .. automethod:: close .. describe:: c[db_name] || c.db_name Get the `db_name` :class:`~pymongo.database.Database` on :class:`MongoClient` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. .. autoattribute:: address .. autoattribute:: is_primary .. autoattribute:: is_mongos .. autoattribute:: max_pool_size .. autoattribute:: nodes .. autoattribute:: max_bson_size .. autoattribute:: max_message_size .. autoattribute:: local_threshold_ms .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. autoattribute:: is_locked .. automethod:: database_names .. automethod:: drop_database .. automethod:: get_default_database .. automethod:: get_database .. automethod:: server_info .. automethod:: close_cursor .. automethod:: kill_cursors .. automethod:: set_cursor_manager .. automethod:: fsync .. automethod:: unlock pymongo-3.2/doc/api/pymongo/mongo_replica_set_client.rst0000644000175000017500000000247312630145074025553 0ustar behackettbehackett00000000000000:mod:`mongo_replica_set_client` -- Tools for connecting to a MongoDB replica set ================================================================================ .. automodule:: pymongo.mongo_replica_set_client :synopsis: Tools for connecting to a MongoDB replica set .. autoclass:: pymongo.mongo_replica_set_client.MongoReplicaSetClient(hosts_or_uri, document_class=dict, tz_aware=False, connect=True, **kwargs) .. automethod:: close .. describe:: c[db_name] || c.db_name Get the `db_name` :class:`~pymongo.database.Database` on :class:`MongoReplicaSetClient` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. .. autoattribute:: primary .. autoattribute:: secondaries .. autoattribute:: arbiters .. autoattribute:: max_pool_size .. autoattribute:: max_bson_size .. autoattribute:: max_message_size .. autoattribute:: local_threshold_ms .. autoattribute:: codec_options .. autoattribute:: read_preference .. autoattribute:: write_concern .. automethod:: database_names .. automethod:: drop_database .. automethod:: get_default_database .. automethod:: get_database .. automethod:: close_cursor .. automethod:: kill_cursors .. automethod:: set_cursor_manager pymongo-3.2/doc/api/pymongo/command_cursor.rst0000644000175000017500000000041112507072032023521 0ustar behackettbehackett00000000000000:mod:`command_cursor` -- Tools for iterating over MongoDB command results ========================================================================= .. automodule:: pymongo.command_cursor :synopsis: Tools for iterating over MongoDB command results :members: pymongo-3.2/doc/api/pymongo/errors.rst0000644000175000017500000000034612507072032022031 0ustar behackettbehackett00000000000000:mod:`errors` -- Exceptions raised by the :mod:`pymongo` package ================================================================ .. automodule:: pymongo.errors :synopsis: Exceptions raised by the pymongo package :members: pymongo-3.2/doc/api/pymongo/son_manipulator.rst0000644000175000017500000000052012507072032023721 0ustar behackettbehackett00000000000000:mod:`son_manipulator` -- Manipulators that can edit SON documents as they are saved or retrieved ================================================================================================= .. automodule:: pymongo.son_manipulator :synopsis: Manipulators that can edit SON documents as they are saved or retrieved :members: pymongo-3.2/doc/tools.rst0000644000175000017500000001544012622440614017420 0ustar behackettbehackett00000000000000Tools ===== Many tools have been written for working with **PyMongo**. If you know of or have created a tool for working with MongoDB from Python please list it here. .. note:: We try to keep this list current. As such, projects that have not been updated recently or appear to be unmaintained will occasionally be removed from the list or moved to the back (to keep the list from becoming too intimidating). If a project gets removed that is still being developed or is in active use please let us know or add it back. ORM-like Layers --------------- Some people have found that they prefer to work with a layer that has more features than PyMongo provides. Often, things like models and validation are desired. To that end, several different ORM-like layers have been written by various authors. It is our recommendation that new users begin by working directly with PyMongo, as described in the rest of this documentation. Many people have found that the features of PyMongo are enough for their needs. Even if you eventually come to the decision to use one of these layers, the time spent working directly with the driver will have increased your understanding of how MongoDB actually works. Humongolus `Humongolus `_ is a lightweight ORM framework for Python and MongoDB. The name comes from the combination of MongoDB and `Homunculus `_ (the concept of a miniature though fully formed human body). Humongolus allows you to create models/schemas with robust validation. It attempts to be as pythonic as possible and exposes the pymongo cursor objects whenever possible. The code is available for download `at github `_. Tutorials and usage examples are also available at GitHub. MongoKit The `MongoKit `_ framework is an ORM-like layer on top of PyMongo. There is also a MongoKit `google group `_. Ming `Ming `_ (the Merciless) is a library that allows you to enforce schemas on a MongoDB database in your Python application. It was developed by `SourceForge `_ in the course of their migration to MongoDB. See the `introductory blog post `_ for more details. MongoAlchemy `MongoAlchemy `_ is another ORM-like layer on top of PyMongo. Its API is inspired by `SQLAlchemy `_. The code is available `on github `_; for more information, see `the tutorial `_. MongoEngine `MongoEngine `_ is another ORM-like layer on top of PyMongo. It allows you to define schemas for documents and query collections using syntax inspired by the Django ORM. The code is available on `github `_; for more information, see the `tutorial `_. Minimongo `minimongo `_ is a lightweight, pythonic interface to MongoDB. It retains pymongo's query and update API, and provides a number of additional features, including a simple document-oriented interface, connection pooling, index management, and collection & database naming helpers. The `source is on github `_. Manga `Manga `_ aims to be a simpler ORM-like layer on top of PyMongo. The syntax for defining schema is inspired by the Django ORM, but Pymongo's query language is maintained. The source `is on github `_. MotorEngine `MotorEngine `_ is a port of MongoEngine to Motor, for asynchronous access with Tornado. It implements the same modeling APIs to be data-portable, meaning that a model defined in MongoEngine can be read in MotorEngine. The source is `available on github `_. Framework Tools --------------- This section lists tools and adapters that have been designed to work with various Python frameworks and libraries. * `Django MongoDB Engine `_ is a MongoDB database backend for Django that completely integrates with its ORM. For more information `see the tutorial `_. * `mango `_ provides MongoDB backends for Django sessions and authentication (bypassing :mod:`django.db` entirely). * `Django MongoEngine `_ is a MongoDB backend for Django, an `example: `_. For more information ``_ * `mongodb_beaker `_ is a project to enable using MongoDB as a backend for `beaker's `_ caching / session system. `The source is on github `_. * `Log4Mongo `_ is a flexible Python logging handler that can store logs in MongoDB using normal and capped collections. * `MongoLog `_ is a Python logging handler that stores logs in MongoDB using a capped collection. * `c5t `_ is a content-management system using TurboGears and MongoDB. * `rod.recipe.mongodb `_ is a ZC Buildout recipe for downloading and installing MongoDB. * `repoze-what-plugins-mongodb `_ is a project working to support a plugin for using MongoDB as a backend for :mod:`repoze.what`. * `mongobox `_ is a tool to run a sandboxed MongoDB instance from within a python app. * `Flask-MongoAlchemy `_ Add Flask support for MongoDB using MongoAlchemy. * `Flask-MongoKit `_ Flask extension to better integrate MongoKit into Flask. * `Flask-PyMongo `_ Flask-PyMongo bridges Flask and PyMongo. Alternative Drivers ------------------- These are alternatives to PyMongo. * `Motor `_ is a full-featured, non-blocking MongoDB driver for Python Tornado applications. * `TxMongo `_ is an asynchronous Twisted Python driver for MongoDB. pymongo-3.2/doc/index.rst0000644000175000017500000000516312630145074017371 0ustar behackettbehackett00000000000000PyMongo |release| Documentation =============================== Overview -------- **PyMongo** is a Python distribution containing tools for working with `MongoDB `_, and is the recommended way to work with MongoDB from Python. This documentation attempts to explain everything you need to know to use **PyMongo**. .. todo:: a list of PyMongo's features :doc:`installation` Instructions on how to get the distribution. :doc:`tutorial` Start here for a quick overview. :doc:`examples/index` Examples of how to perform specific tasks. :doc:`faq` Some questions that come up often. :doc:`migrate-to-pymongo3` A PyMongo 2.x to 3.x migration guide. :doc:`python3` Frequently asked questions about python 3 support. :doc:`api/index` The complete API documentation, organized by module. :doc:`tools` A listing of Python tools and libraries that have been written for MongoDB. :doc:`developer/index` Developer guide for contributors to PyMongo. Getting Help ------------ If you're having trouble or have questions about PyMongo, the best place to ask is the `MongoDB user group `_. Once you get an answer, it'd be great if you could work it back into this documentation and contribute! Issues ------ All issues should be reported (and can be tracked / voted for / commented on) at the main `MongoDB JIRA bug tracker `_, in the "Python Driver" project. Contributing ------------ **PyMongo** has a large :doc:`community ` and contributions are always encouraged. Contributions can be as simple as minor tweaks to this documentation. To contribute, fork the project on `github `_ and send a pull request. Changes ------- See the :doc:`changelog` for a full list of changes to PyMongo. For older versions of the documentation please see the `archive list `_. About This Documentation ------------------------ This documentation is generated using the `Sphinx `_ documentation generator. The source files for the documentation are located in the *doc/* directory of the **PyMongo** distribution. To generate the docs locally run the following command from the root directory of the **PyMongo** source: .. code-block:: bash $ python setup.py doc Indices and tables ------------------ * :ref:`genindex` * :ref:`modindex` * :ref:`search` .. toctree:: :hidden: installation tutorial examples/index faq api/index tools contributors changelog python3 migrate-to-pymongo3 developer/index pymongo-3.2/doc/developer/0000755000175000017500000000000012631423130017501 5ustar behackettbehackett00000000000000pymongo-3.2/doc/developer/periodic_executor.rst0000644000175000017500000001223712630145074023763 0ustar behackettbehackett00000000000000Periodic Executors ================== .. currentmodule:: pymongo PyMongo implements a :class:`~periodic_executor.PeriodicExecutor` for two purposes: as the background thread for :class:`~monitor.Monitor`, and to regularly check if there are `OP_KILL_CURSORS` messages that must be sent to the server. Killing Cursors --------------- An incompletely iterated :class:`~cursor.Cursor` on the client represents an open cursor object on the server. In code like this, we lose a reference to the cursor before finishing iteration:: for doc in collection.find(): raise Exception() We try to send an `OP_KILL_CURSORS` to the server to tell it to clean up the server-side cursor. But we must not take any locks directly from the cursor's destructor (see `PYTHON-799`_), so we cannot safely use the PyMongo data structures required to send a message. The solution is to add the cursor's id to an array on the :class:`~mongo_client.MongoClient` without taking any locks. Each client has a :class:`~periodic_executor.PeriodicExecutor` devoted to checking the array for cursor ids. Any it sees are the result of cursors that were freed while the server-side cursor was still open. The executor can safely take the locks it needs in order to send the `OP_KILL_CURSORS` message. .. _PYTHON-799: https://jira.mongodb.org/browse/PYTHON-799 Stopping Executors ------------------ Just as :class:`~cursor.Cursor` must not take any locks from its destructor, neither can :class:`~mongo_client.MongoClient` and :class:`~topology.Topology`. Thus, although the client calls :meth:`close` on its kill-cursors thread, and the topology calls :meth:`close` on all its monitor threads, the :meth:`close` method cannot actually call :meth:`wake` on the executor, since :meth:`wake` takes a lock. Instead, executors wake periodically to check if ``self.close`` is set, and if so they exit. A thread can log spurious errors if it wakes late in the Python interpreter's shutdown sequence, so we try to join threads before then. Each periodic executor (either a monitor or a kill-cursors thread) adds a weakref to itself to a set called ``_EXECUTORS``, in the ``periodic_executor`` module. An `exit handler`_ runs on shutdown and tells all executors to stop, then tries (with a short timeout) to join all executor threads. .. _exit handler: https://docs.python.org/2/library/atexit.html Monitoring ---------- For each server in the topology, :class:`~topology.Topology` uses a periodic executor to launch a monitor thread. This thread must not prevent the topology from being freed, so it weakrefs the topology. Furthermore, it uses a weakref callback to terminate itself soon after the topology is freed. Solid lines represent strong references, dashed lines weak ones: .. generated with graphviz: "dot -Tpng periodic-executor-refs.dot > periodic-executor-refs.png" .. image:: ../static/periodic-executor-refs.png See `Stopping Executors`_ above for an explanation of the ``_EXECUTORS`` set. It is a requirement of the `Server Discovery And Monitoring Spec`_ that a sleeping monitor can be awakened early. Aside from infrequent wakeups to do their appointed chores, and occasional interruptions, periodic executors also wake periodically to check if they should terminate. Our first implementation of this idea was the obvious one: use the Python standard library's threading.Condition.wait with a timeout. Another thread wakes the executor early by signaling the condition variable. A topology cannot signal the condition variable to tell the executor to terminate, because it would risk a deadlock in the garbage collector: no destructor or weakref callback can take a lock to signal the condition variable (see `PYTHON-863`_); thus the only way for a dying object to terminate a periodic executor is to set its "stopped" flag and let the executor see the flag next time it wakes. We erred on the side of prompt cleanup, and set the check interval at 100ms. We assumed that checking a flag and going back to sleep 10 times a second was cheap on modern machines. Starting in Python 3.2, the builtin C implementation of lock.acquire takes a timeout parameter, so Python 3.2+ Condition variables sleep simply by calling lock.acquire; they are implemented as efficiently as expected. But in Python 2, lock.acquire has no timeout. To wait with a timeout, a Python 2 condition variable sleeps a millisecond, tries to acquire the lock, sleeps twice as long, and tries again. This exponential backoff reaches a maximum sleep time of 50ms. If PyMongo calls the condition variable's "wait" method with a short timeout, the exponential backoff is restarted frequently. Overall, the condition variable is not waking a few times a second, but hundreds of times. (See `PYTHON-983`_.) Thus the current design of periodic executors is surprisingly simple: they do a simple `time.sleep` for a half-second, check if it is time to wake or terminate, and sleep again. .. _Server Discovery And Monitoring Spec: https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring.rst#requesting-an-immediate-check .. _PYTHON-863: https://jira.mongodb.org/browse/PYTHON-863 .. _PYTHON-983: https://jira.mongodb.org/browse/PYTHON-983 pymongo-3.2/doc/developer/index.rst0000644000175000017500000000020212630145074021343 0ustar behackettbehackett00000000000000Developer Guide =============== Technical guide for contributors to PyMongo. .. toctree:: :maxdepth: 1 periodic_executor pymongo-3.2/doc/examples/0000755000175000017500000000000012631423130017332 5ustar behackettbehackett00000000000000pymongo-3.2/doc/examples/tls.rst0000644000175000017500000000740212630145074020700 0ustar behackettbehackett00000000000000TLS/SSL and PyMongo =================== PyMongo supports connecting to MongoDB over TLS/SSL. This guide covers the configuration options supported by PyMongo. See `the server documentation `_ to configure MongoDB. Basic configuration ................... In many cases connecting to MongoDB over TLS/SSL requires nothing more than passing ``ssl=True`` as a keyword argument to :class:`~pymongo.mongo_client.MongoClient`:: >>> client = pymongo.MongoClient('example.com', ssl=True) Or passing ``ssl=true`` in the URI:: >>> client = pymongo.MongoClient('mongodb://example.com/?ssl=true') This configures PyMongo to connect to the server using TLS, verify the server's certificate and verify that the host you are attempting to connect to is listed by that certificate. PyMongo attempts to use the operating system's CA certificates to verify the server's certificate when possible. Some versions of python may require an extra third party module for this to work properly. Users of Python 2 on Windows are encouraged to upgrade to python 2.7.9 or newer. Users of Python 3 on Windows should upgrade to python 3.4.0 or newer. If upgrading is not possible `wincertstore `_ can be used with older python versions. Users of operating systems other than Windows that are stuck on python versions older than 2.7.9 can install `certifi `_ to use the Mozilla CA bundle for certificate verification. Certificate verification policy ............................... By default, PyMongo is configured to require a certificate from the server when TLS is enabled. This is configurable using the `ssl_cert_reqs` option. To disable this requirement pass ``ssl.CERT_NONE`` as a keyword parameter:: >>> import ssl >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_cert_reqs=ssl.CERT_NONE) Or, in the URI:: >>> uri = 'mongodb://example.com/?ssl=true&ssl_cert_reqs=CERT_NONE' >>> client = pymongo.MongoClient(uri) You can also configure optional certificate verification, if a certificate is provided by the server:: >>> import ssl >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_cert_reqs=ssl.CERT_OPTIONAL) >>> >>> uri = 'mongodb://example.com/?ssl=true&ssl_cert_reqs=CERT_OPTIONAL' >>> client = pymongo.MongoClient(uri) Specifying a CA file .................... In some cases you may want to configure PyMongo to use a specific set of CA certificates. This is most often the case when using "self-signed" server certificates. The `ssl_ca_certs` option takes a path to a CA file. It can be passed as a keyword argument:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_ca_certs='/path/to/ca.pem') Or, in the URI:: >>> uri = 'mongodb://example.com/?ssl=true&ssl_ca_certs=/path/to/ca.pem' >>> client = pymongo.MongoClient(uri) Client certificates ................... PyMongo can be configured to present a client certificate using the `ssl_certfile` option:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem') If the private key for the client certificate is stored in a separate file use the `ssl_keyfile` option:: >>> client = pymongo.MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_keyfile='/path/to/key.pem') These options can also be passed as part of the MongoDB URI. pymongo-3.2/doc/examples/aggregation.rst0000644000175000017500000001520212630145074022362 0ustar behackettbehackett00000000000000Aggregation Examples ==================== There are several methods of performing aggregations in MongoDB. These examples cover the new aggregation framework, using map reduce and using the group method. .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('aggregation_example') Setup ----- To start, we'll insert some example data which we can perform aggregations on: .. doctest:: >>> from pymongo import MongoClient >>> db = MongoClient().aggregation_example >>> result = db.things.insert_many([{"x": 1, "tags": ["dog", "cat"]}, ... {"x": 2, "tags": ["cat"]}, ... {"x": 2, "tags": ["mouse", "cat", "dog"]}, ... {"x": 3, "tags": []}]) >>> result.inserted_ids [ObjectId('...'), ObjectId('...'), ObjectId('...'), ObjectId('...')] .. _aggregate-examples: Aggregation Framework --------------------- This example shows how to use the :meth:`~pymongo.collection.Collection.aggregate` method to use the aggregation framework. We'll perform a simple aggregation to count the number of occurrences for each tag in the ``tags`` array, across the entire collection. To achieve this we need to pass in three operations to the pipeline. First, we need to unwind the ``tags`` array, then group by the tags and sum them up, finally we sort by count. As python dictionaries don't maintain order you should use :class:`~bson.son.SON` or :class:`collections.OrderedDict` where explicit ordering is required eg "$sort": .. note:: aggregate requires server version **>= 2.1.0**. .. doctest:: >>> from bson.son import SON >>> pipeline = [ ... {"$unwind": "$tags"}, ... {"$group": {"_id": "$tags", "count": {"$sum": 1}}}, ... {"$sort": SON([("count", -1), ("_id", -1)])} ... ] >>> list(db.things.aggregate(pipeline)) [{u'count': 3, u'_id': u'cat'}, {u'count': 2, u'_id': u'dog'}, {u'count': 1, u'_id': u'mouse'}] To run an explain plan for this aggregation use the :meth:`~pymongo.database.Database.command` method:: >>> db.command('aggregate', 'things', pipeline=pipeline, explain=True) {u'ok': 1.0, u'stages': [...]} As well as simple aggregations the aggregation framework provides projection capabilities to reshape the returned data. Using projections and aggregation, you can add computed fields, create new virtual sub-objects, and extract sub-fields into the top-level of results. .. seealso:: The full documentation for MongoDB's `aggregation framework `_ Map/Reduce ---------- Another option for aggregation is to use the map reduce framework. Here we will define **map** and **reduce** functions to also count the number of occurrences for each tag in the ``tags`` array, across the entire collection. Our **map** function just emits a single `(key, 1)` pair for each tag in the array: .. doctest:: >>> from bson.code import Code >>> mapper = Code(""" ... function () { ... this.tags.forEach(function(z) { ... emit(z, 1); ... }); ... } ... """) The **reduce** function sums over all of the emitted values for a given key: .. doctest:: >>> reducer = Code(""" ... function (key, values) { ... var total = 0; ... for (var i = 0; i < values.length; i++) { ... total += values[i]; ... } ... return total; ... } ... """) .. note:: We can't just return ``values.length`` as the **reduce** function might be called iteratively on the results of other reduce steps. Finally, we call :meth:`~pymongo.collection.Collection.map_reduce` and iterate over the result collection: .. doctest:: >>> result = db.things.map_reduce(mapper, reducer, "myresults") >>> for doc in result.find(): ... print doc ... {u'_id': u'cat', u'value': 3.0} {u'_id': u'dog', u'value': 2.0} {u'_id': u'mouse', u'value': 1.0} Advanced Map/Reduce ------------------- PyMongo's API supports all of the features of MongoDB's map/reduce engine. One interesting feature is the ability to get more detailed results when desired, by passing `full_response=True` to :meth:`~pymongo.collection.Collection.map_reduce`. This returns the full response to the map/reduce command, rather than just the result collection: .. doctest:: >>> db.things.map_reduce(mapper, reducer, "myresults", full_response=True) {u'counts': {u'input': 4, u'reduce': 2, u'emit': 6, u'output': 3}, u'timeMillis': ..., u'ok': ..., u'result': u'...'} All of the optional map/reduce parameters are also supported, simply pass them as keyword arguments. In this example we use the `query` parameter to limit the documents that will be mapped over: .. doctest:: >>> result = db.things.map_reduce(mapper, reducer, "myresults", query={"x": {"$lt": 2}}) >>> for doc in result.find(): ... print doc ... {u'_id': u'cat', u'value': 1.0} {u'_id': u'dog', u'value': 1.0} With MongoDB 1.8.0 or newer you can use :class:`~bson.son.SON` or :class:`collections.OrderedDict` to specify a different database to store the result collection: .. doctest:: >>> from bson.son import SON >>> db.things.map_reduce(mapper, reducer, out=SON([("replace", "results"), ("db", "outdb")]), full_response=True) {u'counts': {u'input': 4, u'reduce': 2, u'emit': 6, u'output': 3}, u'timeMillis': ..., u'ok': ..., u'result': {u'db': ..., u'collection': ...}} .. seealso:: The full list of options for MongoDB's `map reduce engine `_ Group ----- The :meth:`~pymongo.collection.Collection.group` method provides some of the same functionality as SQL's GROUP BY. Simpler than a map reduce you need to provide a key to group by, an initial value for the aggregation and a reduce function. .. note:: Doesn't work with sharded MongoDB configurations, use aggregation or map/reduce instead of group(). Here we are doing a simple group and count of the occurrences of ``x`` values: .. doctest:: >>> from bson.code import Code >>> reducer = Code(""" ... function(obj, prev){ ... prev.count++; ... } ... """) ... >>> results = db.things.group(key={"x":1}, condition={}, initial={"count": 0}, reduce=reducer) >>> for doc in results: ... print doc {u'count': 1.0, u'x': 1.0} {u'count': 2.0, u'x': 2.0} {u'count': 1.0, u'x': 3.0} .. seealso:: The full list of options for MongoDB's `group method `_ pymongo-3.2/doc/examples/mod_wsgi.rst0000644000175000017500000000450412507072032021702 0ustar behackettbehackett00000000000000.. _pymongo-and-mod_wsgi: PyMongo and mod_wsgi ==================== If you run your application under `mod_wsgi `_ and you use PyMongo with its C extensions enabled, follow these guidelines for best performance: * Run ``mod_wsgi`` in daemon mode with the ``WSGIDaemon`` directive. * Assign each application to a separate daemon with ``WSGIProcessGroup``. * Use ``WSGIApplicationGroup %{GLOBAL}`` to ensure your application is running in the daemon's main Python interpreter, not a sub interpreter. For example, this ``mod_wsgi`` configuration ensures an application runs in the main interpreter:: WSGIDaemonProcess my_process WSGIScriptAlias /my_app /path/to/app.wsgi WSGIProcessGroup my_process WSGIApplicationGroup %{GLOBAL} If you have multiple applications that use PyMongo, put each in a separate daemon, still in the global application group:: WSGIDaemonProcess my_process WSGIScriptAlias /my_app /path/to/app.wsgi WSGIProcessGroup my_process WSGIDaemonProcess my_other_process WSGIScriptAlias /my_other_app /path/to/other_app.wsgi WSGIProcessGroup my_other_process WSGIApplicationGroup %{GLOBAL} Background: Python C extensions in general have issues running in multiple Python sub interpreters. These difficulties are explained in the documentation for `Py_NewInterpreter `_ and in the `Multiple Python Sub Interpreters `_ section of the ``mod_wsgi`` documentation. Beginning with PyMongo 2.7, the C extension for BSON detects when it is running in a sub interpreter and activates a workaround, which adds a small cost to BSON decoding. To avoid this cost, use ``WSGIApplicationGroup %{GLOBAL}`` to ensure your application runs in the main interpreter. Since your program runs in the main interpreter it should not share its process with any other applications, lest they interfere with each other's state. Each application should have its own daemon process, as shown in the example above.pymongo-3.2/doc/examples/datetimes.rst0000644000175000017500000000745112630145074022061 0ustar behackettbehackett00000000000000Datetimes and Timezones ======================= .. testsetup:: import datetime from pymongo import MongoClient from bson.codec_options import CodecOptions client = MongoClient() client.drop_database('dt_example') db = client.dt_example These examples show how to handle Python :class:`datetime.datetime` objects correctly in PyMongo. Basic Usage ----------- PyMongo uses :class:`datetime.datetime` objects for representing dates and times in MongoDB documents. Because MongoDB assumes that dates and times are in UTC, care should be taken to ensure that dates and times written to the database reflect UTC. For example, the following code stores the current UTC date and time into MongoDB: .. doctest:: >>> result = db.objects.insert_one( ... {"last_modified": datetime.datetime.utcnow()}) Always use :meth:`datetime.datetime.utcnow`, which returns the current time in UTC, instead of :meth:`datetime.datetime.now`, which returns the current local time. Avoid doing this: .. doctest:: >>> result = db.objects.insert_one( ... {"last_modified": datetime.datetime.now()}) The value for `last_modified` is very different between these two examples, even though both documents were stored at around the same local time. This will be confusing to the application that reads them: .. doctest:: >>> [doc['last_modified'] for doc in db.objects.find()] # doctest: +SKIP [datetime.datetime(2015, 7, 8, 18, 17, 28, 324000), datetime.datetime(2015, 7, 8, 11, 17, 42, 911000)] :class:`bson.codec_options.CodecOptions` has a `tz_aware` option that enables "aware" :class:`datetime.datetime` objects, i.e., datetimes that know what timezone they're in. By default, PyMongo retrieves naive datetimes: .. doctest:: >>> result = db.tzdemo.insert_one( ... {'date': datetime.datetime(2002, 10, 27, 6, 0, 0)}) >>> db.tzdemo.find_one()['date'] datetime.datetime(2002, 10, 27, 6, 0) >>> options = CodecOptions(tz_aware=True) >>> db.get_collection('tzdemo', codec_options=options).find_one()['date'] # doctest: +SKIP datetime.datetime(2002, 10, 27, 6, 0, tzinfo=) Saving Datetimes with Timezones ------------------------------- When storing :class:`datetime.datetime` objects that specify a timezone (i.e. they have a `tzinfo` property that isn't ``None``), PyMongo will convert those datetimes to UTC automatically: .. doctest:: >>> import pytz >>> pacific = pytz.timezone('US/Pacific') >>> aware_datetime = pacific.localize( ... datetime.datetime(2002, 10, 27, 6, 0, 0)) >>> result = db.times.insert_one({"date": aware_datetime}) >>> db.times.find_one()['date'] datetime.datetime(2002, 10, 27, 14, 0) Reading Time ------------ As previously mentioned, by default all :class:`datetime.datetime` objects returned by PyMongo will be naive but reflect UTC (i.e. the time as stored in MongoDB). By setting the `tz_aware` option on :class:`~bson.codec_options.CodecOptions`, :class:`datetime.datetime` objects will be timezone-aware and have a `tzinfo` property that reflects the UTC timezone. PyMongo 3.1 introduced a `tzinfo` property that can be set on :class:`~bson.codec_options.CodecOptions` to convert :class:`datetime.datetime` objects to local time automatically. For example, if we wanted to read all times out of MongoDB in US/Pacific time: >>> from bson.codec_options import CodecOptions >>> db.times.find_one()['date'] datetime.datetime(2002, 10, 27, 14, 0) >>> aware_times = db.times.with_options(codec_options=CodecOptions( ... tz_aware=True, ... tzinfo=pytz.timezone('US/Pacific'))) >>> result = aware_times.find_one() datetime.datetime(2002, 10, 27, 6, 0, # doctest: +NORMALIZE_WHITESPACE tzinfo=) pymongo-3.2/doc/examples/bulk.rst0000644000175000017500000001330712630145074021034 0ustar behackettbehackett00000000000000Bulk Write Operations ===================== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('bulk_example') This tutorial explains how to take advantage of PyMongo's bulk write operation features. Executing write operations in batches reduces the number of network round trips, increasing write throughput. Bulk Insert ----------- .. versionadded:: 2.6 A batch of documents can be inserted by passing a list to the :meth:`~pymongo.collection.Collection.insert_many` method. PyMongo will automatically split the batch into smaller sub-batches based on the maximum message size accepted by MongoDB, supporting very large bulk insert operations. .. doctest:: >>> import pymongo >>> db = pymongo.MongoClient().bulk_example >>> db.test.insert_many([{'i': i} for i in xrange(10000)]).inserted_ids [...] >>> db.test.count() 10000 Mixed Bulk Write Operations --------------------------- .. versionadded:: 2.7 PyMongo also supports executing mixed bulk write operations. A batch of insert, update, and remove operations can be executed together using the bulk write operations API. .. _ordered_bulk: Ordered Bulk Write Operations ............................. Ordered bulk write operations are batched and sent to the server in the order provided for serial execution. The return value is a document describing the type and count of operations performed. .. doctest:: >>> from pprint import pprint >>> >>> bulk = db.test.initialize_ordered_bulk_op() >>> # Remove all documents from the previous example. ... >>> bulk.find({}).remove() >>> bulk.insert({'_id': 1}) >>> bulk.insert({'_id': 2}) >>> bulk.insert({'_id': 3}) >>> bulk.find({'_id': 1}).update({'$set': {'foo': 'bar'}}) >>> bulk.find({'_id': 4}).upsert().update({'$inc': {'j': 1}}) >>> bulk.find({'j': 1}).replace_one({'j': 2}) >>> result = bulk.execute() >>> pprint(result) {'nInserted': 3, 'nMatched': 2, 'nModified': 2, 'nRemoved': 10000, 'nUpserted': 1, 'upserted': [{u'_id': 4, u'index': 5}], 'writeConcernErrors': [], 'writeErrors': []} .. warning:: ``nModified`` is only reported by MongoDB 2.6 and later. When connected to an earlier server version, or in certain mixed version sharding configurations, PyMongo omits this field from the results of a bulk write operation. The first write failure that occurs (e.g. duplicate key error) aborts the remaining operations, and PyMongo raises :class:`~pymongo.errors.BulkWriteError`. The :attr:`details` attibute of the exception instance provides the execution results up until the failure occurred and details about the failure - including the operation that caused the failure. .. doctest:: >>> from pymongo.errors import BulkWriteError >>> bulk = db.test.initialize_ordered_bulk_op() >>> bulk.find({'j': 2}).replace_one({'i': 5}) >>> # Violates the unique key constraint on _id. ... >>> bulk.insert({'_id': 4}) >>> bulk.find({'i': 5}).remove_one() >>> try: ... bulk.execute() ... except BulkWriteError as bwe: ... pprint(bwe.details) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... {'nInserted': 0, 'nMatched': 1, 'nModified': 1, 'nRemoved': 0, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [{u'code': 11000, u'errmsg': u'...E11000 duplicate key error...', u'index': 1, u'op': {'_id': 4}}]} .. _unordered_bulk: Unordered Bulk Write Operations ............................... Unordered bulk write operations are batched and sent to the server in **arbitrary order** where they may be executed in parallel. Any errors that occur are reported after all operations are attempted. In the next example the first and third operations fail due to the unique constraint on _id. Since we are doing unordered execution the second and fourth operations succeed. .. doctest:: >>> bulk = db.test.initialize_unordered_bulk_op() >>> bulk.insert({'_id': 1}) >>> bulk.find({'_id': 2}).remove_one() >>> bulk.insert({'_id': 3}) >>> bulk.find({'_id': 4}).replace_one({'i': 1}) >>> try: ... bulk.execute() ... except BulkWriteError as bwe: ... pprint(bwe.details) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... {'nInserted': 0, 'nMatched': 1, 'nModified': 1, 'nRemoved': 1, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [{u'code': 11000, u'errmsg': u'...E11000 duplicate key error...', u'index': 0, u'op': {'_id': 1}}, {u'code': 11000, u'errmsg': u'...E11000 duplicate key error...', u'index': 2, u'op': {'_id': 3}}]} Write Concern ............. By default bulk operations are executed with the :attr:`~pymongo.collection.Collection.write_concern` of the collection they are executed against. A custom write concern can be passed to the :meth:`~pymongo.bulk.BulkOperationBuilder.execute` method. Write concern errors (e.g. wtimeout) will be reported after all operations are attempted, regardless of execution order. .. doctest:: >>> bulk = db.test.initialize_ordered_bulk_op() >>> bulk.insert({'a': 0}) >>> bulk.insert({'a': 1}) >>> bulk.insert({'a': 2}) >>> bulk.insert({'a': 3}) >>> try: ... bulk.execute({'w': 3, 'wtimeout': 1}) ... except BulkWriteError as bwe: ... pprint(bwe.details) ... {'nInserted': 4, 'nMatched': 0, 'nModified': 0, 'nRemoved': 0, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [{u'code': 64, u'errInfo': {u'wtimeout': True}, u'errmsg': u'waiting for replication timed out'}], 'writeErrors': []} pymongo-3.2/doc/examples/geo.rst0000644000175000017500000000536412630145074020655 0ustar behackettbehackett00000000000000Geospatial Indexing Example =========================== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('geo_example') This example shows how to create and use a :data:`~pymongo.GEO2D` index in PyMongo. .. mongodoc:: geo Creating a Geospatial Index --------------------------- Creating a geospatial index in pymongo is easy: .. doctest:: >>> from pymongo import MongoClient, GEO2D >>> db = MongoClient().geo_example >>> db.places.create_index([("loc", GEO2D)]) u'loc_2d' Inserting Places ---------------- Locations in MongoDB are represented using either embedded documents or lists where the first two elements are coordinates. Here, we'll insert a couple of example locations: .. doctest:: >>> result = db.places.insert_many([{"loc": [2, 5]}, ... {"loc": [30, 5]}, ... {"loc": [1, 2]}, ... {"loc": [4, 4]}]) # doctest: +ELLIPSIS >>> result.inserted_ids [ObjectId('...'), ObjectId('...'), ObjectId('...'), ObjectId('...')] Querying -------- Using the geospatial index we can find documents near another point: .. doctest:: >>> for doc in db.places.find({"loc": {"$near": [3, 6]}}).limit(3): ... repr(doc) # doctest: +ELLIPSIS ... "{u'loc': [2, 5], u'_id': ObjectId('...')}" "{u'loc': [4, 4], u'_id': ObjectId('...')}" "{u'loc': [1, 2], u'_id': ObjectId('...')}" The $maxDistance operator requires the use of :class:`~bson.son.SON`: .. doctest:: >>> from bson.son import SON >>> query = {"loc": SON([("$near", [3, 6]), ("$maxDistance", 100)])} >>> for doc in db.places.find(query).limit(3): ... repr(doc) # doctest: +ELLIPSIS ... "{u'loc': [2, 5], u'_id': ObjectId('...')}" "{u'loc': [4, 4], u'_id': ObjectId('...')}" "{u'loc': [1, 2], u'_id': ObjectId('...')}" It's also possible to query for all items within a given rectangle (specified by lower-left and upper-right coordinates): .. doctest:: >>> query = {"loc": {"$within": {"$box": [[2, 2], [5, 6]]}}} >>> for doc in db.places.find(query).sort('_id'): ... repr(doc) "{u'loc': [2, 5], u'_id': ObjectId('...')}" "{u'loc': [4, 4], u'_id': ObjectId('...')}" Or circle (specified by center point and radius): .. doctest:: >>> query = {"loc": {"$within": {"$center": [[0, 0], 6]}}} >>> for doc in db.places.find(query).sort('_id'): ... repr(doc) # doctest: +ELLIPSIS ... "{u'loc': [2, 5], u'_id': ObjectId('...')}" "{u'loc': [1, 2], u'_id': ObjectId('...')}" "{u'loc': [4, 4], u'_id': ObjectId('...')}" geoNear queries are also supported using :class:`~bson.son.SON`:: >>> from bson.son import SON >>> db.command(SON([('geoNear', 'places'), ('near', [1, 2])])) {u'ok': 1.0, u'stats': ...} pymongo-3.2/doc/examples/index.rst0000644000175000017500000000113612630145074021203 0ustar behackettbehackett00000000000000Examples ======== The examples in this section are intended to give in depth overviews of how to accomplish specific tasks with MongoDB and PyMongo. Unless otherwise noted, all examples assume that a MongoDB instance is running on the default host and port. Assuming you have `downloaded and installed `_ MongoDB, you can start it like so: .. code-block:: bash $ mongod .. toctree:: :maxdepth: 1 aggregation authentication copydb bulk custom_type datetimes geo gevent gridfs high_availability mod_wsgi tls pymongo-3.2/doc/examples/high_availability.rst0000644000175000017500000003307112630145074023550 0ustar behackettbehackett00000000000000High Availability and PyMongo ============================= PyMongo makes it easy to write highly available applications whether you use a `single replica set `_ or a `large sharded cluster `_. Connecting to a Replica Set --------------------------- PyMongo makes working with `replica sets `_ easy. Here we'll launch a new replica set and show how to handle both initialization and normal connections with PyMongo. .. mongodoc:: rs Starting a Replica Set ~~~~~~~~~~~~~~~~~~~~~~ The main `replica set documentation `_ contains extensive information about setting up a new replica set or migrating an existing MongoDB setup, be sure to check that out. Here, we'll just do the bare minimum to get a three node replica set setup locally. .. warning:: Replica sets should always use multiple nodes in production - putting all set members on the same physical node is only recommended for testing and development. We start three ``mongod`` processes, each on a different port and with a different dbpath, but all using the same replica set name "foo". .. code-block:: bash $ mkdir -p /data/db0 /data/db1 /data/db2 $ mongod --port 27017 --dbpath /data/db0 --replSet foo .. code-block:: bash $ mongod --port 27018 --dbpath /data/db1 --replSet foo .. code-block:: bash $ mongod --port 27019 --dbpath /data/db2 --replSet foo Initializing the Set ~~~~~~~~~~~~~~~~~~~~ At this point all of our nodes are up and running, but the set has yet to be initialized. Until the set is initialized no node will become the primary, and things are essentially "offline". To initialize the set we need to connect to a single node and run the initiate command:: >>> from pymongo import MongoClient >>> c = MongoClient('localhost', 27017) .. note:: We could have connected to any of the other nodes instead, but only the node we initiate from is allowed to contain any initial data. After connecting, we run the initiate command to get things started:: >>> config = {'_id': 'foo', 'members': [ ... {'_id': 0, 'host': 'localhost:27017'}, ... {'_id': 1, 'host': 'localhost:27018'}, ... {'_id': 2, 'host': 'localhost:27019'}]} >>> c.admin.command("replSetInitiate", config) {'info': 'Config now saved locally. Should come online in about a minute.', 'ok': 1.0} The three ``mongod`` servers we started earlier will now coordinate and come online as a replica set. Connecting to a Replica Set ~~~~~~~~~~~~~~~~~~~~~~~~~~~ The initial connection as made above is a special case for an uninitialized replica set. Normally we'll want to connect differently. A connection to a replica set can be made using the :meth:`~pymongo.mongo_client.MongoClient` constructor, specifying one or more members of the set, along with the replica set name. Any of the following connects to the replica set we just created:: >>> MongoClient('localhost', replicaset='foo') MongoClient('localhost', 27017) >>> MongoClient('localhost:27018', replicaset='foo') MongoClient('localhost', 27018) >>> MongoClient('localhost', 27019, replicaset='foo') MongoClient('localhost', 27019) >>> MongoClient('mongodb://localhost:27017,localhost:27018/?replicaSet=foo') MongoClient(['localhost:27017', 'localhost:27018']) The addresses passed to :meth:`~pymongo.mongo_client.MongoClient` are called the *seeds*. As long as at least one of the seeds is online, MongoClient discovers all the members in the replica set, and determines which is the current primary and which are secondaries or arbiters. The :class:`~pymongo.mongo_client.MongoClient` constructor is non-blocking: the constructor returns immediately while the client connects to the replica set using background threads. Note how, if you create a client and immediately print its string representation, the client only prints the single host it knows about. If you wait a moment, it discovers the whole replica set: >>> from time import sleep >>> c = MongoClient(replicaset='foo'); print c; sleep(0.1); print c MongoClient('localhost', 27017) MongoClient([u'localhost:27019', u'localhost:27017', u'localhost:27018']) You need not wait for replica set discovery in your application, however. If you need to do any operation with a MongoClient, such as a :meth:`~pymongo.collection.Collection.find` or an :meth:`~pymongo.collection.Collection.insert_one`, the client waits to discover a suitable member before it attempts the operation. Handling Failover ~~~~~~~~~~~~~~~~~ When a failover occurs, PyMongo will automatically attempt to find the new primary node and perform subsequent operations on that node. This can't happen completely transparently, however. Here we'll perform an example failover to illustrate how everything behaves. First, we'll connect to the replica set and perform a couple of basic operations:: >>> db = MongoClient("localhost", replicaSet='foo').test >>> db.test.insert_one({"x": 1}).inserted_id ObjectId('...') >>> db.test.find_one() {u'x': 1, u'_id': ObjectId('...')} By checking the host and port, we can see that we're connected to *localhost:27017*, which is the current primary:: >>> db.client.address ('localhost', 27017) Now let's bring down that node and see what happens when we run our query again:: >>> db.test.find_one() Traceback (most recent call last): pymongo.errors.AutoReconnect: ... We get an :class:`~pymongo.errors.AutoReconnect` exception. This means that the driver was not able to connect to the old primary (which makes sense, as we killed the server), but that it will attempt to automatically reconnect on subsequent operations. When this exception is raised our application code needs to decide whether to retry the operation or to simply continue, accepting the fact that the operation might have failed. On subsequent attempts to run the query we might continue to see this exception. Eventually, however, the replica set will failover and elect a new primary (this should take no more than a couple of seconds in general). At that point the driver will connect to the new primary and the operation will succeed:: >>> db.test.find_one() {u'x': 1, u'_id': ObjectId('...')} >>> db.client.address ('localhost', 27018) Bring the former primary back up. It will rejoin the set as a secondary. Now we can move to the next section: distributing reads to secondaries. .. _secondary-reads: Secondary Reads ~~~~~~~~~~~~~~~ By default an instance of MongoClient sends queries to the primary member of the replica set. To use secondaries for queries we have to change the read preference:: >>> client = MongoClient( ... 'localhost:27017', ... replicaSet='foo', ... readPreference='secondaryPreferred') >>> client.read_preference SecondaryPreferred(tag_sets=None) Now all queries will be sent to the secondary members of the set. If there are no secondary members the primary will be used as a fallback. If you have queries you would prefer to never send to the primary you can specify that using the ``secondary`` read preference. By default the read preference of a :class:`~pymongo.database.Database` is inherited from its MongoClient, and the read preference of a :class:`~pymongo.collection.Collection` is inherited from its Database. To use a different read preference use the :meth:`~pymongo.mongo_client.MongoClient.get_database` method, or the :meth:`~pymongo.database.Database.get_collection` method:: >>> from pymongo import ReadPreference >>> client.read_preference SecondaryPreferred(tag_sets=None) >>> db = client.get_database('test', read_preference=ReadPreference.SECONDARY) >>> db.read_preference Secondary(tag_sets=None) >>> coll = db.get_collection('test', read_preference=ReadPreference.PRIMARY) >>> coll.read_preference Primary() You can also change the read preference of an existing :class:`~pymongo.collection.Collection` with the :meth:`~pymongo.collection.Collection.with_options` method:: >>> coll2 = coll.with_options(read_preference=ReadPreference.NEAREST) >>> coll.read_preference Primary() >>> coll2.read_preference Nearest(tag_sets=None) Note that since most database commands can only be sent to the primary of a replica set, the :meth:`~pymongo.database.Database.command` method does not obey the Database's :attr:`~pymongo.database.Database.read_preference`, but you can pass an explicit read preference to the method:: >>> db.command('dbstats', read_preference=ReadPreference.NEAREST) {...} Reads are configured using three options: **read preference**, **tag sets**, and **local threshold**. **Read preference**: Read preference is configured using one of the classes from :mod:`~pymongo.read_preferences` (:class:`~pymongo.read_preferences.Primary`, :class:`~pymongo.read_preferences.PrimaryPreferred`, :class:`~pymongo.read_preferences.Secondary`, :class:`~pymongo.read_preferences.SecondaryPreferred`, or :class:`~pymongo.read_preferences.Nearest`). For convenience, we also provide :class:`~pymongo.read_preferences.ReadPreference` with the following attributes: - ``PRIMARY``: Read from the primary. This is the default read preference, and provides the strongest consistency. If no primary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``PRIMARY_PREFERRED``: Read from the primary if available, otherwise read from a secondary. - ``SECONDARY``: Read from a secondary. If no matching secondary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``SECONDARY_PREFERRED``: Read from a secondary if available, otherwise from the primary. - ``NEAREST``: Read from any available member. **Tag sets**: Replica-set members can be `tagged `_ according to any criteria you choose. By default, PyMongo ignores tags when choosing a member to read from, but your read preference can be configured with a ``tag_sets`` parameter. ``tag_sets`` must be a list of dictionaries, each dict providing tag values that the replica set member must match. PyMongo tries each set of tags in turn until it finds a set of tags with at least one matching member. For example, to prefer reads from the New York data center, but fall back to the San Francisco data center, tag your replica set members according to their location and create a MongoClient like so:: >>> from pymongo.read_preferences import Secondary >>> db = client.get_database( ... 'test', read_preference=Secondary([{'dc': 'ny'}, {'dc': 'sf'}])) >>> db.read_preference Secondary(tag_sets=[{'dc': 'ny'}, {'dc': 'sf'}]) MongoClient tries to find secondaries in New York, then San Francisco, and raises :class:`~pymongo.errors.AutoReconnect` if none are available. As an additional fallback, specify a final, empty tag set, ``{}``, which means "read from any member that matches the mode, ignoring tags." See :mod:`~pymongo.read_preferences` for more information. .. _distributes reads to secondaries: **Local threshold**: If multiple members match the read preference and tag sets, PyMongo reads from among the nearest members, chosen according to ping time. By default, only members whose ping times are within 15 milliseconds of the nearest are used for queries. You can choose to distribute reads among members with higher latencies by setting ``localThresholdMS`` to a larger number:: >>> client = pymongo.MongoClient( ... replicaSet='repl0', ... readPreference='secondaryPreferred', ... localThresholdMS=35) In this case, PyMongo distributes reads among matching members within 35 milliseconds of the closest member's ping time. .. note:: ``localThresholdMS`` is ignored when talking to a replica set *through* a mongos. The equivalent is the localThreshold_ command line option. .. _localThreshold: http://docs.mongodb.org/manual/reference/mongos/#cmdoption--localThreshold .. _health-monitoring: Health Monitoring ''''''''''''''''' When MongoClient is initialized it launches background threads to monitor the replica set for changes in: * Health: detect when a member goes down or comes up, or if a different member becomes primary * Configuration: detect when members are added or removed, and detect changes in members' tags * Latency: track a moving average of each member's ping time Replica-set monitoring ensures queries are continually routed to the proper members as the state of the replica set changes. .. _mongos-load-balancing: mongos Load Balancing --------------------- An instance of :class:`~pymongo.mongo_client.MongoClient` can be configured with a list of mongos servers: >>> client = MongoClient('mongodb://host1,host2,host3') Each member of the list must be a mongos server. The client continuously monitors all the mongoses' availability, and its network latency to each. PyMongo distributes operations evenly among the set of mongoses within its ``localThresholdMS`` (similar to how it `distributes reads to secondaries`_ in a replica set). By default the threshold is 15 ms. The lowest-latency server, and all servers with latencies no more than ``localThresholdMS`` beyond the lowest-latency server's, receive operations equally. For example, if we have three mongoses: - host1: 20 ms - host2: 35 ms - host3: 40 ms By default the ``localThresholdMS`` is 15 ms, so PyMongo uses host1 and host2 evenly. It uses host1 because its network latency to the driver is shortest. It uses host2 because its latency is within 15 ms of the lowest-latency server's. But it excuses host3: host3 is 20ms beyond the lowest-latency server. If we set ``localThresholdMS`` to 30 ms all servers are within the threshold: >>> client = MongoClient('mongodb://host1,host2,host3/?localThresholdMS=30') pymongo-3.2/doc/examples/gridfs.rst0000644000175000017500000000437212507072032021353 0ustar behackettbehackett00000000000000GridFS Example ============== .. testsetup:: from pymongo import MongoClient client = MongoClient() client.drop_database('gridfs_example') This example shows how to use :mod:`gridfs` to store large binary objects (e.g. files) in MongoDB. .. seealso:: The API docs for :mod:`gridfs`. .. seealso:: `This blog post `_ for some motivation behind this API. Setup ----- We start by creating a :class:`~gridfs.GridFS` instance to use: .. doctest:: >>> from pymongo import MongoClient >>> import gridfs >>> >>> db = MongoClient().gridfs_example >>> fs = gridfs.GridFS(db) Every :class:`~gridfs.GridFS` instance is created with and will operate on a specific :class:`~pymongo.database.Database` instance. Saving and Retrieving Data -------------------------- The simplest way to work with :mod:`gridfs` is to use its key/value interface (the :meth:`~gridfs.GridFS.put` and :meth:`~gridfs.GridFS.get` methods). To write data to GridFS, use :meth:`~gridfs.GridFS.put`: .. doctest:: >>> a = fs.put("hello world") :meth:`~gridfs.GridFS.put` creates a new file in GridFS, and returns the value of the file document's ``"_id"`` key. Given that ``"_id"`` we can use :meth:`~gridfs.GridFS.get` to get back the contents of the file: .. doctest:: >>> fs.get(a).read() 'hello world' :meth:`~gridfs.GridFS.get` returns a file-like object, so we get the file's contents by calling :meth:`~gridfs.grid_file.GridOut.read`. In addition to putting a :class:`str` as a GridFS file, we can also put any file-like object (an object with a :meth:`read` method). GridFS will handle reading the file in chunk-sized segments automatically. We can also add additional attributes to the file as keyword arguments: .. doctest:: >>> b = fs.put(fs.get(a), filename="foo", bar="baz") >>> out = fs.get(b) >>> out.read() 'hello world' >>> out.filename u'foo' >>> out.bar u'baz' >>> out.upload_date datetime.datetime(...) The attributes we set in :meth:`~gridfs.GridFS.put` are stored in the file document, and retrievable after calling :meth:`~gridfs.GridFS.get`. Some attributes (like ``"filename"``) are special and are defined in the GridFS specification - see that document for more details. pymongo-3.2/doc/examples/authentication.rst0000644000175000017500000002006612630145074023116 0ustar behackettbehackett00000000000000Authentication Examples ======================= MongoDB supports several different authentication mechanisms. These examples cover all authentication methods currently supported by PyMongo, documenting Python module and MongoDB version dependencies. Support For Special Characters In Usernames And Passwords --------------------------------------------------------- If your username or password contains special characters (e.g. '/', ' ', or '@') you must ``%xx`` escape them for use in the MongoDB URI. PyMongo uses :meth:`~urllib.unquote_plus` to decode them. For example:: >>> import urllib >>> password = urllib.quote_plus('pass/word') >>> password 'pass%2Fword' >>> MongoClient('mongodb://user:' + password + '@127.0.0.1') MongoClient('127.0.0.1', 27017) SCRAM-SHA-1 (RFC 5802) ---------------------- .. versionadded:: 2.8 SCRAM-SHA-1 is the default authentication mechanism supported by a cluster configured for authentication with MongoDB 3.0 or later. Authentication is per-database and credentials can be specified through the MongoDB URI or passed to the :meth:`~pymongo.database.Database.authenticate` method:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com') >>> client.the_database.authenticate('user', 'password', mechanism='SCRAM-SHA-1') True >>> >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=SCRAM-SHA-1" >>> client = MongoClient(uri) For best performance install `backports.pbkdf2`_, especially on Python older than 2.7.8, or on Python 3 before Python 3.4. .. _backports.pbkdf2: https://pypi.python.org/pypi/backports.pbkdf2/ MONGODB-CR ---------- Before MongoDB 3.0 the default authentication mechanism was MONGODB-CR, the "MongoDB Challenge-Response" protocol:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com') >>> client.the_database.authenticate('user', 'password', mechanism='MONGODB-CR') True >>> >>> uri = "mongodb://user:password@example.com/the_database?authMechanism=MONGODB-CR" >>> client = MongoClient(uri) Default Authentication Mechanism -------------------------------- If no mechanism is specified, PyMongo automatically uses MONGODB-CR when connected to a pre-3.0 version of MongoDB, and SCRAM-SHA-1 when connected to a recent version. Delegated Authentication ------------------------ .. versionadded: 2.5 In MongoDB 2.4.x a separate authentication source can be specified. This feature was introduced in MongoDB 2.4 and removed in 2.6:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com') >>> client.the_database.authenticate('user', ... 'password', ... source='source_database') True >>> >>> uri = "mongodb://user:password@example.com/?authSource=source_database" >>> client = MongoClient(uri) >>> MONGODB-X509 ------------ .. versionadded:: 2.6 The MONGODB-X509 mechanism authenticates a username derived from the distinguished subject name of the X.509 certificate presented by the driver during SSL negotiation. This authentication method requires the use of SSL connections with certificate validation and is available in MongoDB 2.5.1 and newer:: >>> import ssl >>> from pymongo import MongoClient >>> client = MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> client.the_database.authenticate("", ... mechanism='MONGODB-X509') True >>> MONGODB-X509 authenticates against the $external virtual database, so you do not have to specify a database in the URI:: >>> uri = "mongodb://@example.com/?authMechanism=MONGODB-X509" >>> client = MongoClient(uri, ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> .. _use_kerberos: GSSAPI (Kerberos) ----------------- .. versionadded:: 2.5 GSSAPI (Kerberos) authentication is available in the Enterprise Edition of MongoDB, version 2.4 and newer. To authenticate using GSSAPI you must first install the python `kerberos`_ or `pykerberos`_ module using easy_install or pip. Make sure you run kinit before using the following authentication methods:: $ kinit mongodbuser@EXAMPLE.COM mongodbuser@EXAMPLE.COM's Password: $ klist Credentials cache: FILE:/tmp/krb5cc_1000 Principal: mongodbuser@EXAMPLE.COM Issued Expires Principal Feb 9 13:48:51 2013 Feb 9 23:48:51 2013 krbtgt/EXAMPLE.COM@EXAMPLE.COM Now authenticate using the MongoDB URI. GSSAPI authenticates against the $external virtual database so you do not have to specify a database in the URI:: >>> # Note: the kerberos principal must be url encoded. >>> from pymongo import MongoClient >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI" >>> client = MongoClient(uri) >>> or using :meth:`~pymongo.database.Database.authenticate`:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com') >>> db = client.test >>> db.authenticate('mongodbuser@EXAMPLE.COM', mechanism='GSSAPI') True The default service name used by MongoDB and PyMongo is `mongodb`. You can specify a custom service name with the ``authMechanismProperties`` option:: >>> from pymongo import MongoClient >>> uri = "mongodb://mongodbuser%40EXAMPLE.COM@example.com/?authMechanism=GSSAPI&authMechanismProperties=SERVICE_NAME:myservicename" >>> client = MongoClient(uri) >>> >>> client = MongoClient('example.com') >>> db = client.test >>> db.authenticate( ... 'mongodbuser@EXAMPLE.COM', mechanism='GSSAPI', ... authMechanismProperties='SERVICE_NAME:myservicename') True .. note:: Kerberos support is only provided in environments supported by the python `kerberos`_ or `pykerberos`_ modules. This currently limits support to CPython and Unix environments. .. _kerberos: http://pypi.python.org/pypi/kerberos .. _pykerberos: https://pypi.python.org/pypi/pykerberos SASL PLAIN (RFC 4616) --------------------- .. versionadded:: 2.6 MongoDB Enterprise Edition versions 2.5.0 and newer support the SASL PLAIN authentication mechanism, initially intended for delegating authentication to an LDAP server. Using the PLAIN mechanism is very similar to MONGODB-CR. These examples use the $external virtual database for LDAP support:: >>> from pymongo import MongoClient >>> client = MongoClient('example.com') >>> client.the_database.authenticate('user', ... 'password', ... source='$external', ... mechanism='PLAIN') True >>> >>> uri = "mongodb://user:password@example.com/?authMechanism=PLAIN&authSource=$external" >>> client = MongoClient(uri) >>> SASL PLAIN is a clear-text authentication mechanism. We **strongly** recommend that you connect to MongoDB using SSL with certificate validation when using the SASL PLAIN mechanism:: >>> import ssl >>> from pymongo import MongoClient >>> client = MongoClient('example.com', ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> client.the_database.authenticate('user', ... 'password', ... source='$external', ... mechanism='PLAIN') True >>> >>> uri = "mongodb://user:password@example.com/?authMechanism=PLAIN&authSource=$external" >>> client = MongoClient(uri, ... ssl=True, ... ssl_certfile='/path/to/client.pem', ... ssl_cert_reqs=ssl.CERT_REQUIRED, ... ssl_ca_certs='/path/to/ca.pem') >>> pymongo-3.2/doc/examples/custom_type.rst0000644000175000017500000001516312507072032022450 0ustar behackettbehackett00000000000000Custom Type Example =================== This is an example of using a custom type with PyMongo. The example here is a bit contrived, but shows how to use a :class:`~pymongo.son_manipulator.SONManipulator` to manipulate documents as they are saved or retrieved from MongoDB. More specifically, it shows a couple different mechanisms for working with custom datatypes in PyMongo. Setup ----- We'll start by getting a clean database to use for the example: .. doctest:: >>> from pymongo.mongo_client import MongoClient >>> client = MongoClient() >>> client.drop_database("custom_type_example") >>> db = client.custom_type_example Since the purpose of the example is to demonstrate working with custom types, we'll need a custom datatype to use. Here we define the aptly named :class:`Custom` class, which has a single method, :meth:`x`: .. doctest:: >>> class Custom(object): ... def __init__(self, x): ... self.__x = x ... ... def x(self): ... return self.__x ... >>> foo = Custom(10) >>> foo.x() 10 When we try to save an instance of :class:`Custom` with PyMongo, we'll get an :class:`~bson.errors.InvalidDocument` exception: .. doctest:: >>> db.test.insert({"custom": Custom(5)}) Traceback (most recent call last): InvalidDocument: cannot convert value of type to bson Manual Encoding --------------- One way to work around this is to manipulate our data into something we *can* save with PyMongo. To do so we define two methods, :meth:`encode_custom` and :meth:`decode_custom`: .. doctest:: >>> def encode_custom(custom): ... return {"_type": "custom", "x": custom.x()} ... >>> def decode_custom(document): ... assert document["_type"] == "custom" ... return Custom(document["x"]) ... We can now manually encode and decode :class:`Custom` instances and use them with PyMongo: .. doctest:: >>> db.test.insert({"custom": encode_custom(Custom(5))}) ObjectId('...') >>> db.test.find_one() {u'_id': ObjectId('...'), u'custom': {u'x': 5, u'_type': u'custom'}} >>> decode_custom(db.test.find_one()["custom"]) >>> decode_custom(db.test.find_one()["custom"]).x() 5 Automatic Encoding and Decoding ------------------------------- Needless to say, that was a little unwieldy. Let's make this a bit more seamless by creating a new :class:`~pymongo.son_manipulator.SONManipulator`. :class:`~pymongo.son_manipulator.SONManipulator` instances allow you to specify transformations to be applied automatically by PyMongo: .. doctest:: >>> from pymongo.son_manipulator import SONManipulator >>> class Transform(SONManipulator): ... def transform_incoming(self, son, collection): ... for (key, value) in son.items(): ... if isinstance(value, Custom): ... son[key] = encode_custom(value) ... elif isinstance(value, dict): # Make sure we recurse into sub-docs ... son[key] = self.transform_incoming(value, collection) ... return son ... ... def transform_outgoing(self, son, collection): ... for (key, value) in son.items(): ... if isinstance(value, dict): ... if "_type" in value and value["_type"] == "custom": ... son[key] = decode_custom(value) ... else: # Again, make sure to recurse into sub-docs ... son[key] = self.transform_outgoing(value, collection) ... return son ... Now we add our manipulator to the :class:`~pymongo.database.Database`: .. doctest:: >>> db.add_son_manipulator(Transform()) After doing so we can save and restore :class:`Custom` instances seamlessly: .. doctest:: >>> db.test.remove() # remove whatever has already been saved {...} >>> db.test.insert({"custom": Custom(5)}) ObjectId('...') >>> db.test.find_one() {u'_id': ObjectId('...'), u'custom': } >>> db.test.find_one()["custom"].x() 5 If we get a new :class:`~pymongo.database.Database` instance we'll clear out the :class:`~pymongo.son_manipulator.SONManipulator` instance we added: .. doctest:: >>> db = client.custom_type_example This allows us to see what was actually saved to the database: .. doctest:: >>> db.test.find_one() {u'_id': ObjectId('...'), u'custom': {u'x': 5, u'_type': u'custom'}} which is the same format that we encode to with our :meth:`encode_custom` method! Binary Encoding --------------- We can take this one step further by encoding to binary, using a user defined subtype. This allows us to identify what to decode without resorting to tricks like the ``_type`` field used above. We'll start by defining the methods :meth:`to_binary` and :meth:`from_binary`, which convert :class:`Custom` instances to and from :class:`~bson.binary.Binary` instances: .. note:: You could just pickle the instance and save that. What we do here is a little more lightweight. .. doctest:: >>> from bson.binary import Binary >>> def to_binary(custom): ... return Binary(str(custom.x()), 128) ... >>> def from_binary(binary): ... return Custom(int(binary)) ... Next we'll create another :class:`~pymongo.son_manipulator.SONManipulator`, this time using the methods we just defined: .. doctest:: >>> class TransformToBinary(SONManipulator): ... def transform_incoming(self, son, collection): ... for (key, value) in son.items(): ... if isinstance(value, Custom): ... son[key] = to_binary(value) ... elif isinstance(value, dict): ... son[key] = self.transform_incoming(value, collection) ... return son ... ... def transform_outgoing(self, son, collection): ... for (key, value) in son.items(): ... if isinstance(value, Binary) and value.subtype == 128: ... son[key] = from_binary(value) ... elif isinstance(value, dict): ... son[key] = self.transform_outgoing(value, collection) ... return son ... Now we'll empty the :class:`~pymongo.database.Database` and add the new manipulator: .. doctest:: >>> db.test.remove() {...} >>> db.add_son_manipulator(TransformToBinary()) After doing so we can save and restore :class:`Custom` instances seamlessly: .. doctest:: >>> db.test.insert({"custom": Custom(5)}) ObjectId('...') >>> db.test.find_one() {u'_id': ObjectId('...'), u'custom': } >>> db.test.find_one()["custom"].x() 5 We can see what's actually being saved to the database (and verify that it is using a :class:`~bson.binary.Binary` instance) by clearing out the manipulators and repeating our :meth:`~pymongo.collection.Collection.find_one`: .. doctest:: >>> db = client.custom_type_example >>> db.test.find_one() {u'_id': ObjectId('...'), u'custom': Binary('5', 128)} pymongo-3.2/doc/examples/gevent.rst0000644000175000017500000000362712630145074021373 0ustar behackettbehackett00000000000000Gevent ====== PyMongo supports `Gevent `_. Simply call Gevent's ``monkey.patch_all()`` before loading any other modules: .. doctest:: >>> # You must call patch_all() *before* importing any other modules >>> from gevent import monkey >>> monkey.patch_all() >>> from pymongo import MongoClient >>> client = MongoClient() PyMongo uses thread and socket functions from the Python standard library. Gevent's monkey-patching replaces those standard functions so that PyMongo does asynchronous I/O with non-blocking sockets, and schedules operations on greenlets instead of threads. Avoid blocking in Hub.join -------------------------- By default, PyMongo uses threads to discover and monitor your servers' topology (see :ref:`health-monitoring`). If you execute ``monkey.patch_all()`` when your application first begins, PyMongo automatically uses greenlets instead of threads. When shutting down, if your application calls :meth:`~gevent.hub.Hub.join` on Gevent's :class:`~gevent.hub.Hub` without first terminating these background greenlets, the call to :meth:`~gevent.hub.Hub.join` blocks indefinitely. You therefore **must close or dereference** any active :class:`~pymongo.mongo_client.MongoClient` before exiting. An example solution to this issue in some application frameworks is a signal handler to end background greenlets when your application receives SIGHUP: .. code-block:: python import signal def graceful_reload(signum, traceback): """Explicitly close some global MongoClient object.""" client.close() signal.signal(signal.SIGHUP, graceful_reload) Applications using uWSGI prior to 1.9.16 are affected by this issue, or newer uWSGI versions with the ``-gevent-wait-for-hub`` option. See `the uWSGI changelog for details `_. pymongo-3.2/doc/examples/copydb.rst0000644000175000017500000000265412630145074021362 0ustar behackettbehackett00000000000000Copying a Database ================== To copy a database within a single mongod process, or between mongod servers, simply connect to the target mongod and use the :meth:`~pymongo.database.Database.command` method:: >>> from pymongo import MongoClient >>> client = MongoClient('target.example.com') >>> client.admin.command('copydb', fromdb='source_db_name', todb='target_db_name') To copy from a different mongod server that is not password-protected:: >>> client.admin.command('copydb', fromdb='source_db_name', todb='target_db_name', fromhost='source.example.com') If the target server is password-protected, authenticate to the "admin" database first:: >>> client.admin.authenticate('administrator', 'pwd') True >>> client.admin.command('copydb', fromdb='source_db_name', todb='target_db_name', fromhost='source.example.com') See the :doc:`authentication examples `. If the **source** server is password-protected, use the `copyDatabase function in the mongo shell`_. Versions of PyMongo before 3.0 included a ``copy_database`` helper method, but it has been removed. .. _copyDatabase function in the mongo shell: http://docs.mongodb.org/manual/reference/method/db.copyDatabase/ pymongo-3.2/doc/mongo_extensions.py0000644000175000017500000000613612560753776021521 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """MongoDB specific extensions to Sphinx.""" from docutils import nodes from sphinx import addnodes from sphinx.util.compat import (Directive, make_admonition) class mongodoc(nodes.Admonition, nodes.Element): pass class mongoref(nodes.reference): pass def visit_mongodoc_node(self, node): self.visit_admonition(node, "seealso") def depart_mongodoc_node(self, node): self.depart_admonition(node) def visit_mongoref_node(self, node): atts = {"class": "reference external", "href": node["refuri"], "name": node["name"]} self.body.append(self.starttag(node, 'a', '', **atts)) def depart_mongoref_node(self, node): self.body.append('') if not isinstance(node.parent, nodes.TextElement): self.body.append('\n') class MongodocDirective(Directive): has_content = True required_arguments = 0 optional_arguments = 0 final_argument_whitespace = False option_spec = {} def run(self): return make_admonition(mongodoc, self.name, ['See general MongoDB documentation'], self.options, self.content, self.lineno, self.content_offset, self.block_text, self.state, self.state_machine) def process_mongodoc_nodes(app, doctree, fromdocname): for node in doctree.traverse(mongodoc): anchor = None for name in node.parent.parent.traverse(addnodes.desc_signature): anchor = name["ids"][0] break if not anchor: for name in node.parent.traverse(nodes.section): anchor = name["ids"][0] break for para in node.traverse(nodes.paragraph): tag = str(para.traverse()[1]) link = mongoref("", "") link["refuri"] = "http://dochub.mongodb.org/core/%s" % tag link["name"] = anchor link.append(nodes.emphasis(tag, tag)) new_para = nodes.paragraph() new_para += link node.replace(para, new_para) def setup(app): app.add_node(mongodoc, html=(visit_mongodoc_node, depart_mongodoc_node), latex=(visit_mongodoc_node, depart_mongodoc_node), text=(visit_mongodoc_node, depart_mongodoc_node)) app.add_node(mongoref, html=(visit_mongoref_node, depart_mongoref_node)) app.add_directive("mongodoc", MongodocDirective) app.connect("doctree-resolved", process_mongodoc_nodes) pymongo-3.2/doc/python3.rst0000644000175000017500000001222612630145074017664 0ustar behackettbehackett00000000000000Python 3 FAQ ============ .. contents:: What Python 3 versions are supported? ------------------------------------- PyMongo supports Python 3.x where x >= 2. Are there any PyMongo behavior changes with Python 3? ----------------------------------------------------- Only one intentional change. Instances of :class:`bytes` are encoded as BSON type 5 (Binary data) with subtype 0. In Python 3 they are decoded back to :class:`bytes`. In Python 2 they will be decoded to :class:`~bson.binary.Binary` with subtype 0. For example, let's insert a :class:`bytes` instance using Python 3 then read it back. Notice the byte string is decoded back to :class:`bytes`:: Python 3.2.5 (default, Feb 26 2014, 12:40:25) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pymongo >>> c = pymongo.MongoClient() >>> c.test.bintest.insert_one({'binary': b'this is a byte string'}).inserted_id ObjectId('4f9086b1fba5222021000000') >>> c.test.bintest.find_one() {'binary': b'this is a byte string', '_id': ObjectId('4f9086b1fba5222021000000')} Now retrieve the same document in Python 2. Notice the byte string is decoded to :class:`~bson.binary.Binary`:: Python 2.7.6 (default, Feb 26 2014, 10:36:22) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pymongo >>> c = pymongo.MongoClient() >>> c.test.bintest.find_one() {u'binary': Binary('this is a byte string', 0), u'_id': ObjectId('4f9086b1fba5222021000000')} Why can't I share pickled ObjectIds between some versions of Python 2 and 3? ---------------------------------------------------------------------------- Instances of :class:`~bson.objectid.ObjectId` pickled using Python 2 can always be unpickled using Python 3. Due to `http://bugs.python.org/issue13505 `_ you must use Python 3.2.3 or newer to pickle instances of ObjectId if you need to unpickle them in Python 2. If you pickled an ObjectId using Python 2 and want to unpickle it using Python 3 you must pass ``encoding='latin-1'`` to pickle.loads:: Python 2.7.6 (default, Feb 26 2014, 10:36:22) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> from bson.objectid import ObjectId >>> oid = ObjectId() >>> oid ObjectId('4f919ba2fba5225b84000000') >>> pickle.dumps(oid) 'ccopy_reg\n_reconstructor\np0\n(cbson.objectid\...' Python 3.2.5 (default, Feb 26 2014, 12:40:25) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> pickle.loads(b'ccopy_reg\n_reconstructor\np0\n(cbson.objectid\...', encoding='latin-1') ObjectId('4f919ba2fba5225b84000000') If you need to pickle ObjectIds using Python 3 and unpickle them using Python 2 you must use Python 3.2.3 or newer and ``protocol <= 2``:: Python 3.2.3 (v3.2.3:3d0686d90f55, Apr 10 2012, 11:25:50) [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> from bson.objectid import ObjectId >>> oid = ObjectId() >>> oid ObjectId('4f96f20c430ee6bd06000000') >>> pickle.dumps(oid, protocol=2) b'\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c_codecs\nencode\...' Python 2.6.9 (unknown, Feb 26 2014, 12:39:10) [GCC 4.7.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> pickle.loads('\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c_codecs\nencode\...') ObjectId('4f96f20c430ee6bd06000000') Unfortunately this won't work if you pickled the ObjectId using a Python 3 version older than 3.2.3:: Python 3.2.2 (default, Mar 21 2012, 14:32:23) [GCC 4.5.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> from bson.objectid import ObjectId >>> oid = ObjectId() >>> pickle.dumps(oid, protocol=2) b'\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c__builtin__\nbytes\...' Python 2.4.6 (#1, Apr 12 2012, 14:48:24) [GCC 4.5.3] on linux3 Type "help", "copyright", "credits" or "license" for more information. >>> import pickle >>> pickle.loads('\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c__builtin__\nbytes\...') Traceback (most recent call last): File "", line 1, in ? File "/usr/lib/python2.4/pickle.py", line 1394, in loads return Unpickler(file).load() File "/usr/lib/python2.4/pickle.py", line 872, in load dispatch[key](self) File "/usr/lib/python2.4/pickle.py", line 1104, in load_global klass = self.find_class(module, name) File "/usr/lib/python2.4/pickle.py", line 1140, in find_class klass = getattr(mod, name) AttributeError: 'module' object has no attribute 'bytes' .. warning:: Unpickling in Python 2.6 or 2.7 an ObjectId pickled in a Python 3 version older than 3.2.3 will seem to succeed but the resulting ObjectId instance will contain garbage data. >>> pickle.loads('\x80\x02cbson.objectid\nObjectId\nq\x00)\x81q\x01c__builtin__\nbytes\...) ObjectId('5b37392c203135302c203234362c2034352c203235312c203136352c2033342c203532...') pymongo-3.2/ez_setup.py0000644000175000017500000002757312507072032017200 0ustar behackettbehackett00000000000000#!python """Bootstrap setuptools installation If you want to use setuptools in your package's setup.py, just include this file in the same directory with it, and add this to the top of your setup.py:: from ez_setup import use_setuptools use_setuptools() If you want to require a specific version of setuptools, set a download mirror, or use an alternate download directory, you can do so by supplying the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import os import shutil import sys import tempfile import tarfile import optparse import subprocess import platform from distutils import log try: from site import USER_SITE except ImportError: USER_SITE = None DEFAULT_VERSION = "1.4.2" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" def _python_cmd(*args): args = (sys.executable,) + args return subprocess.call(args) == 0 def _check_call_py24(cmd, *args, **kwargs): res = subprocess.call(cmd, *args, **kwargs) class CalledProcessError(Exception): pass if not res == 0: msg = "Command '%s' return non-zero exit status %d" % (cmd, res) raise CalledProcessError(msg) vars(subprocess).setdefault('check_call', _check_call_py24) def _install(tarball, install_args=()): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _build_egg(egg, tarball, to_dir): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) finally: os.chdir(old_wd) shutil.rmtree(tmpdir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') def _do_download(version, download_base, to_dir, download_delay): egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): tarball = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, tarball, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: del sys.modules['pkg_resources'] import setuptools setuptools.bootstrap_install_from = egg def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15): # making sure we use the absolute path to_dir = os.path.abspath(to_dir) was_imported = 'pkg_resources' in sys.modules or \ 'setuptools' in sys.modules try: import pkg_resources except ImportError: return _do_download(version, download_base, to_dir, download_delay) try: pkg_resources.require("setuptools>=" + version) return except pkg_resources.VersionConflict: e = sys.exc_info()[1] if was_imported: sys.stderr.write( "The required version of setuptools (>=%s) is not available,\n" "and can't be installed while this script is running. Please\n" "install a more recent version first, using\n" "'easy_install -U setuptools'." "\n\n(Currently using %r)\n" % (version, e.args[0])) sys.exit(2) else: del pkg_resources, sys.modules['pkg_resources'] # reload ok return _do_download(version, download_base, to_dir, download_delay) except pkg_resources.DistributionNotFound: return _do_download(version, download_base, to_dir, download_delay) def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell (which will validate trust). Raise an exception if the command cannot complete. """ target = os.path.abspath(target) cmd = [ 'powershell', '-Command', "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), ] _clean_check(cmd, target) def has_powershell(): if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """ Use Python to download the file, even though it cannot authenticate the connection. """ try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen src = dst = None try: src = urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = src.read() dst = open(target, "wb") dst.write(data) finally: if src: src.close() if dst: dst.close() download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = [ download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ] for dl in downloaders: if dl.viable(): return dl def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): """Download setuptools from a specified location and return its filename `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) tgz_name = "setuptools-%s.tar.gz" % version url = download_base + tgz_name saveto = os.path.join(to_dir, tgz_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _extractall(self, path=".", members=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on directories afterwards. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by getmembers(). """ import copy import operator from tarfile import ExtractError directories = [] if members is None: members = self for tarinfo in members: if tarinfo.isdir(): # Extract directories with a safe mode. directories.append(tarinfo) tarinfo = copy.copy(tarinfo) tarinfo.mode = 448 # decimal for oct 0700 self.extract(tarinfo, path) # Reverse sort directories. if sys.version_info < (2, 4): def sorter(dir1, dir2): return cmp(dir1.name, dir2.name) directories.sort(sorter) directories.reverse() else: directories.sort(key=operator.attrgetter('name'), reverse=True) # Set correct owner, mtime and filemode on directories. for tarinfo in directories: dirpath = os.path.join(path, tarinfo.name) try: self.chown(tarinfo, dirpath) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError: e = sys.exc_info()[1] if self.errorlevel > 1: raise else: self._dbg(1, "tarfile: %s" % e) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package """ install_args = [] if options.user_install: if sys.version_info < (2, 6): log.warn("--user requires Python 2.6 or later") raise SystemExit(1) install_args.append('--user') return install_args def _parse_args(): """ Parse the command line for options """ parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package (requires Python 2.6 or later)') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) options, args = parser.parse_args() # positional arguments are ignored return options def main(version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" options = _parse_args() tarball = download_setuptools(download_base=options.download_base, downloader_factory=options.downloader_factory) return _install(tarball, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) pymongo-3.2/test/0000755000175000017500000000000012631423130015726 5ustar behackettbehackett00000000000000pymongo-3.2/test/test_topology.py0000644000175000017500000005377012630145074021236 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the topology module.""" import sys sys.path[0:0] = [""] import threading from bson.py3compat import imap from pymongo import common from pymongo.read_preferences import ReadPreference, Secondary from pymongo.server_type import SERVER_TYPE from pymongo.topology import Topology from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure) from pymongo.ismaster import IsMaster from pymongo.monitor import Monitor from pymongo.pool import PoolOptions from pymongo.server_description import ServerDescription from pymongo.server_selectors import (any_server_selector, writable_server_selector) from pymongo.settings import TopologySettings from test import client_knobs, unittest from test.utils import wait_until class MockSocketInfo(object): def close(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass class MockPool(object): def __init__(self, *args, **kwargs): self.pool_id = 0 self._lock = threading.Lock() def get_socket(self, all_credentials): return MockSocketInfo() def return_socket(self, _): pass def reset(self): with self._lock: self.pool_id += 1 class MockMonitor(object): def __init__(self, server_description, topology, pool, topology_settings): self._server_description = server_description self._topology = topology def open(self): pass def request_check(self): pass def close(self): pass class SetNameDiscoverySettings(TopologySettings): def get_topology_type(self): return TOPOLOGY_TYPE.ReplicaSetNoPrimary address = ('a', 27017) def create_mock_topology( seeds=None, replica_set_name=None, monitor_class=MockMonitor): partitioned_seeds = list(imap(common.partition_node, seeds or ['a'])) topology_settings = TopologySettings( partitioned_seeds, replica_set_name=replica_set_name, pool_class=MockPool, monitor_class=monitor_class) t = Topology(topology_settings) t.open() return t def got_ismaster(topology, server_address, ismaster_response): server_description = ServerDescription( server_address, IsMaster(ismaster_response), 0) topology.on_change(server_description) def disconnected(topology, server_address): # Create new description of server type Unknown. topology.on_change(ServerDescription(server_address)) def get_type(topology, hostname): description = topology.get_server_by_address((hostname, 27017)).description return description.server_type class TopologyTest(unittest.TestCase): """Disables periodic monitoring, to make tests deterministic.""" def setUp(self): super(TopologyTest, self).setUp() self.client_knobs = client_knobs(heartbeat_frequency=999999) self.client_knobs.enable() self.addCleanup(self.client_knobs.disable) # Use assertRaisesRegex if available, otherwise use Python 2.7's # deprecated assertRaisesRegexp, with a 'p'. if not hasattr(unittest.TestCase, 'assertRaisesRegex'): TopologyTest.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp class TestTopologyConfiguration(TopologyTest): def test_timeout_configuration(self): pool_options = PoolOptions(connect_timeout=1, socket_timeout=2) topology_settings = TopologySettings(pool_options=pool_options) t = Topology(topology_settings=topology_settings) t.open() # Get the default server. server = t.get_server_by_address(('localhost', 27017)) # The pool for application operations obeys our settings. self.assertEqual(1, server._pool.opts.connect_timeout) self.assertEqual(2, server._pool.opts.socket_timeout) # The pool for monitoring operations uses our connect_timeout as both # its connect_timeout and its socket_timeout. monitor = server._monitor self.assertEqual(1, monitor._pool.opts.connect_timeout) self.assertEqual(1, monitor._pool.opts.socket_timeout) # The monitor, not its pool, is responsible for calling ismaster. self.assertFalse(monitor._pool.handshake) class TestSingleServerTopology(TopologyTest): def test_direct_connection(self): for server_type, ismaster_response in [ (SERVER_TYPE.RSPrimary, { 'ok': 1, 'ismaster': True, 'hosts': ['a'], 'setName': 'rs'}), (SERVER_TYPE.RSSecondary, { 'ok': 1, 'ismaster': False, 'secondary': True, 'hosts': ['a'], 'setName': 'rs'}), (SERVER_TYPE.Mongos, { 'ok': 1, 'ismaster': True, 'msg': 'isdbgrid'}), (SERVER_TYPE.RSArbiter, { 'ok': 1, 'ismaster': False, 'arbiterOnly': True, 'hosts': ['a'], 'setName': 'rs'}), (SERVER_TYPE.Standalone, { 'ok': 1, 'ismaster': True}), # Slave. (SERVER_TYPE.Standalone, { 'ok': 1, 'ismaster': False}), ]: t = create_mock_topology() # Can't select a server while the only server is of type Unknown. with self.assertRaisesRegex(ConnectionFailure, 'No servers found yet'): t.select_servers(any_server_selector, server_selection_timeout=0) got_ismaster(t, address, ismaster_response) # Topology type never changes. self.assertEqual(TOPOLOGY_TYPE.Single, t.description.topology_type) # No matter whether the server is writable, # select_servers() returns it. s = t.select_server(writable_server_selector) self.assertEqual(server_type, s.description.server_type) def test_reopen(self): t = create_mock_topology() # Additional calls are permitted. t.open() t.open() def test_unavailable_seed(self): t = create_mock_topology() disconnected(t, address) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) def test_round_trip_time(self): round_trip_time = 125 available = True class TestMonitor(Monitor): def _check_with_socket(self, sock_info): if available: return IsMaster({'ok': 1}), round_trip_time else: raise AutoReconnect('mock monitor error') t = create_mock_topology(monitor_class=TestMonitor) s = t.select_server(writable_server_selector) self.assertEqual(125, s.description.round_trip_time) round_trip_time = 25 t.request_check_all() # Exponential weighted average: .8 * 125 + .2 * 25 = 105. self.assertAlmostEqual(105, s.description.round_trip_time) # The server is temporarily down. available = False t.request_check_all() def raises_err(): try: t.select_server(writable_server_selector, server_selection_timeout=0.1) except ConnectionFailure: return True else: return False wait_until(raises_err, 'discover server is down') self.assertIsNone(s.description.round_trip_time) # Bring it back, RTT is now 20 milliseconds. available = True round_trip_time = 20 def new_average(): # We reset the average to the most recent measurement. description = s.description return (description.round_trip_time is not None and round(abs(20 - description.round_trip_time), 7) == 0) tries = 0 while not new_average(): t.request_check_all() tries += 1 if tries > 10: self.fail("Didn't ever calculate correct new average") class TestMultiServerTopology(TopologyTest): def test_close(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.RSSecondary, get_type(t, 'b')) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) t.close() self.assertEqual(2, len(t.description.server_descriptions())) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'b')) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetNoPrimary, t.description.topology_type) got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'b')) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) def test_reset_server(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b']}) t.reset_server(('a', 27017)) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.RSSecondary, get_type(t, 'b')) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetNoPrimary, t.description.topology_type) got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b']}) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) t.reset_server(('b', 27017)) self.assertEqual(SERVER_TYPE.RSPrimary, get_type(t, 'a')) self.assertEqual(SERVER_TYPE.Unknown, get_type(t, 'b')) self.assertEqual('rs', t.description.replica_set_name) self.assertEqual(TOPOLOGY_TYPE.ReplicaSetWithPrimary, t.description.topology_type) def test_reset_removed_server(self): t = create_mock_topology(replica_set_name='rs') # No error resetting a server not in the TopologyDescription. t.reset_server(('b', 27017)) # Server was *not* added as type Unknown. self.assertFalse(t.has_server(('b', 27017))) def test_discover_set_name_from_primary(self): # Discovering a replica set without the setName supplied by the user # is not yet supported by MongoClient, but Topology can do it. topology_settings = SetNameDiscoverySettings( seeds=[address], pool_class=MockPool, monitor_class=MockMonitor) t = Topology(topology_settings) self.assertEqual(t.description.replica_set_name, None) self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetNoPrimary) got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) self.assertEqual(t.description.replica_set_name, 'rs') self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetWithPrimary) # Another response from the primary. Tests the code that processes # primary response when topology type is already ReplicaSetWithPrimary. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) # No change. self.assertEqual(t.description.replica_set_name, 'rs') self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetWithPrimary) def test_discover_set_name_from_secondary(self): # Discovering a replica set without the setName supplied by the user # is not yet supported by MongoClient, but Topology can do it. topology_settings = SetNameDiscoverySettings( seeds=[address], pool_class=MockPool, monitor_class=MockMonitor) t = Topology(topology_settings) self.assertEqual(t.description.replica_set_name, None) self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetNoPrimary) got_ismaster(t, address, { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a']}) self.assertEqual(t.description.replica_set_name, 'rs') self.assertEqual(t.description.topology_type, TOPOLOGY_TYPE.ReplicaSetNoPrimary) def test_wire_version(self): t = create_mock_topology(replica_set_name='rs') t.description.check_compatible() # No error. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) # Use defaults. server = t.get_server_by_address(address) self.assertEqual(server.description.min_wire_version, 0) self.assertEqual(server.description.max_wire_version, 0) got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a'], 'minWireVersion': 1, 'maxWireVersion': 5}) self.assertEqual(server.description.min_wire_version, 1) self.assertEqual(server.description.max_wire_version, 5) # Incompatible. got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a'], 'minWireVersion': 11, 'maxWireVersion': 12}) try: t.select_servers(any_server_selector) except ConfigurationError as e: # Error message should say which server failed and why. self.assertTrue('a:27017' in str(e)) self.assertTrue('wire protocol versions 11 through 12' in str(e)) else: self.fail('No error with incompatible wire version') def test_max_write_batch_size(self): t = create_mock_topology(seeds=['a', 'b'], replica_set_name='rs') def write_batch_size(): s = t.select_server(writable_server_selector) return s.description.max_write_batch_size got_ismaster(t, ('a', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'maxWriteBatchSize': 1}) got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'maxWriteBatchSize': 2}) # Uses primary's max batch size. self.assertEqual(1, write_batch_size()) # b becomes primary. got_ismaster(t, ('b', 27017), { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a', 'b'], 'maxWriteBatchSize': 2}) self.assertEqual(2, write_batch_size()) def wait_for_master(topology): """Wait for a Topology to discover a writable server. If the monitor is currently calling ismaster, a blocking call to select_server from this thread can trigger a spurious wake of the monitor thread. In applications this is harmless but it would break some tests, so we pass server_selection_timeout=0 and poll instead. """ def get_master(): try: return topology.select_server(writable_server_selector, 0) except ConnectionFailure: return None return wait_until(get_master, 'find master') class TestTopologyErrors(TopologyTest): # Errors when calling ismaster. def test_pool_reset(self): # ismaster succeeds at first, then always raises socket error. ismaster_count = [0] class TestMonitor(Monitor): def _check_with_socket(self, sock_info): ismaster_count[0] += 1 if ismaster_count[0] == 1: return IsMaster({'ok': 1}), 0 else: raise AutoReconnect('mock monitor error') t = create_mock_topology(monitor_class=TestMonitor) server = wait_for_master(t) self.assertEqual(1, ismaster_count[0]) pool_id = server.pool.pool_id # Pool is reset by ismaster failure. t.request_check_all() self.assertNotEqual(pool_id, server.pool.pool_id) def test_ismaster_retry(self): # ismaster succeeds at first, then raises socket error, then succeeds. ismaster_count = [0] class TestMonitor(Monitor): def _check_with_socket(self, sock_info): ismaster_count[0] += 1 if ismaster_count[0] in (1, 3): return IsMaster({'ok': 1}), 0 else: raise AutoReconnect('mock monitor error') t = create_mock_topology(monitor_class=TestMonitor) server = wait_for_master(t) self.assertEqual(1, ismaster_count[0]) self.assertEqual(SERVER_TYPE.Standalone, server.description.server_type) # Second ismaster call, then immediately the third. t.request_check_all() self.assertEqual(3, ismaster_count[0]) self.assertEqual(SERVER_TYPE.Standalone, get_type(t, 'a')) def test_internal_monitor_error(self): exception = AssertionError('internal error') class TestMonitor(Monitor): def _check_with_socket(self, sock_info): raise exception t = create_mock_topology(monitor_class=TestMonitor) with self.assertRaisesRegex(ConnectionFailure, 'internal error'): t.select_server(any_server_selector, server_selection_timeout=0.5) class TestServerSelectionErrors(TopologyTest): def assertMessage(self, message, topology, selector=any_server_selector): with self.assertRaises(ConnectionFailure) as context: topology.select_server(selector, server_selection_timeout=0) self.assertEqual(message, str(context.exception)) def test_no_primary(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, address, { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs', 'hosts': ['a']}) self.assertMessage('No replica set members match selector "Primary()"', t, ReadPreference.PRIMARY) self.assertMessage('No primary available for writes', t, writable_server_selector) def test_no_secondary(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, address, { 'ok': 1, 'ismaster': True, 'setName': 'rs', 'hosts': ['a']}) self.assertMessage( 'No replica set members match selector' ' "Secondary(tag_sets=None)"', t, ReadPreference.SECONDARY) self.assertMessage( "No replica set members match selector" " \"Secondary(tag_sets=[{'dc': 'ny'}])\"", t, Secondary(tag_sets=[{'dc': 'ny'}])) def test_bad_replica_set_name(self): t = create_mock_topology(replica_set_name='rs') got_ismaster(t, address, { 'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'wrong', 'hosts': ['a']}) self.assertMessage( 'No replica set members available for replica set name "rs"', t) def test_multiple_standalones(self): # Standalones are removed from a topology with multiple seeds. t = create_mock_topology(seeds=['a', 'b']) got_ismaster(t, ('a', 27017), {'ok': 1}) got_ismaster(t, ('b', 27017), {'ok': 1}) self.assertMessage('No servers available', t) def test_no_mongoses(self): # Standalones are removed from a topology with multiple seeds. t = create_mock_topology(seeds=['a', 'b']) # Discover a mongos and change topology type to Sharded. got_ismaster(t, ('a', 27017), {'ok': 1, 'msg': 'isdbgrid'}) # Oops, both servers are standalone now. Remove them. got_ismaster(t, ('a', 27017), {'ok': 1}) got_ismaster(t, ('b', 27017), {'ok': 1}) self.assertMessage('No mongoses available', t) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_replica_set_client.py0000644000175000017500000003226512630145074023206 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the mongo_replica_set_client module.""" import socket import sys import warnings import time sys.path[0:0] = [""] from bson.codec_options import CodecOptions from bson.py3compat import u from bson.son import SON from pymongo.common import partition_node from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, NetworkTimeout, NotMasterError, OperationFailure) from pymongo.mongo_client import MongoClient from pymongo.mongo_replica_set_client import MongoReplicaSetClient from pymongo.read_preferences import ReadPreference, Secondary, Nearest from pymongo.write_concern import WriteConcern from test import (client_context, client_knobs, host, IntegrationTest, pair, port, unittest, db_pwd, db_user, MockClientTest) from test.pymongo_mocks import MockClient from test.utils import (connected, delay, ignore_deprecations, one, rs_client, single_client, wait_until) class TestReplicaSetClientBase(IntegrationTest): @classmethod @client_context.require_replica_set def setUpClass(cls): super(TestReplicaSetClientBase, cls).setUpClass() cls.name = client_context.replica_set_name cls.w = client_context.w ismaster = client_context.ismaster cls.hosts = set(partition_node(h) for h in ismaster['hosts']) cls.arbiters = set(partition_node(h) for h in ismaster.get("arbiters", [])) repl_set_status = client_context.client.admin.command( 'replSetGetStatus') primary_info = [ m for m in repl_set_status['members'] if m['stateStr'] == 'PRIMARY' ][0] cls.primary = partition_node(primary_info['name']) cls.secondaries = set( partition_node(m['name']) for m in repl_set_status['members'] if m['stateStr'] == 'SECONDARY') class TestReplicaSetClient(TestReplicaSetClientBase): def test_deprecated(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) with self.assertRaises(DeprecationWarning): MongoReplicaSetClient() def test_connect(self): client = MongoClient(pair, replicaSet='fdlksjfdslkjfd', serverSelectionTimeoutMS=100) with self.assertRaises(ConnectionFailure): client.test.test.find_one() def test_repr(self): with ignore_deprecations(): client = MongoReplicaSetClient(host, port, replicaSet=self.name) self.assertIn("MongoReplicaSetClient(host=[", repr(client)) self.assertIn(pair, repr(client)) def test_properties(self): c = client_context.rs_client c.admin.command('ping') wait_until(lambda: c.primary == self.primary, "discover primary") wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") wait_until(lambda: c.secondaries == self.secondaries, "discover secondaries") self.assertEqual(c.primary, self.primary) self.assertEqual(c.secondaries, self.secondaries) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 100) # Make sure MongoClient's properties are copied to Database and # Collection. for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions()) self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, WriteConcern()) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.PRIMARY, cursor._Cursor__read_preference) tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets=tag_sets) c = MongoClient( pair, replicaSet=self.name, maxPoolSize=25, document_class=SON, tz_aware=True, read_preference=secondary, localThresholdMS=77, j=True) self.assertEqual(c.max_pool_size, 25) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions(SON, True)) self.assertEqual(obj.read_preference, secondary) self.assertEqual(obj.write_concern, WriteConcern(j=True)) cursor = c.pymongo_test.test.find() self.assertEqual( secondary, cursor._Cursor__read_preference) nearest = Nearest(tag_sets=[{'dc': 'ny'}, {}]) cursor = c.pymongo_test.get_collection( "test", read_preference=nearest).find() self.assertEqual(nearest, cursor._Cursor__read_preference) self.assertEqual(c.max_bson_size, 16777216) c.close() def test_auto_reconnect_exception_when_read_preference_is_secondary(self): c = MongoClient(pair, replicaSet=self.name, serverSelectionTimeoutMS=100) db = c.pymongo_test def raise_socket_error(*args, **kwargs): raise socket.error old_sendall = socket.socket.sendall socket.socket.sendall = raise_socket_error try: cursor = db.get_collection( "test", read_preference=ReadPreference.SECONDARY).find() self.assertRaises(AutoReconnect, cursor.next) finally: socket.socket.sendall = old_sendall def test_timeout_does_not_mark_member_down(self): # If a query times out, the client shouldn't mark the member "down". # Disable background refresh. with client_knobs(heartbeat_frequency=999999): c = rs_client(socketTimeoutMS=3000, w=self.w) collection = c.pymongo_test.test collection.insert_one({}) # Query the primary. self.assertRaises( NetworkTimeout, collection.find_one, {'$where': delay(5)}) self.assertTrue(c.primary) collection.find_one() # No error. coll = collection.with_options( read_preference=ReadPreference.SECONDARY) # Query the secondary. self.assertRaises( NetworkTimeout, coll.find_one, {'$where': delay(5)}) self.assertTrue(c.secondaries) # No error. coll.find_one() @client_context.require_replica_set @client_context.require_ipv6 def test_ipv6(self): c = MongoClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name) # Client switches to IPv4 once it has first ismaster response. msg = 'discovered primary with IPv4 address "%r"' % (self.primary,) wait_until(lambda: c.primary == self.primary, msg) # Same outcome with both IPv4 and IPv6 seeds. c = MongoClient("[::1]:%d,localhost:%d" % (port, port), replicaSet=self.name) wait_until(lambda: c.primary == self.primary, msg) if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port) client = MongoClient(uri, replicaSet=self.name) client.pymongo_test.test.insert_one({"dummy": u("object")}) client.pymongo_test_bernie.test.insert_one({"dummy": u("object")}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) client.close() def _test_kill_cursor_explicit(self, read_pref): with client_knobs(kill_cursor_frequency=0.01): c = rs_client(read_preference=read_pref, w=self.w) db = c.pymongo_test db.drop_collection("test") test = db.test test.insert_many([{"i": i} for i in range(20)]) # Partially evaluate cursor so it's left alive, then kill it cursor = test.find().batch_size(10) next(cursor) self.assertNotEqual(0, cursor.cursor_id) if read_pref == ReadPreference.PRIMARY: msg = "Expected cursor's address to be %s, got %s" % ( c.primary, cursor.address) self.assertEqual(cursor.address, c.primary, msg) else: self.assertNotEqual( cursor.address, c.primary, "Expected cursor's address not to be primary") cursor_id = cursor.cursor_id # Cursor dead on server - trigger a getMore on the same cursor_id # and check that the server returns an error. cursor2 = cursor.clone() cursor2._Cursor__id = cursor_id if sys.platform.startswith('java') or 'PyPy' in sys.version: # Explicitly kill cursor. cursor.close() else: # Implicitly kill it in CPython. del cursor time.sleep(5) self.assertRaises(OperationFailure, lambda: list(cursor2)) def test_kill_cursor_explicit_primary(self): self._test_kill_cursor_explicit(ReadPreference.PRIMARY) def test_kill_cursor_explicit_secondary(self): self._test_kill_cursor_explicit(ReadPreference.SECONDARY) def test_not_master_error(self): secondary_address = one(self.secondaries) direct_client = single_client(*secondary_address) with self.assertRaises(NotMasterError): direct_client.pymongo_test.collection.insert_one({}) db = direct_client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)) with self.assertRaises(NotMasterError): db.collection.insert_one({}) class TestReplicaSetWireVersion(MockClientTest): @client_context.require_connection @client_context.require_no_auth def test_wire_version(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1', replicaSet='rs', connect=False) c.set_wire_version_range('a:1', 1, 5) c.set_wire_version_range('b:2', 0, 1) c.set_wire_version_range('c:3', 1, 2) c.db.command('ismaster') # Connect. c.set_wire_version_range('a:1', 2, 2) # A secondary doesn't overlap with us. c.set_wire_version_range('b:2', 5, 6) def raises_configuration_error(): try: c.db.collection.find_one() return False except ConfigurationError: return True wait_until(raises_configuration_error, 'notice we are incompatible with server') self.assertRaises(ConfigurationError, c.db.collection.insert_one, {}) class TestReplicaSetClientInternalIPs(MockClientTest): @client_context.require_connection def test_connect_with_internal_ips(self): # Client is passed an IP it can reach, 'a:1', but the RS config # only contains unreachable IPs like 'internal-ip'. PYTHON-608. with self.assertRaises(AutoReconnect) as context: connected(MockClient( standalones=[], members=['a:1'], mongoses=[], ismaster_hosts=['internal-ip:27017'], host='a:1', replicaSet='rs', serverSelectionTimeoutMS=100)) self.assertEqual( "Could not reach any servers in [('internal-ip', 27017)]." " Replica set is configured with internal hostnames or IPs?", str(context.exception)) class TestReplicaSetClientMaxWriteBatchSize(MockClientTest): @client_context.require_connection def test_max_write_batch_size(self): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs', connect=False) c.set_max_write_batch_size('a:1', 1) c.set_max_write_batch_size('b:2', 2) # Uses primary's max batch size. self.assertEqual(c.max_write_batch_size, 1) # b becomes primary. c.mock_primary = 'b:2' wait_until(lambda: c.max_write_batch_size == 2, 'update max_write_batch_size') if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_gridfs.py0000644000175000017500000004540412631351537020637 0ustar behackettbehackett00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the gridfs package. """ import sys sys.path[0:0] = [""] import datetime import threading import time import gridfs from bson.binary import Binary from bson.py3compat import u, StringIO, string_type from pymongo.mongo_client import MongoClient from pymongo.errors import (ConfigurationError, ConnectionFailure, ServerSelectionTimeoutError) from pymongo.read_preferences import ReadPreference from gridfs.errors import CorruptGridFile, FileExists, NoFile from test.test_replica_set_client import TestReplicaSetClientBase from test import (client_context, unittest, host, port, IntegrationTest) from test.utils import (joinall, single_client, one, rs_client, rs_or_single_client) class JustWrite(threading.Thread): def __init__(self, fs, n): threading.Thread.__init__(self) self.fs = fs self.n = n self.setDaemon(True) def run(self): for _ in range(self.n): file = self.fs.new_file(filename="test") file.write(b"hello") file.close() class JustRead(threading.Thread): def __init__(self, fs, n, results): threading.Thread.__init__(self) self.fs = fs self.n = n self.results = results self.setDaemon(True) def run(self): for _ in range(self.n): file = self.fs.get("test") data = file.read() self.results.append(data) assert data == b"hello" class TestGridfsNoConnect(unittest.TestCase): @classmethod def setUpClass(cls): client = MongoClient(host, port, connect=False) cls.db = client.pymongo_test def test_gridfs(self): self.assertRaises(TypeError, gridfs.GridFS, "foo") self.assertRaises(TypeError, gridfs.GridFS, self.db, 5) class TestGridfs(IntegrationTest): @classmethod def setUpClass(cls): super(TestGridfs, cls).setUpClass() cls.fs = gridfs.GridFS(cls.db) cls.alt = gridfs.GridFS(cls.db, "alt") def setUp(self): self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") self.db.drop_collection("alt.files") self.db.drop_collection("alt.chunks") def test_basic(self): oid = self.fs.put(b"hello world") self.assertEqual(b"hello world", self.fs.get(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(1, self.db.fs.chunks.count()) self.fs.delete(oid) self.assertRaises(NoFile, self.fs.get, oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) self.assertRaises(NoFile, self.fs.get, "foo") oid = self.fs.put(b"hello world", _id="foo") self.assertEqual("foo", oid) self.assertEqual(b"hello world", self.fs.get("foo").read()) def test_multi_chunk_delete(self): self.db.fs.drop() self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) gfs = gridfs.GridFS(self.db) oid = gfs.put(b"hello", chunkSize=1) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(5, self.db.fs.chunks.count()) gfs.delete(oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) def test_list(self): self.assertEqual([], self.fs.list()) self.fs.put(b"hello world") self.assertEqual([], self.fs.list()) # PYTHON-598: in server versions before 2.5.x, creating an index on # filename, uploadDate causes list() to include None. self.fs.get_last_version() self.assertEqual([], self.fs.list()) self.fs.put(b"", filename="mike") self.fs.put(b"foo", filename="test") self.fs.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set(self.fs.list())) def test_empty_file(self): oid = self.fs.put(b"") self.assertEqual(b"", self.fs.get(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) raw = self.db.fs.files.find_one() self.assertEqual(0, raw["length"]) self.assertEqual(oid, raw["_id"]) self.assertTrue(isinstance(raw["uploadDate"], datetime.datetime)) self.assertEqual(255 * 1024, raw["chunkSize"]) self.assertTrue(isinstance(raw["md5"], string_type)) def test_corrupt_chunk(self): files_id = self.fs.put(b'foobar') self.db.fs.chunks.update_one({'files_id': files_id}, {'$set': {'data': Binary(b'foo', 0)}}) try: out = self.fs.get(files_id) self.assertRaises(CorruptGridFile, out.read) out = self.fs.get(files_id) self.assertRaises(CorruptGridFile, out.readline) finally: self.fs.delete(files_id) def test_put_ensures_index(self): # setUp has dropped collections. names = self.db.collection_names() self.assertFalse([name for name in names if name.startswith('fs')]) chunks = self.db.fs.chunks files = self.db.fs.files self.fs.put(b"junk") self.assertTrue(any( info.get('key') == [('files_id', 1), ('n', 1)] for info in chunks.index_information().values())) self.assertTrue(any( info.get('key') == [('filename', 1), ('uploadDate', 1)] for info in files.index_information().values())) def test_alt_collection(self): oid = self.alt.put(b"hello world") self.assertEqual(b"hello world", self.alt.get(oid).read()) self.assertEqual(1, self.db.alt.files.count()) self.assertEqual(1, self.db.alt.chunks.count()) self.alt.delete(oid) self.assertRaises(NoFile, self.alt.get, oid) self.assertEqual(0, self.db.alt.files.count()) self.assertEqual(0, self.db.alt.chunks.count()) self.assertRaises(NoFile, self.alt.get, "foo") oid = self.alt.put(b"hello world", _id="foo") self.assertEqual("foo", oid) self.assertEqual(b"hello world", self.alt.get("foo").read()) self.alt.put(b"", filename="mike") self.alt.put(b"foo", filename="test") self.alt.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set(self.alt.list())) def test_threaded_reads(self): self.fs.put(b"hello", _id="test") threads = [] results = [] for i in range(10): threads.append(JustRead(self.fs, 10, results)) threads[i].start() joinall(threads) self.assertEqual( 100 * [b'hello'], results ) def test_threaded_writes(self): threads = [] for i in range(10): threads.append(JustWrite(self.fs, 10)) threads[i].start() joinall(threads) f = self.fs.get_last_version("test") self.assertEqual(f.read(), b"hello") # Should have created 100 versions of 'test' file self.assertEqual( 100, self.db.fs.files.find({'filename': 'test'}).count() ) def test_get_last_version(self): one = self.fs.put(b"foo", filename="test") time.sleep(0.01) two = self.fs.new_file(filename="test") two.write(b"bar") two.close() time.sleep(0.01) two = two._id three = self.fs.put(b"baz", filename="test") self.assertEqual(b"baz", self.fs.get_last_version("test").read()) self.fs.delete(three) self.assertEqual(b"bar", self.fs.get_last_version("test").read()) self.fs.delete(two) self.assertEqual(b"foo", self.fs.get_last_version("test").read()) self.fs.delete(one) self.assertRaises(NoFile, self.fs.get_last_version, "test") def test_get_last_version_with_metadata(self): one = self.fs.put(b"foo", filename="test", author="author") time.sleep(0.01) two = self.fs.put(b"bar", filename="test", author="author") self.assertEqual(b"bar", self.fs.get_last_version(author="author").read()) self.fs.delete(two) self.assertEqual(b"foo", self.fs.get_last_version(author="author").read()) self.fs.delete(one) one = self.fs.put(b"foo", filename="test", author="author1") time.sleep(0.01) two = self.fs.put(b"bar", filename="test", author="author2") self.assertEqual(b"foo", self.fs.get_last_version(author="author1").read()) self.assertEqual(b"bar", self.fs.get_last_version(author="author2").read()) self.assertEqual(b"bar", self.fs.get_last_version(filename="test").read()) self.assertRaises(NoFile, self.fs.get_last_version, author="author3") self.assertRaises(NoFile, self.fs.get_last_version, filename="nottest", author="author1") self.fs.delete(one) self.fs.delete(two) def test_get_version(self): self.fs.put(b"foo", filename="test") time.sleep(0.01) self.fs.put(b"bar", filename="test") time.sleep(0.01) self.fs.put(b"baz", filename="test") time.sleep(0.01) self.assertEqual(b"foo", self.fs.get_version("test", 0).read()) self.assertEqual(b"bar", self.fs.get_version("test", 1).read()) self.assertEqual(b"baz", self.fs.get_version("test", 2).read()) self.assertEqual(b"baz", self.fs.get_version("test", -1).read()) self.assertEqual(b"bar", self.fs.get_version("test", -2).read()) self.assertEqual(b"foo", self.fs.get_version("test", -3).read()) self.assertRaises(NoFile, self.fs.get_version, "test", 3) self.assertRaises(NoFile, self.fs.get_version, "test", -4) def test_get_version_with_metadata(self): one = self.fs.put(b"foo", filename="test", author="author1") time.sleep(0.01) two = self.fs.put(b"bar", filename="test", author="author1") time.sleep(0.01) three = self.fs.put(b"baz", filename="test", author="author2") self.assertEqual(b"foo", self.fs.get_version(filename="test", author="author1", version=-2).read()) self.assertEqual(b"bar", self.fs.get_version(filename="test", author="author1", version=-1).read()) self.assertEqual(b"foo", self.fs.get_version(filename="test", author="author1", version=0).read()) self.assertEqual(b"bar", self.fs.get_version(filename="test", author="author1", version=1).read()) self.assertEqual(b"baz", self.fs.get_version(filename="test", author="author2", version=0).read()) self.assertEqual(b"baz", self.fs.get_version(filename="test", version=-1).read()) self.assertEqual(b"baz", self.fs.get_version(filename="test", version=2).read()) self.assertRaises(NoFile, self.fs.get_version, filename="test", author="author3") self.assertRaises(NoFile, self.fs.get_version, filename="test", author="author1", version=2) self.fs.delete(one) self.fs.delete(two) self.fs.delete(three) def test_put_filelike(self): oid = self.fs.put(StringIO(b"hello world"), chunk_size=1) self.assertEqual(11, self.db.fs.chunks.count()) self.assertEqual(b"hello world", self.fs.get(oid).read()) def test_file_exists(self): oid = self.fs.put(b"hello") self.assertRaises(FileExists, self.fs.put, b"world", _id=oid) one = self.fs.new_file(_id=123) one.write(b"some content") one.close() two = self.fs.new_file(_id=123) self.assertRaises(FileExists, two.write, b'x' * 262146) def test_exists(self): oid = self.fs.put(b"hello") self.assertTrue(self.fs.exists(oid)) self.assertTrue(self.fs.exists({"_id": oid})) self.assertTrue(self.fs.exists(_id=oid)) self.assertFalse(self.fs.exists(filename="mike")) self.assertFalse(self.fs.exists("mike")) oid = self.fs.put(b"hello", filename="mike", foo=12) self.assertTrue(self.fs.exists(oid)) self.assertTrue(self.fs.exists({"_id": oid})) self.assertTrue(self.fs.exists(_id=oid)) self.assertTrue(self.fs.exists(filename="mike")) self.assertTrue(self.fs.exists({"filename": "mike"})) self.assertTrue(self.fs.exists(foo=12)) self.assertTrue(self.fs.exists({"foo": 12})) self.assertTrue(self.fs.exists(foo={"$gt": 11})) self.assertTrue(self.fs.exists({"foo": {"$gt": 11}})) self.assertFalse(self.fs.exists(foo=13)) self.assertFalse(self.fs.exists({"foo": 13})) self.assertFalse(self.fs.exists(foo={"$gt": 12})) self.assertFalse(self.fs.exists({"foo": {"$gt": 12}})) def test_put_unicode(self): self.assertRaises(TypeError, self.fs.put, u("hello")) oid = self.fs.put(u("hello"), encoding="utf-8") self.assertEqual(b"hello", self.fs.get(oid).read()) self.assertEqual("utf-8", self.fs.get(oid).encoding) oid = self.fs.put(u("aé"), encoding="iso-8859-1") self.assertEqual(u("aé").encode("iso-8859-1"), self.fs.get(oid).read()) self.assertEqual("iso-8859-1", self.fs.get(oid).encoding) def test_missing_length_iter(self): # Test fix that guards against PHP-237 self.fs.put(b"", filename="empty") doc = self.db.fs.files.find_one({"filename": "empty"}) doc.pop("length") self.db.fs.files.replace_one({"_id": doc["_id"]}, doc) f = self.fs.get_last_version(filename="empty") def iterate_file(grid_file): for chunk in grid_file: pass return True self.assertTrue(iterate_file(f)) def test_gridfs_lazy_connect(self): client = MongoClient('badhost', connect=False, serverSelectionTimeoutMS=10) db = client.db gfs = gridfs.GridFS(db) self.assertRaises(ServerSelectionTimeoutError, gfs.list) fs = gridfs.GridFS(db) f = fs.new_file() self.assertRaises(ServerSelectionTimeoutError, f.close) def test_gridfs_find(self): self.fs.put(b"test2", filename="two") time.sleep(0.01) self.fs.put(b"test2+", filename="two") time.sleep(0.01) self.fs.put(b"test1", filename="one") time.sleep(0.01) self.fs.put(b"test2++", filename="two") self.assertEqual(3, self.fs.find({"filename": "two"}).count()) self.assertEqual(4, self.fs.find().count()) cursor = self.fs.find( no_cursor_timeout=False).sort("uploadDate", -1).skip(1).limit(2) gout = next(cursor) self.assertEqual(b"test1", gout.read()) cursor.rewind() gout = next(cursor) self.assertEqual(b"test1", gout.read()) gout = next(cursor) self.assertEqual(b"test2+", gout.read()) self.assertRaises(StopIteration, cursor.__next__) cursor.close() self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) def test_gridfs_find_one(self): self.assertEqual(None, self.fs.find_one()) id1 = self.fs.put(b'test1', filename='file1') self.assertEqual(b'test1', self.fs.find_one().read()) id2 = self.fs.put(b'test2', filename='file2', meta='data') self.assertEqual(b'test1', self.fs.find_one(id1).read()) self.assertEqual(b'test2', self.fs.find_one(id2).read()) self.assertEqual(b'test1', self.fs.find_one({'filename': 'file1'}).read()) self.assertEqual('data', self.fs.find_one(id2).meta) def test_grid_in_non_int_chunksize(self): # Lua, and perhaps other buggy GridFS clients, store size as a float. data = b'data' self.fs.put(data, filename='f') self.db.fs.files.update_one({'filename': 'f'}, {'$set': {'chunkSize': 100.0}}) self.assertEqual(data, self.fs.get_version('f').read()) def test_unacknowledged(self): # w=0 is prohibited. with self.assertRaises(ConfigurationError): gridfs.GridFS(rs_or_single_client(w=0).pymongo_test) class TestGridfsReplicaSet(TestReplicaSetClientBase): def test_gridfs_replica_set(self): rsc = rs_client( w=self.w, read_preference=ReadPreference.SECONDARY) fs = gridfs.GridFS(rsc.pymongo_test) gin = fs.new_file() self.assertEqual(gin._coll.read_preference, ReadPreference.PRIMARY) oid = fs.put(b'foo') content = fs.get(oid).read() self.assertEqual(b'foo', content) def test_gridfs_secondary(self): primary_host, primary_port = self.primary primary_connection = single_client(primary_host, primary_port) secondary_host, secondary_port = one(self.secondaries) secondary_connection = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY) primary_connection.pymongo_test.drop_collection("fs.files") primary_connection.pymongo_test.drop_collection("fs.chunks") # Should detect it's connected to secondary and not attempt to # create index fs = gridfs.GridFS(secondary_connection.pymongo_test) # This won't detect secondary, raises error self.assertRaises(ConnectionFailure, fs.put, b'foo') def test_gridfs_secondary_lazy(self): # Should detect it's connected to secondary and not attempt to # create index. secondary_host, secondary_port = one(self.secondaries) client = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY, connect=False) # Still no connection. fs = gridfs.GridFS(client.test_gridfs_secondary_lazy) # Connects, doesn't create index. self.assertRaises(NoFile, fs.get_last_version) self.assertRaises(ConnectionFailure, fs.put, 'data') def tearDown(self): rsc = client_context.rs_client rsc.pymongo_test.drop_collection('fs.files') rsc.pymongo_test.drop_collection('fs.chunks') if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_database.py0000644000175000017500000010052612630145074021116 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the database module.""" import datetime import re import sys import warnings sys.path[0:0] = [""] from bson.code import Code from bson.codec_options import CodecOptions from bson.int64 import Int64 from bson.regex import Regex from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import u, string_type, text_type, PY3 from bson.son import SON from pymongo import (MongoClient, ALL, auth, OFF, SLOW_ONLY, helpers) from pymongo.collection import Collection from pymongo.database import Database from pymongo.errors import (CollectionInvalid, ConfigurationError, ExecutionTimeout, InvalidName, OperationFailure) from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference from pymongo.write_concern import WriteConcern from test import (client_context, SkipTest, unittest, host, port, IntegrationTest) from test.utils import (ignore_deprecations, remove_all_users, rs_or_single_client_noauth, rs_or_single_client, server_started_with_auth) if PY3: long = int class TestDatabaseNoConnect(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = MongoClient(host, port, connect=False) def test_name(self): self.assertRaises(TypeError, Database, self.client, 4) self.assertRaises(InvalidName, Database, self.client, "my db") self.assertRaises(InvalidName, Database, self.client, "my\x00db") self.assertRaises(InvalidName, Database, self.client, u("my\u0000db")) self.assertEqual("name", Database(self.client, "name").name) def test_equality(self): self.assertNotEqual(Database(self.client, "test"), Database(self.client, "mike")) self.assertEqual(Database(self.client, "test"), Database(self.client, "test")) # Explicitly test inequality self.assertFalse(Database(self.client, "test") != Database(self.client, "test")) def test_get_coll(self): db = Database(self.client, "pymongo_test") self.assertEqual(db.test, db["test"]) self.assertEqual(db.test, Collection(db, "test")) self.assertNotEqual(db.test, Collection(db, "mike")) self.assertEqual(db.test.mike, db["test.mike"]) def test_get_collection(self): codec_options = CodecOptions(tz_aware=True) write_concern = WriteConcern(w=2, j=True) read_concern = ReadConcern('majority') coll = self.client.pymongo_test.get_collection( 'foo', codec_options, ReadPreference.SECONDARY, write_concern, read_concern) self.assertEqual('foo', coll.name) self.assertEqual(codec_options, coll.codec_options) self.assertEqual(ReadPreference.SECONDARY, coll.read_preference) self.assertEqual(write_concern, coll.write_concern) self.assertEqual(read_concern, coll.read_concern) def test_getattr(self): db = self.client.pymongo_test self.assertTrue(isinstance(db['_does_not_exist'], Collection)) with self.assertRaises(AttributeError) as context: db._does_not_exist # Message should be: "AttributeError: Database has no attribute # '_does_not_exist'. To access the _does_not_exist collection, # use database['_does_not_exist']". self.assertIn("has no attribute '_does_not_exist'", str(context.exception)) def test_iteration(self): self.assertRaises(TypeError, next, self.client.pymongo_test) class TestDatabase(IntegrationTest): def test_repr(self): self.assertEqual(repr(Database(self.client, "pymongo_test")), "Database(%r, %s)" % (self.client, repr(u("pymongo_test")))) def test_create_collection(self): db = Database(self.client, "pymongo_test") db.test.insert_one({"hello": "world"}) self.assertRaises(CollectionInvalid, db.create_collection, "test") db.drop_collection("test") self.assertRaises(TypeError, db.create_collection, 5) self.assertRaises(TypeError, db.create_collection, None) self.assertRaises(InvalidName, db.create_collection, "coll..ection") test = db.create_collection("test") self.assertTrue(u("test") in db.collection_names()) test.insert_one({"hello": u("world")}) self.assertEqual(db.test.find_one()["hello"], "world") db.drop_collection("test.foo") db.create_collection("test.foo") self.assertTrue(u("test.foo") in db.collection_names()) self.assertRaises(CollectionInvalid, db.create_collection, "test.foo") def test_collection_names(self): db = Database(self.client, "pymongo_test") db.test.insert_one({"dummy": u("object")}) db.test.mike.insert_one({"dummy": u("object")}) colls = db.collection_names() self.assertTrue("test" in colls) self.assertTrue("test.mike" in colls) for coll in colls: self.assertTrue("$" not in coll) colls_without_systems = db.collection_names(False) for coll in colls_without_systems: self.assertTrue(not coll.startswith("system.")) # Force more than one batch. db = self.client.many_collections for i in range(101): db["coll" + str(i)].insert_one({}) # No Error try: db.collection_names() finally: self.client.drop_database("many_collections") def test_collection_names_single_socket(self): # Test that Database.collection_names only requires one socket. client = rs_or_single_client(maxPoolSize=1) client.drop_database('test_collection_names_single_socket') db = client.test_collection_names_single_socket for i in range(200): db.create_collection(str(i)) db.collection_names() # Must not hang. client.drop_database('test_collection_names_single_socket') def test_drop_collection(self): db = Database(self.client, "pymongo_test") self.assertRaises(TypeError, db.drop_collection, 5) self.assertRaises(TypeError, db.drop_collection, None) db.test.insert_one({"dummy": u("object")}) self.assertTrue("test" in db.collection_names()) db.drop_collection("test") self.assertFalse("test" in db.collection_names()) db.test.insert_one({"dummy": u("object")}) self.assertTrue("test" in db.collection_names()) db.drop_collection(u("test")) self.assertFalse("test" in db.collection_names()) db.test.insert_one({"dummy": u("object")}) self.assertTrue("test" in db.collection_names()) db.drop_collection(db.test) self.assertFalse("test" in db.collection_names()) db.test.insert_one({"dummy": u("object")}) self.assertTrue("test" in db.collection_names()) db.test.drop() self.assertFalse("test" in db.collection_names()) db.test.drop() db.drop_collection(db.test.doesnotexist) def test_validate_collection(self): db = self.client.pymongo_test self.assertRaises(TypeError, db.validate_collection, 5) self.assertRaises(TypeError, db.validate_collection, None) db.test.insert_one({"dummy": u("object")}) self.assertRaises(OperationFailure, db.validate_collection, "test.doesnotexist") self.assertRaises(OperationFailure, db.validate_collection, db.test.doesnotexist) self.assertTrue(db.validate_collection("test")) self.assertTrue(db.validate_collection(db.test)) self.assertTrue(db.validate_collection(db.test, full=True)) self.assertTrue(db.validate_collection(db.test, scandata=True)) self.assertTrue(db.validate_collection(db.test, scandata=True, full=True)) self.assertTrue(db.validate_collection(db.test, True, True)) @client_context.require_no_mongos def test_profiling_levels(self): db = self.client.pymongo_test self.assertEqual(db.profiling_level(), OFF) # default self.assertRaises(ValueError, db.set_profiling_level, 5.5) self.assertRaises(ValueError, db.set_profiling_level, None) self.assertRaises(ValueError, db.set_profiling_level, -1) self.assertRaises(TypeError, db.set_profiling_level, SLOW_ONLY, 5.5) self.assertRaises(TypeError, db.set_profiling_level, SLOW_ONLY, '1') db.set_profiling_level(SLOW_ONLY) self.assertEqual(db.profiling_level(), SLOW_ONLY) db.set_profiling_level(ALL) self.assertEqual(db.profiling_level(), ALL) db.set_profiling_level(OFF) self.assertEqual(db.profiling_level(), OFF) db.set_profiling_level(SLOW_ONLY, 50) self.assertEqual(50, db.command("profile", -1)['slowms']) db.set_profiling_level(ALL, -1) self.assertEqual(-1, db.command("profile", -1)['slowms']) db.set_profiling_level(OFF, 100) # back to default self.assertEqual(100, db.command("profile", -1)['slowms']) @client_context.require_no_mongos def test_profiling_info(self): db = self.client.pymongo_test db.system.profile.drop() db.set_profiling_level(ALL) db.test.find_one() db.set_profiling_level(OFF) info = db.profiling_info() self.assertTrue(isinstance(info, list)) # Check if we're going to fail because of SERVER-4754, in which # profiling info isn't collected if mongod was started with --auth if server_started_with_auth(self.client): raise SkipTest( "We need SERVER-4754 fixed for the rest of this test to pass" ) self.assertTrue(len(info) >= 1) # These basically clue us in to server changes. self.assertTrue(isinstance(info[0]['responseLength'], int)) self.assertTrue(isinstance(info[0]['millis'], int)) self.assertTrue(isinstance(info[0]['client'], string_type)) self.assertTrue(isinstance(info[0]['user'], string_type)) self.assertTrue(isinstance(info[0]['ns'], string_type)) self.assertTrue(isinstance(info[0]['op'], string_type)) self.assertTrue(isinstance(info[0]["ts"], datetime.datetime)) @client_context.require_no_mongos def test_errors(self): with ignore_deprecations(): # We must call getlasterror, etc. on same socket as last operation. db = rs_or_single_client(maxPoolSize=1).pymongo_test db.reset_error_history() self.assertEqual(None, db.error()) self.assertEqual(None, db.previous_error()) db.command("forceerror", check=False) self.assertTrue(db.error()) self.assertTrue(db.previous_error()) db.command("forceerror", check=False) self.assertTrue(db.error()) prev_error = db.previous_error() self.assertEqual(prev_error["nPrev"], 1) del prev_error["nPrev"] prev_error.pop("lastOp", None) error = db.error() error.pop("lastOp", None) # getLastError includes "connectionId" in recent # server versions, getPrevError does not. error.pop("connectionId", None) self.assertEqual(error, prev_error) db.test.find_one() self.assertEqual(None, db.error()) self.assertTrue(db.previous_error()) self.assertEqual(db.previous_error()["nPrev"], 2) db.reset_error_history() self.assertEqual(None, db.error()) self.assertEqual(None, db.previous_error()) def test_command(self): db = self.client.admin self.assertEqual(db.command("buildinfo"), db.command({"buildinfo": 1})) # We use 'aggregate' as our example command, since it's an easy way to # retrieve a BSON regex from a collection using a command. But until # MongoDB 2.3.2, aggregation turned regexes into strings: SERVER-6470. @client_context.require_version_min(2, 3, 2) def test_command_with_regex(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({'r': re.compile('.*')}) db.test.insert_one({'r': Regex('.*')}) result = db.command('aggregate', 'test', pipeline=[]) for doc in result['result']: self.assertTrue(isinstance(doc['r'], Regex)) def test_password_digest(self): self.assertRaises(TypeError, auth._password_digest, 5) self.assertRaises(TypeError, auth._password_digest, True) self.assertRaises(TypeError, auth._password_digest, None) self.assertTrue(isinstance(auth._password_digest("mike", "password"), text_type)) self.assertEqual(auth._password_digest("mike", "password"), u("cd7e45b3b2767dc2fa9b6b548457ed00")) self.assertEqual(auth._password_digest("mike", "password"), auth._password_digest(u("mike"), u("password"))) self.assertEqual(auth._password_digest("Gustave", u("Dor\xe9")), u("81e0e2364499209f466e75926a162d73")) @client_context.require_auth def test_authenticate_add_remove_user(self): # "self.client" is logged in as root. auth_db = self.client.pymongo_test db = rs_or_single_client_noauth().pymongo_test # Configuration errors self.assertRaises(ValueError, auth_db.add_user, "user", '') self.assertRaises(TypeError, auth_db.add_user, "user", 'password', 15) self.assertRaises(TypeError, auth_db.add_user, "user", 'password', 'True') self.assertRaises(ConfigurationError, auth_db.add_user, "user", 'password', True, roles=['read']) if client_context.version.at_least(2, 5, 3, -1): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) self.assertRaises(DeprecationWarning, auth_db.add_user, "user", "password") self.assertRaises(DeprecationWarning, auth_db.add_user, "user", "password", True) with ignore_deprecations(): self.assertRaises(ConfigurationError, auth_db.add_user, "user", "password", digestPassword=True) # Add / authenticate / remove auth_db.add_user("mike", "password", roles=["dbOwner"]) self.addCleanup(remove_all_users, auth_db) self.assertRaises(TypeError, db.authenticate, 5, "password") self.assertRaises(TypeError, db.authenticate, "mike", 5) self.assertRaises(OperationFailure, db.authenticate, "mike", "not a real password") self.assertRaises(OperationFailure, db.authenticate, "faker", "password") db.authenticate("mike", "password") db.logout() # Unicode name and password. db.authenticate(u("mike"), u("password")) db.logout() auth_db.remove_user("mike") self.assertRaises(OperationFailure, db.authenticate, "mike", "password") # Add / authenticate / change password self.assertRaises(OperationFailure, db.authenticate, "Gustave", u("Dor\xe9")) auth_db.add_user("Gustave", u("Dor\xe9"), roles=["dbOwner"]) db.authenticate("Gustave", u("Dor\xe9")) # Change password. auth_db.add_user("Gustave", "password", roles=["dbOwner"]) db.logout() self.assertRaises(OperationFailure, db.authenticate, "Gustave", u("Dor\xe9")) self.assertTrue(db.authenticate("Gustave", u("password"))) if not client_context.version.at_least(2, 5, 3, -1): # Add a readOnly user with ignore_deprecations(): auth_db.add_user("Ross", "password", read_only=True) db.logout() db.authenticate("Ross", u("password")) self.assertTrue( auth_db.system.users.find({"readOnly": True}).count()) @client_context.require_auth def test_make_user_readonly(self): # "self.client" is logged in as root. auth_db = self.client.pymongo_test db = rs_or_single_client_noauth().pymongo_test # Make a read-write user. auth_db.add_user('jesse', 'pw') self.addCleanup(remove_all_users, auth_db) # Check that we're read-write by default. db.authenticate('jesse', 'pw') db.collection.insert_one({}) db.logout() # Make the user read-only. auth_db.add_user('jesse', 'pw', read_only=True) db.authenticate('jesse', 'pw') self.assertRaises(OperationFailure, db.collection.insert_one, {}) @client_context.require_version_min(2, 5, 3, -1) @client_context.require_auth def test_default_roles(self): # "self.client" is logged in as root. auth_admin = self.client.admin auth_admin.add_user('test_default_roles', 'pass') self.addCleanup(auth_admin.remove_user, 'test_default_roles') info = auth_admin.command( 'usersInfo', 'test_default_roles')['users'][0] self.assertEqual("root", info['roles'][0]['role']) # Read only "admin" user auth_admin.add_user('ro-admin', 'pass', read_only=True) self.addCleanup(auth_admin.remove_user, 'ro-admin') info = auth_admin.command('usersInfo', 'ro-admin')['users'][0] self.assertEqual("readAnyDatabase", info['roles'][0]['role']) # "Non-admin" user auth_db = self.client.pymongo_test auth_db.add_user('user', 'pass') self.addCleanup(remove_all_users, auth_db) info = auth_db.command('usersInfo', 'user')['users'][0] self.assertEqual("dbOwner", info['roles'][0]['role']) # Read only "Non-admin" user auth_db.add_user('ro-user', 'pass', read_only=True) info = auth_db.command('usersInfo', 'ro-user')['users'][0] self.assertEqual("read", info['roles'][0]['role']) @client_context.require_version_min(2, 5, 3, -1) @client_context.require_auth def test_new_user_cmds(self): # "self.client" is logged in as root. auth_db = self.client.pymongo_test auth_db.add_user("amalia", "password", roles=["userAdmin"]) self.addCleanup(auth_db.remove_user, "amalia") db = rs_or_single_client_noauth().pymongo_test db.authenticate("amalia", "password") # This tests the ability to update user attributes. db.add_user("amalia", "new_password", customData={"secret": "koalas"}) user_info = db.command("usersInfo", "amalia") self.assertTrue(user_info["users"]) amalia_user = user_info["users"][0] self.assertEqual(amalia_user["user"], "amalia") self.assertEqual(amalia_user["customData"], {"secret": "koalas"}) @client_context.require_auth def test_authenticate_multiple(self): # "self.client" is logged in as root. self.client.drop_database("pymongo_test") self.client.drop_database("pymongo_test1") admin_db_auth = self.client.admin users_db_auth = self.client.pymongo_test # Non-root client. client = rs_or_single_client_noauth() admin_db = client.admin users_db = client.pymongo_test other_db = client.pymongo_test1 self.assertRaises(OperationFailure, users_db.test.find_one) if client_context.version.at_least(2, 5, 3, -1): admin_db_auth.add_user('ro-admin', 'pass', roles=["userAdmin", "readAnyDatabase"]) else: admin_db_auth.add_user('ro-admin', 'pass', read_only=True) self.addCleanup(admin_db_auth.remove_user, 'ro-admin') users_db_auth.add_user('user', 'pass', roles=["userAdmin", "readWrite"]) self.addCleanup(remove_all_users, users_db_auth) # Regular user should be able to query its own db, but # no other. users_db.authenticate('user', 'pass') self.assertEqual(0, users_db.test.count()) self.assertRaises(OperationFailure, other_db.test.find_one) # Admin read-only user should be able to query any db, # but not write. admin_db.authenticate('ro-admin', 'pass') self.assertEqual(None, other_db.test.find_one()) self.assertRaises(OperationFailure, other_db.test.insert_one, {}) # Close all sockets. client.close() # We should still be able to write to the regular user's db. self.assertTrue(users_db.test.delete_many({})) # And read from other dbs... self.assertEqual(0, other_db.test.count()) # But still not write to other dbs. self.assertRaises(OperationFailure, other_db.test.insert_one, {}) def test_id_ordering(self): # PyMongo attempts to have _id show up first # when you iterate key/value pairs in a document. # This isn't reliable since python dicts don't # guarantee any particular order. This will never # work right in Jython or any Python or environment # with hash randomization enabled (e.g. tox). db = self.client.pymongo_test db.test.drop() db.test.insert_one(SON([("hello", "world"), ("_id", 5)])) db = self.client.get_database( "pymongo_test", codec_options=CodecOptions(document_class=SON)) cursor = db.test.find() for x in cursor: for (k, v) in x.items(): self.assertEqual(k, "_id") break def test_deref(self): db = self.client.pymongo_test db.test.drop() self.assertRaises(TypeError, db.dereference, 5) self.assertRaises(TypeError, db.dereference, "hello") self.assertRaises(TypeError, db.dereference, None) self.assertEqual(None, db.dereference(DBRef("test", ObjectId()))) obj = {"x": True} key = db.test.insert_one(obj).inserted_id self.assertEqual(obj, db.dereference(DBRef("test", key))) self.assertEqual(obj, db.dereference(DBRef("test", key, "pymongo_test"))) self.assertRaises(ValueError, db.dereference, DBRef("test", key, "foo")) self.assertEqual(None, db.dereference(DBRef("test", 4))) obj = {"_id": 4} db.test.insert_one(obj) self.assertEqual(obj, db.dereference(DBRef("test", 4))) def test_deref_kwargs(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({"_id": 4, "foo": "bar"}) db = self.client.get_database( "pymongo_test", codec_options=CodecOptions(document_class=SON)) self.assertEqual(SON([("foo", "bar")]), db.dereference(DBRef("test", 4), projection={"_id": False})) @client_context.require_no_auth def test_eval(self): db = self.client.pymongo_test db.test.drop() self.assertRaises(TypeError, db.eval, None) self.assertRaises(TypeError, db.eval, 5) self.assertRaises(TypeError, db.eval, []) self.assertEqual(3, db.eval("function (x) {return x;}", 3)) self.assertEqual(3, db.eval(u("function (x) {return x;}"), 3)) self.assertEqual(None, db.eval("function (x) {db.test.save({y:x});}", 5)) self.assertEqual(db.test.find_one()["y"], 5) self.assertEqual(5, db.eval("function (x, y) {return x + y;}", 2, 3)) self.assertEqual(5, db.eval("function () {return 5;}")) self.assertEqual(5, db.eval("2 + 3;")) self.assertEqual(5, db.eval(Code("2 + 3;"))) self.assertRaises(OperationFailure, db.eval, Code("return i;")) self.assertEqual(2, db.eval(Code("return i;", {"i": 2}))) self.assertEqual(5, db.eval(Code("i + 3;", {"i": 2}))) self.assertRaises(OperationFailure, db.eval, "5 ++ 5;") # TODO some of these tests belong in the collection level testing. def test_insert_find_one(self): db = self.client.pymongo_test db.test.drop() a_doc = SON({"hello": u("world")}) a_key = db.test.insert_one(a_doc).inserted_id self.assertTrue(isinstance(a_doc["_id"], ObjectId)) self.assertEqual(a_doc["_id"], a_key) self.assertEqual(a_doc, db.test.find_one({"_id": a_doc["_id"]})) self.assertEqual(a_doc, db.test.find_one(a_key)) self.assertEqual(None, db.test.find_one(ObjectId())) self.assertEqual(a_doc, db.test.find_one({"hello": u("world")})) self.assertEqual(None, db.test.find_one({"hello": u("test")})) b = db.test.find_one() b["hello"] = u("mike") db.test.replace_one({"_id": b["_id"]}, b) self.assertNotEqual(a_doc, db.test.find_one(a_key)) self.assertEqual(b, db.test.find_one(a_key)) self.assertEqual(b, db.test.find_one()) count = 0 for _ in db.test.find(): count += 1 self.assertEqual(count, 1) def test_long(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({"x": long(9223372036854775807)}) retrieved = db.test.find_one()['x'] self.assertEqual(Int64(9223372036854775807), retrieved) self.assertIsInstance(retrieved, Int64) db.test.delete_many({}) db.test.insert_one({"x": Int64(1)}) retrieved = db.test.find_one()['x'] self.assertEqual(Int64(1), retrieved) self.assertIsInstance(retrieved, Int64) def test_delete(self): db = self.client.pymongo_test db.test.drop() db.test.insert_one({"x": 1}) db.test.insert_one({"x": 2}) db.test.insert_one({"x": 3}) length = 0 for _ in db.test.find(): length += 1 self.assertEqual(length, 3) db.test.delete_one({"x": 1}) length = 0 for _ in db.test.find(): length += 1 self.assertEqual(length, 2) db.test.delete_one(db.test.find_one()) db.test.delete_one(db.test.find_one()) self.assertEqual(db.test.find_one(), None) db.test.insert_one({"x": 1}) db.test.insert_one({"x": 2}) db.test.insert_one({"x": 3}) self.assertTrue(db.test.find_one({"x": 2})) db.test.delete_one({"x": 2}) self.assertFalse(db.test.find_one({"x": 2})) self.assertTrue(db.test.find_one()) db.test.delete_many({}) self.assertFalse(db.test.find_one()) @client_context.require_no_auth def test_system_js(self): db = self.client.pymongo_test db.system.js.delete_many({}) self.assertEqual(0, db.system.js.count()) db.system_js.add = "function(a, b) { return a + b; }" self.assertEqual('add', db.system.js.find_one()['_id']) self.assertEqual(1, db.system.js.count()) self.assertEqual(6, db.system_js.add(1, 5)) del db.system_js.add self.assertEqual(0, db.system.js.count()) db.system_js['add'] = "function(a, b) { return a + b; }" self.assertEqual('add', db.system.js.find_one()['_id']) self.assertEqual(1, db.system.js.count()) self.assertEqual(6, db.system_js['add'](1, 5)) del db.system_js['add'] self.assertEqual(0, db.system.js.count()) self.assertRaises(OperationFailure, db.system_js.add, 1, 5) # TODO right now CodeWScope doesn't work w/ system js # db.system_js.scope = Code("return hello;", {"hello": 8}) # self.assertEqual(8, db.system_js.scope()) self.assertRaises(OperationFailure, db.system_js.non_existant) # XXX: Broken in V8, works in SpiderMonkey if not client_context.version.at_least(2, 3, 0): db.system_js.no_param = Code("return 5;") self.assertEqual(5, db.system_js.no_param()) def test_system_js_list(self): db = self.client.pymongo_test db.system.js.delete_many({}) self.assertEqual([], db.system_js.list()) db.system_js.foo = "function() { return 'blah'; }" self.assertEqual(["foo"], db.system_js.list()) db.system_js.bar = "function() { return 'baz'; }" self.assertEqual(set(["foo", "bar"]), set(db.system_js.list())) del db.system_js.foo self.assertEqual(["bar"], db.system_js.list()) def test_command_response_without_ok(self): # Sometimes (SERVER-10891) the server's response to a badly-formatted # command document will have no 'ok' field. We should raise # OperationFailure instead of KeyError. self.assertRaises(OperationFailure, helpers._check_command_response, {}) try: helpers._check_command_response({'$err': 'foo'}) except OperationFailure as e: self.assertEqual(e.args[0], 'foo') else: self.fail("_check_command_response didn't raise OperationFailure") def test_mongos_response(self): error_document = { 'ok': 0, 'errmsg': 'outer', 'raw': {'shard0/host0,host1': {'ok': 0, 'errmsg': 'inner'}}} with self.assertRaises(OperationFailure) as context: helpers._check_command_response(error_document) self.assertEqual('inner', str(context.exception)) # If a shard has no primary and you run a command like dbstats, which # cannot be run on a secondary, mongos's response includes empty "raw" # errors. See SERVER-15428. error_document = { 'ok': 0, 'errmsg': 'outer', 'raw': {'shard0/host0,host1': {}}} with self.assertRaises(OperationFailure) as context: helpers._check_command_response(error_document) self.assertEqual('outer', str(context.exception)) # Raw error has ok: 0 but no errmsg. Not a known case, but test it. error_document = { 'ok': 0, 'errmsg': 'outer', 'raw': {'shard0/host0,host1': {'ok': 0}}} with self.assertRaises(OperationFailure) as context: helpers._check_command_response(error_document) self.assertEqual('outer', str(context.exception)) @client_context.require_version_min(2, 5, 3, -1) @client_context.require_test_commands def test_command_max_time_ms(self): self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: db = self.client.pymongo_test db.command('count', 'test') self.assertRaises(ExecutionTimeout, db.command, 'count', 'test', maxTimeMS=1) pipeline = [{'$project': {'name': 1, 'count': 1}}] # Database command helper. db.command('aggregate', 'test', pipeline=pipeline) self.assertRaises(ExecutionTimeout, db.command, 'aggregate', 'test', pipeline=pipeline, maxTimeMS=1) # Collection helper. db.test.aggregate(pipeline=pipeline) self.assertRaises(ExecutionTimeout, db.test.aggregate, pipeline, maxTimeMS=1) finally: self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") if __name__ == "__main__": unittest.main() pymongo-3.2/test/qcheck.py0000644000175000017500000001677612630145074017566 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import random import traceback import datetime import re import sys sys.path[0:0] = [""] from bson.binary import Binary from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import MAXSIZE, PY3, iteritems, u from bson.son import SON if PY3: unichr = chr gen_target = 100 reduction_attempts = 10 examples = 5 def lift(value): return lambda: value def choose_lifted(generator_list): return lambda: random.choice(generator_list) def my_map(generator, function): return lambda: function(generator()) def choose(list): return lambda: random.choice(list)() def gen_range(start, stop): return lambda: random.randint(start, stop) def gen_int(): max_int = 2147483647 return lambda: random.randint(-max_int - 1, max_int) def gen_float(): return lambda: (random.random() - 0.5) * MAXSIZE def gen_boolean(): return lambda: random.choice([True, False]) def gen_printable_char(): return lambda: chr(random.randint(32, 126)) def gen_printable_string(gen_length): return lambda: "".join(gen_list(gen_printable_char(), gen_length)()) if PY3: def gen_char(set=None): return lambda: bytes([random.randint(0, 255)]) else: def gen_char(set=None): return lambda: chr(random.randint(0, 255)) def gen_string(gen_length): return lambda: b"".join(gen_list(gen_char(), gen_length)()) def gen_unichar(): return lambda: unichr(random.randint(1, 0xFFF)) def gen_unicode(gen_length): return lambda: u("").join([x for x in gen_list(gen_unichar(), gen_length)() if x not in ".$"]) def gen_list(generator, gen_length): return lambda: [generator() for _ in range(gen_length())] def gen_datetime(): return lambda: datetime.datetime(random.randint(1970, 2037), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59), random.randint(0, 999) * 1000) def gen_dict(gen_key, gen_value, gen_length): def a_dict(gen_key, gen_value, length): result = {} for _ in range(length): result[gen_key()] = gen_value() return result return lambda: a_dict(gen_key, gen_value, gen_length()) def gen_regexp(gen_length): # TODO our patterns only consist of one letter. # this is because of a bug in CPython's regex equality testing, # which I haven't quite tracked down, so I'm just ignoring it... pattern = lambda: u("").join(gen_list(choose_lifted(u("a")), gen_length)()) def gen_flags(): flags = 0 if random.random() > 0.5: flags = flags | re.IGNORECASE if random.random() > 0.5: flags = flags | re.MULTILINE if random.random() > 0.5: flags = flags | re.VERBOSE return flags return lambda: re.compile(pattern(), gen_flags()) def gen_objectid(): return lambda: ObjectId() def gen_dbref(): collection = gen_unicode(gen_range(0, 20)) return lambda: DBRef(collection(), gen_mongo_value(1, True)()) def gen_mongo_value(depth, ref): bintype = Binary if PY3: # If we used Binary in python3 tests would fail since we # decode BSON binary subtype 0 to bytes. Testing this with # bytes in python3 makes a lot more sense. bintype = bytes choices = [gen_unicode(gen_range(0, 50)), gen_printable_string(gen_range(0, 50)), my_map(gen_string(gen_range(0, 1000)), bintype), gen_int(), gen_float(), gen_boolean(), gen_datetime(), gen_objectid(), lift(None)] if ref: choices.append(gen_dbref()) if depth > 0: choices.append(gen_mongo_list(depth, ref)) choices.append(gen_mongo_dict(depth, ref)) return choose(choices) def gen_mongo_list(depth, ref): return gen_list(gen_mongo_value(depth - 1, ref), gen_range(0, 10)) def gen_mongo_dict(depth, ref=True): return my_map(gen_dict(gen_unicode(gen_range(0, 20)), gen_mongo_value(depth - 1, ref), gen_range(0, 10)), SON) def simplify(case): # TODO this is a hack if isinstance(case, SON) and "$ref" not in case: simplified = SON(case) # make a copy! if random.choice([True, False]): # delete simplified_keys = list(simplified) if not len(simplified_keys): return (False, case) simplified.pop(random.choice(simplified_keys)) return (True, simplified) else: # simplify a value simplified_items = list(iteritems(simplified)) if not len(simplified_items): return (False, case) (key, value) = random.choice(simplified_items) (success, value) = simplify(value) simplified[key] = value return (success, success and simplified or case) if isinstance(case, list): simplified = list(case) if random.choice([True, False]): # delete if not len(simplified): return (False, case) simplified.pop(random.randrange(len(simplified))) return (True, simplified) else: # simplify an item if not len(simplified): return (False, case) index = random.randrange(len(simplified)) (success, value) = simplify(simplified[index]) simplified[index] = value return (success, success and simplified or case) return (False, case) def reduce(case, predicate, reductions=0): for _ in range(reduction_attempts): (reduced, simplified) = simplify(case) if reduced and not predicate(simplified): return reduce(simplified, predicate, reductions + 1) return (reductions, case) def isnt(predicate): return lambda x: not predicate(x) def check(predicate, generator): counter_examples = [] for _ in range(gen_target): case = generator() try: if not predicate(case): reduction = reduce(case, predicate) counter_examples.append("after %s reductions: %r" % reduction) except: counter_examples.append("%r : %s" % (case, traceback.format_exc())) return counter_examples def check_unittest(test, predicate, generator): counter_examples = check(predicate, generator) if counter_examples: failures = len(counter_examples) message = "\n".join([" -> %s" % f for f in counter_examples[:examples]]) message = ("found %d counter examples, displaying first %d:\n%s" % (failures, min(failures, examples), message)) test.fail(message) pymongo-3.2/test/test_gridfs_spec.py0000644000175000017500000001772112630145074021646 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test GridFSBucket class.""" import copy import datetime import os import sys import re import gridfs sys.path[0:0] = [""] from bson import Binary from bson.json_util import loads from bson.py3compat import bytes_from_hex from gridfs.errors import NoFile, CorruptGridFile from test import (unittest, IntegrationTest) # Commands. _COMMANDS = {"delete": lambda coll, doc: [coll.delete_many(d["q"]) for d in doc['deletes']], "insert": lambda coll, doc: coll.insert_many(doc['documents']), "update": lambda coll, doc: [coll.update_many(u["q"], u["u"]) for u in doc['updates']] } # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'gridfs') def camel_to_snake(camel): # Regex to convert CamelCase to snake_case. Special case for _id. if camel == "id": return "file_id" snake = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', snake).lower() class TestAllScenarios(IntegrationTest): @classmethod def setUpClass(cls): super(TestAllScenarios, cls).setUpClass() cls.fs = gridfs.GridFSBucket(cls.db) cls.str_to_cmd = { "upload": cls.fs.upload_from_stream, "download": cls.fs.open_download_stream, "delete": cls.fs.delete, "download_by_name": cls.fs.open_download_stream_by_name} def init_db(self, data, test): self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") self.db.drop_collection("expected.files") self.db.drop_collection("expected.chunks") # Read in data. if data['files']: self.db.fs.files.insert_many(data['files']) self.db.expected.files.insert_many(data['files']) if data['chunks']: self.db.fs.chunks.insert_many(data['chunks']) self.db.expected.chunks.insert_many(data['chunks']) # Make initial modifications. if "arrange" in test: for cmd in test['arrange'].get('data', []): for key in cmd.keys(): if key in _COMMANDS: coll = self.db.get_collection(cmd[key]) _COMMANDS[key](coll, cmd) def init_expected_db(self, test, result): # Modify outcome DB. for cmd in test['assert'].get('data', []): for key in cmd.keys(): if key in _COMMANDS: # Replace wildcards in inserts. for doc in cmd.get('documents', []): keylist = doc.keys() for dockey in copy.deepcopy(list(keylist)): if "result" in str(doc[dockey]): doc[dockey] = result if "actual" in str(doc[dockey]): # Avoid duplicate doc.pop(dockey) # Move contentType to metadata. if dockey == "contentType": doc["metadata"] = {dockey: doc.pop(dockey)} coll = self.db.get_collection(cmd[key]) _COMMANDS[key](coll, cmd) if test['assert'].get('result') == "&result": test['assert']['result'] = result def sorted_list(self, coll, ignore_id): to_sort = [] for doc in coll.find(): docstr = "{" if ignore_id: # Cannot compare _id in chunks collection. doc.pop("_id") for k in sorted(doc.keys()): if k == "uploadDate": # Can't compare datetime. self.assertTrue(isinstance(doc[k], datetime.datetime)) else: docstr += "%s:%s " % (k, repr(doc[k])) to_sort.append(docstr + "}") return to_sort def create_test(scenario_def): def run_scenario(self): # Run tests. self.assertTrue(scenario_def['tests'], "tests cannot be empty") for test in scenario_def['tests']: self.init_db(scenario_def['data'], test) # Run GridFs Operation. operation = self.str_to_cmd[test['act']['operation']] args = test['act']['arguments'] extra_opts = args.pop("options", {}) if "contentType" in extra_opts: extra_opts["metadata"] = { "contentType": extra_opts.pop("contentType")} args.update(extra_opts) converted_args = dict((camel_to_snake(c), v) for c, v in args.items()) error = None try: result = operation(**converted_args) if 'download' in test['act']['operation']: result = Binary(result.read()) except Exception as exc: error = exc self.init_expected_db(test, result) # Asserts. errors = {"FileNotFound": NoFile, "ChunkIsMissing": CorruptGridFile, "ExtraChunk": CorruptGridFile, "ChunkIsWrongSize": CorruptGridFile, "RevisionNotFound": NoFile} if test['assert'].get("error", False): self.assertIsNotNone(error) self.assertTrue(isinstance(error, errors[test['assert']['error']])) else: self.assertIsNone(error) if 'result' in test['assert']: if test['assert']['result'] == 'void': test['assert']['result'] = None self.assertEqual(result, test['assert'].get('result')) if 'data' in test['assert']: # Create alphabetized list self.assertEqual( set(self.sorted_list(self.db.fs.chunks, True)), set(self.sorted_list(self.db.expected.chunks, True))) self.assertEqual( set(self.sorted_list(self.db.fs.files, False)), set(self.sorted_list(self.db.expected.files, False))) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = loads(scenario_stream.read()) # Because object_hook is already defined by bson.json_util, # and everything is named 'data' def str2hex(jsn): for key, val in jsn.items(): if key in ("data", "source", "result"): if "$hex" in val: jsn[key] = Binary(bytes_from_hex(val['$hex'])) if isinstance(jsn[key], dict): str2hex(jsn[key]) if isinstance(jsn[key], list): for k in jsn[key]: str2hex(k) str2hex(scenario_def) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s' % ( os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_read_concern.py0000644000175000017500000001400012630145074021763 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the read_concern module.""" import pymongo from bson.son import SON from pymongo import monitoring from pymongo.errors import ConfigurationError, OperationFailure from pymongo.read_concern import ReadConcern from test import client_context, pair, unittest from test.utils import single_client, EventListener class TestReadConcern(unittest.TestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS # Don't use any global subscribers. monitoring._LISTENERS = monitoring._Listeners([]) cls.client = single_client(event_listeners=[cls.listener]) cls.db = cls.client.pymongo_test @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def tearDown(self): self.db.coll.drop() self.listener.results.clear() def test_read_concern(self): rc = ReadConcern() self.assertIsNone(rc.level) self.assertTrue(rc.ok_for_legacy) rc = ReadConcern('majority') self.assertEqual('majority', rc.level) self.assertFalse(rc.ok_for_legacy) rc = ReadConcern('local') self.assertEqual('local', rc.level) self.assertTrue(rc.ok_for_legacy) self.assertRaises(TypeError, ReadConcern, 42) def test_read_concern_uri(self): uri = 'mongodb://%s/?readConcernLevel=majority' % (pair,) client = pymongo.MongoClient(uri) self.assertEqual(ReadConcern('majority'), client.read_concern) @client_context.require_version_max(3, 1) def test_invalid_read_concern(self): coll = self.db.get_collection( 'coll', read_concern=ReadConcern('majority')) self.assertRaisesRegexp( ConfigurationError, 'read concern level of majority is not valid ' 'with a max wire version of [0-3]', coll.count) @client_context.require_version_min(3, 1, 9, -1) def test_find_command(self): # readConcern not sent in command if not specified. coll = self.db.coll tuple(coll.find({'field': 'value'})) self.assertNotIn('readConcern', self.listener.results['started'][0].command) self.listener.results.clear() # Explicitly set readConcern to 'local'. coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) tuple(coll.find({'field': 'value'})) self.assertEqual( SON([('find', 'coll'), ('filter', {'field': 'value'}), ('readConcern', {'level': 'local'})]), self.listener.results['started'][0].command) @client_context.require_version_min(3, 1, 9, -1) def test_command_cursor(self): # readConcern not sent in command if not specified. coll = self.db.coll tuple(coll.aggregate([{'$match': {'field': 'value'}}])) self.assertNotIn('readConcern', self.listener.results['started'][0].command) self.listener.results.clear() # Explicitly set readConcern to 'local'. coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) tuple(coll.aggregate([{'$match': {'field': 'value'}}])) self.assertEqual( {'level': 'local'}, self.listener.results['started'][0].command['readConcern']) def test_aggregate_out(self): coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) try: tuple(coll.aggregate([{'$match': {'field': 'value'}}, {'$out': 'output_collection'}])) except OperationFailure: # "ns doesn't exist" pass self.assertNotIn('readConcern', self.listener.results['started'][0].command) def test_map_reduce_out(self): coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) try: tuple(coll.map_reduce('function() { emit(this._id, this.value); }', 'function(key, values) { return 42; }', out='output_collection')) except OperationFailure: # "ns doesn't exist" pass self.assertNotIn('readConcern', self.listener.results['started'][0].command) if client_context.version.at_least(3, 1, 9, -1): self.listener.results.clear() try: tuple(coll.map_reduce( 'function() { emit(this._id, this.value); }', 'function(key, values) { return 42; }', out={'inline': 1})) except OperationFailure: # "ns doesn't exist" pass self.assertEqual( {'level': 'local'}, self.listener.results['started'][0].command['readConcern']) @client_context.require_version_min(3, 1, 9, -1) def test_inline_map_reduce(self): coll = self.db.get_collection('coll', read_concern=ReadConcern('local')) try: tuple(coll.inline_map_reduce( 'function() { emit(this._id, this.value); }', 'function(key, values) { return 42; }')) except OperationFailure: # "ns doesn't exist" pass self.assertEqual( {'level': 'local'}, self.listener.results['started'][0].command['readConcern']) pymongo-3.2/test/test_server_description.py0000644000175000017500000001217612630145074023266 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the server_description module.""" import sys sys.path[0:0] = [""] from pymongo.server_type import SERVER_TYPE from pymongo.ismaster import IsMaster from pymongo.server_description import ServerDescription from test import unittest address = ('localhost', 27017) def parse_ismaster_response(doc): ismaster_response = IsMaster(doc) return ServerDescription(address, ismaster_response) class TestServerDescription(unittest.TestCase): def test_unknown(self): # Default, no ismaster_response. s = ServerDescription(address) self.assertEqual(SERVER_TYPE.Unknown, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_mongos(self): s = parse_ismaster_response({'ok': 1, 'msg': 'isdbgrid'}) self.assertEqual(SERVER_TYPE.Mongos, s.server_type) self.assertTrue(s.is_writable) self.assertTrue(s.is_readable) def test_primary(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSPrimary, s.server_type) self.assertTrue(s.is_writable) self.assertTrue(s.is_readable) def test_secondary(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': False, 'secondary': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSSecondary, s.server_type) self.assertFalse(s.is_writable) self.assertTrue(s.is_readable) def test_arbiter(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': False, 'arbiterOnly': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSArbiter, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_other(self): s = parse_ismaster_response( {'ok': 1, 'ismaster': False, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSOther, s.server_type) s = parse_ismaster_response({ 'ok': 1, 'ismaster': False, 'secondary': True, 'hidden': True, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSOther, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_ghost(self): s = parse_ismaster_response({'ok': 1, 'isreplicaset': True}) self.assertEqual(SERVER_TYPE.RSGhost, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_fields(self): s = parse_ismaster_response({ 'ok': 1, 'ismaster': False, 'secondary': True, 'primary': 'a:27017', 'tags': {'a': 'foo', 'b': 'baz'}, 'maxMessageSizeBytes': 1, 'maxBsonObjectSize': 2, 'maxWriteBatchSize': 3, 'minWireVersion': 4, 'maxWireVersion': 5, 'setName': 'rs'}) self.assertEqual(SERVER_TYPE.RSSecondary, s.server_type) self.assertEqual(('a', 27017), s.primary) self.assertEqual({'a': 'foo', 'b': 'baz'}, s.tags) self.assertEqual(1, s.max_message_size) self.assertEqual(2, s.max_bson_size) self.assertEqual(3, s.max_write_batch_size) self.assertEqual(4, s.min_wire_version) self.assertEqual(5, s.max_wire_version) def test_default_max_message_size(self): s = parse_ismaster_response({ 'ok': 1, 'ismaster': True, 'maxBsonObjectSize': 2}) # Twice max_bson_size. self.assertEqual(4, s.max_message_size) def test_standalone(self): s = parse_ismaster_response({'ok': 1, 'ismaster': True}) self.assertEqual(SERVER_TYPE.Standalone, s.server_type) # Mongod started with --slave. s = parse_ismaster_response({'ok': 1, 'ismaster': False}) self.assertEqual(SERVER_TYPE.Standalone, s.server_type) self.assertTrue(s.is_writable) self.assertTrue(s.is_readable) def test_ok_false(self): s = parse_ismaster_response({'ok': 0, 'ismaster': True}) self.assertEqual(SERVER_TYPE.Unknown, s.server_type) self.assertFalse(s.is_writable) self.assertFalse(s.is_readable) def test_all_hosts(self): s = parse_ismaster_response({ 'ok': 1, 'ismaster': True, 'hosts': ['a'], 'passives': ['b:27018'], 'arbiters': ['c'] }) self.assertEqual( [('a', 27017), ('b', 27018), ('c', 27017)], sorted(s.all_hosts)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_uri_parser.py0000644000175000017500000005157312630145074021534 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo uri_parser module.""" import copy import sys import warnings sys.path[0:0] = [""] from pymongo.uri_parser import (_partition, _rpartition, parse_userinfo, split_hosts, split_options, parse_uri) from pymongo.errors import ConfigurationError, InvalidURI from pymongo import ReadPreference from bson.binary import JAVA_LEGACY from bson.py3compat import string_type, _unicode from test import unittest class TestURI(unittest.TestCase): def test_partition(self): self.assertEqual(('foo', ':', 'bar'), _partition('foo:bar', ':')) self.assertEqual(('foobar', '', ''), _partition('foobar', ':')) def test_rpartition(self): self.assertEqual(('fo:o:', ':', 'bar'), _rpartition('fo:o::bar', ':')) self.assertEqual(('', '', 'foobar'), _rpartition('foobar', ':')) def test_validate_userinfo(self): self.assertRaises(InvalidURI, parse_userinfo, 'foo@') self.assertRaises(InvalidURI, parse_userinfo, ':password') self.assertRaises(InvalidURI, parse_userinfo, 'fo::o:p@ssword') self.assertRaises(InvalidURI, parse_userinfo, ':') self.assertTrue(parse_userinfo('user:password')) self.assertEqual(('us:r', 'p@ssword'), parse_userinfo('us%3Ar:p%40ssword')) self.assertEqual(('us er', 'p ssword'), parse_userinfo('us+er:p+ssword')) self.assertEqual(('us er', 'p ssword'), parse_userinfo('us%20er:p%20ssword')) self.assertEqual(('us+er', 'p+ssword'), parse_userinfo('us%2Ber:p%2Bssword')) self.assertEqual(('dev1@FOO.COM', ''), parse_userinfo('dev1%40FOO.COM')) self.assertEqual(('dev1@FOO.COM', ''), parse_userinfo('dev1%40FOO.COM:')) def test_split_hosts(self): self.assertRaises(ConfigurationError, split_hosts, 'localhost:27017,') self.assertRaises(ConfigurationError, split_hosts, ',localhost:27017') self.assertRaises(ConfigurationError, split_hosts, 'localhost:27017,,localhost:27018') self.assertEqual([('localhost', 27017), ('example.com', 27017)], split_hosts('localhost,example.com')) self.assertEqual([('localhost', 27018), ('example.com', 27019)], split_hosts('localhost:27018,example.com:27019')) self.assertEqual([('/tmp/mongodb-27017.sock', None)], split_hosts('/tmp/mongodb-27017.sock')) self.assertEqual([('/tmp/mongodb-27017.sock', None), ('example.com', 27017)], split_hosts('/tmp/mongodb-27017.sock,' 'example.com:27017')) self.assertEqual([('example.com', 27017), ('/tmp/mongodb-27017.sock', None)], split_hosts('example.com:27017,' '/tmp/mongodb-27017.sock')) self.assertRaises(ValueError, split_hosts, '::1', 27017) self.assertRaises(ValueError, split_hosts, '[::1:27017') self.assertRaises(ValueError, split_hosts, '::1') self.assertRaises(ValueError, split_hosts, '::1]:27017') self.assertEqual([('::1', 27017)], split_hosts('[::1]:27017')) self.assertEqual([('::1', 27017)], split_hosts('[::1]')) def test_split_options(self): self.assertRaises(ConfigurationError, split_options, 'foo') self.assertRaises(ConfigurationError, split_options, 'foo=bar;foo') self.assertTrue(split_options('ssl=true')) self.assertTrue(split_options('ssl_match_hostname=true')) # Test Invalid URI options that should throw warnings. with warnings.catch_warnings(): warnings.filterwarnings('error') self.assertRaises(Warning, split_options, 'foo=bar', warn=True) self.assertRaises(Warning, split_options, 'socketTimeoutMS=foo', warn=True) self.assertRaises(Warning, split_options, 'socketTimeoutMS=0.0', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=foo', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=0.0', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=1e100000', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=-1e100000', warn=True) self.assertRaises(Warning, split_options, 'ssl=foo', warn=True) self.assertRaises(Warning, split_options, 'ssl_match_hostname=foo', warn=True) # On most platforms float('inf') and float('-inf') represent # +/- infinity, although on Python 2.4 and 2.5 on Windows those # expressions are invalid if not (sys.platform == "win32" and sys.version_info <= (2, 5)): self.assertRaises(Warning, split_options, 'connectTimeoutMS=inf', warn=True) self.assertRaises(Warning, split_options, 'connectTimeoutMS=-inf', warn=True) self.assertRaises(Warning, split_options, 'wtimeoutms=foo', warn=True) self.assertRaises(Warning, split_options, 'wtimeoutms=5.5', warn=True) self.assertRaises(Warning, split_options, 'fsync=foo', warn=True) self.assertRaises(Warning, split_options, 'fsync=5.5', warn=True) self.assertRaises(Warning, split_options, 'authMechanism=foo', warn=True) # Test invalid options with warn=False. self.assertRaises(ConfigurationError, split_options, 'foo=bar') self.assertRaises(ValueError, split_options, 'socketTimeoutMS=foo') self.assertRaises(ValueError, split_options, 'socketTimeoutMS=0.0') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=foo') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=0.0') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=1e100000') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=-1e100000') self.assertRaises(ValueError, split_options, 'ssl=foo') self.assertRaises(ValueError, split_options, 'ssl_match_hostname=foo') if not (sys.platform == "win32" and sys.version_info <= (2, 5)): self.assertRaises(ValueError, split_options, 'connectTimeoutMS=inf') self.assertRaises(ValueError, split_options, 'connectTimeoutMS=-inf') self.assertRaises(ValueError, split_options, 'wtimeoutms=foo') self.assertRaises(ValueError, split_options, 'wtimeoutms=5.5') self.assertRaises(ValueError, split_options, 'fsync=foo') self.assertRaises(ValueError, split_options, 'fsync=5.5') self.assertRaises(ValueError, split_options, 'authMechanism=foo') # Test splitting options works when valid. self.assertTrue(split_options('socketTimeoutMS=300')) self.assertTrue(split_options('connectTimeoutMS=300')) self.assertEqual({'sockettimeoutms': 0.3}, split_options('socketTimeoutMS=300')) self.assertEqual({'sockettimeoutms': 0.0001}, split_options('socketTimeoutMS=0.1')) self.assertEqual({'connecttimeoutms': 0.3}, split_options('connectTimeoutMS=300')) self.assertEqual({'connecttimeoutms': 0.0001}, split_options('connectTimeoutMS=0.1')) self.assertTrue(split_options('connectTimeoutMS=300')) self.assertTrue(isinstance(split_options('w=5')['w'], int)) self.assertTrue(isinstance(split_options('w=5.5')['w'], string_type)) self.assertTrue(split_options('w=foo')) self.assertTrue(split_options('w=majority')) self.assertTrue(split_options('wtimeoutms=500')) self.assertEqual({'fsync': True}, split_options('fsync=true')) self.assertEqual({'fsync': False}, split_options('fsync=false')) self.assertEqual({'authmechanism': 'GSSAPI'}, split_options('authMechanism=GSSAPI')) self.assertEqual({'authmechanism': 'MONGODB-CR'}, split_options('authMechanism=MONGODB-CR')) self.assertEqual({'authmechanism': 'SCRAM-SHA-1'}, split_options('authMechanism=SCRAM-SHA-1')) self.assertEqual({'authsource': 'foobar'}, split_options('authSource=foobar')) self.assertEqual({'maxpoolsize': 50}, split_options('maxpoolsize=50')) def test_parse_uri(self): self.assertRaises(InvalidURI, parse_uri, "http://foobar.com") self.assertRaises(InvalidURI, parse_uri, "http://foo@foobar.com") self.assertRaises(ValueError, parse_uri, "mongodb://::1", 27017) orig = { 'nodelist': [("localhost", 27017)], 'username': None, 'password': None, 'database': None, 'collection': None, 'options': {} } res = copy.deepcopy(orig) self.assertEqual(res, parse_uri("mongodb://localhost")) res.update({'username': 'fred', 'password': 'foobar'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost")) res.update({'database': 'baz'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost/baz")) res = copy.deepcopy(orig) res['nodelist'] = [("example1.com", 27017), ("example2.com", 27017)] self.assertEqual(res, parse_uri("mongodb://example1.com:27017," "example2.com:27017")) res = copy.deepcopy(orig) res['nodelist'] = [("localhost", 27017), ("localhost", 27018), ("localhost", 27019)] self.assertEqual(res, parse_uri("mongodb://localhost," "localhost:27018,localhost:27019")) res = copy.deepcopy(orig) res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://localhost/foo")) res = copy.deepcopy(orig) self.assertEqual(res, parse_uri("mongodb://localhost/")) res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://" "localhost/test.yield_historical.in")) res.update({'username': 'fred', 'password': 'foobar'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost/" "test.yield_historical.in")) res = copy.deepcopy(orig) res['nodelist'] = [("example1.com", 27017), ("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://example1.com:27017,example2.com" ":27017/test.yield_historical.in")) # Test socket path without escaped characters. self.assertRaises(InvalidURI, parse_uri, "mongodb:///tmp/mongodb-27017.sock") # Test with escaped characters. res = copy.deepcopy(orig) res['nodelist'] = [("example2.com", 27017), ("/tmp/mongodb-27017.sock", None)] self.assertEqual(res, parse_uri("mongodb://example2.com," "%2Ftmp%2Fmongodb-27017.sock")) res = copy.deepcopy(orig) res['nodelist'] = [("shoe.sock.pants.co.uk", 27017), ("/tmp/mongodb-27017.sock", None)] res['database'] = "nethers_db" self.assertEqual(res, parse_uri("mongodb://shoe.sock.pants.co.uk," "%2Ftmp%2Fmongodb-27017.sock/nethers_db")) res = copy.deepcopy(orig) res['nodelist'] = [("/tmp/mongodb-27017.sock", None), ("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27017.sock," "example2.com:27017" "/test.yield_historical.in")) res = copy.deepcopy(orig) res['nodelist'] = [("/tmp/mongodb-27017.sock", None), ("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.sock'}) self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27017.sock," "example2.com:27017/test.yield_historical" ".sock")) res = copy.deepcopy(orig) res['nodelist'] = [("example2.com", 27017)] res.update({'database': 'test', 'collection': 'yield_historical.sock'}) self.assertEqual(res, parse_uri("mongodb://example2.com:27017" "/test.yield_historical.sock")) res = copy.deepcopy(orig) res['nodelist'] = [("/tmp/mongodb-27017.sock", None)] res.update({'database': 'test', 'collection': 'mongodb-27017.sock'}) self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27017.sock" "/test.mongodb-27017.sock")) res = copy.deepcopy(orig) res['nodelist'] = [('/tmp/mongodb-27020.sock', None), ("::1", 27017), ("2001:0db8:85a3:0000:0000:8a2e:0370:7334", 27018), ("192.168.0.212", 27019), ("localhost", 27018)] self.assertEqual(res, parse_uri("mongodb://%2Ftmp%2Fmongodb-27020.sock" ",[::1]:27017,[2001:0db8:" "85a3:0000:0000:8a2e:0370:7334]," "192.168.0.212:27019,localhost", 27018)) res = copy.deepcopy(orig) res.update({'username': 'fred', 'password': 'foobar'}) res.update({'database': 'test', 'collection': 'yield_historical.in'}) self.assertEqual(res, parse_uri("mongodb://fred:foobar@localhost/" "test.yield_historical.in")) res = copy.deepcopy(orig) res['options'] = {'readpreference': ReadPreference.SECONDARY.mode} self.assertEqual(res, parse_uri( "mongodb://localhost/?readPreference=secondary")) # Various authentication tests res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'MONGODB-CR'} res['username'] = 'user' res['password'] = 'password' self.assertEqual(res, parse_uri("mongodb://user:password@localhost/" "?authMechanism=MONGODB-CR")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'MONGODB-CR', 'authsource': 'bar'} res['username'] = 'user' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user:password@localhost/foo" "?authSource=bar;authMechanism=MONGODB-CR")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'MONGODB-CR'} res['username'] = 'user' res['password'] = '' self.assertEqual(res, parse_uri("mongodb://user:@localhost/" "?authMechanism=MONGODB-CR")) res = copy.deepcopy(orig) res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'GSSAPI'} res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?authMechanism=GSSAPI")) res = copy.deepcopy(orig) res['options'] = {'authmechanism': 'GSSAPI'} res['username'] = 'user@domain.com' res['password'] = '' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com" "@localhost/foo?authMechanism=GSSAPI")) res = copy.deepcopy(orig) res['options'] = {'readpreference': ReadPreference.SECONDARY.mode, 'readpreferencetags': [ {'dc': 'west', 'use': 'website'}, {'dc': 'east', 'use': 'website'}]} res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?readpreference=secondary&" "readpreferencetags=dc:west,use:website&" "readpreferencetags=dc:east,use:website")) res = copy.deepcopy(orig) res['options'] = {'readpreference': ReadPreference.SECONDARY.mode, 'readpreferencetags': [ {'dc': 'west', 'use': 'website'}, {'dc': 'east', 'use': 'website'}, {}]} res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?readpreference=secondary&" "readpreferencetags=dc:west,use:website&" "readpreferencetags=dc:east,use:website&" "readpreferencetags=")) res = copy.deepcopy(orig) res['options'] = {'uuidrepresentation': JAVA_LEGACY} res['username'] = 'user@domain.com' res['password'] = 'password' res['database'] = 'foo' self.assertEqual(res, parse_uri("mongodb://user%40domain.com:password" "@localhost/foo?uuidrepresentation=" "javaLegacy")) with warnings.catch_warnings(): warnings.filterwarnings('error') self.assertRaises(Warning, parse_uri, "mongodb://user%40domain.com:password" "@localhost/foo?uuidrepresentation=notAnOption", warn=True) self.assertRaises(ValueError, parse_uri, "mongodb://user%40domain.com:password" "@localhost/foo?uuidrepresentation=notAnOption") def test_parse_uri_unicode(self): # Ensure parsing a unicode returns option names that can be passed # as kwargs. In Python 2.4, keyword argument names must be ASCII. # In all Pythons, str is the type of valid keyword arg names. res = parse_uri(_unicode("mongodb://localhost/?fsync=true")) for key in res['options']: self.assertTrue(isinstance(key, str)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_bulk.py0000644000175000017500000013031012630145074020301 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the bulk API.""" import sys sys.path[0:0] = [""] from bson import InvalidDocument, SON from bson.objectid import ObjectId from bson.py3compat import string_type from pymongo import MongoClient from pymongo.operations import * from pymongo.common import partition_node from pymongo.errors import (BulkWriteError, ConfigurationError, InvalidOperation, OperationFailure) from pymongo.write_concern import WriteConcern from test import (client_context, unittest, host, port, IntegrationTest, SkipTest) from test.utils import oid_generated_on_client, remove_all_users, wait_until class BulkTestBase(IntegrationTest): @classmethod def setUpClass(cls): super(BulkTestBase, cls).setUpClass() cls.coll = cls.db.test ismaster = client_context.client.admin.command('ismaster') cls.has_write_commands = (ismaster.get("maxWireVersion", 0) > 1) def setUp(self): super(BulkTestBase, self).setUp() self.coll.drop() def assertEqualResponse(self, expected, actual): """Compare response from bulk.execute() to expected response.""" for key, value in expected.items(): if key == 'nModified': if self.has_write_commands: self.assertEqual(value, actual['nModified']) else: # Legacy servers don't include nModified in the response. self.assertFalse('nModified' in actual) elif key == 'upserted': expected_upserts = value actual_upserts = actual['upserted'] self.assertEqual( len(expected_upserts), len(actual_upserts), 'Expected %d elements in "upserted", got %d' % ( len(expected_upserts), len(actual_upserts))) for e, a in zip(expected_upserts, actual_upserts): self.assertEqualUpsert(e, a) elif key == 'writeErrors': expected_errors = value actual_errors = actual['writeErrors'] self.assertEqual( len(expected_errors), len(actual_errors), 'Expected %d elements in "writeErrors", got %d' % ( len(expected_errors), len(actual_errors))) for e, a in zip(expected_errors, actual_errors): self.assertEqualWriteError(e, a) else: self.assertEqual( actual.get(key), value, '%r value of %r does not match expected %r' % (key, actual.get(key), value)) def assertEqualUpsert(self, expected, actual): """Compare bulk.execute()['upserts'] to expected value. Like: {'index': 0, '_id': ObjectId()} """ self.assertEqual(expected['index'], actual['index']) if expected['_id'] == '...': # Unspecified value. self.assertTrue('_id' in actual) else: self.assertEqual(expected['_id'], actual['_id']) def assertEqualWriteError(self, expected, actual): """Compare bulk.execute()['writeErrors'] to expected value. Like: {'index': 0, 'code': 123, 'errmsg': '...', 'op': { ... }} """ self.assertEqual(expected['index'], actual['index']) self.assertEqual(expected['code'], actual['code']) if expected['errmsg'] == '...': # Unspecified value. self.assertTrue('errmsg' in actual) else: self.assertEqual(expected['errmsg'], actual['errmsg']) expected_op = expected['op'].copy() actual_op = actual['op'].copy() if expected_op.get('_id') == '...': # Unspecified _id. self.assertTrue('_id' in actual_op) actual_op.pop('_id') expected_op.pop('_id') self.assertEqual(expected_op, actual_op) class TestBulk(BulkTestBase): def test_empty(self): bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(InvalidOperation, bulk.execute) def test_find(self): # find() requires a selector. bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(TypeError, bulk.find) self.assertRaises(TypeError, bulk.find, 'foo') # No error. bulk.find({}) @client_context.require_version_min(3, 1, 9, -1) @client_context.require_no_auth def test_bypass_document_validation_bulk_op(self): # Test insert self.coll.insert_one({"z": 0}) self.db.command(SON([("collMod", "test"), ("validator", {"z": {"$gte": 0}})])) bulk = self.coll.initialize_ordered_bulk_op( bypass_document_validation=False) bulk.insert({"z": -1}) # error self.assertRaises(BulkWriteError, bulk.execute) self.assertEqual(0, self.coll.count({"z": -1})) bulk = self.coll.initialize_ordered_bulk_op( bypass_document_validation=True) bulk.insert({"z": -1}) bulk.execute() self.assertEqual(1, self.coll.count({"z": -1})) self.coll.insert_one({"z": 0}) self.db.command(SON([("collMod", "test"), ("validator", {"z": {"$gte": 0}})])) bulk = self.coll.initialize_unordered_bulk_op( bypass_document_validation=False) bulk.insert({"z": -1}) # error self.assertRaises(BulkWriteError, bulk.execute) self.assertEqual(1, self.coll.count({"z": -1})) bulk = self.coll.initialize_unordered_bulk_op( bypass_document_validation=True) bulk.insert({"z": -1}) bulk.execute() self.assertEqual(2, self.coll.count({"z": -1})) self.coll.drop() def test_insert(self): expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(TypeError, bulk.insert, 1) # find() before insert() is prohibited. self.assertRaises(AttributeError, lambda: bulk.find({}).insert({})) # We don't allow multiple documents per call. self.assertRaises(TypeError, bulk.insert, [{}, {}]) self.assertRaises(TypeError, bulk.insert, ({} for _ in range(2))) bulk.insert({}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(1, self.coll.count()) doc = self.coll.find_one() self.assertTrue(oid_generated_on_client(doc['_id'])) bulk = self.coll.initialize_unordered_bulk_op() bulk.insert({}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(2, self.coll.count()) result = self.coll.bulk_write([InsertOne({})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.inserted_count) self.assertEqual(3, self.coll.count()) def test_insert_check_keys(self): bulk = self.coll.initialize_ordered_bulk_op() bulk.insert({'$dollar': 1}) self.assertRaises(InvalidDocument, bulk.execute) bulk = self.coll.initialize_ordered_bulk_op() bulk.insert({'a.b': 1}) self.assertRaises(InvalidDocument, bulk.execute) def test_update(self): expected = { 'nMatched': 2, 'nModified': 2, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() # update() requires find() first. self.assertRaises( AttributeError, lambda: bulk.update({'$set': {'x': 1}})) self.assertRaises(TypeError, bulk.find({}).update, 1) self.assertRaises(ValueError, bulk.find({}).update, {}) # All fields must be $-operators. self.assertRaises(ValueError, bulk.find({}).update, {'foo': 'bar'}) bulk.find({}).update({'$set': {'foo': 'bar'}}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 2) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([UpdateMany({}, {'$set': {'foo': 'bar'}})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(2, result.matched_count) self.assertTrue(result.modified_count in (2, None)) # All fields must be $-operators -- validated server-side. bulk = self.coll.initialize_ordered_bulk_op() updates = SON([('$set', {'x': 1}), ('y', 1)]) bulk.find({}).update(updates) self.assertRaises(BulkWriteError, bulk.execute) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).update({'$set': {'bim': 'baz'}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 2, 'nModified': 2, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 2) self.coll.insert_one({'x': 1}) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 1}).update({'$set': {'x': 42}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(1, self.coll.find({'x': 42}).count()) # Second time, x is already 42 so nModified is 0. bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 42}).update({'$set': {'x': 42}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) def test_update_one(self): expected = { 'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() # update_one() requires find() first. self.assertRaises( AttributeError, lambda: bulk.update_one({'$set': {'x': 1}})) self.assertRaises(TypeError, bulk.find({}).update_one, 1) self.assertRaises(ValueError, bulk.find({}).update_one, {}) self.assertRaises(ValueError, bulk.find({}).update_one, {'foo': 'bar'}) bulk.find({}).update_one({'$set': {'foo': 'bar'}}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 1) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([UpdateOne({}, {'$set': {'foo': 'bar'}})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (1, None)) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).update_one({'$set': {'bim': 'baz'}}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 1) # All fields must be $-operators -- validated server-side. bulk = self.coll.initialize_ordered_bulk_op() updates = SON([('$set', {'x': 1}), ('y', 1)]) bulk.find({}).update_one(updates) self.assertRaises(BulkWriteError, bulk.execute) def test_replace_one(self): expected = { 'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() self.assertRaises(TypeError, bulk.find({}).replace_one, 1) self.assertRaises(ValueError, bulk.find({}).replace_one, {'$set': {'foo': 'bar'}}) bulk.find({}).replace_one({'foo': 'bar'}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 1) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([ReplaceOne({}, {'foo': 'bar'})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (1, None)) self.coll.delete_many({}) self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).replace_one({'bim': 'baz'}) result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 1) def test_remove(self): # Test removing all documents, ordered. expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_ordered_bulk_op() # remove() must be preceded by find(). self.assertRaises(AttributeError, lambda: bulk.remove()) bulk.find({}).remove() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.count(), 0) self.coll.insert_many([{}, {}]) result = self.coll.bulk_write([DeleteMany({})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(2, result.deleted_count) # Test removing some documents, ordered. self.coll.insert_many([{}, {'x': 1}, {}, {'x': 1}]) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({'x': 1}).remove() result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.count(), 2) self.coll.delete_many({}) # Test removing all documents, unordered. self.coll.insert_many([{}, {}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).remove() result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) # Test removing some documents, unordered. self.assertEqual(self.coll.count(), 0) self.coll.insert_many([{}, {'x': 1}, {}, {'x': 1}]) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 1}).remove() result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 2, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.count(), 2) self.coll.delete_many({}) def test_remove_one(self): bulk = self.coll.initialize_ordered_bulk_op() # remove_one() must be preceded by find(). self.assertRaises(AttributeError, lambda: bulk.remove_one()) # Test removing one document, empty selector. # First ordered, then unordered. self.coll.insert_many([{}, {}]) expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 1, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': [] } bulk.find({}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.count(), 1) self.coll.insert_one({}) result = self.coll.bulk_write([DeleteOne({})]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.deleted_count) self.assertEqual(self.coll.count(), 1) self.coll.insert_one({}) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual(self.coll.count(), 1) # Test removing one document, with a selector. # First ordered, then unordered. self.coll.insert_one({'x': 1}) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({'x': 1}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual([{}], list(self.coll.find({}, {'_id': False}))) self.coll.insert_one({'x': 1}) bulk = self.coll.initialize_unordered_bulk_op() bulk.find({'x': 1}).remove_one() result = bulk.execute() self.assertEqualResponse(expected, result) self.assertEqual([{}], list(self.coll.find({}, {'_id': False}))) def test_upsert(self): bulk = self.coll.initialize_ordered_bulk_op() # upsert() requires find() first. self.assertRaises( AttributeError, lambda: bulk.upsert()) expected = { 'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': '...'}] } # Note, in MongoDB 2.4 the server won't return the # "upserted" field unless _id is an ObjectId bulk.find({}).upsert().replace_one({'foo': 'bar'}) result = bulk.execute() self.assertEqualResponse(expected, result) self.coll.delete_many({}) result = self.coll.bulk_write([ReplaceOne({}, {'foo': 'bar'}, upsert=True)]) self.assertEqualResponse(expected, result.bulk_api_result) self.assertEqual(1, result.upserted_count) self.assertEqual(1, len(result.upserted_ids)) self.assertTrue(isinstance(result.upserted_ids.get(0), ObjectId)) self.assertEqual(self.coll.find({'foo': 'bar'}).count(), 1) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({}).upsert().update_one({'$set': {'bim': 'baz'}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.find({'bim': 'baz'}).count(), 1) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({}).upsert().update({'$set': {'bim': 'bop'}}) # Non-upsert, no matches. bulk.find({'x': 1}).update({'$set': {'x': 2}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 0, 'nInserted': 0, 'nRemoved': 0, 'upserted': [], 'writeErrors': [], 'writeConcernErrors': []}, result) self.assertEqual(self.coll.find({'bim': 'bop'}).count(), 1) self.assertEqual(self.coll.find({'x': 2}).count(), 0) def test_upsert_large(self): big = 'a' * (client_context.client.max_bson_size - 37) bulk = self.coll.initialize_ordered_bulk_op() bulk.find({'x': 1}).upsert().update({'$set': {'s': big}}) result = bulk.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': '...'}]}, result) self.assertEqual(1, self.coll.find({'x': 1}).count()) def test_client_generated_upsert_id(self): batch = self.coll.initialize_ordered_bulk_op() batch.find({'_id': 0}).upsert().update_one({'$set': {'a': 0}}) batch.find({'a': 1}).upsert().replace_one({'_id': 1}) if not client_context.version.at_least(2, 6, 0): # This case is only possible in MongoDB versions before 2.6. batch.find({'_id': 3}).upsert().replace_one({'_id': 2}) else: # This is just here to make the counts right in all cases. batch.find({'_id': 2}).upsert().replace_one({'_id': 2}) result = batch.execute() self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 3, 'nInserted': 0, 'nRemoved': 0, 'upserted': [{'index': 0, '_id': 0}, {'index': 1, '_id': 1}, {'index': 2, '_id': 2}]}, result) def test_single_ordered_batch(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 1}).update_one({'$set': {'b': 1}}) batch.find({'a': 2}).upsert().update_one({'$set': {'b': 2}}) batch.insert({'a': 3}) batch.find({'a': 3}).remove() result = batch.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 1, 'nInserted': 2, 'nRemoved': 1, 'upserted': [{'index': 2, '_id': '...'}]}, result) def test_single_error_ordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.insert({'b': 3, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [ {'index': 1, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 2}, 'u': {'$set': {'a': 1}}, 'multi': False, 'upsert': True}}]}, result) def test_multiple_error_ordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.find({'b': 3}).upsert().update_one({'$set': {'a': 2}}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.insert({'b': 4, 'a': 3}) batch.insert({'b': 5, 'a': 1}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 1, 'nRemoved': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [ {'index': 1, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 2}, 'u': {'$set': {'a': 1}}, 'multi': False, 'upsert': True}}]}, result) def test_single_unordered_batch(self): batch = self.coll.initialize_unordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 1}).update_one({'$set': {'b': 1}}) batch.find({'a': 2}).upsert().update_one({'$set': {'b': 2}}) batch.insert({'a': 3}) batch.find({'a': 3}).remove() result = batch.execute() self.assertEqualResponse( {'nMatched': 1, 'nModified': 1, 'nUpserted': 1, 'nInserted': 2, 'nRemoved': 1, 'upserted': [{'index': 2, '_id': '...'}], 'writeErrors': [], 'writeConcernErrors': []}, result) def test_single_error_unordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_unordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) batch.insert({'b': 3, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 2, 'nRemoved': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [ {'index': 1, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 2}, 'u': {'$set': {'a': 1}}, 'multi': False, 'upsert': True}}]}, result) def test_multiple_error_unordered_batch(self): self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) batch = self.coll.initialize_unordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.find({'b': 2}).upsert().update_one({'$set': {'a': 3}}) batch.find({'b': 3}).upsert().update_one({'$set': {'a': 4}}) batch.find({'b': 4}).upsert().update_one({'$set': {'a': 3}}) batch.insert({'b': 5, 'a': 2}) batch.insert({'b': 6, 'a': 1}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") # Assume the update at index 1 runs before the update at index 3, # although the spec does not require it. Same for inserts. self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 2, 'nInserted': 2, 'nRemoved': 0, 'upserted': [ {'index': 1, '_id': '...'}, {'index': 2, '_id': '...'}], 'writeConcernErrors': [], 'writeErrors': [ {'index': 3, 'code': 11000, 'errmsg': '...', 'op': {'q': {'b': 4}, 'u': {'$set': {'a': 3}}, 'multi': False, 'upsert': True}}, {'index': 5, 'code': 11000, 'errmsg': '...', 'op': {'_id': '...', 'b': 6, 'a': 1}}]}, result) def test_large_inserts_ordered(self): big = 'x' * self.coll.database.client.max_bson_size batch = self.coll.initialize_ordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.insert({'big': big}) batch.insert({'b': 2, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(1, result['nInserted']) self.coll.delete_many({}) big = 'x' * (1024 * 1024 * 4) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1, 'big': big}) batch.insert({'a': 2, 'big': big}) batch.insert({'a': 3, 'big': big}) batch.insert({'a': 4, 'big': big}) batch.insert({'a': 5, 'big': big}) batch.insert({'a': 6, 'big': big}) result = batch.execute() self.assertEqual(6, result['nInserted']) self.assertEqual(6, self.coll.count()) def test_large_inserts_unordered(self): big = 'x' * self.coll.database.client.max_bson_size batch = self.coll.initialize_unordered_bulk_op() batch.insert({'b': 1, 'a': 1}) batch.insert({'big': big}) batch.insert({'b': 2, 'a': 2}) try: batch.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(2, result['nInserted']) self.coll.delete_many({}) big = 'x' * (1024 * 1024 * 4) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1, 'big': big}) batch.insert({'a': 2, 'big': big}) batch.insert({'a': 3, 'big': big}) batch.insert({'a': 4, 'big': big}) batch.insert({'a': 5, 'big': big}) batch.insert({'a': 6, 'big': big}) result = batch.execute() self.assertEqual(6, result['nInserted']) self.assertEqual(6, self.coll.count()) def test_numerous_inserts(self): # Ensure we don't exceed server's 1000-document batch size limit. n_docs = 2100 batch = self.coll.initialize_unordered_bulk_op() for _ in range(n_docs): batch.insert({}) result = batch.execute() self.assertEqual(n_docs, result['nInserted']) self.assertEqual(n_docs, self.coll.count()) # Same with ordered bulk. self.coll.delete_many({}) batch = self.coll.initialize_ordered_bulk_op() for _ in range(n_docs): batch.insert({}) result = batch.execute() self.assertEqual(n_docs, result['nInserted']) self.assertEqual(n_docs, self.coll.count()) def test_multiple_execution(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({}) batch.execute() self.assertRaises(InvalidOperation, batch.execute) def test_generator_insert(self): def gen(): yield {'a': 1, 'b': 1} yield {'a': 1, 'b': 2} yield {'a': 2, 'b': 3} yield {'a': 3, 'b': 5} yield {'a': 5, 'b': 8} result = self.coll.insert_many(gen()) self.assertEqual(5, len(result.inserted_ids)) class TestBulkWriteConcern(BulkTestBase): @classmethod def setUpClass(cls): super(TestBulkWriteConcern, cls).setUpClass() cls.w = client_context.w cls.secondary = None if cls.w > 1: for member in client_context.ismaster['hosts']: if member != client_context.ismaster['primary']: cls.secondary = MongoClient(*partition_node(member)) break # We tested wtimeout errors by specifying a write concern greater than # the number of members, but in MongoDB 2.7.8+ this causes a different # sort of error, "Not enough data-bearing nodes". In recent servers we # use a failpoint to pause replication on a secondary. cls.need_replication_stopped = client_context.version.at_least(2, 7, 8) def cause_wtimeout(self, batch): if self.need_replication_stopped: if not client_context.test_commands_enabled: raise SkipTest("Test commands must be enabled.") self.secondary.admin.command('configureFailPoint', 'rsSyncApplyStop', mode='alwaysOn') try: return batch.execute({'w': self.w, 'wtimeout': 1}) finally: self.secondary.admin.command('configureFailPoint', 'rsSyncApplyStop', mode='off') else: return batch.execute({'w': self.w + 1, 'wtimeout': 1}) def test_fsync_and_j(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) self.assertRaises( ConfigurationError, batch.execute, {'fsync': True, 'j': True}) @client_context.require_replica_set def test_write_concern_failure_ordered(self): # Ensure we don't raise on wnote. batch = self.coll.initialize_ordered_bulk_op() batch.find({"something": "that does no exist"}).remove() self.assertTrue(batch.execute({"w": self.w})) batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) batch.insert({'a': 2}) # Replication wtimeout is a 'soft' error. # It shouldn't stop batch processing. try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 0, 'nInserted': 2, 'nRemoved': 0, 'upserted': [], 'writeErrors': []}, result) # When talking to legacy servers there will be a # write concern error for each operation. self.assertTrue(len(result['writeConcernErrors']) > 0) failed = result['writeConcernErrors'][0] self.assertEqual(64, failed['code']) self.assertTrue(isinstance(failed['errmsg'], string_type)) self.coll.delete_many({}) self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) # Fail due to write concern support as well # as duplicate key error on ordered batch. batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 3}).upsert().replace_one({'b': 1}) batch.insert({'a': 1}) batch.insert({'a': 2}) try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqualResponse( {'nMatched': 0, 'nModified': 0, 'nUpserted': 1, 'nInserted': 1, 'nRemoved': 0, 'upserted': [{'index': 1, '_id': '...'}], 'writeErrors': [ {'index': 2, 'code': 11000, 'errmsg': '...', 'op': {'_id': '...', 'a': 1}}]}, result) self.assertTrue(len(result['writeConcernErrors']) > 1) failed = result['writeErrors'][0] self.assertTrue("duplicate" in failed['errmsg']) @client_context.require_replica_set def test_write_concern_failure_unordered(self): # Ensure we don't raise on wnote. batch = self.coll.initialize_unordered_bulk_op() batch.find({"something": "that does no exist"}).remove() self.assertTrue(batch.execute({"w": self.w})) batch = self.coll.initialize_unordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3, 'b': 1}}) batch.insert({'a': 2}) # Replication wtimeout is a 'soft' error. # It shouldn't stop batch processing. try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(2, result['nInserted']) self.assertEqual(1, result['nUpserted']) self.assertEqual(0, len(result['writeErrors'])) # When talking to legacy servers there will be a # write concern error for each operation. self.assertTrue(len(result['writeConcernErrors']) > 1) self.coll.delete_many({}) self.coll.create_index('a', unique=True) self.addCleanup(self.coll.drop_index, [('a', 1)]) # Fail due to write concern support as well # as duplicate key error on unordered batch. batch = self.coll.initialize_unordered_bulk_op() batch.insert({'a': 1}) batch.find({'a': 3}).upsert().update_one({'$set': {'a': 3, 'b': 1}}) batch.insert({'a': 1}) batch.insert({'a': 2}) try: self.cause_wtimeout(batch) except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(2, result['nInserted']) self.assertEqual(1, result['nUpserted']) self.assertEqual(1, len(result['writeErrors'])) # When talking to legacy servers there will be a # write concern error for each operation. self.assertTrue(len(result['writeConcernErrors']) > 1) failed = result['writeErrors'][0] self.assertEqual(2, failed['index']) self.assertEqual(11000, failed['code']) self.assertTrue(isinstance(failed['errmsg'], string_type)) self.assertEqual(1, failed['op']['a']) failed = result['writeConcernErrors'][0] self.assertEqual(64, failed['code']) self.assertTrue(isinstance(failed['errmsg'], string_type)) upserts = result['upserted'] self.assertEqual(1, len(upserts)) self.assertEqual(1, upserts[0]['index']) self.assertTrue(upserts[0].get('_id')) class TestBulkNoResults(BulkTestBase): def test_no_results_ordered_success(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 2 == self.coll.count(), 'insert 2 documents') def test_no_results_ordered_failure(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) batch.insert({'_id': 1}) batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 3 == self.coll.count(), 'insert 3 documents') def test_no_results_unordered_success(self): batch = self.coll.initialize_unordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 2 == self.coll.count(), 'insert 2 documents') def test_no_results_unordered_failure(self): batch = self.coll.initialize_unordered_bulk_op() batch.insert({'_id': 1}) batch.find({'_id': 3}).upsert().update_one({'$set': {'b': 1}}) batch.insert({'_id': 2}) batch.insert({'_id': 1}) batch.find({'_id': 1}).remove_one() self.assertTrue(batch.execute({'w': 0}) is None) wait_until(lambda: 2 == self.coll.count(), 'insert 2 documents') self.assertTrue(self.coll.find_one({'_id': 1}) is None) def test_bulk_write_no_results(self): coll = self.coll.with_options(write_concern=WriteConcern(w=0)) result = coll.bulk_write([InsertOne({})]) self.assertFalse(result.acknowledged) self.assertRaises(InvalidOperation, lambda: result.inserted_count) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.deleted_count) self.assertRaises(InvalidOperation, lambda: result.upserted_count) self.assertRaises(InvalidOperation, lambda: result.upserted_ids) class TestBulkAuthorization(BulkTestBase): @classmethod @client_context.require_auth @client_context.require_version_min(2, 5, 3) def setUpClass(cls): super(TestBulkAuthorization, cls).setUpClass() def setUp(self): super(TestBulkAuthorization, self).setUp() self.db.add_user('readonly', 'pw', roles=['read']) self.db.command( 'createRole', 'noremove', privileges=[{ 'actions': ['insert', 'update', 'find'], 'resource': {'db': 'pymongo_test', 'collection': 'test'} }], roles=[]) self.db.add_user('noremove', 'pw', roles=['noremove']) def tearDown(self): self.db.command('dropRole', 'noremove') remove_all_users(self.db) def test_readonly(self): # We test that an authorization failure aborts the batch and is raised # as OperationFailure. cli = MongoClient(host, port) db = cli.pymongo_test coll = db.test db.authenticate('readonly', 'pw') bulk = coll.initialize_ordered_bulk_op() bulk.insert({'x': 1}) self.assertRaises(OperationFailure, bulk.execute) def test_no_remove(self): # We test that an authorization failure aborts the batch and is raised # as OperationFailure. cli = MongoClient(host, port) db = cli.pymongo_test coll = db.test db.authenticate('noremove', 'pw') bulk = coll.initialize_ordered_bulk_op() bulk.insert({'x': 1}) bulk.find({'x': 2}).upsert().replace_one({'x': 2}) bulk.find({}).remove() # Prohibited. bulk.insert({'x': 3}) # Never attempted. self.assertRaises(OperationFailure, bulk.execute) self.assertEqual(set([1, 2]), set(self.coll.distinct('x'))) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_monitoring.py0000644000175000017500000017610212630145074021542 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import sys import time import warnings sys.path[0:0] = [""] from bson.objectid import ObjectId from bson.py3compat import text_type from bson.son import SON from pymongo import CursorType, monitoring, InsertOne, UpdateOne, DeleteOne from pymongo.command_cursor import CommandCursor from pymongo.errors import NotMasterError, OperationFailure from pymongo.read_preferences import ReadPreference from pymongo.write_concern import WriteConcern from test import unittest, client_context, client_knobs from test.utils import single_client, wait_until, EventListener class TestCommandMonitoring(unittest.TestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS # Don't use any global subscribers. monitoring._LISTENERS = monitoring._Listeners([]) cls.client = single_client(event_listeners=[cls.listener]) @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def tearDown(self): self.listener.results.clear() def test_started_simple(self): self.client.pymongo_test.command('ismaster') results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual(SON([('ismaster', 1)]), started.command) self.assertEqual('ismaster', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) def test_succeeded_simple(self): self.client.pymongo_test.command('ismaster') results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertEqual('ismaster', succeeded.command_name) self.assertEqual(self.client.address, succeeded.connection_id) self.assertEqual(1, succeeded.reply.get('ok')) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertTrue(isinstance(succeeded.duration_micros, int)) def test_failed_simple(self): try: self.client.pymongo_test.command('oops!') except OperationFailure: pass results = self.listener.results started = results['started'][0] failed = results['failed'][0] self.assertEqual(0, len(results['succeeded'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertTrue( isinstance(failed, monitoring.CommandFailedEvent)) self.assertEqual('oops!', failed.command_name) self.assertEqual(self.client.address, failed.connection_id) self.assertEqual(0, failed.failure.get('ok')) self.assertTrue(isinstance(failed.request_id, int)) self.assertTrue(isinstance(failed.duration_micros, int)) def test_find_one(self): self.client.pymongo_test.test.find_one() results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('find', 'test'), ('filter', {}), ('limit', 1), ('singleBatch', True)]), started.command) self.assertEqual('find', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) def test_find_and_get_more(self): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{} for _ in range(10)]) self.listener.results.clear() cursor = self.client.pymongo_test.test.find( projection={'_id': False}, batch_size=4) for _ in range(4): next(cursor) cursor_id = cursor.cursor_id results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('find', 'test'), ('filter', {}), ('projection', {'_id': False}), ('batchSize', 4)]), started.command) self.assertEqual('find', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('find', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) csr = succeeded.reply["cursor"] self.assertEqual(csr["id"], cursor_id) self.assertEqual(csr["ns"], "pymongo_test.test") self.assertEqual(csr["firstBatch"], [{} for _ in range(4)]) self.listener.results.clear() # Next batch. Exhausting the cursor could cause a getMore # that returns id of 0 and no results. next(cursor) try: results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('getMore', cursor_id), ('collection', 'test'), ('batchSize', 4)]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('getMore', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': cursor_id, 'ns': 'pymongo_test.test', 'nextBatch': [{} for _ in range(4)]}, 'ok': 1} self.assertEqual(expected_result, succeeded.reply) finally: # Exhaust the cursor to avoid kill cursors. tuple(cursor) def test_find_with_explain(self): cmd = SON([('explain', SON([('find', 'test'), ('filter', {})]))]) self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_one({}) self.listener.results.clear() coll = self.client.pymongo_test.test # Test that we publish the unwrapped command. if self.client.is_mongos and client_context.version.at_least(2, 4, 0): coll = coll.with_options( read_preference=ReadPreference.PRIMARY_PREFERRED) res = coll.find().explain() results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual(cmd, started.command) self.assertEqual('explain', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('explain', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(self.client.address, succeeded.connection_id) self.assertEqual(res, succeeded.reply) def test_find_options(self): cmd = SON([('find', 'test'), ('filter', {}), ('comment', 'this is a test'), ('sort', SON([('_id', 1)])), ('projection', {'x': False}), ('skip', 1), ('batchSize', 2), ('noCursorTimeout', True), ('allowPartialResults', True)]) self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{'x': i} for i in range(5)]) self.listener.results.clear() coll = self.client.pymongo_test.test # Test that we publish the unwrapped command. if self.client.is_mongos and client_context.version.at_least(2, 4, 0): coll = coll.with_options( read_preference=ReadPreference.PRIMARY_PREFERRED) cursor = coll.find( filter={}, projection={'x': False}, skip=1, no_cursor_timeout=True, sort=[('_id', 1)], allow_partial_results=True, modifiers=SON([('$comment', 'this is a test')]), batch_size=2) next(cursor) try: results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual(cmd, started.command) self.assertEqual('find', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('find', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(self.client.address, succeeded.connection_id) finally: # Exhaust the cursor to avoid kill cursors. tuple(cursor) @client_context.require_version_min(2, 6, 0) def test_command_and_get_more(self): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many( [{'x': 1} for _ in range(10)]) self.listener.results.clear() coll = self.client.pymongo_test.test # Test that we publish the unwrapped command. if self.client.is_mongos and client_context.version.at_least(2, 4, 0): coll = coll.with_options( read_preference=ReadPreference.PRIMARY_PREFERRED) cursor = coll.aggregate( [{'$project': {'_id': False, 'x': 1}}], batchSize=4) for _ in range(4): next(cursor) cursor_id = cursor.cursor_id results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('aggregate', 'test'), ('pipeline', [{'$project': {'_id': False, 'x': 1}}]), ('cursor', {'batchSize': 4})]), started.command) self.assertEqual('aggregate', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('aggregate', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_cursor = {'id': cursor_id, 'ns': 'pymongo_test.test', 'firstBatch': [{'x': 1} for _ in range(4)]} self.assertEqual(expected_cursor, succeeded.reply.get('cursor')) self.listener.results.clear() next(cursor) try: results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('getMore', cursor_id), ('collection', 'test'), ('batchSize', 4)]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('getMore', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': cursor_id, 'ns': 'pymongo_test.test', 'nextBatch': [{'x': 1} for _ in range(4)]}, 'ok': 1} self.assertEqual(expected_result, succeeded.reply) finally: # Exhaust the cursor to avoid kill cursors. tuple(cursor) def test_get_more_failure(self): address = self.client.address coll = self.client.pymongo_test.test cursor_doc = {"id": 12345, "firstBatch": [], "ns": coll.full_name} cursor = CommandCursor(coll, cursor_doc, address) try: next(cursor) except Exception: pass results = self.listener.results started = results['started'][0] self.assertEqual(0, len(results['succeeded'])) failed = results['failed'][0] self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('getMore', 12345), ('collection', 'test')]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(failed, monitoring.CommandFailedEvent)) self.assertTrue(isinstance(failed.duration_micros, int)) self.assertEqual('getMore', failed.command_name) self.assertTrue(isinstance(failed.request_id, int)) self.assertEqual(cursor.address, failed.connection_id) self.assertEqual(0, failed.failure.get("ok")) @client_context.require_replica_set def test_not_master_error(self): address = next(iter(client_context.rs_client.secondaries)) client = single_client(*address, event_listeners=[self.listener]) # Clear authentication command results from the listener. client.admin.command('ismaster') self.listener.results.clear() error = None try: client.pymongo_test.test.find_one_and_delete({}) except NotMasterError as exc: error = exc.errors results = self.listener.results started = results['started'][0] failed = results['failed'][0] self.assertEqual(0, len(results['succeeded'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertTrue( isinstance(failed, monitoring.CommandFailedEvent)) self.assertEqual('findAndModify', failed.command_name) self.assertEqual(address, failed.connection_id) self.assertEqual(0, failed.failure.get('ok')) self.assertTrue(isinstance(failed.request_id, int)) self.assertTrue(isinstance(failed.duration_micros, int)) self.assertEqual(error, failed.failure) @client_context.require_no_mongos def test_exhaust(self): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{} for _ in range(10)]) self.listener.results.clear() cursor = self.client.pymongo_test.test.find( projection={'_id': False}, batch_size=5, cursor_type=CursorType.EXHAUST) next(cursor) cursor_id = cursor.cursor_id results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('find', 'test'), ('filter', {}), ('projection', {'_id': False}), ('batchSize', 5)]), started.command) self.assertEqual('find', started.command_name) self.assertEqual(cursor.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('find', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': cursor_id, 'ns': 'pymongo_test.test', 'firstBatch': [{} for _ in range(5)]}, 'ok': 1} self.assertEqual(expected_result, succeeded.reply) self.listener.results.clear() tuple(cursor) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual( SON([('getMore', cursor_id), ('collection', 'test'), ('batchSize', 5)]), started.command) self.assertEqual('getMore', started.command_name) self.assertEqual(cursor.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('getMore', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) expected_result = { 'cursor': {'id': 0, 'ns': 'pymongo_test.test', 'nextBatch': [{} for _ in range(5)]}, 'ok': 1} self.assertEqual(expected_result, succeeded.reply) def test_kill_cursors(self): with client_knobs(kill_cursor_frequency=0.01): self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_many([{} for _ in range(10)]) cursor = self.client.pymongo_test.test.find().batch_size(5) next(cursor) cursor_id = cursor.cursor_id self.listener.results.clear() cursor.close() time.sleep(2) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) # There could be more than one cursor_id here depending on # when the thread last ran. self.assertIn(cursor_id, started.command['cursors']) self.assertEqual('killCursors', started.command_name) self.assertEqual(cursor.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue(isinstance(succeeded.duration_micros, int)) self.assertEqual('killCursors', succeeded.command_name) self.assertTrue(isinstance(succeeded.request_id, int)) self.assertEqual(cursor.address, succeeded.connection_id) # There could be more than one cursor_id here depending on # when the thread last ran. self.assertTrue(cursor_id in succeeded.reply['cursorsUnknown'] or cursor_id in succeeded.reply['cursorsKilled']) def test_non_bulk_writes(self): coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() # Implied write concern insert_one res = coll.insert_one({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': res.inserted_id, 'x': 1}])]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # Unacknowledged insert_one self.listener.results.clear() coll = coll.with_options(write_concern=WriteConcern(w=0)) res = coll.insert_one({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': res.inserted_id, 'x': 1}]), ('writeConcern', {'w': 0})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertEqual(succeeded.reply, {'ok': 1}) # Explicit write concern insert_one self.listener.results.clear() coll = coll.with_options(write_concern=WriteConcern(w=1)) res = coll.insert_one({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': res.inserted_id, 'x': 1}]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # delete_many self.listener.results.clear() res = coll.delete_many({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 1}), ('limit', 0)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(res.deleted_count, reply.get('n')) # replace_one self.listener.results.clear() oid = ObjectId() res = coll.replace_one({'_id': oid}, {'_id': oid, 'x': 1}, upsert=True) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'_id': oid}), ('u', {'_id': oid, 'x': 1}), ('multi', False), ('upsert', True)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual([{'index': 0, '_id': oid}], reply.get('upserted')) # update_one self.listener.results.clear() res = coll.update_one({'x': 1}, {'$inc': {'x': 1}}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 1}), ('u', {'$inc': {'x': 1}}), ('multi', False), ('upsert', False)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # update_many self.listener.results.clear() res = coll.update_many({'x': 2}, {'$inc': {'x': 1}}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 2}), ('u', {'$inc': {'x': 1}}), ('multi', True), ('upsert', False)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # delete_one self.listener.results.clear() res = coll.delete_one({'x': 3}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 3}), ('limit', 1)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual(0, coll.count()) # write errors coll.insert_one({'_id': 1}) try: self.listener.results.clear() coll.insert_one({'_id': 1}) except OperationFailure: pass results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': 1}]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(0, reply.get('n')) errors = reply.get('writeErrors') self.assertIsInstance(errors, list) error = errors[0] self.assertEqual(0, error.get('index')) self.assertIsInstance(error.get('code'), int) self.assertIsInstance(error.get('errmsg'), text_type) def test_legacy_writes(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() # Implied write concern insert _id = coll.insert({'x': 1}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': _id, 'x': 1}])]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # Unacknowledged insert self.listener.results.clear() _id = coll.insert({'x': 1}, w=0) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': _id, 'x': 1}]), ('writeConcern', {'w': 0})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertEqual(succeeded.reply, {'ok': 1}) # Explicit write concern insert self.listener.results.clear() _id = coll.insert({'x': 1}, w=1) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': _id, 'x': 1}]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('insert', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # remove all self.listener.results.clear() res = coll.remove({'x': 1}, w=1) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 1}), ('limit', 0)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(res['n'], reply.get('n')) # upsert self.listener.results.clear() oid = ObjectId() coll.update({'_id': oid}, {'_id': oid, 'x': 1}, upsert=True, w=1) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'_id': oid}), ('u', {'_id': oid, 'x': 1}), ('multi', False), ('upsert', True)])]), ('writeConcern', {'w': 1})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual([{'index': 0, '_id': oid}], reply.get('upserted')) # update one self.listener.results.clear() coll.update({'x': 1}, {'$inc': {'x': 1}}) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 1}), ('u', {'$inc': {'x': 1}}), ('multi', False), ('upsert', False)])])]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # update many self.listener.results.clear() coll.update({'x': 2}, {'$inc': {'x': 1}}, multi=True) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'x': 2}), ('u', {'$inc': {'x': 1}}), ('multi', True), ('upsert', False)])])]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('update', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) # remove one self.listener.results.clear() coll.remove({'x': 3}, multi=False) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'x': 3}), ('limit', 1)])])]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('delete', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) reply = succeeded.reply self.assertEqual(1, reply.get('ok')) self.assertEqual(1, reply.get('n')) self.assertEqual(0, coll.count()) def test_insert_many(self): # This always uses the bulk API. coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() big = 'x' * (1024 * 1024 * 4) docs = [{'_id': i, 'big': big} for i in range(6)] coll.insert_many(docs) results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) documents = [] count = 0 operation_id = started[0].operation_id self.assertIsInstance(operation_id, int) for start, succeed in zip(started, succeeded): self.assertIsInstance(start, monitoring.CommandStartedEvent) cmd = start.command self.assertEqual(['insert', 'ordered', 'documents'], list(cmd.keys())) self.assertEqual(coll.name, cmd['insert']) self.assertIs(True, cmd['ordered']) documents.extend(cmd['documents']) self.assertEqual('pymongo_test', start.database_name) self.assertEqual('insert', start.command_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) reply = succeed.reply self.assertEqual(1, reply.get('ok')) count += reply.get('n', 0) self.assertEqual(documents, docs) self.assertEqual(6, count) def test_legacy_insert_many(self): # On legacy servers this uses bulk OP_INSERT. with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() # Force two batches on legacy servers. big = 'x' * (1024 * 1024 * 12) docs = [{'_id': i, 'big': big} for i in range(6)] coll.insert(docs) results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) documents = [] count = 0 operation_id = started[0].operation_id self.assertIsInstance(operation_id, int) for start, succeed in zip(started, succeeded): self.assertIsInstance(start, monitoring.CommandStartedEvent) cmd = start.command self.assertEqual(['insert', 'ordered', 'documents'], list(cmd.keys())) self.assertEqual(coll.name, cmd['insert']) self.assertIs(True, cmd['ordered']) documents.extend(cmd['documents']) self.assertEqual('pymongo_test', start.database_name) self.assertEqual('insert', start.command_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) reply = succeed.reply self.assertEqual(1, reply.get('ok')) count += reply.get('n', 0) self.assertEqual(documents, docs) self.assertEqual(6, count) def test_bulk_write(self): coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() coll.bulk_write([InsertOne({'_id': 1}), UpdateOne({'_id': 1}, {'$set': {'x': 1}}), DeleteOne({'_id': 1})]) results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) operation_id = started[0].operation_id pairs = list(zip(started, succeeded)) self.assertEqual(3, len(pairs)) for start, succeed in pairs: self.assertIsInstance(start, monitoring.CommandStartedEvent) self.assertEqual('pymongo_test', start.database_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) expected = SON([('insert', coll.name), ('ordered', True), ('documents', [{'_id': 1}])]) self.assertEqual(expected, started[0].command) expected = SON([('update', coll.name), ('ordered', True), ('updates', [SON([('q', {'_id': 1}), ('u', {'$set': {'x': 1}}), ('multi', False), ('upsert', False)])])]) self.assertEqual(expected, started[1].command) expected = SON([('delete', coll.name), ('ordered', True), ('deletes', [SON([('q', {'_id': 1}), ('limit', 1)])])]) self.assertEqual(expected, started[2].command) def test_write_errors(self): coll = self.client.pymongo_test.test coll.drop() self.listener.results.clear() try: coll.bulk_write([InsertOne({'_id': 1}), InsertOne({'_id': 1}), InsertOne({'_id': 1}), DeleteOne({'_id': 1})], ordered=False) except OperationFailure: pass results = self.listener.results started = results['started'] succeeded = results['succeeded'] self.assertEqual(0, len(results['failed'])) operation_id = started[0].operation_id pairs = list(zip(started, succeeded)) errors = [] for start, succeed in pairs: self.assertIsInstance(start, monitoring.CommandStartedEvent) self.assertEqual('pymongo_test', start.database_name) self.assertIsInstance(start.request_id, int) self.assertEqual(self.client.address, start.connection_id) self.assertIsInstance(succeed, monitoring.CommandSucceededEvent) self.assertIsInstance(succeed.duration_micros, int) self.assertEqual(start.command_name, succeed.command_name) self.assertEqual(start.request_id, succeed.request_id) self.assertEqual(start.connection_id, succeed.connection_id) self.assertEqual(start.operation_id, operation_id) self.assertEqual(succeed.operation_id, operation_id) if 'writeErrors' in succeed.reply: errors.extend(succeed.reply['writeErrors']) self.assertEqual(2, len(errors)) fields = set(['index', 'code', 'errmsg']) for error in errors: self.assertEqual(fields, set(error)) def test_first_batch_helper(self): # Regardless of server version and use of helpers._first_batch # this test should still pass. self.listener.results.clear() self.client.pymongo_test.collection_names() results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('listCollections', 1), ('cursor', {})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('listCollections', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.listener.results.clear() tuple(self.client.pymongo_test.test.list_indexes()) started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('listIndexes', 'test'), ('cursor', {})]) self.assertEqual(expected, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('listIndexes', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.listener.results.clear() self.client.pymongo_test.current_op(True) started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = SON([('currentOp', 1), ('$all', True)]) self.assertEqual(expected, started.command) self.assertEqual('admin', started.database_name) self.assertEqual('currentOp', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) if not client_context.is_mongos: self.client.fsync(lock=True) self.listener.results.clear() self.client.unlock() # Wait for async unlock... wait_until( lambda: not self.client.is_locked, "unlock the database") started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) expected = {'fsyncUnlock': 1} self.assertEqual(expected, started.command) self.assertEqual('admin', started.database_name) self.assertEqual('fsyncUnlock', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertIsInstance(succeeded.duration_micros, int) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) def test_sensitive_commands(self): listeners = self.client._event_listeners self.listener.results.clear() cmd = SON([("getnonce", 1)]) listeners.publish_command_start( cmd, "pymongo_test", 12345, self.client.address) delta = datetime.timedelta(milliseconds=100) listeners.publish_command_success( delta, {'nonce': 'e474f4561c5eb40b', 'ok': 1.0}, "getnonce", 12345, self.client.address) results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertIsInstance(started, monitoring.CommandStartedEvent) self.assertEqual({}, started.command) self.assertEqual('pymongo_test', started.database_name) self.assertEqual('getnonce', started.command_name) self.assertIsInstance(started.request_id, int) self.assertEqual(self.client.address, started.connection_id) self.assertIsInstance(succeeded, monitoring.CommandSucceededEvent) self.assertEqual(succeeded.duration_micros, 100000) self.assertEqual(started.command_name, succeeded.command_name) self.assertEqual(started.request_id, succeeded.request_id) self.assertEqual(started.connection_id, succeeded.connection_id) self.assertEqual({}, succeeded.reply) class TestGlobalListener(unittest.TestCase): @classmethod @client_context.require_connection def setUpClass(cls): cls.listener = EventListener() cls.saved_listeners = monitoring._LISTENERS monitoring.register(cls.listener) cls.client = single_client() # Get one (authenticated) socket in the pool. cls.client.pymongo_test.command('ismaster') @classmethod def tearDownClass(cls): monitoring._LISTENERS = cls.saved_listeners def setUp(self): self.listener.results.clear() def test_simple(self): self.client.pymongo_test.command('ismaster') results = self.listener.results started = results['started'][0] succeeded = results['succeeded'][0] self.assertEqual(0, len(results['failed'])) self.assertTrue( isinstance(succeeded, monitoring.CommandSucceededEvent)) self.assertTrue( isinstance(started, monitoring.CommandStartedEvent)) self.assertEqual(SON([('ismaster', 1)]), started.command) self.assertEqual('ismaster', started.command_name) self.assertEqual(self.client.address, started.connection_id) self.assertEqual('pymongo_test', started.database_name) self.assertTrue(isinstance(started.request_id, int)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_replica_set_reconfig.py0000644000175000017500000001316712630145074023524 0ustar behackettbehackett00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test clients and replica set configuration changes, using mocks.""" import sys sys.path[0:0] = [""] from pymongo.errors import ConnectionFailure, AutoReconnect from pymongo import ReadPreference from test import unittest, client_context, client_knobs, MockClientTest from test.pymongo_mocks import MockClient from test.utils import wait_until @client_context.require_connection def setUpModule(): pass class TestSecondaryBecomesStandalone(MockClientTest): # An administrator removes a secondary from a 3-node set and # brings it back up as standalone, without updating the other # members' config. Verify we don't continue using it. def test_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1,b:2,c:3', replicaSet='rs', serverSelectionTimeoutMS=100) # MongoClient connects to primary by default. wait_until(lambda: c.address is not None, 'connect to primary') self.assertEqual(c.address, ('a', 1)) # C is brought up as a standalone. c.mock_members.remove('c:3') c.mock_standalones.append('c:3') # Fail over. c.kill_host('a:1') c.kill_host('b:2') # Force reconnect. c.close() with self.assertRaises(AutoReconnect): c.db.command('ismaster') self.assertEqual(c.address, None) def test_replica_set_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1,b:2,c:3', replicaSet='rs') wait_until(lambda: ('b', 2) in c.secondaries, 'discover host "b"') wait_until(lambda: ('c', 3) in c.secondaries, 'discover host "c"') # C is brought up as a standalone. c.mock_members.remove('c:3') c.mock_standalones.append('c:3') wait_until(lambda: set([('b', 2)]) == c.secondaries, 'update the list of secondaries') self.assertEqual(('a', 1), c.primary) class TestSecondaryRemoved(MockClientTest): # An administrator removes a secondary from a 3-node set *without* # restarting it as standalone. def test_replica_set_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2', 'c:3'], mongoses=[], host='a:1,b:2,c:3', replicaSet='rs') wait_until(lambda: ('b', 2) in c.secondaries, 'discover host "b"') wait_until(lambda: ('c', 3) in c.secondaries, 'discover host "c"') # C is removed. c.mock_ismaster_hosts.remove('c:3') wait_until(lambda: set([('b', 2)]) == c.secondaries, 'update list of secondaries') self.assertEqual(('a', 1), c.primary) class TestSocketError(MockClientTest): def test_socket_error_marks_member_down(self): # Disable background refresh. with client_knobs(heartbeat_frequency=999999): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs') wait_until(lambda: len(c.nodes) == 2, 'discover both nodes') # b now raises socket.error. c.mock_down_hosts.append('b:2') self.assertRaises( ConnectionFailure, c.db.collection.with_options( read_preference=ReadPreference.SECONDARY).find_one) self.assertEqual(1, len(c.nodes)) class TestSecondaryAdded(MockClientTest): def test_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs') wait_until(lambda: len(c.nodes) == 2, 'discover both nodes') # MongoClient connects to primary by default. self.assertEqual(c.address, ('a', 1)) self.assertEqual(set([('a', 1), ('b', 2)]), c.nodes) # C is added. c.mock_members.append('c:3') c.mock_ismaster_hosts.append('c:3') c.close() c.db.command('ismaster') self.assertEqual(c.address, ('a', 1)) wait_until(lambda: set([('a', 1), ('b', 2), ('c', 3)]) == c.nodes, 'reconnect to both secondaries') def test_replica_set_client(self): c = MockClient( standalones=[], members=['a:1', 'b:2'], mongoses=[], host='a:1', replicaSet='rs') wait_until(lambda: ('a', 1) == c.primary, 'discover the primary') wait_until(lambda: set([('b', 2)]) == c.secondaries, 'discover the secondary') # C is added. c.mock_members.append('c:3') c.mock_ismaster_hosts.append('c:3') wait_until(lambda: set([('b', 2), ('c', 3)]) == c.secondaries, 'discover the new secondary') self.assertEqual(('a', 1), c.primary) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_monitor.py0000644000175000017500000000270012630145074021034 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the monitor module.""" import gc import sys from functools import partial sys.path[0:0] = [""] from pymongo.periodic_executor import _EXECUTORS from test import unittest, port, host, IntegrationTest from test.utils import single_client, one, connected, wait_until def unregistered(ref): gc.collect() return ref not in _EXECUTORS class TestMonitor(IntegrationTest): def test_atexit_hook(self): client = single_client(host, port) executor = one(client._topology._servers.values())._monitor._executor connected(client) # The executor stores a weakref to itself in _EXECUTORS. ref = one([r for r in _EXECUTORS.copy() if r() is executor]) del executor del client wait_until(partial(unregistered, ref), 'unregister executor', timeout=5) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_server_selection_rtt.py0000644000175000017500000000402012630145074023606 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the topology module.""" import json import os import sys sys.path[0:0] = [""] from test import unittest from pymongo.read_preferences import MovingAverage # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'server_selection/rtt') class TestAllScenarios(unittest.TestCase): pass def create_test(scenario_def): def run_scenario(self): moving_average = MovingAverage() if scenario_def['avg_rtt_ms'] != "NULL": moving_average.add_sample(scenario_def['avg_rtt_ms']) if scenario_def['new_rtt_ms'] != "NULL": moving_average.add_sample(scenario_def['new_rtt_ms']) self.assertAlmostEqual(moving_average.get(), scenario_def['new_avg_rtt']) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath)[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load(scenario_stream) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s_%s' % ( dirname, os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_binary.py0000644000175000017500000003250212630145074020634 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Binary wrapper.""" import base64 import copy import pickle import sys import uuid sys.path[0:0] = [""] import bson from bson.binary import * from bson.codec_options import CodecOptions from bson.py3compat import u from bson.son import SON from test import client_context, unittest from pymongo.mongo_client import MongoClient class TestBinary(unittest.TestCase): @classmethod def setUpClass(cls): # Generated by the Java driver from_java = ( b'bAAAAAdfaWQAUCBQxkVm+XdxJ9tOBW5ld2d1aWQAEAAAAAMIQkfACFu' b'Z/0RustLOU/G6Am5ld2d1aWRzdHJpbmcAJQAAAGZmOTk1YjA4LWMwND' b'ctNDIwOC1iYWYxLTUzY2VkMmIyNmU0NAAAbAAAAAdfaWQAUCBQxkVm+' b'XdxJ9tPBW5ld2d1aWQAEAAAAANgS/xhRXXv8kfIec+dYdyCAm5ld2d1' b'aWRzdHJpbmcAJQAAAGYyZWY3NTQ1LTYxZmMtNGI2MC04MmRjLTYxOWR' b'jZjc5Yzg0NwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tQBW5ld2d1aWQAEA' b'AAAAPqREIbhZPUJOSdHCJIgaqNAm5ld2d1aWRzdHJpbmcAJQAAADI0Z' b'DQ5Mzg1LTFiNDItNDRlYS04ZGFhLTgxNDgyMjFjOWRlNAAAbAAAAAdf' b'aWQAUCBQxkVm+XdxJ9tRBW5ld2d1aWQAEAAAAANjQBn/aQuNfRyfNyx' b'29COkAm5ld2d1aWRzdHJpbmcAJQAAADdkOGQwYjY5LWZmMTktNDA2My' b'1hNDIzLWY0NzYyYzM3OWYxYwAAbAAAAAdfaWQAUCBQxkVm+XdxJ9tSB' b'W5ld2d1aWQAEAAAAAMtSv/Et1cAQUFHUYevqxaLAm5ld2d1aWRzdHJp' b'bmcAJQAAADQxMDA1N2I3LWM0ZmYtNGEyZC04YjE2LWFiYWY4NzUxNDc' b'0MQAA') cls.java_data = base64.b64decode(from_java) # Generated by the .net driver from_csharp = ( b'ZAAAABBfaWQAAAAAAAVuZXdndWlkABAAAAAD+MkoCd/Jy0iYJ7Vhl' b'iF3BAJuZXdndWlkc3RyaW5nACUAAAAwOTI4YzlmOC1jOWRmLTQ4Y2' b'ItOTgyNy1iNTYxOTYyMTc3MDQAAGQAAAAQX2lkAAEAAAAFbmV3Z3V' b'pZAAQAAAAA9MD0oXQe6VOp7mK4jkttWUCbmV3Z3VpZHN0cmluZwAl' b'AAAAODVkMjAzZDMtN2JkMC00ZWE1LWE3YjktOGFlMjM5MmRiNTY1A' b'ABkAAAAEF9pZAACAAAABW5ld2d1aWQAEAAAAAPRmIO2auc/Tprq1Z' b'oQ1oNYAm5ld2d1aWRzdHJpbmcAJQAAAGI2ODM5OGQxLWU3NmEtNGU' b'zZi05YWVhLWQ1OWExMGQ2ODM1OAAAZAAAABBfaWQAAwAAAAVuZXdn' b'dWlkABAAAAADISpriopuTEaXIa7arYOCFAJuZXdndWlkc3RyaW5nA' b'CUAAAA4YTZiMmEyMS02ZThhLTQ2NGMtOTcyMS1hZWRhYWQ4MzgyMT' b'QAAGQAAAAQX2lkAAQAAAAFbmV3Z3VpZAAQAAAAA98eg0CFpGlPihP' b'MwOmYGOMCbmV3Z3VpZHN0cmluZwAlAAAANDA4MzFlZGYtYTQ4NS00' b'ZjY5LThhMTMtY2NjMGU5OTgxOGUzAAA=') cls.csharp_data = base64.b64decode(from_csharp) def test_binary(self): a_string = "hello world" a_binary = Binary(b"hello world") self.assertTrue(a_binary.startswith(b"hello")) self.assertTrue(a_binary.endswith(b"world")) self.assertTrue(isinstance(a_binary, Binary)) self.assertFalse(isinstance(a_string, Binary)) def test_exceptions(self): self.assertRaises(TypeError, Binary, None) self.assertRaises(TypeError, Binary, u("hello")) self.assertRaises(TypeError, Binary, 5) self.assertRaises(TypeError, Binary, 10.2) self.assertRaises(TypeError, Binary, b"hello", None) self.assertRaises(TypeError, Binary, b"hello", "100") self.assertRaises(ValueError, Binary, b"hello", -1) self.assertRaises(ValueError, Binary, b"hello", 256) self.assertTrue(Binary(b"hello", 0)) self.assertTrue(Binary(b"hello", 255)) def test_subtype(self): one = Binary(b"hello") self.assertEqual(one.subtype, 0) two = Binary(b"hello", 2) self.assertEqual(two.subtype, 2) three = Binary(b"hello", 100) self.assertEqual(three.subtype, 100) def test_equality(self): two = Binary(b"hello") three = Binary(b"hello", 100) self.assertNotEqual(two, three) self.assertEqual(three, Binary(b"hello", 100)) self.assertEqual(two, Binary(b"hello")) self.assertNotEqual(two, Binary(b"hello ")) self.assertNotEqual(b"hello", Binary(b"hello")) # Explicitly test inequality self.assertFalse(three != Binary(b"hello", 100)) self.assertFalse(two != Binary(b"hello")) def test_repr(self): one = Binary(b"hello world") self.assertEqual(repr(one), "Binary(%s, 0)" % (repr(b"hello world"),)) two = Binary(b"hello world", 2) self.assertEqual(repr(two), "Binary(%s, 2)" % (repr(b"hello world"),)) three = Binary(b"\x08\xFF") self.assertEqual(repr(three), "Binary(%s, 0)" % (repr(b"\x08\xFF"),)) four = Binary(b"\x08\xFF", 2) self.assertEqual(repr(four), "Binary(%s, 2)" % (repr(b"\x08\xFF"),)) five = Binary(b"test", 100) self.assertEqual(repr(five), "Binary(%s, 100)" % (repr(b"test"),)) def test_hash(self): one = Binary(b"hello world") two = Binary(b"hello world", 42) self.assertEqual(hash(Binary(b"hello world")), hash(one)) self.assertNotEqual(hash(one), hash(two)) self.assertEqual(hash(Binary(b"hello world", 42)), hash(two)) def test_legacy_java_uuid(self): # Test decoding data = self.java_data docs = bson.decode_all(data, CodecOptions(SON, False, PYTHON_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, STANDARD)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY)) for d in docs: self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) # Test encoding encoded = b''.join([ bson.BSON.encode(doc, False, CodecOptions(uuid_representation=PYTHON_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=STANDARD)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=CSHARP_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=JAVA_LEGACY)) for doc in docs]) self.assertEqual(data, encoded) @client_context.require_connection def test_legacy_java_uuid_roundtrip(self): data = self.java_data docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY)) client_context.client.pymongo_test.drop_collection('java_uuid') db = client_context.client.pymongo_test coll = db.get_collection( 'java_uuid', CodecOptions(uuid_representation=JAVA_LEGACY)) coll.insert_many(docs) self.assertEqual(5, coll.count()) for d in coll.find(): self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) coll = db.get_collection( 'java_uuid', CodecOptions(uuid_representation=PYTHON_LEGACY)) for d in coll.find(): self.assertNotEqual(d['newguid'], d['newguidstring']) client_context.client.pymongo_test.drop_collection('java_uuid') def test_legacy_csharp_uuid(self): data = self.csharp_data # Test decoding docs = bson.decode_all(data, CodecOptions(SON, False, PYTHON_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, STANDARD)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY)) for d in docs: self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring'])) docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY)) for d in docs: self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) # Test encoding encoded = b''.join([ bson.BSON.encode(doc, False, CodecOptions(uuid_representation=PYTHON_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join([ bson.BSON.encode(doc, False, CodecOptions(uuid_representation=STANDARD)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=JAVA_LEGACY)) for doc in docs]) self.assertNotEqual(data, encoded) encoded = b''.join( [bson.BSON.encode(doc, False, CodecOptions(uuid_representation=CSHARP_LEGACY)) for doc in docs]) self.assertEqual(data, encoded) @client_context.require_connection def test_legacy_csharp_uuid_roundtrip(self): data = self.csharp_data docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY)) client_context.client.pymongo_test.drop_collection('csharp_uuid') db = client_context.client.pymongo_test coll = db.get_collection( 'csharp_uuid', CodecOptions(uuid_representation=CSHARP_LEGACY)) coll.insert_many(docs) self.assertEqual(5, coll.count()) for d in coll.find(): self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring'])) coll = db.get_collection( 'csharp_uuid', CodecOptions(uuid_representation=PYTHON_LEGACY)) for d in coll.find(): self.assertNotEqual(d['newguid'], d['newguidstring']) client_context.client.pymongo_test.drop_collection('csharp_uuid') def test_uri_to_uuid(self): uri = "mongodb://foo/?uuidrepresentation=csharpLegacy" client = MongoClient(uri, connect=False) self.assertEqual( client.pymongo_test.test.codec_options.uuid_representation, CSHARP_LEGACY) @client_context.require_connection def test_uuid_queries(self): db = client_context.client.pymongo_test coll = db.test coll.drop() uu = uuid.uuid4() coll.insert_one({'uuid': Binary(uu.bytes, 3)}) self.assertEqual(1, coll.count()) # Test UUIDLegacy queries. coll = db.get_collection("test", CodecOptions(uuid_representation=STANDARD)) self.assertEqual(0, coll.find({'uuid': uu}).count()) cur = coll.find({'uuid': UUIDLegacy(uu)}) self.assertEqual(1, cur.count()) retrieved = next(cur) self.assertEqual(uu, retrieved['uuid']) # Test regular UUID queries (using subtype 4). coll.insert_one({'uuid': uu}) self.assertEqual(2, coll.count()) cur = coll.find({'uuid': uu}) self.assertEqual(1, cur.count()) retrieved = next(cur) self.assertEqual(uu, retrieved['uuid']) # Test both. cur = coll.find({'uuid': {'$in': [uu, UUIDLegacy(uu)]}}) self.assertEqual(2, cur.count()) coll.drop() def test_pickle(self): b1 = Binary(b'123', 2) # For testing backwards compatibility with pre-2.4 pymongo if PY3: p = (b"\x80\x03cbson.binary\nBinary\nq\x00C\x03123q\x01\x85q" b"\x02\x81q\x03}q\x04X\x10\x00\x00\x00_Binary__subtypeq" b"\x05K\x02sb.") else: p = (b"ccopy_reg\n_reconstructor\np0\n(cbson.binary\nBinary\np1\nc" b"__builtin__\nstr\np2\nS'123'\np3\ntp4\nRp5\n(dp6\nS'_Binary" b"__subtype'\np7\nI2\nsb.") if not sys.version.startswith('3.0'): self.assertEqual(b1, pickle.loads(p)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.assertEqual(b1, pickle.loads(pickle.dumps(b1, proto))) uu = uuid.uuid4() uul = UUIDLegacy(uu) self.assertEqual(uul, copy.copy(uul)) self.assertEqual(uul, copy.deepcopy(uul)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.assertEqual(uul, pickle.loads(pickle.dumps(uul, proto))) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_objectid.py0000644000175000017500000001546512630145074021144 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the objectid module.""" import datetime import pickle import sys sys.path[0:0] = [""] from bson.errors import InvalidId from bson.objectid import ObjectId from bson.py3compat import PY3, u, _unicode from bson.tz_util import (FixedOffset, utc) from test import SkipTest, unittest from test.utils import oid_generated_on_client def oid(x): return ObjectId() class TestObjectId(unittest.TestCase): def test_creation(self): self.assertRaises(TypeError, ObjectId, 4) self.assertRaises(TypeError, ObjectId, 175.0) self.assertRaises(TypeError, ObjectId, {"test": 4}) self.assertRaises(TypeError, ObjectId, ["something"]) self.assertRaises(InvalidId, ObjectId, "") self.assertRaises(InvalidId, ObjectId, "12345678901") self.assertRaises(InvalidId, ObjectId, "1234567890123") self.assertTrue(ObjectId()) self.assertTrue(ObjectId(b"123456789012")) a = ObjectId() self.assertTrue(ObjectId(a)) def test_unicode(self): a = ObjectId() self.assertEqual(a, ObjectId(_unicode(a))) self.assertEqual(ObjectId("123456789012123456789012"), ObjectId(u("123456789012123456789012"))) self.assertRaises(InvalidId, ObjectId, u("hello")) def test_from_hex(self): ObjectId("123456789012123456789012") self.assertRaises(InvalidId, ObjectId, "123456789012123456789G12") self.assertRaises(InvalidId, ObjectId, u("123456789012123456789G12")) def test_repr_str(self): self.assertEqual(repr(ObjectId("1234567890abcdef12345678")), "ObjectId('1234567890abcdef12345678')") self.assertEqual(str(ObjectId("1234567890abcdef12345678")), "1234567890abcdef12345678") self.assertEqual(str(ObjectId(b"123456789012")), "313233343536373839303132") self.assertEqual(ObjectId("1234567890abcdef12345678").binary, b'\x124Vx\x90\xab\xcd\xef\x124Vx') self.assertEqual(str(ObjectId(b'\x124Vx\x90\xab\xcd\xef\x124Vx')), "1234567890abcdef12345678") def test_equality(self): a = ObjectId() self.assertEqual(a, ObjectId(a)) self.assertEqual(ObjectId(b"123456789012"), ObjectId(b"123456789012")) self.assertNotEqual(ObjectId(), ObjectId()) self.assertNotEqual(ObjectId(b"123456789012"), b"123456789012") # Explicitly test inequality self.assertFalse(a != ObjectId(a)) self.assertFalse(ObjectId(b"123456789012") != ObjectId(b"123456789012")) def test_binary_str_equivalence(self): a = ObjectId() self.assertEqual(a, ObjectId(a.binary)) self.assertEqual(a, ObjectId(str(a))) def test_pid(self): self.assertTrue(oid_generated_on_client(ObjectId())) def test_generation_time(self): d1 = datetime.datetime.utcnow() d2 = ObjectId().generation_time self.assertEqual(utc, d2.tzinfo) d2 = d2.replace(tzinfo=None) self.assertTrue(d2 - d1 < datetime.timedelta(seconds=2)) def test_from_datetime(self): if 'PyPy 1.8.0' in sys.version: # See https://bugs.pypy.org/issue1092 raise SkipTest("datetime.timedelta is broken in pypy 1.8.0") d = datetime.datetime.utcnow() d = d - datetime.timedelta(microseconds=d.microsecond) oid = ObjectId.from_datetime(d) self.assertEqual(d, oid.generation_time.replace(tzinfo=None)) self.assertEqual("0" * 16, str(oid)[8:]) aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) oid = ObjectId.from_datetime(aware) self.assertEqual(as_utc, oid.generation_time) def test_pickling(self): orig = ObjectId() for protocol in [0, 1, 2, -1]: pkl = pickle.dumps(orig, protocol=protocol) self.assertEqual(orig, pickle.loads(pkl)) def test_pickle_backwards_compatability(self): # For a full discussion see http://bugs.python.org/issue6137 if sys.version.startswith('3.0'): raise SkipTest("Python 3.0.x can't unpickle " "objects pickled in Python 2.x.") # This string was generated by pickling an ObjectId in pymongo # version 1.9 pickled_with_1_9 = ( b"ccopy_reg\n_reconstructor\np0\n" b"(cbson.objectid\nObjectId\np1\nc__builtin__\n" b"object\np2\nNtp3\nRp4\n" b"(dp5\nS'_ObjectId__id'\np6\n" b"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np7\nsb.") # We also test against a hardcoded "New" pickle format so that we # make sure we're backward compatible with the current version in # the future as well. pickled_with_1_10 = ( b"ccopy_reg\n_reconstructor\np0\n" b"(cbson.objectid\nObjectId\np1\nc__builtin__\n" b"object\np2\nNtp3\nRp4\n" b"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np5\nb.") if PY3: # Have to load using 'latin-1' since these were pickled in python2.x. oid_1_9 = pickle.loads(pickled_with_1_9, encoding='latin-1') oid_1_10 = pickle.loads(pickled_with_1_10, encoding='latin-1') else: oid_1_9 = pickle.loads(pickled_with_1_9) oid_1_10 = pickle.loads(pickled_with_1_10) self.assertEqual(oid_1_9, ObjectId("4d9a66561376c00b88000000")) self.assertEqual(oid_1_9, oid_1_10) def test_is_valid(self): self.assertFalse(ObjectId.is_valid(None)) self.assertFalse(ObjectId.is_valid(4)) self.assertFalse(ObjectId.is_valid(175.0)) self.assertFalse(ObjectId.is_valid({"test": 4})) self.assertFalse(ObjectId.is_valid(["something"])) self.assertFalse(ObjectId.is_valid("")) self.assertFalse(ObjectId.is_valid("12345678901")) self.assertFalse(ObjectId.is_valid("1234567890123")) self.assertTrue(ObjectId.is_valid(b"123456789012")) self.assertTrue(ObjectId.is_valid("123456789012123456789012")) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_bson.py0000644000175000017500000012022012630145074020304 0ustar behackettbehackett00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the bson module.""" import collections import datetime import re import sys import uuid sys.path[0:0] = [""] import bson from bson import (BSON, decode_all, decode_file_iter, decode_iter, EPOCH_AWARE, is_valid, Regex) from bson.binary import Binary, UUIDLegacy from bson.code import Code from bson.codec_options import CodecOptions from bson.int64 import Int64 from bson.objectid import ObjectId from bson.dbref import DBRef from bson.py3compat import PY3, u, text_type, iteritems, StringIO from bson.son import SON from bson.timestamp import Timestamp from bson.tz_util import FixedOffset from bson.errors import (InvalidBSON, InvalidDocument, InvalidStringData) from bson.max_key import MaxKey from bson.min_key import MinKey from bson.tz_util import (FixedOffset, utc) from test import qcheck, SkipTest, unittest if PY3: long = int class NotADict(collections.MutableMapping): """Non-dict type that implements the mapping protocol.""" def __init__(self, initial=None): if not initial: self._dict = {} else: self._dict = initial def __iter__(self): return iter(self._dict) def __getitem__(self, item): return self._dict[item] def __delitem__(self, item): del self._dict[item] def __setitem__(self, item, value): self._dict[item] = value def __len__(self): return len(self._dict) def __eq__(self, other): if isinstance(other, collections.Mapping): return all(self.get(k) == other.get(k) for k in self) return NotImplemented def __repr__(self): return "NotADict(%s)" % repr(self._dict) class DSTAwareTimezone(datetime.tzinfo): def __init__(self, offset, name, dst_start_month, dst_end_month): self.__offset = offset self.__dst_start_month = dst_start_month self.__dst_end_month = dst_end_month self.__name = name def _is_dst(self, dt): return self.__dst_start_month <= dt.month <= self.__dst_end_month def utcoffset(self, dt): return datetime.timedelta(minutes=self.__offset) + self.dst(dt) def dst(self, dt): if self._is_dst(dt): return datetime.timedelta(hours=1) return datetime.timedelta(0) def tzname(self, dt): return self.__name class TestBSON(unittest.TestCase): def assertInvalid(self, data): self.assertRaises(InvalidBSON, bson.BSON(data).decode) def check_encode_then_decode(self, doc_class=dict): # Work around http://bugs.jython.org/issue1728 if sys.platform.startswith('java'): doc_class = SON def helper(doc): self.assertEqual(doc, (BSON.encode(doc_class(doc))).decode()) helper({}) helper({"test": u("hello")}) self.assertTrue(isinstance(BSON.encode({"hello": "world"}) .decode()["hello"], text_type)) helper({"mike": -10120}) helper({"long": Int64(10)}) helper({"really big long": 2147483648}) helper({u("hello"): 0.0013109}) helper({"something": True}) helper({"false": False}) helper({"an array": [1, True, 3.8, u("world")]}) helper({"an object": doc_class({"test": u("something")})}) helper({"a binary": Binary(b"test", 100)}) helper({"a binary": Binary(b"test", 128)}) helper({"a binary": Binary(b"test", 254)}) helper({"another binary": Binary(b"test", 2)}) helper(SON([(u('test dst'), datetime.datetime(1993, 4, 4, 2))])) helper(SON([(u('test negative dst'), datetime.datetime(1, 1, 1, 1, 1, 1))])) helper({"big float": float(10000000000)}) helper({"ref": DBRef("coll", 5)}) helper({"ref": DBRef("coll", 5, foo="bar", bar=4)}) helper({"ref": DBRef("coll", 5, "foo")}) helper({"ref": DBRef("coll", 5, "foo", foo="bar")}) helper({"ref": Timestamp(1, 2)}) helper({"foo": MinKey()}) helper({"foo": MaxKey()}) helper({"$field": Code("function(){ return true; }")}) helper({"$field": Code("return function(){ return x; }", scope={'x': False})}) def encode_then_decode(doc): return doc_class(doc) == BSON.encode(doc).decode( CodecOptions(document_class=doc_class)) qcheck.check_unittest(self, encode_then_decode, qcheck.gen_mongo_dict(3)) def test_encode_then_decode(self): self.check_encode_then_decode() def test_encode_then_decode_any_mapping(self): self.check_encode_then_decode(doc_class=NotADict) def test_encoding_defaultdict(self): dct = collections.defaultdict(dict, [('foo', 'bar')]) BSON.encode(dct) self.assertEqual(dct, collections.defaultdict(dict, [('foo', 'bar')])) def test_basic_validation(self): self.assertRaises(TypeError, is_valid, 100) self.assertRaises(TypeError, is_valid, u("test")) self.assertRaises(TypeError, is_valid, 10.4) self.assertInvalid(b"test") # the simplest valid BSON document self.assertTrue(is_valid(b"\x05\x00\x00\x00\x00")) self.assertTrue(is_valid(BSON(b"\x05\x00\x00\x00\x00"))) # failure cases self.assertInvalid(b"\x04\x00\x00\x00\x00") self.assertInvalid(b"\x05\x00\x00\x00\x01") self.assertInvalid(b"\x05\x00\x00\x00") self.assertInvalid(b"\x05\x00\x00\x00\x00\x00") self.assertInvalid(b"\x07\x00\x00\x00\x02a\x00\x78\x56\x34\x12") self.assertInvalid(b"\x09\x00\x00\x00\x10a\x00\x05\x00") self.assertInvalid(b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00") self.assertInvalid(b"\x13\x00\x00\x00\x02foo\x00" b"\x04\x00\x00\x00bar\x00\x00") self.assertInvalid(b"\x18\x00\x00\x00\x03foo\x00\x0f\x00\x00" b"\x00\x10bar\x00\xff\xff\xff\x7f\x00\x00") self.assertInvalid(b"\x15\x00\x00\x00\x03foo\x00\x0c" b"\x00\x00\x00\x08bar\x00\x01\x00\x00") self.assertInvalid(b"\x1c\x00\x00\x00\x03foo\x00" b"\x12\x00\x00\x00\x02bar\x00" b"\x05\x00\x00\x00baz\x00\x00\x00") self.assertInvalid(b"\x10\x00\x00\x00\x02a\x00" b"\x04\x00\x00\x00abc\xff\x00") def test_bad_string_lengths(self): self.assertInvalid( b"\x0c\x00\x00\x00\x02\x00" b"\x00\x00\x00\x00\x00\x00") self.assertInvalid( b"\x12\x00\x00\x00\x02\x00" b"\xff\xff\xff\xfffoobar\x00\x00") self.assertInvalid( b"\x0c\x00\x00\x00\x0e\x00" b"\x00\x00\x00\x00\x00\x00") self.assertInvalid( b"\x12\x00\x00\x00\x0e\x00" b"\xff\xff\xff\xfffoobar\x00\x00") self.assertInvalid( b"\x18\x00\x00\x00\x0c\x00" b"\x00\x00\x00\x00\x00RY\xb5j" b"\xfa[\xd8A\xd6X]\x99\x00") self.assertInvalid( b"\x1e\x00\x00\x00\x0c\x00" b"\xff\xff\xff\xfffoobar\x00" b"RY\xb5j\xfa[\xd8A\xd6X]\x99\x00") self.assertInvalid( b"\x0c\x00\x00\x00\r\x00" b"\x00\x00\x00\x00\x00\x00") self.assertInvalid( b"\x0c\x00\x00\x00\r\x00" b"\xff\xff\xff\xff\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\x00\x00" b"\x00\x00\x00\x0c\x00\x00" b"\x00\x02\x00\x01\x00\x00" b"\x00\x00\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\xff\xff" b"\xff\xff\x00\x0c\x00\x00" b"\x00\x02\x00\x01\x00\x00" b"\x00\x00\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\x01\x00" b"\x00\x00\x00\x0c\x00\x00" b"\x00\x02\x00\x00\x00\x00" b"\x00\x00\x00\x00") self.assertInvalid( b"\x1c\x00\x00\x00\x0f\x00" b"\x15\x00\x00\x00\x01\x00" b"\x00\x00\x00\x0c\x00\x00" b"\x00\x02\x00\xff\xff\xff" b"\xff\x00\x00\x00") def test_random_data_is_not_bson(self): qcheck.check_unittest(self, qcheck.isnt(is_valid), qcheck.gen_string(qcheck.gen_range(0, 40))) def test_basic_decode(self): self.assertEqual({"test": u("hello world")}, BSON(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74\x00\x0C" b"\x00\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F" b"\x72\x6C\x64\x00\x00").decode()) self.assertEqual([{"test": u("hello world")}, {}], decode_all(b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00")) self.assertEqual([{"test": u("hello world")}, {}], list(decode_iter( b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00"))) self.assertEqual([{"test": u("hello world")}, {}], list(decode_file_iter(StringIO( b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00")))) def test_invalid_decodes(self): # Invalid object size (not enough bytes in document for even # an object size of first object. # NOTE: decode_all and decode_iter don't care, not sure if they should? self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(b"\x1B"))) # An object size that's too small to even include the object size, # but is correctly encoded, along with a correct EOO (and no data). data = b"\x01\x00\x00\x00\x00" self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) # One object, but with object size listed smaller than it is in the # data. data = (b"\x1A\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\x00") self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) # One object, missing the EOO at the end. data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00") self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) # One object, sized correctly, with a spot for an EOO, but the EOO # isn't 0x00. data = (b"\x1B\x00\x00\x00\x0E\x74\x65\x73\x74" b"\x00\x0C\x00\x00\x00\x68\x65\x6C\x6C" b"\x6f\x20\x77\x6F\x72\x6C\x64\x00\x00" b"\x05\x00\x00\x00\xFF") self.assertRaises(InvalidBSON, decode_all, data) self.assertRaises(InvalidBSON, list, decode_iter(data)) self.assertRaises(InvalidBSON, list, decode_file_iter(StringIO(data))) def test_data_timestamp(self): self.assertEqual({"test": Timestamp(4, 20)}, BSON(b"\x13\x00\x00\x00\x11\x74\x65\x73\x74\x00\x14" b"\x00\x00\x00\x04\x00\x00\x00\x00").decode()) def test_basic_encode(self): self.assertRaises(TypeError, BSON.encode, 100) self.assertRaises(TypeError, BSON.encode, "hello") self.assertRaises(TypeError, BSON.encode, None) self.assertRaises(TypeError, BSON.encode, []) self.assertEqual(BSON.encode({}), BSON(b"\x05\x00\x00\x00\x00")) self.assertEqual(BSON.encode({"test": u("hello world")}), b"\x1B\x00\x00\x00\x02\x74\x65\x73\x74\x00\x0C\x00" b"\x00\x00\x68\x65\x6C\x6C\x6F\x20\x77\x6F\x72\x6C" b"\x64\x00\x00") self.assertEqual(BSON.encode({u("mike"): 100}), b"\x0F\x00\x00\x00\x10\x6D\x69\x6B\x65\x00\x64\x00" b"\x00\x00\x00") self.assertEqual(BSON.encode({"hello": 1.5}), b"\x14\x00\x00\x00\x01\x68\x65\x6C\x6C\x6F\x00\x00" b"\x00\x00\x00\x00\x00\xF8\x3F\x00") self.assertEqual(BSON.encode({"true": True}), b"\x0C\x00\x00\x00\x08\x74\x72\x75\x65\x00\x01\x00") self.assertEqual(BSON.encode({"false": False}), b"\x0D\x00\x00\x00\x08\x66\x61\x6C\x73\x65\x00\x00" b"\x00") self.assertEqual(BSON.encode({"empty": []}), b"\x11\x00\x00\x00\x04\x65\x6D\x70\x74\x79\x00\x05" b"\x00\x00\x00\x00\x00") self.assertEqual(BSON.encode({"none": {}}), b"\x10\x00\x00\x00\x03\x6E\x6F\x6E\x65\x00\x05\x00" b"\x00\x00\x00\x00") self.assertEqual(BSON.encode({"test": Binary(b"test", 0)}), b"\x14\x00\x00\x00\x05\x74\x65\x73\x74\x00\x04\x00" b"\x00\x00\x00\x74\x65\x73\x74\x00") self.assertEqual(BSON.encode({"test": Binary(b"test", 2)}), b"\x18\x00\x00\x00\x05\x74\x65\x73\x74\x00\x08\x00" b"\x00\x00\x02\x04\x00\x00\x00\x74\x65\x73\x74\x00") self.assertEqual(BSON.encode({"test": Binary(b"test", 128)}), b"\x14\x00\x00\x00\x05\x74\x65\x73\x74\x00\x04\x00" b"\x00\x00\x80\x74\x65\x73\x74\x00") self.assertEqual(BSON.encode({"test": None}), b"\x0B\x00\x00\x00\x0A\x74\x65\x73\x74\x00\x00") self.assertEqual(BSON.encode({"date": datetime.datetime(2007, 1, 8, 0, 30, 11)}), b"\x13\x00\x00\x00\x09\x64\x61\x74\x65\x00\x38\xBE" b"\x1C\xFF\x0F\x01\x00\x00\x00") self.assertEqual(BSON.encode({"regex": re.compile(b"a*b", re.IGNORECASE)}), b"\x12\x00\x00\x00\x0B\x72\x65\x67\x65\x78\x00\x61" b"\x2A\x62\x00\x69\x00\x00") self.assertEqual(BSON.encode({"$where": Code("test")}), b"\x16\x00\x00\x00\r$where\x00\x05\x00\x00\x00test" b"\x00\x00") self.assertEqual(BSON.encode({"$field": Code("function(){ return true;}", scope=None)}), b"+\x00\x00\x00\r$field\x00\x1a\x00\x00\x00" b"function(){ return true;}\x00\x00") self.assertEqual(BSON.encode({"$field": Code("return function(){ return x; }", scope={'x': False})}), b"=\x00\x00\x00\x0f$field\x000\x00\x00\x00\x1f\x00" b"\x00\x00return function(){ return x; }\x00\t\x00" b"\x00\x00\x08x\x00\x00\x00\x00") a = ObjectId(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B") self.assertEqual(BSON.encode({"oid": a}), b"\x16\x00\x00\x00\x07\x6F\x69\x64\x00\x00\x01\x02" b"\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x00") self.assertEqual(BSON.encode({"ref": DBRef("coll", a)}), b"\x2F\x00\x00\x00\x03ref\x00\x25\x00\x00\x00\x02" b"$ref\x00\x05\x00\x00\x00coll\x00\x07$id\x00\x00" b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x00" b"\x00") def test_dbpointer(self): # *Note* - DBPointer and DBRef are *not* the same thing. DBPointer # is a deprecated BSON type. DBRef is a convention that does not # exist in the BSON spec, meant to replace DBPointer. PyMongo does # not support creation of the DBPointer type, but will decode # DBPointer to DBRef. bs = (b"\x18\x00\x00\x00\x0c\x00\x01\x00\x00" b"\x00\x00RY\xb5j\xfa[\xd8A\xd6X]\x99\x00") self.assertEqual({'': DBRef('', ObjectId('5259b56afa5bd841d6585d99'))}, bson.BSON(bs).decode()) def test_bad_dbref(self): ref_only = {'ref': {'$ref': 'collection'}} id_only = {'ref': {'$id': ObjectId()}} self.assertEqual(DBRef('collection', id=None), BSON.encode(ref_only).decode()['ref']) self.assertEqual(id_only, BSON.encode(id_only).decode()) def test_bytes_as_keys(self): doc = {b"foo": 'bar'} # Since `bytes` are stored as Binary you can't use them # as keys in python 3.x. Using binary data as a key makes # no sense in BSON anyway and little sense in python. if PY3: self.assertRaises(InvalidDocument, BSON.encode, doc) else: self.assertTrue(BSON.encode(doc)) def test_datetime_encode_decode(self): # Negative timestamps dt1 = datetime.datetime(1, 1, 1, 1, 1, 1, 111000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) dt1 = datetime.datetime(1959, 6, 25, 12, 16, 59, 999000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) # Positive timestamps dt1 = datetime.datetime(9999, 12, 31, 23, 59, 59, 999000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) dt1 = datetime.datetime(2011, 6, 14, 10, 47, 53, 444000) dt2 = BSON.encode({"date": dt1}).decode()["date"] self.assertEqual(dt1, dt2) def test_aware_datetime(self): aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) self.assertEqual(datetime.datetime(1993, 4, 3, 16, 45, tzinfo=utc), as_utc) after = BSON.encode({"date": aware}).decode( CodecOptions(tz_aware=True))["date"] self.assertEqual(utc, after.tzinfo) self.assertEqual(as_utc, after) def test_local_datetime(self): # Timezone -60 minutes of UTC, with DST between April and July. tz = DSTAwareTimezone(60, "sixty-minutes", 4, 7) # It's not DST. local = datetime.datetime(year=2025, month=12, hour=2, day=1, tzinfo=tz) options = CodecOptions(tz_aware=True, tzinfo=tz) # Encode with this timezone, then decode to UTC. encoded = BSON.encode({'date': local}, codec_options=options) self.assertEqual(local.replace(hour=1, tzinfo=None), encoded.decode()['date']) # It's DST. local = datetime.datetime(year=2025, month=4, hour=1, day=1, tzinfo=tz) encoded = BSON.encode({'date': local}, codec_options=options) self.assertEqual(local.replace(month=3, day=31, hour=23, tzinfo=None), encoded.decode()['date']) # Encode UTC, then decode in a different timezone. encoded = BSON.encode({'date': local.replace(tzinfo=utc)}) decoded = encoded.decode(options)['date'] self.assertEqual(local.replace(hour=3), decoded) self.assertEqual(tz, decoded.tzinfo) # Test round-tripping. self.assertEqual( local, (BSON .encode({'date': local}, codec_options=options) .decode(options)['date'])) # Test around the Unix Epoch. epochs = ( EPOCH_AWARE, EPOCH_AWARE.astimezone(FixedOffset(120, 'one twenty')), EPOCH_AWARE.astimezone(FixedOffset(-120, 'minus one twenty')) ) utc_co = CodecOptions(tz_aware=True) for epoch in epochs: doc = {'epoch': epoch} # We always retrieve datetimes in UTC unless told to do otherwise. self.assertEqual( EPOCH_AWARE, BSON.encode(doc).decode(codec_options=utc_co)['epoch']) # Round-trip the epoch. local_co = CodecOptions(tz_aware=True, tzinfo=epoch.tzinfo) self.assertEqual( epoch, BSON.encode(doc).decode(codec_options=local_co)['epoch']) def test_naive_decode(self): aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) naive_utc = (aware - aware.utcoffset()).replace(tzinfo=None) self.assertEqual(datetime.datetime(1993, 4, 3, 16, 45), naive_utc) after = BSON.encode({"date": aware}).decode()["date"] self.assertEqual(None, after.tzinfo) self.assertEqual(naive_utc, after) def test_dst(self): d = {"x": datetime.datetime(1993, 4, 4, 2)} self.assertEqual(d, BSON.encode(d).decode()) def test_bad_encode(self): if not PY3: # Python3 treats this as a unicode string which won't raise # an exception. If we passed the string as bytes instead we # still wouldn't get an error since we store bytes as BSON # binary subtype 0. self.assertRaises(InvalidStringData, BSON.encode, {"lalala": '\xf4\xe0\xf0\xe1\xc0 Color Touch'}) evil_list = {'a': []} evil_list['a'].append(evil_list) evil_dict = {} evil_dict['a'] = evil_dict # Work around what seems like a regression in python 3.5.0. # See http://bugs.python.org/issue25222 # 100 is an arbitrary choice. The default is 1000 on the machines # I have access to. depth = sys.getrecursionlimit() sys.setrecursionlimit(100) try: for evil_data in [evil_dict, evil_list]: self.assertRaises(RuntimeError, BSON.encode, evil_data) finally: sys.setrecursionlimit(depth) def test_overflow(self): self.assertTrue(BSON.encode({"x": long(9223372036854775807)})) self.assertRaises(OverflowError, BSON.encode, {"x": long(9223372036854775808)}) self.assertTrue(BSON.encode({"x": long(-9223372036854775808)})) self.assertRaises(OverflowError, BSON.encode, {"x": long(-9223372036854775809)}) def test_small_long_encode_decode(self): encoded1 = BSON.encode({'x': 256}) decoded1 = BSON.decode(encoded1)['x'] self.assertEqual(256, decoded1) self.assertEqual(type(256), type(decoded1)) encoded2 = BSON.encode({'x': Int64(256)}) decoded2 = BSON.decode(encoded2)['x'] expected = Int64(256) self.assertEqual(expected, decoded2) self.assertEqual(type(expected), type(decoded2)) self.assertNotEqual(type(decoded1), type(decoded2)) def test_tuple(self): self.assertEqual({"tuple": [1, 2]}, BSON.encode({"tuple": (1, 2)}).decode()) def test_uuid(self): id = uuid.uuid4() transformed_id = (BSON.encode({"id": id})).decode()["id"] self.assertTrue(isinstance(transformed_id, uuid.UUID)) self.assertEqual(id, transformed_id) self.assertNotEqual(uuid.uuid4(), transformed_id) def test_uuid_legacy(self): id = uuid.uuid4() legacy = UUIDLegacy(id) self.assertEqual(3, legacy.subtype) transformed = (BSON.encode({"uuid": legacy})).decode()["uuid"] self.assertTrue(isinstance(transformed, uuid.UUID)) self.assertEqual(id, transformed) self.assertNotEqual(UUIDLegacy(uuid.uuid4()), UUIDLegacy(transformed)) # The C extension was segfaulting on unicode RegExs, so we have this test # that doesn't really test anything but the lack of a segfault. def test_unicode_regex(self): regex = re.compile(u('revisi\xf3n')) BSON.encode({"regex": regex}).decode() def test_non_string_keys(self): self.assertRaises(InvalidDocument, BSON.encode, {8.9: "test"}) def test_utf8(self): w = {u("aéあ"): u("aéあ")} self.assertEqual(w, BSON.encode(w).decode()) # b'a\xe9' == u"aé".encode("iso-8859-1") iso8859_bytes = b'a\xe9' y = {"hello": iso8859_bytes} if PY3: # Stored as BSON binary subtype 0. out = BSON.encode(y).decode() self.assertTrue(isinstance(out['hello'], bytes)) self.assertEqual(out['hello'], iso8859_bytes) else: # Python 2. try: BSON.encode(y) except InvalidStringData as e: self.assertTrue(repr(iso8859_bytes) in str(e)) # The next two tests only make sense in python 2.x since # you can't use `bytes` type as document keys in python 3.x. x = {u("aéあ").encode("utf-8"): u("aéあ").encode("utf-8")} self.assertEqual(w, BSON.encode(x).decode()) z = {iso8859_bytes: "hello"} self.assertRaises(InvalidStringData, BSON.encode, z) def test_null_character(self): doc = {"a": "\x00"} self.assertEqual(doc, BSON.encode(doc).decode()) # This test doesn't make much sense in Python2 # since {'a': '\x00'} == {'a': u'\x00'}. # Decoding here actually returns {'a': '\x00'} doc = {"a": u("\x00")} self.assertEqual(doc, BSON.encode(doc).decode()) self.assertRaises(InvalidDocument, BSON.encode, {b"\x00": "a"}) self.assertRaises(InvalidDocument, BSON.encode, {u("\x00"): "a"}) self.assertRaises(InvalidDocument, BSON.encode, {"a": re.compile(b"ab\x00c")}) self.assertRaises(InvalidDocument, BSON.encode, {"a": re.compile(u("ab\x00c"))}) def test_move_id(self): self.assertEqual(b"\x19\x00\x00\x00\x02_id\x00\x02\x00\x00\x00a\x00" b"\x02a\x00\x02\x00\x00\x00a\x00\x00", BSON.encode(SON([("a", "a"), ("_id", "a")]))) self.assertEqual(b"\x2c\x00\x00\x00" b"\x02_id\x00\x02\x00\x00\x00b\x00" b"\x03b\x00" b"\x19\x00\x00\x00\x02a\x00\x02\x00\x00\x00a\x00" b"\x02_id\x00\x02\x00\x00\x00a\x00\x00\x00", BSON.encode(SON([("b", SON([("a", "a"), ("_id", "a")])), ("_id", "b")]))) def test_dates(self): doc = {"early": datetime.datetime(1686, 5, 5), "late": datetime.datetime(2086, 5, 5)} try: self.assertEqual(doc, BSON.encode(doc).decode()) except ValueError: # Ignore ValueError when no C ext, since it's probably # a problem w/ 32-bit Python - we work around this in the # C ext, though. if bson.has_c(): raise def test_custom_class(self): self.assertIsInstance(BSON.encode({}).decode(), dict) self.assertNotIsInstance(BSON.encode({}).decode(), SON) self.assertIsInstance( BSON.encode({}).decode(CodecOptions(document_class=SON)), SON) self.assertEqual( 1, BSON.encode({"x": 1}).decode( CodecOptions(document_class=SON))["x"]) x = BSON.encode({"x": [{"y": 1}]}) self.assertIsInstance( x.decode(CodecOptions(document_class=SON))["x"][0], SON) def test_subclasses(self): # make sure we can serialize subclasses of native Python types. class _myint(int): pass class _myfloat(float): pass class _myunicode(text_type): pass d = {'a': _myint(42), 'b': _myfloat(63.9), 'c': _myunicode('hello world') } d2 = BSON.encode(d).decode() for key, value in iteritems(d2): orig_value = d[key] orig_type = orig_value.__class__.__bases__[0] self.assertEqual(type(value), orig_type) self.assertEqual(value, orig_type(value)) def test_ordered_dict(self): try: from collections import OrderedDict except ImportError: raise SkipTest("No OrderedDict") d = OrderedDict([("one", 1), ("two", 2), ("three", 3), ("four", 4)]) self.assertEqual( d, BSON.encode(d).decode(CodecOptions(document_class=OrderedDict))) def test_bson_regex(self): # Invalid Python regex, though valid PCRE. bson_re1 = Regex(r'[\w-\.]') self.assertEqual(r'[\w-\.]', bson_re1.pattern) self.assertEqual(0, bson_re1.flags) doc1 = {'r': bson_re1} doc1_bson = ( b'\x11\x00\x00\x00' # document length b'\x0br\x00[\\w-\\.]\x00\x00' # r: regex b'\x00') # document terminator self.assertEqual(doc1_bson, BSON.encode(doc1)) self.assertEqual(doc1, BSON(doc1_bson).decode()) # Valid Python regex, with flags. re2 = re.compile(u('.*'), re.I | re.M | re.S | re.U | re.X) bson_re2 = Regex(u('.*'), re.I | re.M | re.S | re.U | re.X) doc2_with_re = {'r': re2} doc2_with_bson_re = {'r': bson_re2} doc2_bson = ( b"\x11\x00\x00\x00" # document length b"\x0br\x00.*\x00imsux\x00" # r: regex b"\x00") # document terminator self.assertEqual(doc2_bson, BSON.encode(doc2_with_re)) self.assertEqual(doc2_bson, BSON.encode(doc2_with_bson_re)) self.assertEqual(re2.pattern, BSON(doc2_bson).decode()['r'].pattern) self.assertEqual(re2.flags, BSON(doc2_bson).decode()['r'].flags) def test_regex_from_native(self): self.assertEqual('.*', Regex.from_native(re.compile('.*')).pattern) self.assertEqual(0, Regex.from_native(re.compile(b'')).flags) regex = re.compile(b'', re.I | re.L | re.M | re.S | re.X) self.assertEqual( re.I | re.L | re.M | re.S | re.X, Regex.from_native(regex).flags) unicode_regex = re.compile('', re.U) self.assertEqual(re.U, Regex.from_native(unicode_regex).flags) def test_regex_hash(self): self.assertRaises(TypeError, hash, Regex('hello')) def test_exception_wrapping(self): # No matter what exception is raised while trying to decode BSON, # the final exception always matches InvalidBSON. # {'s': '\xff'}, will throw attempting to decode utf-8. bad_doc = b'\x0f\x00\x00\x00\x02s\x00\x03\x00\x00\x00\xff\x00\x00\x00' with self.assertRaises(InvalidBSON) as context: decode_all(bad_doc) self.assertIn("codec can't decode byte 0xff", str(context.exception)) def test_minkey_maxkey_comparison(self): # MinKey's <, <=, >, >=, !=, and ==. self.assertTrue(MinKey() < None) self.assertTrue(MinKey() < 1) self.assertTrue(MinKey() <= 1) self.assertTrue(MinKey() <= MinKey()) self.assertFalse(MinKey() > None) self.assertFalse(MinKey() > 1) self.assertFalse(MinKey() >= 1) self.assertTrue(MinKey() >= MinKey()) self.assertTrue(MinKey() != 1) self.assertFalse(MinKey() == 1) self.assertTrue(MinKey() == MinKey()) # MinKey compared to MaxKey. self.assertTrue(MinKey() < MaxKey()) self.assertTrue(MinKey() <= MaxKey()) self.assertFalse(MinKey() > MaxKey()) self.assertFalse(MinKey() >= MaxKey()) self.assertTrue(MinKey() != MaxKey()) self.assertFalse(MinKey() == MaxKey()) # MaxKey's <, <=, >, >=, !=, and ==. self.assertFalse(MaxKey() < None) self.assertFalse(MaxKey() < 1) self.assertFalse(MaxKey() <= 1) self.assertTrue(MaxKey() <= MaxKey()) self.assertTrue(MaxKey() > None) self.assertTrue(MaxKey() > 1) self.assertTrue(MaxKey() >= 1) self.assertTrue(MaxKey() >= MaxKey()) self.assertTrue(MaxKey() != 1) self.assertFalse(MaxKey() == 1) self.assertTrue(MaxKey() == MaxKey()) # MaxKey compared to MinKey. self.assertFalse(MaxKey() < MinKey()) self.assertFalse(MaxKey() <= MinKey()) self.assertTrue(MaxKey() > MinKey()) self.assertTrue(MaxKey() >= MinKey()) self.assertTrue(MaxKey() != MinKey()) self.assertFalse(MaxKey() == MinKey()) def test_minkey_maxkey_hash(self): self.assertEqual(hash(MaxKey()), hash(MaxKey())) self.assertEqual(hash(MinKey()), hash(MinKey())) self.assertNotEqual(hash(MaxKey()), hash(MinKey())) def test_timestamp_comparison(self): # Timestamp is initialized with time, inc. Time is the more # significant comparand. self.assertTrue(Timestamp(1, 0) < Timestamp(2, 17)) self.assertTrue(Timestamp(2, 0) > Timestamp(1, 0)) self.assertTrue(Timestamp(1, 7) <= Timestamp(2, 0)) self.assertTrue(Timestamp(2, 0) >= Timestamp(1, 1)) self.assertTrue(Timestamp(2, 0) <= Timestamp(2, 0)) self.assertTrue(Timestamp(2, 0) >= Timestamp(2, 0)) self.assertFalse(Timestamp(1, 0) > Timestamp(2, 0)) # Comparison by inc. self.assertTrue(Timestamp(1, 0) < Timestamp(1, 1)) self.assertTrue(Timestamp(1, 1) > Timestamp(1, 0)) self.assertTrue(Timestamp(1, 0) <= Timestamp(1, 0)) self.assertTrue(Timestamp(1, 0) <= Timestamp(1, 1)) self.assertFalse(Timestamp(1, 0) >= Timestamp(1, 1)) self.assertTrue(Timestamp(1, 0) >= Timestamp(1, 0)) self.assertTrue(Timestamp(1, 1) >= Timestamp(1, 0)) self.assertFalse(Timestamp(1, 1) <= Timestamp(1, 0)) self.assertTrue(Timestamp(1, 0) <= Timestamp(1, 0)) self.assertFalse(Timestamp(1, 0) > Timestamp(1, 0)) def test_bad_id_keys(self): self.assertRaises(InvalidDocument, BSON.encode, {"_id": {"$bad": 123}}, True) self.assertRaises(InvalidDocument, BSON.encode, {"_id": {'$oid': "52d0b971b3ba219fdeb4170e"}}, True) BSON.encode({"_id": {'$oid': "52d0b971b3ba219fdeb4170e"}}) class TestCodecOptions(unittest.TestCase): def test_document_class(self): self.assertRaises(TypeError, CodecOptions, document_class=object) self.assertIs(SON, CodecOptions(document_class=SON).document_class) def test_tz_aware(self): self.assertRaises(TypeError, CodecOptions, tz_aware=1) self.assertFalse(CodecOptions().tz_aware) self.assertTrue(CodecOptions(tz_aware=True).tz_aware) def test_uuid_representation(self): self.assertRaises(ValueError, CodecOptions, uuid_representation=None) self.assertRaises(ValueError, CodecOptions, uuid_representation=7) self.assertRaises(ValueError, CodecOptions, uuid_representation=2) def test_tzinfo(self): self.assertRaises(TypeError, CodecOptions, tzinfo='pacific') tz = FixedOffset(42, 'forty-two') self.assertRaises(ValueError, CodecOptions, tzinfo=tz) self.assertEqual(tz, CodecOptions(tz_aware=True, tzinfo=tz).tzinfo) def test_codec_options_repr(self): r = ("CodecOptions(document_class=dict, tz_aware=False, " "uuid_representation=PYTHON_LEGACY, " "unicode_decode_error_handler='strict', " "tzinfo=None)") self.assertEqual(r, repr(CodecOptions())) def test_decode_all_defaults(self): # Test decode_all()'s default document_class is dict and tz_aware is # False. The default uuid_representation is PYTHON_LEGACY but this # decodes same as STANDARD, so all this test proves about UUID decoding # is that it's not CSHARP_LEGACY or JAVA_LEGACY. doc = {'sub_document': {}, 'uuid': uuid.uuid4(), 'dt': datetime.datetime.utcnow()} decoded = bson.decode_all(bson.BSON.encode(doc))[0] self.assertIsInstance(decoded['sub_document'], dict) self.assertEqual(decoded['uuid'], doc['uuid']) self.assertIsNone(decoded['dt'].tzinfo) def test_unicode_decode_error_handler(self): enc = BSON.encode({"keystr": "foobar"}) # Test handling of bad key value. invalid_key = BSON(enc[:7] + b'\xe9' + enc[8:]) replaced_key = b'ke\xef\xbf\xbdstr'.decode('utf-8') dec = BSON.decode(invalid_key, CodecOptions( unicode_decode_error_handler="replace")) self.assertEqual(dec, {replaced_key: u("foobar")}) dec = BSON.decode(invalid_key, CodecOptions( unicode_decode_error_handler="ignore")) self.assertEqual(dec, {"kestr": "foobar"}) self.assertRaises(InvalidBSON, BSON.decode, invalid_key, CodecOptions( unicode_decode_error_handler="strict")) self.assertRaises(InvalidBSON, BSON.decode, invalid_key, CodecOptions()) self.assertRaises(InvalidBSON, BSON.decode, invalid_key) # Test handing of bad string value. invalid_val = BSON(enc[:18] + b'\xe9' + enc[19:]) replaced_val = b'fo\xef\xbf\xbdbar'.decode('utf-8') dec = BSON.decode(invalid_val, CodecOptions( unicode_decode_error_handler="replace")) self.assertEqual(dec, {"keystr": replaced_val}) dec = BSON.decode(invalid_val, CodecOptions( unicode_decode_error_handler="ignore")) self.assertEqual(dec, {"keystr": "fobar"}) self.assertRaises(InvalidBSON, BSON.decode, invalid_val, CodecOptions( unicode_decode_error_handler="strict")) self.assertRaises(InvalidBSON, BSON.decode, invalid_val, CodecOptions()) self.assertRaises(InvalidBSON, BSON.decode, invalid_val) # Test handing bad key + bad value. invalid_both = BSON( enc[:7] + b'\xe9' + enc[8:18] + b'\xe9' + enc[19:]) dec = BSON.decode(invalid_both, CodecOptions( unicode_decode_error_handler="replace")) self.assertEqual(dec, {replaced_key: replaced_val}) dec = BSON.decode(invalid_both, CodecOptions( unicode_decode_error_handler="ignore")) self.assertEqual(dec, {"kestr": "fobar"}) self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions( unicode_decode_error_handler="strict")) self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions()) self.assertRaises(InvalidBSON, BSON.decode, invalid_both) # Test handling bad error mode. dec = BSON.decode(enc, CodecOptions( unicode_decode_error_handler="junk")) self.assertEqual(dec, {"keystr": "foobar"}) self.assertRaises(InvalidBSON, BSON.decode, invalid_both, CodecOptions(unicode_decode_error_handler="junk")) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_pymongo.py0000644000175000017500000000174412630145074021044 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo module itself.""" import sys sys.path[0:0] = [""] import pymongo from test import unittest class TestPyMongo(unittest.TestCase): def test_mongo_client_alias(self): # Testing that pymongo module imports mongo_client.MongoClient self.assertEqual(pymongo.MongoClient, pymongo.mongo_client.MongoClient) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_cursor_manager.py0000644000175000017500000000603412630145074022360 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the cursor_manager module.""" import sys sys.path[0:0] = [""] from pymongo.cursor_manager import CursorManager from pymongo.errors import CursorNotFound from pymongo.message import _CursorAddress from test import (client_context, client_knobs, unittest, IntegrationTest, SkipTest) from test.utils import rs_or_single_client, wait_until class TestCursorManager(IntegrationTest): @classmethod def setUpClass(cls): super(TestCursorManager, cls).setUpClass() cls.collection = cls.db.test cls.collection.drop() # Ensure two batches. cls.collection.insert_many([{'_id': i} for i in range(200)]) @classmethod def tearDownClass(cls): cls.collection.drop() def test_cursor_manager_validation(self): with self.assertRaises(TypeError): client_context.client.set_cursor_manager(1) def test_cursor_manager(self): if (client_context.is_mongos and not client_context.version.at_least(2, 4, 7)): # Old mongos sends incorrectly formatted error response when # cursor isn't found, see SERVER-9738. raise SkipTest("Can't test kill_cursors against old mongos") self.close_was_called = False test_case = self class CM(CursorManager): def __init__(self, client): super(CM, self).__init__(client) def close(self, cursor_id, address): test_case.close_was_called = True super(CM, self).close(cursor_id, address) with client_knobs(kill_cursor_frequency=0.01): client = rs_or_single_client(maxPoolSize=1) client.set_cursor_manager(CM) # Create a cursor on the same client so we're certain the getMore # is sent after the killCursors message. cursor = client.pymongo_test.test.find().batch_size(1) next(cursor) client.close_cursor( cursor.cursor_id, _CursorAddress(self.client.address, self.collection.full_name)) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor') self.assertTrue(self.close_was_called) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_json_util.py0000644000175000017500000002123512630145074021357 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test some utilities for working with JSON and PyMongo.""" import datetime import json import re import sys import uuid sys.path[0:0] = [""] from bson import json_util, EPOCH_AWARE from bson.binary import Binary, MD5_SUBTYPE, USER_DEFINED_SUBTYPE from bson.code import Code from bson.dbref import DBRef from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey from bson.objectid import ObjectId from bson.regex import Regex from bson.timestamp import Timestamp from bson.tz_util import utc from test import unittest, IntegrationTest PY3 = sys.version_info[0] == 3 class TestJsonUtil(unittest.TestCase): def round_tripped(self, doc): return json_util.loads(json_util.dumps(doc)) def round_trip(self, doc): self.assertEqual(doc, self.round_tripped(doc)) def test_basic(self): self.round_trip({"hello": "world"}) def test_objectid(self): self.round_trip({"id": ObjectId()}) def test_dbref(self): self.round_trip({"ref": DBRef("foo", 5)}) self.round_trip({"ref": DBRef("foo", 5, "db")}) self.round_trip({"ref": DBRef("foo", ObjectId())}) # Check order. self.assertEqual( '{"$ref": "collection", "$id": 1, "$db": "db"}', json_util.dumps(DBRef('collection', 1, 'db'))) def test_datetime(self): # only millis, not micros self.round_trip({"date": datetime.datetime(2009, 12, 9, 15, 49, 45, 191000, utc)}) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000+0000"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000+00:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000Z"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) # No explicit offset jsn = '{"dt": { "$date" : "1970-01-01T00:00:00.000"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) # Localtime behind UTC jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000-0800"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1969-12-31T16:00:00.000-08:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) # Localtime ahead of UTC jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000+0100"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) jsn = '{"dt": { "$date" : "1970-01-01T01:00:00.000+01:00"}}' self.assertEqual(EPOCH_AWARE, json_util.loads(jsn)["dt"]) dtm = datetime.datetime(1, 1, 1, 1, 1, 1, 0, utc) jsn = '{"dt": {"$date": -62135593139000}}' self.assertEqual(dtm, json_util.loads(jsn)["dt"]) jsn = '{"dt": {"$date": {"$numberLong": "-62135593139000"}}}' self.assertEqual(dtm, json_util.loads(jsn)["dt"]) def test_regex_object_hook(self): # Extended JSON format regular expression. pat = 'a*b' json_re = '{"$regex": "%s", "$options": "u"}' % pat loaded = json_util.object_hook(json.loads(json_re)) self.assertTrue(isinstance(loaded, Regex)) self.assertEqual(pat, loaded.pattern) self.assertEqual(re.U, loaded.flags) def test_regex(self): for regex_instance in ( re.compile("a*b", re.IGNORECASE), Regex("a*b", re.IGNORECASE)): res = self.round_tripped({"r": regex_instance})["r"] self.assertEqual("a*b", res.pattern) res = self.round_tripped({"r": Regex("a*b", re.IGNORECASE)})["r"] self.assertEqual("a*b", res.pattern) self.assertEqual(re.IGNORECASE, res.flags) unicode_options = re.I|re.M|re.S|re.U|re.X regex = re.compile("a*b", unicode_options) res = self.round_tripped({"r": regex})["r"] self.assertEqual(unicode_options, res.flags) # Some tools may not add $options if no flags are set. res = json_util.loads('{"r": {"$regex": "a*b"}}')['r'] self.assertEqual(0, res.flags) self.assertEqual( Regex('.*', 'ilm'), json_util.loads( '{"r": {"$regex": ".*", "$options": "ilm"}}')['r']) # Check order. self.assertEqual( '{"$regex": ".*", "$options": "mx"}', json_util.dumps(Regex('.*', re.M | re.X))) self.assertEqual( '{"$regex": ".*", "$options": "mx"}', json_util.dumps(re.compile(b'.*', re.M | re.X))) def test_minkey(self): self.round_trip({"m": MinKey()}) def test_maxkey(self): self.round_trip({"m": MaxKey()}) def test_timestamp(self): dct = {"ts": Timestamp(4, 13)} res = json_util.dumps(dct, default=json_util.default) self.assertEqual('{"ts": {"$timestamp": {"t": 4, "i": 13}}}', res) rtdct = json_util.loads(res) self.assertEqual(dct, rtdct) def test_uuid(self): self.round_trip( {'uuid': uuid.UUID('f47ac10b-58cc-4372-a567-0e02b2c3d479')}) def test_binary(self): bin_type_dict = {"bin": Binary(b"\x00\x01\x02\x03\x04")} md5_type_dict = { "md5": Binary(b' n7\x18\xaf\t/\xd1\xd1/\x80\xca\xe7q\xcc\xac', MD5_SUBTYPE)} custom_type_dict = {"custom": Binary(b"hello", USER_DEFINED_SUBTYPE)} self.round_trip(bin_type_dict) self.round_trip(md5_type_dict) self.round_trip(custom_type_dict) # PYTHON-443 ensure old type formats are supported json_bin_dump = json_util.dumps(bin_type_dict) self.assertTrue('"$type": "00"' in json_bin_dump) self.assertEqual(bin_type_dict, json_util.loads('{"bin": {"$type": 0, "$binary": "AAECAwQ="}}')) json_bin_dump = json_util.dumps(md5_type_dict) # Check order. self.assertEqual( '{"md5": {"$binary": "IG43GK8JL9HRL4DK53HMrA==",' + ' "$type": "05"}}', json_bin_dump) self.assertEqual(md5_type_dict, json_util.loads('{"md5": {"$type": 5, "$binary":' ' "IG43GK8JL9HRL4DK53HMrA=="}}')) json_bin_dump = json_util.dumps(custom_type_dict) self.assertTrue('"$type": "80"' in json_bin_dump) self.assertEqual(custom_type_dict, json_util.loads('{"custom": {"$type": 128, "$binary":' ' "aGVsbG8="}}')) # Handle mongoexport where subtype >= 128 self.assertEqual(128, json_util.loads('{"custom": {"$type": "ffffff80", "$binary":' ' "aGVsbG8="}}')['custom'].subtype) self.assertEqual(255, json_util.loads('{"custom": {"$type": "ffffffff", "$binary":' ' "aGVsbG8="}}')['custom'].subtype) def test_code(self): self.round_trip({"code": Code("function x() { return 1; }")}) code = Code("return z", z=2) res = json_util.dumps(code) self.assertEqual(code, json_util.loads(res)) # Check order. self.assertEqual('{"$code": "return z", "$scope": {"z": 2}}', res) def test_undefined(self): json = '{"name": {"$undefined": true}}' self.assertIsNone(json_util.loads(json)['name']) def test_numberlong(self): json = '{"weight": {"$numberLong": 65535}}' self.assertEqual(json_util.loads(json)['weight'], Int64(65535)) class TestJsonUtilRoundtrip(IntegrationTest): def test_cursor(self): db = self.db db.drop_collection("test") docs = [ {'foo': [1, 2]}, {'bar': {'hello': 'world'}}, {'code': Code("function x() { return 1; }")}, {'bin': Binary(b"\x00\x01\x02\x03\x04")}, {'dbref': {'_ref': DBRef('simple', ObjectId('509b8db456c02c5ab7e63c34'))}} ] db.test.insert_many(docs) reloaded_docs = json_util.loads(json_util.dumps(db.test.find())) for doc in docs: self.assertTrue(doc in reloaded_docs) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_read_preferences.py0000644000175000017500000004657512630145074022663 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the replica_set_connection module.""" import contextlib import copy import random import sys import pickle sys.path[0:0] = [""] from bson.py3compat import MAXSIZE from bson.son import SON from pymongo.errors import ConfigurationError from pymongo.message import _maybe_add_read_preference from pymongo.mongo_client import MongoClient from pymongo.read_preferences import (ReadPreference, MovingAverage, Primary, PrimaryPreferred, Secondary, SecondaryPreferred, Nearest, _ServerMode) from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector from pymongo.server_type import SERVER_TYPE from pymongo.write_concern import WriteConcern from test.test_replica_set_client import TestReplicaSetClientBase from test import (SkipTest, client_context, host, port, unittest, db_user, db_pwd) from test.utils import connected, single_client, one, wait_until, rs_client from test.version import Version class TestReadPreferenceObjects(unittest.TestCase): prefs = [Primary(), Secondary(), Nearest(tag_sets=[{'a': 1}, {'b': 2}])] def test_pickle(self): for pref in self.prefs: self.assertEqual(pref, pickle.loads(pickle.dumps(pref))) def test_copy(self): for pref in self.prefs: self.assertEqual(pref, copy.copy(pref)) class TestReadPreferencesBase(TestReplicaSetClientBase): def setUp(self): super(TestReadPreferencesBase, self).setUp() # Insert some data so we can use cursors in read_from_which_host self.client.pymongo_test.test.drop() self.client.get_database( "pymongo_test", write_concern=WriteConcern(w=self.w)).test.insert_many( [{'_id': i} for i in range(10)]) self.addCleanup(self.client.pymongo_test.test.drop) def read_from_which_host(self, client): """Do a find() on the client and return which host was used """ cursor = client.pymongo_test.test.find() next(cursor) return cursor.address def read_from_which_kind(self, client): """Do a find() on the client and return 'primary' or 'secondary' depending on which the client used. """ address = self.read_from_which_host(client) if address == client.primary: return 'primary' elif address in client.secondaries: return 'secondary' else: self.fail( 'Cursor used address %s, expected either primary ' '%s or secondaries %s' % ( address, client.primary, client.secondaries)) def assertReadsFrom(self, expected, **kwargs): c = rs_client(**kwargs) wait_until( lambda: len(c.nodes - c.arbiters) == self.w, "discovered all nodes") used = self.read_from_which_kind(c) self.assertEqual(expected, used, 'Cursor used %s, expected %s' % ( used, expected)) class TestSingleSlaveOk(TestReadPreferencesBase): def test_reads_from_secondary(self): host, port = next(iter(self.client.secondaries)) # Direct connection to a secondary. client = single_client(host, port) self.assertFalse(client.is_primary) # Regardless of read preference, we should be able to do # "reads" with a direct connection to a secondary. # See server-selection.rst#topology-type-single. self.assertEqual(client.read_preference, ReadPreference.PRIMARY) db = client.pymongo_test coll = db.test # Test find and find_one. self.assertIsNotNone(coll.find_one()) self.assertEqual(10, len(list(coll.find()))) # Test some database helpers. self.assertIsNotNone(db.collection_names()) self.assertIsNotNone(db.validate_collection("test")) self.assertIsNotNone(db.command("count", "test")) # Test some collection helpers. self.assertEqual(10, coll.count()) self.assertEqual(10, len(coll.distinct("_id"))) self.assertIsNotNone(coll.aggregate([])) self.assertIsNotNone(coll.index_information()) # Test some "magic" namespace helpers. self.assertIsNotNone(db.current_op()) class TestReadPreferences(TestReadPreferencesBase): def test_mode_validation(self): for mode in (ReadPreference.PRIMARY, ReadPreference.PRIMARY_PREFERRED, ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED, ReadPreference.NEAREST): self.assertEqual( mode, rs_client(read_preference=mode).read_preference) self.assertRaises( TypeError, rs_client, read_preference='foo') def test_tag_sets_validation(self): # Can't use tags with PRIMARY self.assertRaises(ConfigurationError, _ServerMode, 0, tag_sets=[{'k': 'v'}]) # ... but empty tag sets are ok with PRIMARY self.assertRaises(ConfigurationError, _ServerMode, 0, tag_sets=[{}]) S = Secondary(tag_sets=[{}]) self.assertEqual( [{}], rs_client(read_preference=S).read_preference.tag_sets) S = Secondary(tag_sets=[{'k': 'v'}]) self.assertEqual( [{'k': 'v'}], rs_client(read_preference=S).read_preference.tag_sets) S = Secondary(tag_sets=[{'k': 'v'}, {}]) self.assertEqual( [{'k': 'v'}, {}], rs_client(read_preference=S).read_preference.tag_sets) self.assertRaises(ValueError, Secondary, tag_sets=[]) # One dict not ok, must be a list of dicts self.assertRaises(TypeError, Secondary, tag_sets={'k': 'v'}) self.assertRaises(TypeError, Secondary, tag_sets='foo') self.assertRaises(TypeError, Secondary, tag_sets=['foo']) def test_threshold_validation(self): self.assertEqual(17, rs_client( localThresholdMS=17 ).local_threshold_ms) self.assertEqual(42, rs_client( localThresholdMS=42 ).local_threshold_ms) self.assertEqual(666, rs_client( localthresholdms=666 ).local_threshold_ms) self.assertEqual(0, rs_client( localthresholdms=0 ).local_threshold_ms) self.assertRaises(ValueError, rs_client, localthresholdms=-1) def test_zero_latency(self): ping_times = set() # Generate unique ping times. while len(ping_times) < len(self.client.nodes): ping_times.add(random.random()) for ping_time, host in zip(ping_times, self.client.nodes): ServerDescription._host_to_round_trip_time[host] = ping_time try: client = connected( rs_client(readPreference='nearest', localThresholdMS=0)) wait_until( lambda: client.nodes == self.client.nodes, "discovered all nodes") host = self.read_from_which_host(client) for _ in range(5): self.assertEqual(host, self.read_from_which_host(client)) finally: ServerDescription._host_to_round_trip_time.clear() def test_primary(self): self.assertReadsFrom( 'primary', read_preference=ReadPreference.PRIMARY) def test_primary_with_tags(self): # Tags not allowed with PRIMARY self.assertRaises( ConfigurationError, rs_client, tag_sets=[{'dc': 'ny'}]) def test_primary_preferred(self): self.assertReadsFrom( 'primary', read_preference=ReadPreference.PRIMARY_PREFERRED) def test_secondary(self): self.assertReadsFrom( 'secondary', read_preference=ReadPreference.SECONDARY) def test_secondary_preferred(self): self.assertReadsFrom( 'secondary', read_preference=ReadPreference.SECONDARY_PREFERRED) def test_nearest(self): # With high localThresholdMS, expect to read from any # member c = rs_client( read_preference=ReadPreference.NEAREST, localThresholdMS=10000) # 10 seconds data_members = set(self.hosts).difference(set(self.arbiters)) # This is a probabilistic test; track which members we've read from so # far, and keep reading until we've used all the members or give up. # Chance of using only 2 of 3 members 10k times if there's no bug = # 3 * (2/3)**10000, very low. used = set() i = 0 while data_members.difference(used) and i < 10000: address = self.read_from_which_host(c) used.add(address) i += 1 not_used = data_members.difference(used) latencies = ', '.join( '%s: %dms' % (server.description.address, server.description.round_trip_time) for server in c._get_topology().select_servers( readable_server_selector)) self.assertFalse( not_used, "Expected to use primary and all secondaries for mode NEAREST," " but didn't use %s\nlatencies: %s" % (not_used, latencies)) class ReadPrefTester(MongoClient): def __init__(self, *args, **kwargs): self.has_read_from = set() super(ReadPrefTester, self).__init__(*args, **kwargs) @contextlib.contextmanager def _socket_for_reads(self, read_preference): context = super(ReadPrefTester, self)._socket_for_reads(read_preference) with context as (sock_info, slave_ok): self.record_a_read(sock_info.address) yield sock_info, slave_ok def record_a_read(self, address): server = self._get_topology().select_server_by_address(address, 0) self.has_read_from.add(server) _PREF_MAP = [ (Primary, SERVER_TYPE.RSPrimary), (PrimaryPreferred, SERVER_TYPE.RSPrimary), (Secondary, SERVER_TYPE.RSSecondary), (SecondaryPreferred, SERVER_TYPE.RSSecondary), (Nearest, 'any') ] class TestCommandAndReadPreference(TestReplicaSetClientBase): @classmethod def setUpClass(cls): super(TestCommandAndReadPreference, cls).setUpClass() cls.c = ReadPrefTester( '%s:%s' % (host, port), replicaSet=cls.name, # Ignore round trip times, to test ReadPreference modes only. localThresholdMS=1000*1000) if client_context.auth_enabled: cls.c.admin.authenticate(db_user, db_pwd) cls.client_version = Version.from_client(cls.c) # mapReduce and group fail with no collection coll = cls.c.pymongo_test.get_collection( 'test', write_concern=WriteConcern(w=cls.w)) coll.insert_one({}) @classmethod def tearDownClass(cls): cls.c.drop_database('pymongo_test') def executed_on_which_server(self, client, fn, *args, **kwargs): """Execute fn(*args, **kwargs) and return the Server instance used.""" client.has_read_from.clear() fn(*args, **kwargs) self.assertEqual(1, len(client.has_read_from)) return one(client.has_read_from) def assertExecutedOn(self, server_type, client, fn, *args, **kwargs): server = self.executed_on_which_server(client, fn, *args, **kwargs) self.assertEqual(SERVER_TYPE._fields[server_type], SERVER_TYPE._fields[server.description.server_type]) def _test_fn(self, server_type, fn): for _ in range(10): if server_type == 'any': used = set() for _ in range(1000): server = self.executed_on_which_server(self.c, fn) used.add(server.description.address) if len(used) == len(self.c.secondaries) + 1: # Success break unused = self.c.secondaries.union( set([self.c.primary]) ).difference(used) if unused: self.fail( "Some members not used for NEAREST: %s" % ( unused)) else: self.assertExecutedOn(server_type, self.c, fn) def _test_primary_helper(self, func): # Helpers that ignore read preference. self._test_fn(SERVER_TYPE.RSPrimary, func) def _test_coll_helper(self, secondary_ok, coll, meth, *args, **kwargs): for mode, server_type in _PREF_MAP: new_coll = coll.with_options(read_preference=mode()) func = lambda: getattr(new_coll, meth)(*args, **kwargs) if secondary_ok: self._test_fn(server_type, func) else: self._test_fn(SERVER_TYPE.RSPrimary, func) def test_command(self): # Test that the generic command helper obeys the read preference # passed to it. for mode, server_type in _PREF_MAP: func = lambda: self.c.pymongo_test.command('dbStats', read_preference=mode()) self._test_fn(server_type, func) def test_create_collection(self): # Collections should be created on primary, obviously self._test_primary_helper( lambda: self.c.pymongo_test.create_collection( 'some_collection%s' % random.randint(0, MAXSIZE))) def test_drop_collection(self): self._test_primary_helper( lambda: self.c.pymongo_test.drop_collection('some_collection')) self._test_primary_helper( lambda: self.c.pymongo_test.some_collection.drop()) def test_group(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'group', {'a': 1}, {}, {}, 'function() { }') def test_map_reduce(self): self._test_coll_helper(False, self.c.pymongo_test.test, 'map_reduce', 'function() { }', 'function() { }', {'inline': 1}) def test_inline_map_reduce(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'inline_map_reduce', 'function() { }', 'function() { }') def test_count(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'count') def test_distinct(self): self._test_coll_helper(True, self.c.pymongo_test.test, 'distinct', 'a') def test_aggregate(self): if self.client_version.at_least(2, 1, 0): self._test_coll_helper(True, self.c.pymongo_test.test, 'aggregate', [{'$project': {'_id': 1}}]) class TestMovingAverage(unittest.TestCase): def test_moving_average(self): avg = MovingAverage() self.assertIsNone(avg.get()) avg.add_sample(10) self.assertAlmostEqual(10, avg.get()) avg.add_sample(20) self.assertAlmostEqual(12, avg.get()) avg.add_sample(30) self.assertAlmostEqual(15.6, avg.get()) class TestMongosAndReadPreference(unittest.TestCase): def test_maybe_add_read_preference(self): # Primary doesn't add $readPreference out = _maybe_add_read_preference({}, Primary()) self.assertEqual(out, {}) pref = PrimaryPreferred() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = PrimaryPreferred(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Secondary() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Secondary(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) # SecondaryPreferred without tag_sets doesn't add $readPreference pref = SecondaryPreferred() out = _maybe_add_read_preference({}, pref) self.assertEqual(out, {}) pref = SecondaryPreferred(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Nearest() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Nearest(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) criteria = SON([("$query", {}), ("$orderby", SON([("_id", 1)]))]) pref = Nearest() out = _maybe_add_read_preference(criteria, pref) self.assertEqual( out, SON([("$query", {}), ("$orderby", SON([("_id", 1)])), ("$readPreference", pref.document)])) pref = Nearest(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference(criteria, pref) self.assertEqual( out, SON([("$query", {}), ("$orderby", SON([("_id", 1)])), ("$readPreference", pref.document)])) @client_context.require_mongos def test_mongos(self): shard = client_context.client.config.shards.find_one()['host'] num_members = shard.count(',') + 1 if num_members == 1: raise SkipTest("Need a replica set shard to test.") coll = client_context.client.pymongo_test.get_collection( "test", write_concern=WriteConcern(w=num_members)) coll.drop() res = coll.insert_many([{} for _ in range(5)]) first_id = res.inserted_ids[0] last_id = res.inserted_ids[-1] # Note - this isn't a perfect test since there's no way to # tell what shard member a query ran on. for pref in (Primary(), PrimaryPreferred(), Secondary(), SecondaryPreferred(), Nearest()): qcoll = coll.with_options(read_preference=pref) results = list(qcoll.find().sort([("_id", 1)])) self.assertEqual(first_id, results[0]["_id"]) self.assertEqual(last_id, results[-1]["_id"]) results = list(qcoll.find().sort([("_id", -1)])) self.assertEqual(first_id, results[-1]["_id"]) self.assertEqual(last_id, results[0]["_id"]) if __name__ == "__main__": unittest.main() pymongo-3.2/test/version.py0000644000175000017500000000403212630145074017773 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Some tools for running tests based on MongoDB server version.""" class Version(tuple): def __new__(cls, *version): padded_version = cls._padded(version, 4) return super(Version, cls).__new__(cls, tuple(padded_version)) @classmethod def _padded(cls, iter, length, padding=0): l = list(iter) if len(l) < length: for _ in range(length - len(l)): l.append(padding) return l @classmethod def from_string(cls, version_string): mod = 0 if version_string.endswith("+"): version_string = version_string[0:-1] mod = 1 elif version_string.endswith("-pre-"): version_string = version_string[0:-5] mod = -1 elif version_string.endswith("-"): version_string = version_string[0:-1] mod = -1 # Deal with '-rcX' substrings if version_string.find('-rc') != -1: version_string = version_string[0:version_string.find('-rc')] mod = -1 version = [int(part) for part in version_string.split(".")] version = cls._padded(version, 3) version.append(mod) return Version(*version) @classmethod def from_client(cls, client): return cls.from_string(client.server_info()['version']) def at_least(self, *other_version): return self >= Version(*other_version) def __str__(self): return ".".join(map(str, self)) pymongo-3.2/test/utils.py0000644000175000017500000003202712630145074017453 0ustar behackettbehackett00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for testing pymongo """ import contextlib import os import struct import sys import threading import time import warnings from collections import defaultdict from functools import partial from pymongo import MongoClient, monitoring from pymongo.errors import AutoReconnect, OperationFailure from pymongo.server_selectors import (any_server_selector, writable_server_selector) from pymongo.write_concern import WriteConcern from test import (client_context, db_user, db_pwd, host, port) from test.version import Version class EventListener(monitoring.CommandListener): def __init__(self): self.results = defaultdict(list) def started(self, event): self.results['started'].append(event) def succeeded(self, event): self.results['succeeded'].append(event) def failed(self, event): self.results['failed'].append(event) def _connection_string_noauth(h, p): if h.startswith("mongodb://"): return h return "mongodb://%s:%d" % (h, p) def _connection_string(h, p): if h.startswith("mongodb://"): return h elif client_context.auth_enabled: return "mongodb://%s:%s@%s:%d" % (db_user, db_pwd, h, p) else: return _connection_string_noauth(h, p) def single_client_noauth(h=host, p=port, **kwargs): """Make a direct connection. Don't authenticate.""" return MongoClient(_connection_string_noauth(h, p), **kwargs) def single_client(h=host, p=port, **kwargs): """Make a direct connection, and authenticate if necessary.""" return MongoClient(_connection_string(h, p), **kwargs) def rs_client_noauth(h=host, p=port, **kwargs): """Connect to the replica set. Don't authenticate.""" return MongoClient(_connection_string_noauth(h, p), replicaSet=client_context.replica_set_name, **kwargs) def rs_client(h=host, p=port, **kwargs): """Connect to the replica set and authenticate if necessary.""" return MongoClient(_connection_string(h, p), replicaSet=client_context.replica_set_name, **kwargs) def rs_or_single_client_noauth(h=host, p=port, **kwargs): """Connect to the replica set if there is one, otherwise the standalone. Like rs_or_single_client, but does not authenticate. """ if client_context.replica_set_name: return rs_client_noauth(h, p, **kwargs) else: return single_client_noauth(h, p, **kwargs) def rs_or_single_client(h=host, p=port, **kwargs): """Connect to the replica set if there is one, otherwise the standalone. Authenticates if necessary. """ if client_context.replica_set_name: return rs_client(h, p, **kwargs) else: return single_client(h, p, **kwargs) def one(s): """Get one element of a set""" return next(iter(s)) def oid_generated_on_client(oid): """Is this process's PID in this ObjectId?""" pid_from_doc = struct.unpack(">H", oid.binary[7:9])[0] return (os.getpid() % 0xFFFF) == pid_from_doc def delay(sec): return '''function() { sleep(%f * 1000); return true; }''' % sec def get_command_line(client): command_line = client.admin.command('getCmdLineOpts') assert command_line['ok'] == 1, "getCmdLineOpts() failed" return command_line def server_started_with_option(client, cmdline_opt, config_opt): """Check if the server was started with a particular option. :Parameters: - `cmdline_opt`: The command line option (i.e. --nojournal) - `config_opt`: The config file option (i.e. nojournal) """ command_line = get_command_line(client) if 'parsed' in command_line: parsed = command_line['parsed'] if config_opt in parsed: return parsed[config_opt] argv = command_line['argv'] return cmdline_opt in argv def server_started_with_auth(client): try: command_line = get_command_line(client) except OperationFailure as e: msg = e.details.get('errmsg', '') if e.code == 13 or 'unauthorized' in msg or 'login' in msg: # Unauthorized. return True raise # MongoDB >= 2.0 if 'parsed' in command_line: parsed = command_line['parsed'] # MongoDB >= 2.6 if 'security' in parsed: security = parsed['security'] # >= rc3 if 'authorization' in security: return security['authorization'] == 'enabled' # < rc3 return security.get('auth', False) or bool(security.get('keyFile')) return parsed.get('auth', False) or bool(parsed.get('keyFile')) # Legacy argv = command_line['argv'] return '--auth' in argv or '--keyFile' in argv def server_started_with_nojournal(client): command_line = get_command_line(client) # MongoDB 2.6. if 'parsed' in command_line: parsed = command_line['parsed'] if 'storage' in parsed: storage = parsed['storage'] if 'journal' in storage: return not storage['journal']['enabled'] return server_started_with_option(client, '--nojournal', 'nojournal') def server_is_master_with_slave(client): command_line = get_command_line(client) if 'parsed' in command_line: return command_line['parsed'].get('master', False) return '--master' in command_line['argv'] def drop_collections(db): for coll in db.collection_names(): if not coll.startswith('system'): db.drop_collection(coll) def remove_all_users(db): if Version.from_client(db.client).at_least(2, 5, 3, -1): db.command("dropAllUsersFromDatabase", 1, writeConcern={"w": client_context.w}) else: db = db.client.get_database( db.name, write_concern=WriteConcern(w=client_context.w)) db.system.users.delete_many({}) def joinall(threads): """Join threads with a 5-minute timeout, assert joins succeeded""" for t in threads: t.join(300) assert not t.isAlive(), "Thread %s hung" % t def connected(client): """Convenience to wait for a newly-constructed client to connect.""" with warnings.catch_warnings(): # Ignore warning that "ismaster" is always routed to primary even # if client's read preference isn't PRIMARY. warnings.simplefilter("ignore", UserWarning) client.admin.command('ismaster') # Force connection. return client def wait_until(predicate, success_description, timeout=10): """Wait up to 10 seconds (by default) for predicate to be true. E.g.: wait_until(lambda: client.primary == ('a', 1), 'connect to the primary') If the lambda-expression isn't true after 10 seconds, we raise AssertionError("Didn't ever connect to the primary"). Returns the predicate's first true value. """ start = time.time() while True: retval = predicate() if retval: return retval if time.time() - start > timeout: raise AssertionError("Didn't ever %s" % success_description) time.sleep(0.1) def is_mongos(client): res = client.admin.command('ismaster') return res.get('msg', '') == 'isdbgrid' def enable_text_search(client): client.admin.command( 'setParameter', textSearchEnabled=True) for host, port in client.secondaries: client = MongoClient(host, port) if client_context.auth_enabled: client.admin.authenticate(db_user, db_pwd) client.admin.command('setParameter', textSearchEnabled=True) def assertRaisesExactly(cls, fn, *args, **kwargs): """ Unlike the standard assertRaises, this checks that a function raises a specific class of exception, and not a subclass. E.g., check that MongoClient() raises ConnectionFailure but not its subclass, AutoReconnect. """ try: fn(*args, **kwargs) except Exception as e: assert e.__class__ == cls, "got %s, expected %s" % ( e.__class__.__name__, cls.__name__) else: raise AssertionError("%s not raised" % cls) @contextlib.contextmanager def ignore_deprecations(): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) yield def read_from_which_host( client, pref, tag_sets=None, ): """Read from a client with the given Read Preference. Return the 'host:port' which was read from. :Parameters: - `client`: A MongoClient - `mode`: A ReadPreference - `tag_sets`: List of dicts of tags for data-center-aware reads """ db = client.pymongo_test if isinstance(tag_sets, dict): tag_sets = [tag_sets] if tag_sets: tags = tag_sets or pref.tag_sets pref = pref.__class__(tags) db.read_preference = pref cursor = db.test.find() try: try: next(cursor) except StopIteration: # No documents in collection, that's fine pass return cursor.address except AutoReconnect: return None def assertReadFrom(testcase, client, member, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from the expected replica-set member. :Parameters: - `testcase`: A unittest.TestCase - `client`: A MongoClient - `member`: A host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads """ for _ in range(10): testcase.assertEqual(member, read_from_which_host(client, *args, **kwargs)) def assertReadFromAll(testcase, client, members, *args, **kwargs): """Check that a query with the given mode and tag_sets reads from all members in a set, and only members in that set. :Parameters: - `testcase`: A unittest.TestCase - `client`: A MongoClient - `members`: Sequence of host:port expected to be used - `mode`: A ReadPreference - `tag_sets` (optional): List of dicts of tags for data-center-aware reads """ members = set(members) used = set() for _ in range(100): used.add(read_from_which_host(client, *args, **kwargs)) testcase.assertEqual(members, used) def get_pool(client): """Get the standalone, primary, or mongos pool.""" topology = client._get_topology() server = topology.select_server(writable_server_selector) return server.pool def get_pools(client): """Get all pools.""" return [ server.pool for server in client._get_topology().select_servers(any_server_selector)] # Constants for run_threads and lazy_client_trial. NTRIALS = 5 NTHREADS = 10 def run_threads(collection, target): """Run a target function in many threads. target is a function taking a Collection and an integer. """ threads = [] for i in range(NTHREADS): bound_target = partial(target, collection, i) threads.append(threading.Thread(target=bound_target)) for t in threads: t.start() for t in threads: t.join(30) assert not t.isAlive() @contextlib.contextmanager def frequent_thread_switches(): """Make concurrency bugs more likely to manifest.""" interval = None if not sys.platform.startswith('java'): if hasattr(sys, 'getswitchinterval'): interval = sys.getswitchinterval() sys.setswitchinterval(1e-6) else: interval = sys.getcheckinterval() sys.setcheckinterval(1) try: yield finally: if not sys.platform.startswith('java'): if hasattr(sys, 'setswitchinterval'): sys.setswitchinterval(interval) else: sys.setcheckinterval(interval) def lazy_client_trial(reset, target, test, get_client): """Test concurrent operations on a lazily-connecting client. `reset` takes a collection and resets it for the next trial. `target` takes a lazily-connecting collection and an index from 0 to NTHREADS, and performs some operation, e.g. an insert. `test` takes the lazily-connecting collection and asserts a post-condition to prove `target` succeeded. """ collection = client_context.client.pymongo_test.test with frequent_thread_switches(): for i in range(NTRIALS): reset(collection) lazy_client = get_client() lazy_collection = lazy_client.pymongo_test.test run_threads(lazy_collection, target) test(lazy_collection) pymongo-3.2/test/test_raw_bson.py0000644000175000017500000001072312630145074021163 0ustar behackettbehackett00000000000000import datetime import uuid from bson import BSON from bson.binary import JAVA_LEGACY from bson.codec_options import CodecOptions from bson.raw_bson import RawBSONDocument from test import client_context, unittest class TestRawBSONDocument(unittest.TestCase): # {u'_id': ObjectId('556df68b6e32ab21a95e0785'), # u'name': u'Sherlock', # u'addresses': [{u'street': u'Baker Street'}]} bson_string = ( b'Z\x00\x00\x00\x07_id\x00Um\xf6\x8bn2\xab!\xa9^\x07\x85\x02name\x00\t' b'\x00\x00\x00Sherlock\x00\x04addresses\x00&\x00\x00\x00\x030\x00\x1e' b'\x00\x00\x00\x02street\x00\r\x00\x00\x00Baker Street\x00\x00\x00\x00' ) document = RawBSONDocument(bson_string) @classmethod def setUpClass(cls): cls.client = client_context.rs_or_standalone_client def tearDown(self): if client_context.connected: self.client.pymongo_test.test_raw.drop() def test_decode(self): self.assertEqual('Sherlock', self.document['name']) first_address = self.document['addresses'][0] self.assertIsInstance(first_address, RawBSONDocument) self.assertEqual('Baker Street', first_address['street']) def test_raw(self): self.assertEqual(self.bson_string, self.document.raw) @client_context.require_connection def test_round_trip(self): db = self.client.get_database( 'pymongo_test', codec_options=CodecOptions(document_class=RawBSONDocument)) db.test_raw.insert_one(self.document) result = db.test_raw.find_one(self.document['_id']) self.assertIsInstance(result, RawBSONDocument) self.assertEqual(dict(self.document.items()), dict(result.items())) def test_with_codec_options(self): # {u'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000), # u'_id': UUID('026fab8f-975f-4965-9fbf-85ad874c60ff')} # encoded with JAVA_LEGACY uuid representation. bson_string = ( b'-\x00\x00\x00\x05_id\x00\x10\x00\x00\x00\x03eI_\x97\x8f\xabo\x02' b'\xff`L\x87\xad\x85\xbf\x9f\tdate\x00\x8a\xd6\xb9\xbaM' b'\x01\x00\x00\x00' ) document = RawBSONDocument( bson_string, codec_options=CodecOptions(uuid_representation=JAVA_LEGACY)) self.assertEqual(uuid.UUID('026fab8f-975f-4965-9fbf-85ad874c60ff'), document['_id']) @client_context.require_connection def test_round_trip_codec_options(self): doc = { 'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000), '_id': uuid.UUID('026fab8f-975f-4965-9fbf-85ad874c60ff') } db = self.client.pymongo_test coll = db.get_collection( 'test_raw', codec_options=CodecOptions(uuid_representation=JAVA_LEGACY)) coll.insert_one(doc) raw_java_legacy = CodecOptions(uuid_representation=JAVA_LEGACY, document_class=RawBSONDocument) coll = db.get_collection('test_raw', codec_options=raw_java_legacy) self.assertEqual( RawBSONDocument(BSON.encode(doc, codec_options=raw_java_legacy)), coll.find_one()) @client_context.require_connection def test_raw_bson_document_embedded(self): doc = {'embedded': self.document} db = self.client.pymongo_test db.test_raw.insert_one(doc) result = db.test_raw.find_one() self.assertEqual(BSON(self.document.raw).decode(), result['embedded']) # Make sure that CodecOptions are preserved. # {'embedded': [ # {u'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000), # u'_id': UUID('026fab8f-975f-4965-9fbf-85ad874c60ff')} # ]} # encoded with JAVA_LEGACY uuid representation. bson_string = ( b'D\x00\x00\x00\x04embedded\x005\x00\x00\x00\x030\x00-\x00\x00\x00' b'\tdate\x00\x8a\xd6\xb9\xbaM\x01\x00\x00\x05_id\x00\x10\x00\x00' b'\x00\x03eI_\x97\x8f\xabo\x02\xff`L\x87\xad\x85\xbf\x9f\x00\x00' b'\x00' ) rbd = RawBSONDocument( bson_string, codec_options=CodecOptions(uuid_representation=JAVA_LEGACY)) db.test_raw.drop() db.test_raw.insert_one(rbd) result = db.get_collection('test_raw', codec_options=CodecOptions( uuid_representation=JAVA_LEGACY)).find_one() self.assertEqual(rbd['embedded'][0]['_id'], result['embedded'][0]['_id']) pymongo-3.2/test/test_dbref.py0000644000175000017500000001236612630145074020440 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the dbref module.""" import pickle import sys sys.path[0:0] = [""] from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import u from test import unittest from copy import deepcopy class TestDBRef(unittest.TestCase): def test_creation(self): a = ObjectId() self.assertRaises(TypeError, DBRef) self.assertRaises(TypeError, DBRef, "coll") self.assertRaises(TypeError, DBRef, 4, a) self.assertRaises(TypeError, DBRef, 1.5, a) self.assertRaises(TypeError, DBRef, a, a) self.assertRaises(TypeError, DBRef, None, a) self.assertRaises(TypeError, DBRef, "coll", a, 5) self.assertTrue(DBRef("coll", a)) self.assertTrue(DBRef(u("coll"), a)) self.assertTrue(DBRef(u("coll"), 5)) self.assertTrue(DBRef(u("coll"), 5, "database")) def test_read_only(self): a = DBRef("coll", ObjectId()) def foo(): a.collection = "blah" def bar(): a.id = "aoeu" self.assertEqual("coll", a.collection) a.id self.assertEqual(None, a.database) self.assertRaises(AttributeError, foo) self.assertRaises(AttributeError, bar) def test_repr(self): self.assertEqual(repr(DBRef("coll", ObjectId("1234567890abcdef12345678"))), "DBRef('coll', ObjectId('1234567890abcdef12345678'))") self.assertEqual(repr(DBRef(u("coll"), ObjectId("1234567890abcdef12345678"))), "DBRef(%s, ObjectId('1234567890abcdef12345678'))" % (repr(u('coll')),) ) self.assertEqual(repr(DBRef("coll", 5, foo="bar")), "DBRef('coll', 5, foo='bar')") self.assertEqual(repr(DBRef("coll", ObjectId("1234567890abcdef12345678"), "foo")), "DBRef('coll', ObjectId('1234567890abcdef12345678'), " "'foo')") def test_equality(self): obj_id = ObjectId("1234567890abcdef12345678") self.assertEqual(DBRef('foo', 5), DBRef('foo', 5)) self.assertEqual(DBRef("coll", obj_id), DBRef(u("coll"), obj_id)) self.assertNotEqual(DBRef("coll", obj_id), DBRef(u("coll"), obj_id, "foo")) self.assertNotEqual(DBRef("coll", obj_id), DBRef("col", obj_id)) self.assertNotEqual(DBRef("coll", obj_id), DBRef("coll", ObjectId(b"123456789011"))) self.assertNotEqual(DBRef("coll", obj_id), 4) self.assertEqual(DBRef("coll", obj_id, "foo"), DBRef(u("coll"), obj_id, "foo")) self.assertNotEqual(DBRef("coll", obj_id, "foo"), DBRef(u("coll"), obj_id, "bar")) # Explicitly test inequality self.assertFalse(DBRef('foo', 5) != DBRef('foo', 5)) self.assertFalse(DBRef("coll", obj_id) != DBRef(u("coll"), obj_id)) self.assertFalse(DBRef("coll", obj_id, "foo") != DBRef(u("coll"), obj_id, "foo")) def test_kwargs(self): self.assertEqual(DBRef("coll", 5, foo="bar"), DBRef("coll", 5, foo="bar")) self.assertNotEqual(DBRef("coll", 5, foo="bar"), DBRef("coll", 5)) self.assertNotEqual(DBRef("coll", 5, foo="bar"), DBRef("coll", 5, foo="baz")) self.assertEqual("bar", DBRef("coll", 5, foo="bar").foo) self.assertRaises(AttributeError, getattr, DBRef("coll", 5, foo="bar"), "bar") def test_deepcopy(self): a = DBRef('coll', 'asdf', 'db', x=[1]) b = deepcopy(a) self.assertEqual(a, b) self.assertNotEqual(id(a), id(b.x)) self.assertEqual(a.x, b.x) self.assertNotEqual(id(a.x), id(b.x)) b.x[0] = 2 self.assertEqual(a.x, [1]) self.assertEqual(b.x, [2]) def test_pickling(self): dbr = DBRef('coll', 5, foo='bar') for protocol in [0, 1, 2, -1]: pkl = pickle.dumps(dbr, protocol=protocol) dbr2 = pickle.loads(pkl) self.assertEqual(dbr, dbr2) def test_dbref_hash(self): dbref_1a = DBRef('collection', 'id', 'database') dbref_1b = DBRef('collection', 'id', 'database') self.assertEqual(hash(dbref_1a), hash(dbref_1b)) dbref_2a = DBRef('collection', 'id', 'database', custom='custom') dbref_2b = DBRef('collection', 'id', 'database', custom='custom') self.assertEqual(hash(dbref_2a), hash(dbref_2b)) self.assertNotEqual(hash(dbref_1a), hash(dbref_2a)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_server.py0000644000175000017500000000215012630145074020652 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the server module.""" import sys sys.path[0:0] = [""] from pymongo.ismaster import IsMaster from pymongo.server import Server from pymongo.server_description import ServerDescription from test import unittest class TestServer(unittest.TestCase): def test_repr(self): ismaster = IsMaster({'ok': 1}) sd = ServerDescription(('localhost', 27017), ismaster) server = Server(sd, pool=object(), monitor=object()) self.assertTrue('Standalone' in str(server)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_common.py0000644000175000017500000002046512630145074020645 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo common module.""" import sys import uuid sys.path[0:0] = [""] from bson.binary import UUIDLegacy, PYTHON_LEGACY, STANDARD from bson.code import Code from bson.codec_options import CodecOptions from bson.objectid import ObjectId from pymongo.mongo_client import MongoClient from pymongo.errors import OperationFailure from pymongo.write_concern import WriteConcern from test import client_context, pair, unittest, IntegrationTest from test.utils import connected, rs_or_single_client, single_client @client_context.require_connection def setUpModule(): pass class TestCommon(IntegrationTest): def test_uuid_representation(self): coll = self.db.uuid coll.drop() # Test property self.assertEqual(PYTHON_LEGACY, coll.codec_options.uuid_representation) # Test basic query uu = uuid.uuid4() # Insert as binary subtype 3 coll.insert_one({'uu': uu}) self.assertEqual(uu, coll.find_one({'uu': uu})['uu']) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) self.assertEqual(STANDARD, coll.codec_options.uuid_representation) self.assertEqual(None, coll.find_one({'uu': uu})) self.assertEqual(uu, coll.find_one({'uu': UUIDLegacy(uu)})['uu']) # Test Cursor.count self.assertEqual(0, coll.find({'uu': uu}).count()) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual(1, coll.find({'uu': uu}).count()) # Test delete coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) coll.delete_one({'uu': uu}) self.assertEqual(1, coll.count()) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) coll.delete_one({'uu': uu}) self.assertEqual(0, coll.count()) # Test update_one coll.insert_one({'_id': uu, 'i': 1}) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) coll.update_one({'_id': uu}, {'$set': {'i': 2}}) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual(1, coll.find_one({'_id': uu})['i']) coll.update_one({'_id': uu}, {'$set': {'i': 2}}) self.assertEqual(2, coll.find_one({'_id': uu})['i']) # Test Cursor.distinct self.assertEqual([2], coll.find({'_id': uu}).distinct('i')) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) self.assertEqual([], coll.find({'_id': uu}).distinct('i')) # Test findAndModify self.assertEqual(None, coll.find_one_and_update({'_id': uu}, {'$set': {'i': 5}})) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual(2, coll.find_one_and_update({'_id': uu}, {'$set': {'i': 5}})['i']) self.assertEqual(5, coll.find_one({'_id': uu})['i']) # Test command self.assertEqual(5, self.db.command('findAndModify', 'uuid', update={'$set': {'i': 6}}, query={'_id': uu})['value']['i']) self.assertEqual(6, self.db.command( 'findAndModify', 'uuid', update={'$set': {'i': 7}}, query={'_id': UUIDLegacy(uu)})['value']['i']) # Test (inline)_map_reduce coll.drop() coll.insert_one({"_id": uu, "x": 1, "tags": ["dog", "cat"]}) coll.insert_one({"_id": uuid.uuid4(), "x": 3, "tags": ["mouse", "cat", "dog"]}) map = Code("function () {" " this.tags.forEach(function(z) {" " emit(z, 1);" " });" "}") reduce = Code("function (key, values) {" " var total = 0;" " for (var i = 0; i < values.length; i++) {" " total += values[i];" " }" " return total;" "}") coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) q = {"_id": uu} result = coll.inline_map_reduce(map, reduce, query=q) self.assertEqual([], result) result = coll.map_reduce(map, reduce, "results", query=q) self.assertEqual(0, self.db.results.count()) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) q = {"_id": uu} result = coll.inline_map_reduce(map, reduce, query=q) self.assertEqual(2, len(result)) result = coll.map_reduce(map, reduce, "results", query=q) self.assertEqual(2, self.db.results.count()) self.db.drop_collection("result") coll.drop() # Test group coll.insert_one({"_id": uu, "a": 2}) coll.insert_one({"_id": uuid.uuid4(), "a": 1}) reduce = "function (obj, prev) { prev.count++; }" coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=STANDARD)) self.assertEqual([], coll.group([], {"_id": uu}, {"count": 0}, reduce)) coll = self.db.get_collection( "uuid", CodecOptions(uuid_representation=PYTHON_LEGACY)) self.assertEqual([{"count": 1}], coll.group([], {"_id": uu}, {"count": 0}, reduce)) def test_write_concern(self): c = MongoClient(connect=False) self.assertEqual(WriteConcern(), c.write_concern) c = MongoClient(connect=False, w=2, wtimeout=1000) wc = WriteConcern(w=2, wtimeout=1000) self.assertEqual(wc, c.write_concern) db = c.pymongo_test self.assertEqual(wc, db.write_concern) coll = db.test self.assertEqual(wc, coll.write_concern) cwc = WriteConcern(j=True) coll = db.get_collection('test', write_concern=cwc) self.assertEqual(cwc, coll.write_concern) self.assertEqual(wc, db.write_concern) def test_mongo_client(self): m = rs_or_single_client(w=0) coll = m.pymongo_test.write_concern_test coll.drop() doc = {"_id": ObjectId()} coll.insert_one(doc) self.assertTrue(coll.insert_one(doc)) coll = coll.with_options(write_concern=WriteConcern(w=1)) self.assertRaises(OperationFailure, coll.insert_one, doc) m = rs_or_single_client() coll = m.pymongo_test.write_concern_test new_coll = coll.with_options(write_concern=WriteConcern(w=0)) self.assertTrue(new_coll.insert_one(doc)) self.assertRaises(OperationFailure, coll.insert_one, doc) m = MongoClient("mongodb://%s/" % (pair,), replicaSet=client_context.replica_set_name) coll = m.pymongo_test.write_concern_test self.assertRaises(OperationFailure, coll.insert_one, doc) m = MongoClient("mongodb://%s/?w=0" % (pair,), replicaSet=client_context.replica_set_name) coll = m.pymongo_test.write_concern_test coll.insert_one(doc) # Equality tests direct = connected(single_client(w=0)) self.assertEqual(direct, connected(MongoClient("mongodb://%s/?w=0" % (pair,)))) self.assertFalse(direct != connected(MongoClient("mongodb://%s/?w=0" % (pair,)))) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_grid_file.py0000644000175000017500000004763112630145074021305 0ustar behackettbehackett00000000000000# -*- coding: utf-8 -*- # # Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the grid_file module. """ import datetime import sys sys.path[0:0] = [""] from bson.objectid import ObjectId from bson.py3compat import u, StringIO from gridfs import GridFS from gridfs.grid_file import (DEFAULT_CHUNK_SIZE, _SEEK_CUR, _SEEK_END, GridIn, GridOut, GridOutCursor) from gridfs.errors import NoFile from pymongo import MongoClient from pymongo.errors import ConfigurationError, ServerSelectionTimeoutError from test import (IntegrationTest, host, port, unittest, qcheck) from test.utils import rs_or_single_client class TestGridFileNoConnect(unittest.TestCase): @classmethod def setUpClass(cls): client = MongoClient(host, port, connect=False) cls.db = client.pymongo_test def test_grid_in_custom_opts(self): self.assertRaises(TypeError, GridIn, "foo") a = GridIn(self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") self.assertEqual(5, a._id) self.assertEqual("my_file", a.filename) self.assertEqual("my_file", a.name) self.assertEqual("text/html", a.content_type) self.assertEqual(1000, a.chunk_size) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1, "bar": 2}, a.metadata) self.assertEqual(3, a.bar) self.assertEqual("hello", a.baz) self.assertRaises(AttributeError, getattr, a, "mike") b = GridIn(self.db.fs, content_type="text/html", chunk_size=1000, baz=100) self.assertEqual("text/html", b.content_type) self.assertEqual(1000, b.chunk_size) self.assertEqual(100, b.baz) def test_grid_out_cursor_options(self): self.assertRaises(TypeError, GridOutCursor.__init__, self.db.fs, {}, projection={"filename": 1}) cursor = GridOutCursor(self.db.fs, {}) cursor_clone = cursor.clone() self.assertEqual(cursor_clone.__dict__, cursor.__dict__) self.assertRaises(NotImplementedError, cursor.add_option, 0) self.assertRaises(NotImplementedError, cursor.remove_option, 0) class TestGridFile(IntegrationTest): def setUp(self): self.db.drop_collection('fs.files') self.db.drop_collection('fs.chunks') def test_basic(self): f = GridIn(self.db.fs, filename="test") f.write(b"hello world") f.close() self.assertEqual(1, self.db.fs.files.find().count()) self.assertEqual(1, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(b"hello world", g.read()) # make sure it's still there... g = GridOut(self.db.fs, f._id) self.assertEqual(b"hello world", g.read()) f = GridIn(self.db.fs, filename="test") f.close() self.assertEqual(2, self.db.fs.files.find().count()) self.assertEqual(1, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(b"", g.read()) # test that reading 0 returns proper type self.assertEqual(b"", g.read(0)) def test_md5(self): f = GridIn(self.db.fs) f.write(b"hello world\n") f.close() self.assertEqual("6f5902ac237024bdd0c176cb93063dc4", f.md5) def test_alternate_collection(self): self.db.alt.files.delete_many({}) self.db.alt.chunks.delete_many({}) f = GridIn(self.db.alt) f.write(b"hello world") f.close() self.assertEqual(1, self.db.alt.files.find().count()) self.assertEqual(1, self.db.alt.chunks.find().count()) g = GridOut(self.db.alt, f._id) self.assertEqual(b"hello world", g.read()) # test that md5 still works... self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5) def test_grid_in_default_opts(self): self.assertRaises(TypeError, GridIn, "foo") a = GridIn(self.db.fs) self.assertTrue(isinstance(a._id, ObjectId)) self.assertRaises(AttributeError, setattr, a, "_id", 5) self.assertEqual(None, a.filename) self.assertEqual(None, a.name) a.filename = "my_file" self.assertEqual("my_file", a.filename) self.assertEqual("my_file", a.name) self.assertEqual(None, a.content_type) a.content_type = "text/html" self.assertEqual("text/html", a.content_type) self.assertRaises(AttributeError, getattr, a, "length") self.assertRaises(AttributeError, setattr, a, "length", 5) self.assertEqual(255 * 1024, a.chunk_size) self.assertRaises(AttributeError, setattr, a, "chunk_size", 5) self.assertRaises(AttributeError, getattr, a, "upload_date") self.assertRaises(AttributeError, setattr, a, "upload_date", 5) self.assertRaises(AttributeError, getattr, a, "aliases") a.aliases = ["foo"] self.assertEqual(["foo"], a.aliases) self.assertRaises(AttributeError, getattr, a, "metadata") a.metadata = {"foo": 1} self.assertEqual({"foo": 1}, a.metadata) self.assertRaises(AttributeError, setattr, a, "md5", 5) a.close() a.forty_two = 42 self.assertEqual(42, a.forty_two) self.assertTrue(isinstance(a._id, ObjectId)) self.assertRaises(AttributeError, setattr, a, "_id", 5) self.assertEqual("my_file", a.filename) self.assertEqual("my_file", a.name) self.assertEqual("text/html", a.content_type) self.assertEqual(0, a.length) self.assertRaises(AttributeError, setattr, a, "length", 5) self.assertEqual(255 * 1024, a.chunk_size) self.assertRaises(AttributeError, setattr, a, "chunk_size", 5) self.assertTrue(isinstance(a.upload_date, datetime.datetime)) self.assertRaises(AttributeError, setattr, a, "upload_date", 5) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1}, a.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", a.md5) self.assertRaises(AttributeError, setattr, a, "md5", 5) # Make sure custom attributes that were set both before and after # a.close() are reflected in b. PYTHON-411. b = GridFS(self.db).get_last_version(filename=a.filename) self.assertEqual(a.metadata, b.metadata) self.assertEqual(a.aliases, b.aliases) self.assertEqual(a.forty_two, b.forty_two) def test_grid_out_default_opts(self): self.assertRaises(TypeError, GridOut, "foo") gout = GridOut(self.db.fs, 5) with self.assertRaises(NoFile): gout.name a = GridIn(self.db.fs) a.close() b = GridOut(self.db.fs, a._id) self.assertEqual(a._id, b._id) self.assertEqual(0, b.length) self.assertEqual(None, b.content_type) self.assertEqual(None, b.name) self.assertEqual(None, b.filename) self.assertEqual(255 * 1024, b.chunk_size) self.assertTrue(isinstance(b.upload_date, datetime.datetime)) self.assertEqual(None, b.aliases) self.assertEqual(None, b.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5) for attr in ["_id", "name", "content_type", "length", "chunk_size", "upload_date", "aliases", "metadata", "md5"]: self.assertRaises(AttributeError, setattr, b, attr, 5) def test_grid_out_custom_opts(self): one = GridIn(self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") one.write(b"hello world") one.close() two = GridOut(self.db.fs, 5) self.assertEqual("my_file", two.name) self.assertEqual("my_file", two.filename) self.assertEqual(5, two._id) self.assertEqual(11, two.length) self.assertEqual("text/html", two.content_type) self.assertEqual(1000, two.chunk_size) self.assertTrue(isinstance(two.upload_date, datetime.datetime)) self.assertEqual(["foo"], two.aliases) self.assertEqual({"foo": 1, "bar": 2}, two.metadata) self.assertEqual(3, two.bar) self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5) for attr in ["_id", "name", "content_type", "length", "chunk_size", "upload_date", "aliases", "metadata", "md5"]: self.assertRaises(AttributeError, setattr, two, attr, 5) def test_grid_out_file_document(self): one = GridIn(self.db.fs) one.write(b"foo bar") one.close() two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one()) self.assertEqual(b"foo bar", two.read()) three = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one()) self.assertEqual(b"foo bar", three.read()) four = GridOut(self.db.fs, file_document={}) with self.assertRaises(NoFile): four.name def test_write_file_like(self): one = GridIn(self.db.fs) one.write(b"hello world") one.close() two = GridOut(self.db.fs, one._id) three = GridIn(self.db.fs) three.write(two) three.close() four = GridOut(self.db.fs, three._id) self.assertEqual(b"hello world", four.read()) five = GridIn(self.db.fs, chunk_size=2) five.write(b"hello") buffer = StringIO(b" world") five.write(buffer) five.write(b" and mongodb") five.close() self.assertEqual(b"hello world and mongodb", GridOut(self.db.fs, five._id).read()) def test_write_lines(self): a = GridIn(self.db.fs) a.writelines([b"hello ", b"world"]) a.close() self.assertEqual(b"hello world", GridOut(self.db.fs, a._id).read()) def test_close(self): f = GridIn(self.db.fs) f.close() self.assertRaises(ValueError, f.write, "test") f.close() def test_multi_chunk_file(self): random_string = b'a' * (DEFAULT_CHUNK_SIZE + 1000) f = GridIn(self.db.fs) f.write(random_string) f.close() self.assertEqual(1, self.db.fs.files.find().count()) self.assertEqual(2, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(random_string, g.read()) def test_small_chunks(self): self.files = 0 self.chunks = 0 def helper(data): f = GridIn(self.db.fs, chunkSize=1) f.write(data) f.close() self.files += 1 self.chunks += len(data) self.assertEqual(self.files, self.db.fs.files.find().count()) self.assertEqual(self.chunks, self.db.fs.chunks.find().count()) g = GridOut(self.db.fs, f._id) self.assertEqual(data, g.read()) g = GridOut(self.db.fs, f._id) self.assertEqual(data, g.read(10) + g.read(10)) return True qcheck.check_unittest(self, helper, qcheck.gen_string(qcheck.gen_range(0, 20))) def test_seek(self): f = GridIn(self.db.fs, chunkSize=3) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(b"hello world", g.read()) g.seek(0) self.assertEqual(b"hello world", g.read()) g.seek(1) self.assertEqual(b"ello world", g.read()) self.assertRaises(IOError, g.seek, -1) g.seek(-3, _SEEK_END) self.assertEqual(b"rld", g.read()) g.seek(0, _SEEK_END) self.assertEqual(b"", g.read()) self.assertRaises(IOError, g.seek, -100, _SEEK_END) g.seek(3) g.seek(3, _SEEK_CUR) self.assertEqual(b"world", g.read()) self.assertRaises(IOError, g.seek, -100, _SEEK_CUR) def test_tell(self): f = GridIn(self.db.fs, chunkSize=3) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(0, g.tell()) g.read(0) self.assertEqual(0, g.tell()) g.read(1) self.assertEqual(1, g.tell()) g.read(2) self.assertEqual(3, g.tell()) g.read() self.assertEqual(g.length, g.tell()) def test_multiple_reads(self): f = GridIn(self.db.fs, chunkSize=3) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(b"he", g.read(2)) self.assertEqual(b"ll", g.read(2)) self.assertEqual(b"o ", g.read(2)) self.assertEqual(b"wo", g.read(2)) self.assertEqual(b"rl", g.read(2)) self.assertEqual(b"d", g.read(2)) self.assertEqual(b"", g.read(2)) def test_readline(self): f = GridIn(self.db.fs, chunkSize=5) f.write((b"""Hello world, How are you? Hope all is well. Bye""")) f.close() # Try read(), then readline(). g = GridOut(self.db.fs, f._id) self.assertEqual(b"H", g.read(1)) self.assertEqual(b"ello world,\n", g.readline()) self.assertEqual(b"How a", g.readline(5)) self.assertEqual(b"", g.readline(0)) self.assertEqual(b"re you?\n", g.readline()) self.assertEqual(b"Hope all is well.\n", g.readline(1000)) self.assertEqual(b"Bye", g.readline()) self.assertEqual(b"", g.readline()) # Try readline() first, then read(). g = GridOut(self.db.fs, f._id) self.assertEqual(b"He", g.readline(2)) self.assertEqual(b"l", g.read(1)) self.assertEqual(b"lo", g.readline(2)) self.assertEqual(b" world,\n", g.readline()) # Only readline(). g = GridOut(self.db.fs, f._id) self.assertEqual(b"H", g.readline(1)) self.assertEqual(b"e", g.readline(1)) self.assertEqual(b"llo world,\n", g.readline()) def test_iterator(self): f = GridIn(self.db.fs) f.close() g = GridOut(self.db.fs, f._id) self.assertEqual([], list(g)) f = GridIn(self.db.fs) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual([b"hello world"], list(g)) self.assertEqual(b"hello", g.read(5)) self.assertEqual([b"hello world"], list(g)) self.assertEqual(b" worl", g.read(5)) f = GridIn(self.db.fs, chunk_size=2) f.write(b"hello world") f.close() g = GridOut(self.db.fs, f._id) self.assertEqual([b"he", b"ll", b"o ", b"wo", b"rl", b"d"], list(g)) def test_read_unaligned_buffer_size(self): in_data = (b"This is a text that doesn't " b"quite fit in a single 16-byte chunk.") f = GridIn(self.db.fs, chunkSize=16) f.write(in_data) f.close() g = GridOut(self.db.fs, f._id) out_data = b'' while 1: s = g.read(13) if not s: break out_data += s self.assertEqual(in_data, out_data) def test_readchunk(self): in_data = b'a' * 10 f = GridIn(self.db.fs, chunkSize=3) f.write(in_data) f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(3, len(g.readchunk())) self.assertEqual(2, len(g.read(2))) self.assertEqual(1, len(g.readchunk())) self.assertEqual(3, len(g.read(3))) self.assertEqual(1, len(g.readchunk())) self.assertEqual(0, len(g.readchunk())) def test_write_unicode(self): f = GridIn(self.db.fs) self.assertRaises(TypeError, f.write, u("foo")) f = GridIn(self.db.fs, encoding="utf-8") f.write(u("foo")) f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(b"foo", g.read()) f = GridIn(self.db.fs, encoding="iso-8859-1") f.write(u("aé")) f.close() g = GridOut(self.db.fs, f._id) self.assertEqual(u("aé").encode("iso-8859-1"), g.read()) def test_set_after_close(self): f = GridIn(self.db.fs, _id="foo", bar="baz") self.assertEqual("foo", f._id) self.assertEqual("baz", f.bar) self.assertRaises(AttributeError, getattr, f, "baz") self.assertRaises(AttributeError, getattr, f, "uploadDate") self.assertRaises(AttributeError, setattr, f, "_id", 5) f.bar = "foo" f.baz = 5 self.assertEqual("foo", f._id) self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertRaises(AttributeError, getattr, f, "uploadDate") f.close() self.assertEqual("foo", f._id) self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertTrue(f.uploadDate) self.assertRaises(AttributeError, setattr, f, "_id", 5) f.bar = "a" f.baz = "b" self.assertRaises(AttributeError, setattr, f, "upload_date", 5) g = GridOut(self.db.fs, f._id) self.assertEqual("a", g.bar) self.assertEqual("b", g.baz) # Versions 2.0.1 and older saved a _closed field for some reason. self.assertRaises(AttributeError, getattr, g, "_closed") def test_context_manager(self): contents = b"Imagine this is some important data..." with GridIn(self.db.fs, filename="important") as infile: infile.write(contents) with GridOut(self.db.fs, infile._id) as outfile: self.assertEqual(contents, outfile.read()) def test_prechunked_string(self): def write_me(s, chunk_size): buf = StringIO(s) infile = GridIn(self.db.fs) while True: to_write = buf.read(chunk_size) if to_write == b'': break infile.write(to_write) infile.close() buf.close() outfile = GridOut(self.db.fs, infile._id) data = outfile.read() self.assertEqual(s, data) s = b'x' * DEFAULT_CHUNK_SIZE * 4 # Test with default chunk size write_me(s, DEFAULT_CHUNK_SIZE) # Multiple write_me(s, DEFAULT_CHUNK_SIZE * 3) # Custom write_me(s, 262300) def test_grid_out_lazy_connect(self): fs = self.db.fs outfile = GridOut(fs, file_id=-1) self.assertRaises(NoFile, outfile.read) self.assertRaises(NoFile, getattr, outfile, 'filename') infile = GridIn(fs, filename=1) infile.close() outfile = GridOut(fs, infile._id) outfile.read() outfile.filename outfile = GridOut(fs, infile._id) outfile.readchunk() def test_grid_in_lazy_connect(self): client = MongoClient('badhost', connect=False, serverSelectionTimeoutMS=10) fs = client.db.fs infile = GridIn(fs, file_id=-1, chunk_size=1) self.assertRaises(ServerSelectionTimeoutError, infile.write, b'data') self.assertRaises(ServerSelectionTimeoutError, infile.close) def test_unacknowledged(self): # w=0 is prohibited. with self.assertRaises(ConfigurationError): GridIn(rs_or_single_client(w=0).pymongo_test.fs) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_gridfs_bucket.py0000644000175000017500000003732212630145074022170 0ustar behackettbehackett00000000000000# -*- coding: utf-8 -*- # # Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the gridfs package. """ import datetime import threading import time import gridfs from bson.binary import Binary from bson.py3compat import StringIO, string_type from gridfs.errors import NoFile, CorruptGridFile from pymongo.errors import (ConfigurationError, ConnectionFailure, ServerSelectionTimeoutError, OperationFailure) from pymongo.mongo_client import MongoClient from pymongo.read_preferences import ReadPreference from test import (client_context, IntegrationTest) from test.test_replica_set_client import TestReplicaSetClientBase from test.utils import (joinall, single_client, one, rs_client, rs_or_single_client, rs_or_single_client_noauth, remove_all_users) class JustWrite(threading.Thread): def __init__(self, gfs, num): threading.Thread.__init__(self) self.gfs = gfs self.num = num self.setDaemon(True) def run(self): for _ in range(self.num): file = self.gfs.open_upload_stream("test") file.write(b"hello") file.close() class JustRead(threading.Thread): def __init__(self, gfs, num, results): threading.Thread.__init__(self) self.gfs = gfs self.num = num self.results = results self.setDaemon(True) def run(self): for _ in range(self.num): file = self.gfs.open_download_stream_by_name("test") data = file.read() self.results.append(data) assert data == b"hello" class TestGridfs(IntegrationTest): @classmethod def setUpClass(cls): super(TestGridfs, cls).setUpClass() cls.fs = gridfs.GridFSBucket(cls.db) cls.alt = gridfs.GridFSBucket( cls.db, bucket_name="alt") def setUp(self): self.db.drop_collection("fs.files") self.db.drop_collection("fs.chunks") self.db.drop_collection("alt.files") self.db.drop_collection("alt.chunks") def test_basic(self): oid = self.fs.upload_from_stream("test_filename", b"hello world") self.assertEqual(b"hello world", self.fs.open_download_stream(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(1, self.db.fs.chunks.count()) self.fs.delete(oid) self.assertRaises(NoFile, self.fs.open_download_stream, oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) def test_multi_chunk_delete(self): self.db.fs.drop() self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) gfs = gridfs.GridFSBucket(self.db) oid = gfs.upload_from_stream("test_filename", b"hello", chunk_size_bytes=1) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(5, self.db.fs.chunks.count()) gfs.delete(oid) self.assertEqual(0, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) def test_empty_file(self): oid = self.fs.upload_from_stream("test_filename", b"") self.assertEqual(b"", self.fs.open_download_stream(oid).read()) self.assertEqual(1, self.db.fs.files.count()) self.assertEqual(0, self.db.fs.chunks.count()) raw = self.db.fs.files.find_one() self.assertEqual(0, raw["length"]) self.assertEqual(oid, raw["_id"]) self.assertTrue(isinstance(raw["uploadDate"], datetime.datetime)) self.assertEqual(255 * 1024, raw["chunkSize"]) self.assertTrue(isinstance(raw["md5"], string_type)) def test_corrupt_chunk(self): files_id = self.fs.upload_from_stream("test_filename", b'foobar') self.db.fs.chunks.update_one({'files_id': files_id}, {'$set': {'data': Binary(b'foo', 0)}}) try: out = self.fs.open_download_stream(files_id) self.assertRaises(CorruptGridFile, out.read) out = self.fs.open_download_stream(files_id) self.assertRaises(CorruptGridFile, out.readline) finally: self.fs.delete(files_id) def test_upload_ensures_index(self): # setUp has dropped collections. names = self.db.collection_names() self.assertFalse([name for name in names if name.startswith('fs')]) chunks = self.db.fs.chunks files = self.db.fs.files self.fs.upload_from_stream("filename", b"junk") self.assertTrue(any( info.get('key') == [('files_id', 1), ('n', 1)] for info in chunks.index_information().values())) self.assertTrue(any( info.get('key') == [('filename', 1), ('uploadDate', 1)] for info in files.index_information().values())) def test_alt_collection(self): oid = self.alt.upload_from_stream("test_filename", b"hello world") self.assertEqual(b"hello world", self.alt.open_download_stream(oid).read()) self.assertEqual(1, self.db.alt.files.count()) self.assertEqual(1, self.db.alt.chunks.count()) self.alt.delete(oid) self.assertRaises(NoFile, self.alt.open_download_stream, oid) self.assertEqual(0, self.db.alt.files.count()) self.assertEqual(0, self.db.alt.chunks.count()) self.assertRaises(NoFile, self.alt.open_download_stream, "foo") self.alt.upload_from_stream("foo", b"hello world") self.assertEqual(b"hello world", self.alt.open_download_stream_by_name("foo").read()) self.alt.upload_from_stream("mike", b"") self.alt.upload_from_stream("test", b"foo") self.alt.upload_from_stream("hello world", b"") self.assertEqual(set(["mike", "test", "hello world", "foo"]), set(k["filename"] for k in list( self.db.alt.files.find()))) def test_threaded_reads(self): self.fs.upload_from_stream("test", b"hello") threads = [] results = [] for i in range(10): threads.append(JustRead(self.fs, 10, results)) threads[i].start() joinall(threads) self.assertEqual( 100 * [b'hello'], results ) def test_threaded_writes(self): threads = [] for i in range(10): threads.append(JustWrite(self.fs, 10)) threads[i].start() joinall(threads) fstr = self.fs.open_download_stream_by_name("test") self.assertEqual(fstr.read(), b"hello") # Should have created 100 versions of 'test' file self.assertEqual( 100, self.db.fs.files.find({'filename': 'test'}).count() ) def test_get_last_version(self): one = self.fs.upload_from_stream("test", b"foo") time.sleep(0.01) two = self.fs.open_upload_stream("test") two.write(b"bar") two.close() time.sleep(0.01) two = two._id three = self.fs.upload_from_stream("test", b"baz") self.assertEqual(b"baz", self.fs.open_download_stream_by_name("test").read()) self.fs.delete(three) self.assertEqual(b"bar", self.fs.open_download_stream_by_name("test").read()) self.fs.delete(two) self.assertEqual(b"foo", self.fs.open_download_stream_by_name("test").read()) self.fs.delete(one) self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "test") def test_get_version(self): self.fs.upload_from_stream("test", b"foo") time.sleep(0.01) self.fs.upload_from_stream("test", b"bar") time.sleep(0.01) self.fs.upload_from_stream("test", b"baz") time.sleep(0.01) self.assertEqual(b"foo", self.fs.open_download_stream_by_name( "test", revision=0).read()) self.assertEqual(b"bar", self.fs.open_download_stream_by_name( "test", revision=1).read()) self.assertEqual(b"baz", self.fs.open_download_stream_by_name( "test", revision=2).read()) self.assertEqual(b"baz", self.fs.open_download_stream_by_name( "test", revision=-1).read()) self.assertEqual(b"bar", self.fs.open_download_stream_by_name( "test", revision=-2).read()) self.assertEqual(b"foo", self.fs.open_download_stream_by_name( "test", revision=-3).read()) self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "test", revision=3) self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "test", revision=-4) def test_upload_from_stream_filelike(self): oid = self.fs.upload_from_stream("test_file", StringIO(b"hello world"), chunk_size_bytes=1) self.assertEqual(11, self.db.fs.chunks.count()) self.assertEqual(b"hello world", self.fs.open_download_stream(oid).read()) def test_missing_length_iter(self): # Test fix that guards against PHP-237 self.fs.upload_from_stream("empty", b"") doc = self.db.fs.files.find_one({"filename": "empty"}) doc.pop("length") self.db.fs.files.replace_one({"_id": doc["_id"]}, doc) fstr = self.fs.open_download_stream_by_name("empty") def iterate_file(grid_file): for _ in grid_file: pass return True self.assertTrue(iterate_file(fstr)) def test_gridfs_lazy_connect(self): client = MongoClient('badhost', connect=False, serverSelectionTimeoutMS=0) cdb = client.db gfs = gridfs.GridFSBucket(cdb) self.assertRaises(ServerSelectionTimeoutError, gfs.delete, 0) gfs = gridfs.GridFSBucket(cdb) self.assertRaises( ServerSelectionTimeoutError, gfs.upload_from_stream, "test", b"") # Still no connection. def test_gridfs_find(self): self.fs.upload_from_stream("two", b"test2") time.sleep(0.01) self.fs.upload_from_stream("two", b"test2+") time.sleep(0.01) self.fs.upload_from_stream("one", b"test1") time.sleep(0.01) self.fs.upload_from_stream("two", b"test2++") self.assertEqual(3, self.fs.find({"filename": "two"}).count()) self.assertEqual(4, self.fs.find({}).count()) cursor = self.fs.find( {}, no_cursor_timeout=False, sort=[("uploadDate", -1)], skip=1, limit=2) gout = next(cursor) self.assertEqual(b"test1", gout.read()) cursor.rewind() gout = next(cursor) self.assertEqual(b"test1", gout.read()) gout = next(cursor) self.assertEqual(b"test2+", gout.read()) self.assertRaises(StopIteration, cursor.__next__) cursor.close() self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) def test_grid_in_non_int_chunksize(self): # Lua, and perhaps other buggy GridFS clients, store size as a float. data = b'data' self.fs.upload_from_stream('f', data) self.db.fs.files.update_one({'filename': 'f'}, {'$set': {'chunkSize': 100.0}}) self.assertEqual(data, self.fs.open_download_stream_by_name('f').read()) def test_unacknowledged(self): # w=0 is prohibited. with self.assertRaises(ConfigurationError): gridfs.GridFSBucket(rs_or_single_client(w=0).pymongo_test) def test_rename(self): _id = self.fs.upload_from_stream("first_name", b'testing') self.assertEqual(b'testing', self.fs.open_download_stream_by_name( "first_name").read()) self.fs.rename(_id, "second_name") self.assertRaises(NoFile, self.fs.open_download_stream_by_name, "first_name") self.assertEqual(b"testing", self.fs.open_download_stream_by_name( "second_name").read()) def test_abort(self): gin = self.fs.open_upload_stream("test_filename", chunk_size_bytes=5) gin.write(b"test1") gin.write(b"test2") gin.write(b"test3") self.assertEqual(3, self.db.fs.chunks.count( {"files_id": gin._id})) gin.abort() self.assertTrue(gin.closed) self.assertRaises(ValueError, gin.write, b"test4") self.assertEqual(0, self.db.fs.chunks.count( {"files_id": gin._id})) class TestGridfsBucketReplicaSet(TestReplicaSetClientBase): def test_gridfs_replica_set(self): rsc = rs_client( w=self.w, read_preference=ReadPreference.SECONDARY) gfs = gridfs.GridFSBucket(rsc.pymongo_test) oid = gfs.upload_from_stream("test_filename", b'foo') content = gfs.open_download_stream(oid).read() self.assertEqual(b'foo', content) def test_gridfs_secondary(self): primary_host, primary_port = self.primary primary_connection = single_client(primary_host, primary_port) secondary_host, secondary_port = one(self.secondaries) secondary_connection = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY) primary_connection.pymongo_test.drop_collection("fs.files") primary_connection.pymongo_test.drop_collection("fs.chunks") # Should detect it's connected to secondary and not attempt to # create index gfs = gridfs.GridFSBucket(secondary_connection.pymongo_test) # This won't detect secondary, raises error self.assertRaises(ConnectionFailure, gfs.upload_from_stream, "test_filename", b'foo') def test_gridfs_secondary_lazy(self): # Should detect it's connected to secondary and not attempt to # create index. secondary_host, secondary_port = one(self.secondaries) client = single_client( secondary_host, secondary_port, read_preference=ReadPreference.SECONDARY, connect=False) # Still no connection. gfs = gridfs.GridFSBucket(client.test_gridfs_secondary_lazy) # Connects, doesn't create index. self.assertRaises(NoFile, gfs.open_download_stream_by_name, "test_filename") self.assertRaises(ConnectionFailure, gfs.upload_from_stream, "test_filename", b'data') def tearDown(self): rsc = client_context.rs_client rsc.pymongo_test.drop_collection('fs.files') rsc.pymongo_test.drop_collection('fs.chunks') if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_ssl.py0000644000175000017500000006343112630145074020156 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for SSL support.""" import os import socket import sys sys.path[0:0] = [""] try: from urllib.parse import quote_plus except ImportError: # Python 2 from urllib import quote_plus from pymongo import MongoClient, ssl_support from pymongo.errors import (ConfigurationError, ConnectionFailure, OperationFailure) from pymongo.ssl_support import HAVE_SSL, get_ssl_context, validate_cert_reqs from test import (host, pair, port, SkipTest, unittest) from test.utils import server_started_with_auth, remove_all_users, connected from test.version import Version CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'certificates') CLIENT_PEM = os.path.join(CERT_PATH, 'client.pem') CA_PEM = os.path.join(CERT_PATH, 'ca.pem') SIMPLE_SSL = False CERT_SSL = False SERVER_IS_RESOLVABLE = False MONGODB_X509_USERNAME = ( "CN=client,OU=kerneluser,O=10Gen,L=New York City,ST=New York,C=US") # To fully test this start a mongod instance (built with SSL support) like so: # mongod --dbpath /path/to/data/directory --sslOnNormalPorts \ # --sslPEMKeyFile /path/to/pymongo/test/certificates/server.pem \ # --sslCAFile /path/to/pymongo/test/certificates/ca.pem \ # --sslCRLFile /path/to/pymongo/test/certificates/crl.pem \ # --sslWeakCertificateValidation # Also, make sure you have 'server' as an alias for localhost in /etc/hosts # # Note: For all replica set tests to pass, the replica set configuration must # use 'server' for the hostname of all hosts. def is_server_resolvable(): """Returns True if 'server' is resolvable.""" socket_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(1) try: try: socket.gethostbyname('server') return True except socket.error: return False finally: socket.setdefaulttimeout(socket_timeout) # Shared ssl-enabled client for the tests ssl_client = None if HAVE_SSL: import ssl # Check this all once instead of before every test method below. # Is MongoDB configured for SSL? try: connected(MongoClient(host, port, ssl=True, ssl_cert_reqs=ssl.CERT_NONE, serverSelectionTimeoutMS=100)) SIMPLE_SSL = True except ConnectionFailure: pass # Is MongoDB configured with server.pem, ca.pem, and crl.pem from # mongodb jstests/lib? try: ssl_client = connected(MongoClient( host, port, ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_NONE, serverSelectionTimeoutMS=100)) CERT_SSL = True except ConnectionFailure: pass if CERT_SSL: SERVER_IS_RESOLVABLE = is_server_resolvable() class TestClientSSL(unittest.TestCase): def test_no_ssl_module(self): # Test that ConfigurationError is raised if the ssl # module isn't available. if HAVE_SSL: raise SkipTest( "The ssl module is available, can't test what happens " "without it." ) # Explicit self.assertRaises(ConfigurationError, MongoClient, ssl=True) # Implied self.assertRaises(ConfigurationError, MongoClient, ssl_certfile=CLIENT_PEM) def test_config_ssl(self): # Tests various ssl configurations self.assertRaises(ValueError, MongoClient, ssl='foo') self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_certfile=CLIENT_PEM) self.assertRaises(TypeError, MongoClient, ssl=0) self.assertRaises(TypeError, MongoClient, ssl=5.5) self.assertRaises(TypeError, MongoClient, ssl=[]) self.assertRaises(IOError, MongoClient, ssl_certfile="NoSuchFile") self.assertRaises(TypeError, MongoClient, ssl_certfile=True) self.assertRaises(TypeError, MongoClient, ssl_certfile=[]) self.assertRaises(IOError, MongoClient, ssl_keyfile="NoSuchFile") self.assertRaises(TypeError, MongoClient, ssl_keyfile=True) self.assertRaises(TypeError, MongoClient, ssl_keyfile=[]) # Test invalid combinations self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_keyfile=CLIENT_PEM) self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_certfile=CLIENT_PEM) self.assertRaises(ConfigurationError, MongoClient, ssl=False, ssl_keyfile=CLIENT_PEM, ssl_certfile=CLIENT_PEM) self.assertRaises( ValueError, validate_cert_reqs, 'ssl_cert_reqs', 3) self.assertRaises( ValueError, validate_cert_reqs, 'ssl_cert_reqs', -1) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', None), None) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', ssl.CERT_NONE), ssl.CERT_NONE) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', ssl.CERT_OPTIONAL), ssl.CERT_OPTIONAL) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', ssl.CERT_REQUIRED), ssl.CERT_REQUIRED) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 0), ssl.CERT_NONE) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 1), ssl.CERT_OPTIONAL) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 2), ssl.CERT_REQUIRED) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 'CERT_NONE'), ssl.CERT_NONE) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 'CERT_OPTIONAL'), ssl.CERT_OPTIONAL) self.assertEqual( validate_cert_reqs('ssl_cert_reqs', 'CERT_REQUIRED'), ssl.CERT_REQUIRED) class TestSSL(unittest.TestCase): @classmethod def setUpClass(cls): if not HAVE_SSL: raise SkipTest("The ssl module is not available.") if sys.version.startswith('3.0'): raise SkipTest("Python 3.0.x has problems " "with SSL and socket timeouts.") def test_simple_ssl(self): # Expects the server to be running with ssl and with # no --sslPEMKeyFile or with --sslWeakCertificateValidation if not SIMPLE_SSL: raise SkipTest("No simple mongod available over SSL") client = MongoClient(host, port, ssl=True, ssl_cert_reqs=ssl.CERT_NONE) response = client.admin.command('ismaster') if 'setName' in response: client = MongoClient(pair, replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_cert_reqs=ssl.CERT_NONE) db = client.pymongo_ssl_test db.test.drop() db.test.insert_one({'ssl': True}) self.assertTrue(db.test.find_one()['ssl']) client.drop_database('pymongo_ssl_test') def test_cert_ssl(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") client = ssl_client response = ssl_client.admin.command('ismaster') if 'setName' in response: client = MongoClient(pair, replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) db = client.pymongo_ssl_test db.test.drop() db.test.insert_one({'ssl': True}) self.assertTrue(db.test.find_one()['ssl']) client.drop_database('pymongo_ssl_test') def test_cert_ssl_implicitly_set(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") client = MongoClient(host, port, ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) response = ssl_client.admin.command('ismaster') if 'setName' in response: client = MongoClient(pair, replicaSet=response['setName'], w=len(response['hosts']), ssl_cert_reqs=ssl.CERT_NONE, ssl_certfile=CLIENT_PEM) db = client.pymongo_ssl_test db.test.drop() db.test.insert_one({'ssl': True}) self.assertTrue(db.test.find_one()['ssl']) client.drop_database('pymongo_ssl_test') def test_cert_ssl_validation(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") if not SERVER_IS_RESOLVABLE: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") client = MongoClient('server', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM) response = client.admin.command('ismaster') if 'setName' in response: if response['primary'].split(":")[0] != 'server': raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") client = MongoClient('server', replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM) db = client.pymongo_ssl_test db.test.drop() db.test.insert_one({'ssl': True}) self.assertTrue(db.test.find_one()['ssl']) client.drop_database('pymongo_ssl_test') def test_cert_ssl_uri_support(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") if not SERVER_IS_RESOLVABLE: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") uri_fmt = ("mongodb://server/?ssl=true&ssl_certfile=%s&ssl_cert_reqs" "=%s&ssl_ca_certs=%s&ssl_match_hostname=true") client = MongoClient(uri_fmt % (CLIENT_PEM, 'CERT_REQUIRED', CA_PEM)) db = client.pymongo_ssl_test db.test.drop() db.test.insert_one({'ssl': True}) self.assertTrue(db.test.find_one()['ssl']) client.drop_database('pymongo_ssl_test') def test_cert_ssl_validation_optional(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") if not SERVER_IS_RESOLVABLE: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") client = MongoClient('server', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_OPTIONAL, ssl_ca_certs=CA_PEM) response = client.admin.command('ismaster') if 'setName' in response: if response['primary'].split(":")[0] != 'server': raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") client = MongoClient('server', replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_OPTIONAL, ssl_ca_certs=CA_PEM) db = client.pymongo_ssl_test db.test.drop() db.test.insert_one({'ssl': True}) self.assertTrue(db.test.find_one()['ssl']) client.drop_database('pymongo_ssl_test') def test_cert_ssl_validation_hostname_matching(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") response = ssl_client.admin.command('ismaster') with self.assertRaises(ConnectionFailure): connected(MongoClient(pair, ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=100)) connected(MongoClient(pair, ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, ssl_match_hostname=False, serverSelectionTimeoutMS=100)) if 'setName' in response: with self.assertRaises(ConnectionFailure): connected(MongoClient(pair, replicaSet=response['setName'], ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, serverSelectionTimeoutMS=100)) connected(MongoClient(pair, replicaSet=response['setName'], ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, ssl_match_hostname=False, serverSelectionTimeoutMS=100)) def test_validation_with_system_ca_certs(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests eg: # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # --sslWeakCertificateValidation # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") if not SERVER_IS_RESOLVABLE: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") if sys.platform == "win32": raise SkipTest("Can't test system ca certs on Windows.") if sys.version_info < (2, 7, 9): raise SkipTest("Can't load system CA certificates.") # Tell OpenSSL where CA certificates live. os.environ['SSL_CERT_FILE'] = CA_PEM try: with self.assertRaises(ConnectionFailure): # Server cert is verified but hostname matching fails connected(MongoClient(pair, ssl=True, serverSelectionTimeoutMS=100)) # Server cert is verified. Disable hostname matching. connected(MongoClient(pair, ssl=True, ssl_match_hostname=False, serverSelectionTimeoutMS=100)) # Server cert and hostname are verified. connected(MongoClient('server', ssl=True, serverSelectionTimeoutMS=100)) # Server cert and hostname are verified. connected( MongoClient( 'mongodb://server/?ssl=true&serverSelectionTimeoutMS=100')) finally: os.environ.pop('SSL_CERT_FILE') def test_system_certs_config_error(self): ctx = get_ssl_context(None, None, None, ssl.CERT_NONE) if ((sys.platform != "win32" and hasattr(ctx, "set_default_verify_paths")) or hasattr(ctx, "load_default_certs")): raise SkipTest( "Can't test when system CA certificates are loadable.") have_certifi = ssl_support.HAVE_CERTIFI have_wincertstore = ssl_support.HAVE_WINCERTSTORE # Force the test regardless of environment. ssl_support.HAVE_CERTIFI = False ssl_support.HAVE_WINCERTSTORE = False try: with self.assertRaises(ConfigurationError): MongoClient("mongodb://localhost/?ssl=true") finally: ssl_support.HAVE_CERTIFI = have_certifi ssl_support.HAVE_WINCERTSTORE = have_wincertstore def test_certifi_support(self): if hasattr(ssl, "SSLContext"): # SSLSocket doesn't provide ca_certs attribute on pythons # with SSLContext and SSLContext provides no information # about ca_certs. raise SkipTest("Can't test when SSLContext available.") if not ssl_support.HAVE_CERTIFI: raise SkipTest("Need certifi to test certifi support.") have_wincertstore = ssl_support.HAVE_WINCERTSTORE # Force the test on Windows, regardless of environment. ssl_support.HAVE_WINCERTSTORE = False try: ctx = get_ssl_context(None, None, CA_PEM, ssl.CERT_REQUIRED) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, CA_PEM) ctx = get_ssl_context(None, None, None, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, ssl_support.certifi.where()) finally: ssl_support.HAVE_WINCERTSTORE = have_wincertstore def test_wincertstore(self): if sys.platform != "win32": raise SkipTest("Only valid on Windows.") if hasattr(ssl, "SSLContext"): # SSLSocket doesn't provide ca_certs attribute on pythons # with SSLContext and SSLContext provides no information # about ca_certs. raise SkipTest("Can't test when SSLContext available.") if not ssl_support.HAVE_WINCERTSTORE: raise SkipTest("Need wincertstore to test wincertstore.") ctx = get_ssl_context(None, None, CA_PEM, ssl.CERT_REQUIRED) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, CA_PEM) ctx = get_ssl_context(None, None, None, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, ssl_support._WINCERTS.name) def test_mongodb_x509_auth(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests as well as # --auth # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem # --auth if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") if not Version.from_client(ssl_client).at_least(2, 5, 3, -1): raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer") if not server_started_with_auth(ssl_client): raise SkipTest('Authentication is not enabled on server') self.addCleanup(ssl_client['$external'].logout) self.addCleanup(remove_all_users, ssl_client['$external']) # Give admin all necessary privileges. ssl_client['$external'].add_user(MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) coll = ssl_client.pymongo_test.test self.assertRaises(OperationFailure, coll.count) self.assertTrue(ssl_client.admin.authenticate( MONGODB_X509_USERNAME, mechanism='MONGODB-X509')) coll.drop() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), host, port)) # SSL options aren't supported in the URI... self.assertTrue(MongoClient(uri, ssl=True, ssl_certfile=CLIENT_PEM)) # Should require a username uri = ('mongodb://%s:%d/?authMechanism=MONGODB-X509' % (host, port)) client_bad = MongoClient(uri, ssl=True, ssl_certfile=CLIENT_PEM) self.assertRaises(OperationFailure, client_bad.pymongo_test.test.delete_one, {}) # Auth should fail if username and certificate do not match uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus("not the username"), host, port)) bad_client = MongoClient(uri, ssl=True, ssl_certfile=CLIENT_PEM) with self.assertRaises(OperationFailure): bad_client.pymongo_test.test.find_one() self.assertRaises(OperationFailure, ssl_client.admin.authenticate, "not the username", mechanism="MONGODB-X509") # Invalid certificate (using CA certificate as client certificate) uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), host, port)) # These tests will raise SSLError (>= 3.2) or ConnectionFailure # (2.x) depending on where OpenSSL first sees the PEM file. try: connected(MongoClient(uri, ssl=True, ssl_certfile=CA_PEM, serverSelectionTimeoutMS=100)) except (ssl.SSLError, ConnectionFailure): pass else: self.fail("Invalid certificate accepted.") try: connected(MongoClient(pair, ssl=True, ssl_certfile=CA_PEM, serverSelectionTimeoutMS=100)) except (ssl.SSLError, ConnectionFailure): pass else: self.fail("Invalid certificate accepted.") if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_uri_spec.py0000644000175000017500000001111212630145074021153 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the pymongo uri_parser module is up to spec.""" import json import os import sys import warnings sys.path[0:0] = [""] from pymongo.uri_parser import parse_uri from test import unittest # Location of JSON test specifications. _TEST_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), os.path.join('connection_string', 'test')) class TestAllScenarios(unittest.TestCase): pass def create_test(scenario_def): def run_scenario(self): self.assertTrue(scenario_def['tests'], "tests cannot be empty") for test in scenario_def['tests']: dsc = test['description'] warned = False error = False with warnings.catch_warnings(): warnings.filterwarnings('error') try: options = parse_uri(test['uri'], warn=True) except Warning: warned = True except Exception: error = True self.assertEqual(not error, test['valid'], "Test failure '%s'" % dsc) if test.get("warning", False): self.assertTrue(warned, "Expected warning for test '%s'" % (dsc,)) # Redo in the case there were warnings that were not expected. if warned: options = parse_uri(test['uri'], warn=True) # Compare hosts and port. if test['hosts'] is not None: self.assertEqual( len(test['hosts']), len(options['nodelist']), "Incorrect number of hosts parsed from URI") for exp, actual in zip(test['hosts'], options['nodelist']): self.assertEqual(exp['host'], actual[0], "Expected host %s but got %s" % (exp['host'], actual[0])) if exp['port'] is not None: self.assertEqual(exp['port'], actual[1], "Expected port %s but got %s" % (exp['port'], actual)) # Compare auth options. auth = test['auth'] if auth is not None: auth['database'] = auth.pop('db') # db == database # Special case for PyMongo's collection parsing. if options.get('collection') is not None: options['database'] += "." + options['collection'] for elm in auth: if auth[elm] is not None: self.assertEqual(auth[elm], options[elm], "Expected %s but got %s" % (auth[elm], options[elm])) # Compare URI options. if test['options'] is not None: for opt in test['options']: if options.get(opt) is not None: self.assertEqual( options[opt], test['options'][opt], "For option %s expected %s but got %s" % (opt, options[opt], test['options'][opt])) return run_scenario def create_tests(): for dirpath, _, filenames in os.walk(_TEST_PATH): dirname = os.path.split(dirpath) dirname = os.path.split(dirname[-2])[-1] + '_' + dirname[-1] for filename in filenames: with open(os.path.join(dirpath, filename)) as scenario_stream: scenario_def = json.load(scenario_stream) # Construct test from scenario. new_test = create_test(scenario_def) test_name = 'test_%s_%s' % ( dirname, os.path.splitext(filename)[0]) new_test.__name__ = test_name setattr(TestAllScenarios, new_test.__name__, new_test) create_tests() if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_mongos_load_balancing.py0000644000175000017500000001415512630145074023653 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test MongoClient's mongos load balancing using a mock.""" import sys import threading sys.path[0:0] = [""] from pymongo.errors import AutoReconnect, InvalidOperation from pymongo.server_selectors import writable_server_selector from pymongo.topology_description import TOPOLOGY_TYPE from test import unittest, client_context, MockClientTest from test.pymongo_mocks import MockClient from test.utils import connected, wait_until @client_context.require_connection def setUpModule(): pass class SimpleOp(threading.Thread): def __init__(self, client): super(SimpleOp, self).__init__() self.client = client self.passed = False def run(self): self.client.db.command('ismaster') self.passed = True # No exception raised. def do_simple_op(client, nthreads): threads = [SimpleOp(client) for _ in range(nthreads)] for t in threads: t.start() for t in threads: t.join() for t in threads: assert t.passed def writable_addresses(topology): return set(server.description.address for server in topology.select_servers(writable_server_selector)) class TestMongosLoadBalancing(MockClientTest): def mock_client(self, **kwargs): mock_client = MockClient( standalones=[], members=[], mongoses=['a:1', 'b:2', 'c:3'], host='a:1,b:2,c:3', connect=False, **kwargs) # Latencies in seconds. mock_client.mock_rtts['a:1'] = 0.020 mock_client.mock_rtts['b:2'] = 0.025 mock_client.mock_rtts['c:3'] = 0.045 return mock_client def test_lazy_connect(self): # While connected() ensures we can trigger connection from the main # thread and wait for the monitors, this test triggers connection from # several threads at once to check for data races. nthreads = 10 client = self.mock_client() self.assertEqual(0, len(client.nodes)) # Trigger initial connection. do_simple_op(client, nthreads) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') def test_reconnect(self): nthreads = 10 client = connected(self.mock_client()) # connected() ensures we've contacted at least one mongos. Wait for # all of them. wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') # Trigger reconnect. client.close() do_simple_op(client, nthreads) wait_until(lambda: len(client.nodes) == 3, 'reconnect to all mongoses') def test_failover(self): nthreads = 10 client = connected(self.mock_client(localThresholdMS=0.001)) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') # Our chosen mongos goes down. client.kill_host('a:1') # Trigger failover to higher-latency nodes. AutoReconnect should be # raised at most once in each thread. passed = [] def f(): try: client.db.command('ismaster') except AutoReconnect: # Second attempt succeeds. client.db.command('ismaster') passed.append(True) threads = [threading.Thread(target=f) for _ in range(nthreads)] for t in threads: t.start() for t in threads: t.join() self.assertEqual(nthreads, len(passed)) # Down host removed from list. self.assertEqual(2, len(client.nodes)) def test_local_threshold(self): client = connected(self.mock_client(localThresholdMS=30)) self.assertEqual(30, client.local_threshold_ms) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') topology = client._topology # All are within a 30-ms latency window, see self.mock_client(). self.assertEqual(set([('a', 1), ('b', 2), ('c', 3)]), writable_addresses(topology)) # No error client.admin.command('ismaster') client = connected(self.mock_client(localThresholdMS=0)) self.assertEqual(0, client.local_threshold_ms) # No error client.db.command('ismaster') # Our chosen mongos goes down. client.kill_host('%s:%s' % next(iter(client.nodes))) try: client.db.command('ismaster') except: pass # No error client.db.command('ismaster') def test_load_balancing(self): # Although the server selection JSON tests already prove that # select_servers works for sharded topologies, here we do an end-to-end # test of discovering servers' round trip times and configuring # localThresholdMS. client = connected(self.mock_client()) wait_until(lambda: len(client.nodes) == 3, 'connect to all mongoses') # Prohibited for topology type Sharded. with self.assertRaises(InvalidOperation): client.address topology = client._topology self.assertEqual(TOPOLOGY_TYPE.Sharded, topology.description.topology_type) # a and b are within the 15-ms latency window, see self.mock_client(). self.assertEqual(set([('a', 1), ('b', 2)]), writable_addresses(topology)) client.mock_rtts['a:1'] = 0.045 # Discover only b is within latency window. wait_until(lambda: set([('b', 2)]) == writable_addresses(topology), 'discover server "a" is too far') if __name__ == "__main__": unittest.main() pymongo-3.2/test/certificates/0000755000175000017500000000000012631423130020373 5ustar behackettbehackett00000000000000pymongo-3.2/test/certificates/client.pem0000644000175000017500000001304212513020120022343 0ustar behackettbehackett00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 7 (0x7) Signature Algorithm: sha1WithRSAEncryption Issuer: C=US, ST=New York, L=New York City, O=10Gen, OU=Kernel, CN=My Cert Authority/emailAddress=root@lazarus Validity Not Before: Aug 23 14:55:32 2013 GMT Not After : Jan 7 14:55:32 2041 GMT Subject: C=US, ST=New York, L=New York City, O=10Gen, OU=kerneluser, CN=client Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:ba:16:42:d4:8b:3d:5e:8a:67:9e:a7:c0:cd:4a: 9c:9c:fd:95:b9:83:bf:f4:cf:03:8c:2e:db:a9:c1: 35:58:80:f6:e2:e9:87:28:84:e3:d0:9b:68:60:51: 0e:42:84:d8:6f:e8:34:cc:18:97:79:d3:8d:d8:2f: 23:11:25:6f:69:7a:38:bb:8c:b2:29:e9:91:be:79: 8c:cc:1b:56:98:98:d3:83:2a:c5:f9:9c:86:0c:2c: 24:0e:5c:46:3b:a9:95:44:6c:c5:e0:7c:9d:03:ae: 0d:23:99:49:a4:48:dd:0e:35:a2:e5:b4:8b:86:bd: c0:c8:ce:d5:ac:c4:36:f3:9e:5f:17:00:23:8d:53: a1:43:1b:a3:61:96:36:80:4d:35:50:b5:8b:69:31: 39:b4:63:8b:96:59:5c:d1:ea:92:eb:eb:fa:1b:35: 64:44:b3:f6:f3:a6:9d:49:3a:59:e5:e1:c2:cb:98: be:29:b3:22:dd:33:97:d7:50:4f:db:c2:58:64:18: b5:8c:3c:6b:2d:21:f6:bd:8d:e5:d2:da:8d:79:fe: a7:80:75:a8:15:b9:ee:79:7f:01:31:1d:e5:e7:15: 76:53:65:f6:fe:f0:93:7d:20:3d:cc:ff:9b:ca:b2: 50:2c:1b:3a:69:d5:e6:70:cf:ac:be:7e:5c:33:c4: 6e:a7 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: 4A:8B:EE:22:42:E6:F8:62:4C:86:38:8D:C5:78:95:98:C1:10:05:7C X509v3 Authority Key Identifier: keyid:07:41:19:3A:9F:7E:C5:B7:22:4E:B7:BC:D5:DF:E4:FC:09:B8:64:16 Signature Algorithm: sha1WithRSAEncryption 13:13:a8:f0:de:78:c6:b1:e0:85:cc:27:e6:04:28:44:93:1d: f1:ff:5e:81:69:33:1f:f3:76:e0:49:ca:d9:ad:aa:db:f5:a5: f8:a6:50:bb:a1:a7:40:14:e4:2f:8d:b8:21:7f:35:04:60:db: af:f0:9e:dd:a1:ca:0b:7f:03:2e:2f:19:1e:32:6e:1e:2d:87: 68:e3:37:47:a8:5b:93:d1:88:41:73:da:88:21:59:27:d4:35: 1c:6a:27:b5:c0:c6:17:ba:f3:87:c8:e1:f4:8f:43:12:bc:fa: 8d:90:d5:86:83:df:51:a5:c9:e0:92:f0:66:d0:37:61:6f:85: 24:18 -----BEGIN CERTIFICATE----- MIIDdjCCAt+gAwIBAgIBBzANBgkqhkiG9w0BAQUFADCBkjELMAkGA1UEBhMCVVMx ETAPBgNVBAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYD VQQKDAUxMEdlbjEPMA0GA1UECwwGS2VybmVsMRowGAYDVQQDDBFNeSBDZXJ0IEF1 dGhvcml0eTEbMBkGCSqGSIb3DQEJARYMcm9vdEBsYXphcnVzMB4XDTEzMDgyMzE0 NTUzMloXDTQxMDEwNzE0NTUzMlowbjELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE5l dyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYDVQQKDAUxMEdlbjET MBEGA1UECwwKa2VybmVsdXNlcjEPMA0GA1UEAwwGY2xpZW50MIIBIjANBgkqhkiG 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuhZC1Is9XopnnqfAzUqcnP2VuYO/9M8DjC7b qcE1WID24umHKITj0JtoYFEOQoTYb+g0zBiXedON2C8jESVvaXo4u4yyKemRvnmM zBtWmJjTgyrF+ZyGDCwkDlxGO6mVRGzF4HydA64NI5lJpEjdDjWi5bSLhr3AyM7V rMQ2855fFwAjjVOhQxujYZY2gE01ULWLaTE5tGOLlllc0eqS6+v6GzVkRLP286ad STpZ5eHCy5i+KbMi3TOX11BP28JYZBi1jDxrLSH2vY3l0tqNef6ngHWoFbnueX8B MR3l5xV2U2X2/vCTfSA9zP+byrJQLBs6adXmcM+svn5cM8RupwIDAQABo3sweTAJ BgNVHRMEAjAAMCwGCWCGSAGG+EIBDQQfFh1PcGVuU1NMIEdlbmVyYXRlZCBDZXJ0 aWZpY2F0ZTAdBgNVHQ4EFgQUSovuIkLm+GJMhjiNxXiVmMEQBXwwHwYDVR0jBBgw FoAUB0EZOp9+xbciTre81d/k/Am4ZBYwDQYJKoZIhvcNAQEFBQADgYEAExOo8N54 xrHghcwn5gQoRJMd8f9egWkzH/N24EnK2a2q2/Wl+KZQu6GnQBTkL424IX81BGDb r/Ce3aHKC38DLi8ZHjJuHi2HaOM3R6hbk9GIQXPaiCFZJ9Q1HGontcDGF7rzh8jh 9I9DErz6jZDVhoPfUaXJ4JLwZtA3YW+FJBg= -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6FkLUiz1eimee p8DNSpyc/ZW5g7/0zwOMLtupwTVYgPbi6YcohOPQm2hgUQ5ChNhv6DTMGJd5043Y LyMRJW9peji7jLIp6ZG+eYzMG1aYmNODKsX5nIYMLCQOXEY7qZVEbMXgfJ0Drg0j mUmkSN0ONaLltIuGvcDIztWsxDbznl8XACONU6FDG6NhljaATTVQtYtpMTm0Y4uW WVzR6pLr6/obNWREs/bzpp1JOlnl4cLLmL4psyLdM5fXUE/bwlhkGLWMPGstIfa9 jeXS2o15/qeAdagVue55fwExHeXnFXZTZfb+8JN9ID3M/5vKslAsGzpp1eZwz6y+ flwzxG6nAgMBAAECggEBALYw92urjAFVFxCiA8W7aEzYhtAkaztft4R3mD/C19z4 H0CZDeig+3+RuIactY5xDIu8WHz/EseHVlg0BmxSL5ugu4z8uq8IbNaFoVFw7r7m 2ieRKFY0ZpXiXcbllynw5iEhMjeRKhWhQmH5Qb2kTTINV5j4xKa+f9Lblx7Y2Uh4 tsaOtlMwb98D2/KYJdTv5Nj1nyuSqRVhECsd00Cb6JUBGQBx8Ja0wFy9gEygq6kU w3s1XNOSnYNEo4FaVZwp5KZyCyBENcKpNUq4nXt/7ncEfVYdJck0Li3wN4Jr2J9S eHqRzh8QkHxc1Ro8ktcXaUSs9kFuwvVvb4rcGUpOMWkCgYEA9xxp8yDtFVgzMtc/ vS8xgM1Wj4SrgKKYhE2wS05BJh/41oFMzfH1FpZ1GCM983r4QgYWoT71XsBgiOMC yN2p2IbV4V44bMGKJqaVMkB91CVCUWI6piaCQb/1CJTwaXE7zPim6dlUSxxBBnRn LP50NTscRLFcCZELD3Yl7jR8XFUCgYEAwMfkNFmGtBKAwlHZ3Y3XOwPWg+jCll7s 9nhv8TU2IB9pcCRGqyOT7k1YymvYkDT2Je4JUPWEBs4cW7yD61LrQ8w8+DrE9dGo czzGPyjOAANSX0asG74UjkNIQThmyEOltVHIxYMaSqowjHRSPdA+R4Od9EdcDdfS q5SfSVFxmwsCgYBtl1thqUOcCL7EGHQ7KdfxgJ+YDMWmyfWMD4xVCYKZLurD7xop 59nDR7zslIygE/RQC7Uzk+FsQTNO4ibVAIGX9syaI5gwm3DyjURzwehMEq4ju8W4 9DEmicRZJvysNrzHvasA4RKiMQihnTQ43yyYgvuZd3MTBxF5rPNLfll89QKBgQC9 SsmiOZIR+OUjaTmS2bbQBNm7Fm8TNcxZyzKn1wb5jb57VbNqUfnskVgxEqpIFyjn X48YRqtH/1RLI5UpGXdXUBFB8Hr7oM1VsgQ7ejakPp7AXOWcLA2FDz3AhMAvvnTU 0KRihHPpgqk/EOy8M2Ej2XHcrcEO+q+quLmbRXRWtwKBgHacQiwci/2J+v0e9i52 re/2AJHKP5MwNHFe1e01iNc5EEN0G+/Ut8XW19DWf6bsxqie0ChC+xN8TUst8alT F+tXTsHHmt/lRcjTROjT5XVuoqjtU2Q0QeVeGLgvObso+fZy3ZNeQuSJjWukdMZ3 57rGT6p0OuM8qbrTzpv3JMrm -----END PRIVATE KEY----- pymongo-3.2/test/certificates/crl.pem0000644000175000017500000000104412516255237021672 0ustar behackettbehackett00000000000000-----BEGIN X509 CRL----- MIIBazCB1QIBATANBgkqhkiG9w0BAQUFADCBkjELMAkGA1UEBhMCVVMxETAPBgNV BAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYDVQQKDAUx MEdlbjEPMA0GA1UECwwGS2VybmVsMRowGAYDVQQDDBFNeSBDZXJ0IEF1dGhvcml0 eTEbMBkGCSqGSIb3DQEJARYMcm9vdEBsYXphcnVzFw0xMjEyMTIxODQ3NDFaFw00 MDA0MjgxODQ3NDFaoA4wDDAKBgNVHRQEAwIBCzANBgkqhkiG9w0BAQUFAAOBgQAu PlPDGei2q6kdkoHe8vmDuts7Hm/o9LFbBmn0XUcfHisCJCPsJTyGCsgnfIiBcXJY 1LMKsQFnYGv28rE2ZPpFg2qNxL+6qUEzCvqaHLX9q1V0F+f8hHDxucNYu52oo/h0 uNZxB1KPFI2PReG5d3oUYqJ2+EctKkrGtxSPzbN0gg== -----END X509 CRL----- pymongo-3.2/test/certificates/ca.pem0000644000175000017500000000171112513020120021450 0ustar behackettbehackett00000000000000-----BEGIN CERTIFICATE----- MIICnTCCAgYCCQD4+RCKzwZr/zANBgkqhkiG9w0BAQUFADCBkjELMAkGA1UEBhMC VVMxETAPBgNVBAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4w DAYDVQQKDAUxMEdlbjEPMA0GA1UECwwGS2VybmVsMRowGAYDVQQDDBFNeSBDZXJ0 IEF1dGhvcml0eTEbMBkGCSqGSIb3DQEJARYMcm9vdEBsYXphcnVzMB4XDTEzMTEz MDAyMzU0OVoXDTIzMTEyODAyMzU0OVowgZIxCzAJBgNVBAYTAlVTMREwDwYDVQQI DAhOZXcgWW9yazEWMBQGA1UEBwwNTmV3IFlvcmsgQ2l0eTEOMAwGA1UECgwFMTBH ZW4xDzANBgNVBAsMBktlcm5lbDEaMBgGA1UEAwwRTXkgQ2VydCBBdXRob3JpdHkx GzAZBgkqhkiG9w0BCQEWDHJvb3RAbGF6YXJ1czCBnzANBgkqhkiG9w0BAQEFAAOB jQAwgYkCgYEA1xymeY+U/evUuQvxpun9moe4GopN80c1ptmaAHM/1Onwaq54Wt27 nl1wUVme3dh4DdWviYY7mJ333HVEnp/QhVcT4kQhICZqdgPKPdCseQW3H+8x6Gwz hrNRBdz0NkSoFxDlIymfy2Q2xoQpbCGAg+EnRYUTKlHMXNpUDLFhGjcCAwEAATAN BgkqhkiG9w0BAQUFAAOBgQDRQB3c/9osTexEzMPHyMGTzG5nGwy8Wv77GgW3BETM hECoGqueXLa5ZgvealJrnMHNKdj6vrCGgBDzE0K0VdXc4dLtLmx3DRntDOAWKJdB 2XPMvdC7Ec//Fwep/9emz0gDiJrTiEpL4p74+h+sp4Xy8cBokQ3Ss5S9NmnPXT7E qQ== -----END CERTIFICATE----- pymongo-3.2/test/certificates/server.pem0000644000175000017500000000371212516255237022424 0ustar behackettbehackett00000000000000-----BEGIN PRIVATE KEY----- MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAK53miP9GczBWXnq NxHwQkgVqsDuesjwJbWilMK4gf3fjnf2PN3qDpnGbZbPD0ij8975pIKtSPoDycFm A8Mogip0yU2Lv2lL56CWthSBftOFDL2CWIsmuuURFXZPiVLtLytfI9oLASZFlywW Cs83qEDTvdW8VoVhVsxV1JFDnpXLAgMBAAECgYBoGBgxrMt97UazhNkCrPT/CV5t 6lv8E7yMGMrlOyzkCkR4ssQyK3o2qbutJTGbR6czvIM5LKbD9Qqlh3ZrNHokWmTR VQQpJxt8HwP5boQvwRHg9+KSGr4JvRko1qxFs9C7Bzjt4r9VxdjhwZPdy0McGI/z yPXyQHjqBayrHV1EwQJBANorfCKeIxLhH3LAeUZuRS8ACldJ2N1kL6Ov43/v+0S/ OprQeBTODuTds3sv7FCT1aYDTOe6JLNOwN2i4YVOMBsCQQDMuCozrwqftD17D06P 9+lRXUekY5kFBs5j28Xnl8t8jnuxsXtQUTru660LD0QrmDNSauhpEmlpJknicnGt hmwRAkEA12MI6bBPlir0/jgxQqxI1w7mJqj8Vg27zpEuO7dzzLoyJHddpcSNBbwu npaAakiZK42klj26T9+XHvjYRuAbMwJBAJ5WnwWEkGH/pUHGEAyYQdSVojDKe/MA Vae0tzguFswK5C8GyArSGRPsItYYA7D4MlG/sGx8Oh2C6MiFndkJzBECQDcP1y4r Qsek151t1zArLKH4gG5dQAeZ0Lc2VeC4nLMUqVwrHcZDdd1RzLlSaH3j1MekFVfT 6v6rrcNLEVbeuk4= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIC7jCCAlegAwIBAgIBCjANBgkqhkiG9w0BAQUFADCBkjELMAkGA1UEBhMCVVMx ETAPBgNVBAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYD VQQKDAUxMEdlbjEPMA0GA1UECwwGS2VybmVsMRowGAYDVQQDDBFNeSBDZXJ0IEF1 dGhvcml0eTEbMBkGCSqGSIb3DQEJARYMcm9vdEBsYXphcnVzMB4XDTEzMTIwNTEz MjU0MFoXDTQxMDQyMTEzMjU0MFowajELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE5l dyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MQ4wDAYDVQQKDAUxMEdlbjEP MA0GA1UECwwGS2VybmVsMQ8wDQYDVQQDDAZzZXJ2ZXIwgZ8wDQYJKoZIhvcNAQEB BQADgY0AMIGJAoGBAK53miP9GczBWXnqNxHwQkgVqsDuesjwJbWilMK4gf3fjnf2 PN3qDpnGbZbPD0ij8975pIKtSPoDycFmA8Mogip0yU2Lv2lL56CWthSBftOFDL2C WIsmuuURFXZPiVLtLytfI9oLASZFlywWCs83qEDTvdW8VoVhVsxV1JFDnpXLAgMB AAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJh dGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQgCkKiZhUV9/Zo7RwYYwm2cNK6tzAf BgNVHSMEGDAWgBQHQRk6n37FtyJOt7zV3+T8CbhkFjANBgkqhkiG9w0BAQUFAAOB gQCbsfr+Q4pty4Fy38lSxoCgnbB4pX6+Ex3xyw5zxDYR3xUlb/uHBiNZ1dBrXBxU ekU8dEvf+hx4iRDSW/C5N6BGnBBhCHcrPabo2bEEWKVsbUC3xchTB5rNGkvnMt9t G9ol7vanuzjL3S8/2PB33OshkBH570CxqqPflQbdjwt9dg== -----END CERTIFICATE----- pymongo-3.2/test/__init__.py0000644000175000017500000003375712630145074020065 0ustar behackettbehackett00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test suite for pymongo, bson, and gridfs. """ import os import socket import sys from pymongo.common import partition_node if sys.version_info[:2] == (2, 6): import unittest2 as unittest from unittest2 import SkipTest else: import unittest from unittest import SkipTest import warnings from functools import wraps import pymongo import pymongo.errors from bson.py3compat import _unicode from pymongo import common from test.version import Version # hostnames retrieved from isMaster will be of unicode type in Python 2, # so ensure these hostnames are unicodes, too. It makes tests like # `test_repr` predictable. host = _unicode(os.environ.get("DB_IP", 'localhost')) port = int(os.environ.get("DB_PORT", 27017)) pair = '%s:%d' % (host, port) host2 = _unicode(os.environ.get("DB_IP2", 'localhost')) port2 = int(os.environ.get("DB_PORT2", 27018)) host3 = _unicode(os.environ.get("DB_IP3", 'localhost')) port3 = int(os.environ.get("DB_PORT3", 27019)) db_user = _unicode(os.environ.get("DB_USER", "user")) db_pwd = _unicode(os.environ.get("DB_PASSWORD", "password")) class client_knobs(object): def __init__( self, heartbeat_frequency=None, kill_cursor_frequency=None): self.heartbeat_frequency = heartbeat_frequency self.kill_cursor_frequency = kill_cursor_frequency self.old_heartbeat_frequency = None self.old_kill_cursor_frequency = None def enable(self): self.old_heartbeat_frequency = common.HEARTBEAT_FREQUENCY self.old_kill_cursor_frequency = common.KILL_CURSOR_FREQUENCY if self.heartbeat_frequency is not None: common.HEARTBEAT_FREQUENCY = self.heartbeat_frequency if self.kill_cursor_frequency is not None: common.KILL_CURSOR_FREQUENCY = self.kill_cursor_frequency def __enter__(self): self.enable() def disable(self): common.HEARTBEAT_FREQUENCY = self.old_heartbeat_frequency common.KILL_CURSOR_FREQUENCY = self.old_kill_cursor_frequency def __exit__(self, exc_type, exc_val, exc_tb): self.disable() class ClientContext(object): def __init__(self): """Create a client and grab essential information from the server.""" self.connected = False self.ismaster = {} self.w = None self.nodes = set() self.replica_set_name = None self.rs_client = None self.cmd_line = None self.version = Version(-1) # Needs to be comparable with Version self.auth_enabled = False self.test_commands_enabled = False self.is_mongos = False self.is_rs = False self.has_ipv6 = False try: client = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=100) client.admin.command('ismaster') # Can we connect? # If so, then reset client to defaults. self.client = pymongo.MongoClient(host, port) except pymongo.errors.ConnectionFailure: self.client = None else: self.connected = True self.ismaster = self.client.admin.command('ismaster') self.w = len(self.ismaster.get("hosts", [])) or 1 self.nodes = set([(host, port)]) self.replica_set_name = self.ismaster.get('setName', '') self.rs_client = None self.version = Version.from_client(self.client) if self.replica_set_name: self.is_rs = True self.rs_client = pymongo.MongoClient( pair, replicaSet=self.replica_set_name) nodes = [partition_node(node) for node in self.ismaster.get('hosts', [])] nodes.extend([partition_node(node) for node in self.ismaster.get('passives', [])]) nodes.extend([partition_node(node) for node in self.ismaster.get('arbiters', [])]) self.nodes = set(nodes) self.rs_or_standalone_client = self.rs_client or self.client try: self.cmd_line = self.client.admin.command('getCmdLineOpts') except pymongo.errors.OperationFailure as e: msg = e.details.get('errmsg', '') if e.code == 13 or 'unauthorized' in msg or 'login' in msg: # Unauthorized. self.auth_enabled = True else: raise else: self.auth_enabled = self._server_started_with_auth() if self.auth_enabled: # See if db_user already exists. self.user_provided = self._check_user_provided() if not self.user_provided: roles = {} if self.version.at_least(2, 5, 3, -1): roles = {'roles': ['root']} self.client.admin.add_user(db_user, db_pwd, **roles) self.client.admin.authenticate(db_user, db_pwd) if self.rs_client: self.rs_client.admin.authenticate(db_user, db_pwd) # May not have this if OperationFailure was raised earlier. self.cmd_line = self.client.admin.command('getCmdLineOpts') if 'enableTestCommands=1' in self.cmd_line['argv']: self.test_commands_enabled = True elif 'parsed' in self.cmd_line: params = self.cmd_line['parsed'].get('setParameter', []) if 'enableTestCommands=1' in params: self.test_commands_enabled = True self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid') self.has_ipv6 = self._server_started_with_ipv6() def _check_user_provided(self): try: self.client.admin.authenticate(db_user, db_pwd) return True except pymongo.errors.OperationFailure as e: msg = e.details.get('errmsg', '') if e.code == 18 or 'auth fails' in msg: # Auth failed. return False else: raise def _server_started_with_auth(self): # MongoDB >= 2.0 if 'parsed' in self.cmd_line: parsed = self.cmd_line['parsed'] # MongoDB >= 2.6 if 'security' in parsed: security = parsed['security'] # >= rc3 if 'authorization' in security: return security['authorization'] == 'enabled' # < rc3 return (security.get('auth', False) or bool(security.get('keyFile'))) return parsed.get('auth', False) or bool(parsed.get('keyFile')) # Legacy argv = self.cmd_line['argv'] return '--auth' in argv or '--keyFile' in argv def _server_started_with_ipv6(self): if not socket.has_ipv6: return False if 'parsed' in self.cmd_line: if not self.cmd_line['parsed'].get('net', {}).get('ipv6'): return False else: if '--ipv6' not in self.cmd_line['argv']: return False # The server was started with --ipv6. Is there an IPv6 route to it? try: for info in socket.getaddrinfo(host, port): if info[0] == socket.AF_INET6: return True except socket.error: pass return False def _require(self, condition, msg, func=None): def make_wrapper(f): @wraps(f) def wrap(*args, **kwargs): # Always raise SkipTest if we can't connect to MongoDB if not self.connected: raise SkipTest("Cannot connect to MongoDB on %s" % pair) if condition: return f(*args, **kwargs) raise SkipTest(msg) return wrap if func is None: def decorate(f): return make_wrapper(f) return decorate return make_wrapper(func) def require_connection(self, func): """Run a test only if we can connect to MongoDB.""" return self._require(self.connected, "Cannot connect to MongoDB on %s" % pair, func=func) def require_version_min(self, *ver): """Run a test only if the server version is at least ``version``.""" other_version = Version(*ver) return self._require(self.version >= other_version, "Server version must be at least %s" % str(other_version)) def require_version_max(self, *ver): """Run a test only if the server version is at most ``version``.""" other_version = Version(*ver) return self._require(self.version <= other_version, "Server version must be at most %s" % str(other_version)) def require_auth(self, func): """Run a test only if the server is running with auth enabled.""" return self.check_auth_with_sharding( self._require(self.auth_enabled, "Authentication is not enabled on the server", func=func)) def require_no_auth(self, func): """Run a test only if the server is running without auth enabled.""" return self._require(not self.auth_enabled, "Authentication must not be enabled on the server", func=func) def require_replica_set(self, func): """Run a test only if the client is connected to a replica set.""" return self._require(self.is_rs, "Not connected to a replica set", func=func) def require_no_replica_set(self, func): """Run a test if the client is *not* connected to a replica set.""" return self._require( not self.is_rs, "Connected to a replica set, not a standalone mongod", func=func) def require_ipv6(self, func): """Run a test only if the client can connect to a server via IPv6.""" return self._require(self.has_ipv6, "No IPv6", func=func) def require_no_mongos(self, func): """Run a test only if the client is not connected to a mongos.""" return self._require(not self.is_mongos, "Must be connected to a mongod, not a mongos", func=func) def require_mongos(self, func): """Run a test only if the client is connected to a mongos.""" return self._require(self.is_mongos, "Must be connected to a mongos", func=func) def check_auth_with_sharding(self, func): """Skip a test when connected to mongos < 2.0 and running with auth.""" condition = not (self.auth_enabled and self.is_mongos and self.version < (2,)) return self._require(condition, "Auth with sharding requires MongoDB >= 2.0.0", func=func) def require_test_commands(self, func): """Run a test only if the server has test commands enabled.""" return self._require(self.test_commands_enabled, "Test commands must be enabled", func=func) # Reusable client context client_context = ClientContext() class IntegrationTest(unittest.TestCase): """Base class for TestCases that need a connection to MongoDB to pass.""" @classmethod @client_context.require_connection def setUpClass(cls): cls.client = client_context.rs_or_standalone_client cls.db = cls.client.pymongo_test class MockClientTest(unittest.TestCase): """Base class for TestCases that use MockClient. This class is *not* an IntegrationTest: if properly written, MockClient tests do not require a running server. The class temporarily overrides HEARTBEAT_FREQUENCY to speed up tests. """ def setUp(self): super(MockClientTest, self).setUp() self.client_knobs = client_knobs( heartbeat_frequency=0.001) self.client_knobs.enable() def tearDown(self): self.client_knobs.disable() super(MockClientTest, self).tearDown() def setup(): warnings.resetwarnings() warnings.simplefilter("always") def teardown(): c = client_context.client c.drop_database("pymongo-pooling-tests") c.drop_database("pymongo_test") c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.drop_database("pymongo_test_mike") c.drop_database("pymongo_test_bernie") if client_context.auth_enabled and not client_context.user_provided: c.admin.remove_user(db_user) class PymongoTestRunner(unittest.TextTestRunner): def run(self, test): setup() result = super(PymongoTestRunner, self).run(test) try: teardown() finally: return result def test_cases(suite): """Iterator over all TestCases within a TestSuite.""" for suite_or_case in suite._tests: if isinstance(suite_or_case, unittest.TestCase): # unittest.TestCase yield suite_or_case else: # unittest.TestSuite for case in test_cases(suite_or_case): yield case pymongo-3.2/test/test_timestamp.py0000644000175000017500000000527112630145074021356 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Timestamp class.""" import datetime import sys import copy import pickle sys.path[0:0] = [""] from bson.timestamp import Timestamp from bson.tz_util import utc from test import unittest class TestTimestamp(unittest.TestCase): def test_timestamp(self): t = Timestamp(123, 456) self.assertEqual(t.time, 123) self.assertEqual(t.inc, 456) self.assertTrue(isinstance(t, Timestamp)) def test_datetime(self): d = datetime.datetime(2010, 5, 5, tzinfo=utc) t = Timestamp(d, 0) self.assertEqual(1273017600, t.time) self.assertEqual(d, t.as_datetime()) def test_datetime_copy_pickle(self): d = datetime.datetime(2010, 5, 5, tzinfo=utc) t = Timestamp(d, 0) dc = copy.deepcopy(d) self.assertEqual(dc, t.as_datetime()) for protocol in [0, 1, 2, -1]: pkl = pickle.dumps(d, protocol=protocol) dp = pickle.loads(pkl) self.assertEqual(dp, t.as_datetime()) def test_exceptions(self): self.assertRaises(TypeError, Timestamp) self.assertRaises(TypeError, Timestamp, None, 123) self.assertRaises(TypeError, Timestamp, 1.2, 123) self.assertRaises(TypeError, Timestamp, 123, None) self.assertRaises(TypeError, Timestamp, 123, 1.2) self.assertRaises(ValueError, Timestamp, 0, -1) self.assertRaises(ValueError, Timestamp, -1, 0) self.assertTrue(Timestamp(0, 0)) def test_equality(self): t = Timestamp(1, 1) self.assertNotEqual(t, Timestamp(0, 1)) self.assertNotEqual(t, Timestamp(1, 0)) self.assertEqual(t, Timestamp(1, 1)) # Explicitly test inequality self.assertFalse(t != Timestamp(1, 1)) def test_hash(self): self.assertEqual(hash(Timestamp(1, 2)), hash(Timestamp(1, 2))) self.assertNotEqual(hash(Timestamp(1, 2)), hash(Timestamp(1, 3))) self.assertNotEqual(hash(Timestamp(1, 2)), hash(Timestamp(2, 2))) def test_repr(self): t = Timestamp(0, 0) self.assertEqual(repr(t), "Timestamp(0, 0)") if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_son.py0000644000175000017500000001625612630145074020157 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the son module.""" import copy import pickle import re import sys sys.path[0:0] = [""] from bson.py3compat import b from bson.son import SON from test import SkipTest, unittest class TestSON(unittest.TestCase): def test_ordered_dict(self): a1 = SON() a1["hello"] = "world" a1["mike"] = "awesome" a1["hello_"] = "mike" self.assertEqual(list(a1.items()), [("hello", "world"), ("mike", "awesome"), ("hello_", "mike")]) b2 = SON({"hello": "world"}) self.assertEqual(b2["hello"], "world") self.assertRaises(KeyError, lambda: b2["goodbye"]) def test_equality(self): a1 = SON({"hello": "world"}) b2 = SON((('hello', 'world'), ('mike', 'awesome'), ('hello_', 'mike'))) self.assertEqual(a1, SON({"hello": "world"})) self.assertEqual(b2, SON((('hello', 'world'), ('mike', 'awesome'), ('hello_', 'mike')))) self.assertEqual(b2, dict((('hello_', 'mike'), ('mike', 'awesome'), ('hello', 'world')))) self.assertNotEqual(a1, b2) self.assertNotEqual(b2, SON((('hello_', 'mike'), ('mike', 'awesome'), ('hello', 'world')))) # Explicitly test inequality self.assertFalse(a1 != SON({"hello": "world"})) self.assertFalse(b2 != SON((('hello', 'world'), ('mike', 'awesome'), ('hello_', 'mike')))) self.assertFalse(b2 != dict((('hello_', 'mike'), ('mike', 'awesome'), ('hello', 'world')))) # Embedded SON. d4 = SON([('blah', {'foo': SON()})]) self.assertEqual(d4, {'blah': {'foo': {}}}) self.assertEqual(d4, {'blah': {'foo': SON()}}) self.assertNotEqual(d4, {'blah': {'foo': []}}) # Original data unaffected. self.assertEqual(SON, d4['blah']['foo'].__class__) def test_to_dict(self): a1 = SON() b2 = SON([("blah", SON())]) c3 = SON([("blah", [SON()])]) d4 = SON([("blah", {"foo": SON()})]) self.assertEqual({}, a1.to_dict()) self.assertEqual({"blah": {}}, b2.to_dict()) self.assertEqual({"blah": [{}]}, c3.to_dict()) self.assertEqual({"blah": {"foo": {}}}, d4.to_dict()) self.assertEqual(dict, a1.to_dict().__class__) self.assertEqual(dict, b2.to_dict()["blah"].__class__) self.assertEqual(dict, c3.to_dict()["blah"][0].__class__) self.assertEqual(dict, d4.to_dict()["blah"]["foo"].__class__) # Original data unaffected. self.assertEqual(SON, d4['blah']['foo'].__class__) def test_pickle(self): simple_son = SON([]) complex_son = SON([('son', simple_son), ('list', [simple_son, simple_son])]) for protocol in range(pickle.HIGHEST_PROTOCOL + 1): pickled = pickle.loads(pickle.dumps(complex_son, protocol=protocol)) self.assertEqual(pickled['son'], pickled['list'][0]) self.assertEqual(pickled['son'], pickled['list'][1]) def test_pickle_backwards_compatability(self): # For a full discussion see http://bugs.python.org/issue6137 if sys.version.startswith('3.0'): raise SkipTest("Python 3.0.x can't unpickle " "objects pickled in Python 2.x.") # This string was generated by pickling a SON object in pymongo # version 2.1.1 pickled_with_2_1_1 = b( "ccopy_reg\n_reconstructor\np0\n(cbson.son\nSON\np1\n" "c__builtin__\ndict\np2\n(dp3\ntp4\nRp5\n(dp6\n" "S'_SON__keys'\np7\n(lp8\nsb." ) son_2_1_1 = pickle.loads(pickled_with_2_1_1) self.assertEqual(son_2_1_1, SON([])) def test_copying(self): simple_son = SON([]) complex_son = SON([('son', simple_son), ('list', [simple_son, simple_son])]) regex_son = SON([("x", re.compile("^hello.*"))]) reflexive_son = SON([('son', simple_son)]) reflexive_son["reflexive"] = reflexive_son simple_son1 = copy.copy(simple_son) self.assertEqual(simple_son, simple_son1) complex_son1 = copy.copy(complex_son) self.assertEqual(complex_son, complex_son1) regex_son1 = copy.copy(regex_son) self.assertEqual(regex_son, regex_son1) reflexive_son1 = copy.copy(reflexive_son) self.assertEqual(reflexive_son, reflexive_son1) # Test deepcopying simple_son1 = copy.deepcopy(simple_son) self.assertEqual(simple_son, simple_son1) regex_son1 = copy.deepcopy(regex_son) self.assertEqual(regex_son, regex_son1) complex_son1 = copy.deepcopy(complex_son) self.assertEqual(complex_son, complex_son1) reflexive_son1 = copy.deepcopy(reflexive_son) self.assertEqual(list(reflexive_son), list(reflexive_son1)) self.assertEqual(id(reflexive_son1), id(reflexive_son1["reflexive"])) def test_iteration(self): """ Test __iter__ """ # test success case test_son = SON([(1, 100), (2, 200), (3, 300)]) for ele in test_son: self.assertEqual(ele * 100, test_son[ele]) def test_contains_has(self): """ has_key and __contains__ """ test_son = SON([(1, 100), (2, 200), (3, 300)]) self.assertIn(1, test_son) self.assertTrue(2 in test_son, "in failed") self.assertFalse(22 in test_son, "in succeeded when it shouldn't") self.assertTrue(test_son.has_key(2), "has_key failed") self.assertFalse(test_son.has_key(22), "has_key succeeded when it shouldn't") def test_clears(self): """ Test clear() """ test_son = SON([(1, 100), (2, 200), (3, 300)]) test_son.clear() self.assertNotIn(1, test_son) self.assertEqual(0, len(test_son)) self.assertEqual(0, len(test_son.keys())) self.assertEqual({}, test_son.to_dict()) def test_len(self): """ Test len """ test_son = SON() self.assertEqual(0, len(test_son)) test_son = SON([(1, 100), (2, 200), (3, 300)]) self.assertEqual(3, len(test_son)) test_son.popitem() self.assertEqual(2, len(test_son)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_code.py0000644000175000017500000000661512630145074020270 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for the Code wrapper.""" import sys sys.path[0:0] = [""] from bson.code import Code from bson.py3compat import u from test import unittest class TestCode(unittest.TestCase): def test_types(self): self.assertRaises(TypeError, Code, 5) self.assertRaises(TypeError, Code, None) self.assertRaises(TypeError, Code, "aoeu", 5) self.assertRaises(TypeError, Code, u("aoeu"), 5) self.assertTrue(Code("aoeu")) self.assertTrue(Code(u("aoeu"))) self.assertTrue(Code("aoeu", {})) self.assertTrue(Code(u("aoeu"), {})) def test_read_only(self): c = Code("blah") def set_c(): c.scope = 5 self.assertRaises(AttributeError, set_c) def test_code(self): a_string = "hello world" a_code = Code("hello world") self.assertTrue(a_code.startswith("hello")) self.assertTrue(a_code.endswith("world")) self.assertTrue(isinstance(a_code, Code)) self.assertFalse(isinstance(a_string, Code)) self.assertEqual(a_code.scope, {}) a_code.scope["my_var"] = 5 self.assertEqual(a_code.scope, {"my_var": 5}) def test_repr(self): c = Code("hello world") self.assertEqual(repr(c), "Code('hello world', {})") c.scope["foo"] = "bar" self.assertEqual(repr(c), "Code('hello world', {'foo': 'bar'})") c = Code("hello world", {"blah": 3}) self.assertEqual(repr(c), "Code('hello world', {'blah': 3})") c = Code("\x08\xFF") self.assertEqual(repr(c), "Code(%s, {})" % (repr("\x08\xFF"),)) def test_equality(self): b = Code("hello") c = Code("hello", {"foo": 5}) self.assertNotEqual(b, c) self.assertEqual(c, Code("hello", {"foo": 5})) self.assertNotEqual(c, Code("hello", {"foo": 6})) self.assertEqual(b, Code("hello")) self.assertEqual(b, Code("hello", {})) self.assertNotEqual(b, Code("hello ")) self.assertNotEqual("hello", Code("hello")) # Explicitly test inequality self.assertFalse(c != Code("hello", {"foo": 5})) self.assertFalse(b != Code("hello")) self.assertFalse(b != Code("hello", {})) def test_hash(self): self.assertRaises(TypeError, hash, Code("hello world")) def test_scope_preserved(self): a = Code("hello") b = Code("hello", {"foo": 5}) self.assertEqual(a, Code(a)) self.assertEqual(b, Code(b)) self.assertNotEqual(a, Code(b)) self.assertNotEqual(b, Code(a)) def test_scope_kwargs(self): self.assertEqual({"a": 1}, Code("", a=1).scope) self.assertEqual({"a": 1}, Code("", {"a": 2}, a=1).scope) self.assertEqual({"a": 1, "b": 2, "c": 3}, Code("", {"b": 2}, a=1, c=3).scope) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_auth.py0000644000175000017500000004422012630145074020311 0ustar behackettbehackett00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Authentication Tests.""" import os import sys import threading try: from urllib.parse import quote_plus except ImportError: # Python 2 from urllib import quote_plus sys.path[0:0] = [""] from pymongo import MongoClient from pymongo.auth import HAVE_KERBEROS, _build_credentials_tuple from pymongo.errors import OperationFailure from pymongo.read_preferences import ReadPreference from test import client_context, host, port, SkipTest, unittest, Version from test.utils import delay # YOU MUST RUN KINIT BEFORE RUNNING GSSAPI TESTS. GSSAPI_HOST = os.environ.get('GSSAPI_HOST') GSSAPI_PORT = int(os.environ.get('GSSAPI_PORT', '27017')) PRINCIPAL = os.environ.get('PRINCIPAL') SASL_HOST = os.environ.get('SASL_HOST') SASL_PORT = int(os.environ.get('SASL_PORT', '27017')) SASL_USER = os.environ.get('SASL_USER') SASL_PASS = os.environ.get('SASL_PASS') SASL_DB = os.environ.get('SASL_DB', '$external') class AutoAuthenticateThread(threading.Thread): """Used in testing threaded authentication. This does collection.find_one() with a 1-second delay to ensure it must check out and authenticate multiple sockets from the pool concurrently. :Parameters: `collection`: An auth-protected collection containing one document. """ def __init__(self, collection): super(AutoAuthenticateThread, self).__init__() self.collection = collection self.success = False def run(self): assert self.collection.find_one({'$where': delay(1)}) is not None self.success = True class TestGSSAPI(unittest.TestCase): @classmethod def setUpClass(cls): if not HAVE_KERBEROS: raise SkipTest('Kerberos module not available.') if not GSSAPI_HOST or not PRINCIPAL: raise SkipTest('Must set GSSAPI_HOST and PRINCIPAL to test GSSAPI') def test_credentials_hashing(self): # GSSAPI credentials are properly hashed. creds0 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {}) creds1 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {'authmechanismproperties': {'SERVICE_NAME': 'A'}}) creds2 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {'authmechanismproperties': {'SERVICE_NAME': 'A'}}) creds3 = _build_credentials_tuple( 'GSSAPI', '', 'user', 'pass', {'authmechanismproperties': {'SERVICE_NAME': 'B'}}) self.assertEqual(1, len(set([creds1, creds2]))) self.assertEqual(3, len(set([creds0, creds1, creds2, creds3]))) def test_gssapi_simple(self): # Call authenticate() without authMechanismProperties. client = MongoClient(GSSAPI_HOST, GSSAPI_PORT) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) client.test.collection.find_one() # Log in using URI, without authMechanismProperties. uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.test.collection.find_one() # Call authenticate() with authMechanismProperties. self.assertTrue(client.test.authenticate( PRINCIPAL, mechanism='GSSAPI', authMechanismProperties='SERVICE_NAME:mongodb')) client.test.collection.find_one() # Log in using URI, with authMechanismProperties. uri = ('mongodb://%s@%s:%d/?authMechanism=' 'GSSAPI;authMechanismProperties' '=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT)) client = MongoClient(uri) client.test.collection.find_one() set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(GSSAPI_HOST, port=GSSAPI_PORT, replicaSet=set_name) # Without authMechanismProperties self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoClient(uri) client.database_names() # With authMechanismProperties self.assertTrue(client.test.authenticate( PRINCIPAL, mechanism='GSSAPI', authMechanismProperties='SERVICE_NAME:mongodb')) client.database_names() uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet' '=%s;authMechanismProperties' '=SERVICE_NAME:mongodb' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT, str(set_name))) client = MongoClient(uri) client.database_names() def test_gssapi_threaded(self): client = MongoClient(GSSAPI_HOST) self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) # Need one document in the collection. AutoAuthenticateThread does # collection.find_one with a 1-second delay, forcing it to check out # multiple sockets from the pool concurrently, proving that # auto-authentication works with GSSAPI. collection = client.test.collection collection.drop() collection.insert_one({'_id': 1}) threads = [] for _ in range(4): threads.append(AutoAuthenticateThread(collection)) for thread in threads: thread.start() for thread in threads: thread.join() self.assertTrue(thread.success) set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(GSSAPI_HOST, replicaSet=set_name, readPreference='secondary') self.assertTrue(client.test.authenticate(PRINCIPAL, mechanism='GSSAPI')) self.assertTrue(client.test.command('dbstats')) threads = [] for _ in range(4): threads.append(AutoAuthenticateThread(collection)) for thread in threads: thread.start() for thread in threads: thread.join() self.assertTrue(thread.success) class TestSASLPlain(unittest.TestCase): @classmethod def setUpClass(cls): if not SASL_HOST or not SASL_USER or not SASL_PASS: raise SkipTest('Must set SASL_HOST, ' 'SASL_USER, and SASL_PASS to test SASL') def test_sasl_plain(self): client = MongoClient(SASL_HOST, SASL_PORT) self.assertTrue(client.ldap.authenticate(SASL_USER, SASL_PASS, SASL_DB, 'PLAIN')) client.ldap.test.find_one() uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;' 'authSource=%s' % (quote_plus(SASL_USER), quote_plus(SASL_PASS), SASL_HOST, SASL_PORT, SASL_DB)) client = MongoClient(uri) client.ldap.test.find_one() set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(SASL_HOST, port=SASL_PORT, replicaSet=set_name) self.assertTrue(client.ldap.authenticate(SASL_USER, SASL_PASS, SASL_DB, 'PLAIN')) client.ldap.test.find_one() uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;' 'authSource=%s;replicaSet=%s' % (quote_plus(SASL_USER), quote_plus(SASL_PASS), SASL_HOST, SASL_PORT, SASL_DB, str(set_name))) client = MongoClient(uri) client.ldap.test.find_one() def test_sasl_plain_bad_credentials(self): client = MongoClient(SASL_HOST, SASL_PORT) # Bad username self.assertRaises(OperationFailure, client.ldap.authenticate, 'not-user', SASL_PASS, SASL_DB, 'PLAIN') self.assertRaises(OperationFailure, client.ldap.test.find_one) self.assertRaises(OperationFailure, client.ldap.test.insert_one, {"failed": True}) # Bad password self.assertRaises(OperationFailure, client.ldap.authenticate, SASL_USER, 'not-pwd', SASL_DB, 'PLAIN') self.assertRaises(OperationFailure, client.ldap.test.find_one) self.assertRaises(OperationFailure, client.ldap.test.insert_one, {"failed": True}) def auth_string(user, password): uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;' 'authSource=%s' % (quote_plus(user), quote_plus(password), SASL_HOST, SASL_PORT, SASL_DB)) return uri bad_user = MongoClient(auth_string('not-user', SASL_PASS)) bad_pwd = MongoClient(auth_string(SASL_USER, 'not-pwd')) # OperationFailure raised upon connecting. self.assertRaises(OperationFailure, bad_user.admin.command, 'ismaster') self.assertRaises(OperationFailure, bad_pwd.admin.command, 'ismaster') class TestSCRAMSHA1(unittest.TestCase): @client_context.require_auth @client_context.require_version_min(2, 7, 2) def setUp(self): self.replica_set_name = client_context.replica_set_name # Before 2.7.7, SCRAM-SHA-1 had to be enabled from the command line. if client_context.version < Version(2, 7, 7): cmd_line = client_context.cmd_line if 'SCRAM-SHA-1' not in cmd_line.get( 'parsed', {}).get('setParameter', {}).get('authenticationMechanisms', ''): raise SkipTest('SCRAM-SHA-1 mechanism not enabled') client = client_context.rs_or_standalone_client client.pymongo_test.add_user( 'user', 'pass', roles=['userAdmin', 'readWrite'], writeConcern={'w': client_context.w}) def test_scram_sha1(self): client = MongoClient(host, port) self.assertTrue(client.pymongo_test.authenticate( 'user', 'pass', mechanism='SCRAM-SHA-1')) client.pymongo_test.command('dbstats') client = MongoClient('mongodb://user:pass@%s:%d/pymongo_test' '?authMechanism=SCRAM-SHA-1' % (host, port)) client.pymongo_test.command('dbstats') if self.replica_set_name: client = MongoClient(host, port, replicaSet='%s' % (self.replica_set_name,)) self.assertTrue(client.pymongo_test.authenticate( 'user', 'pass', mechanism='SCRAM-SHA-1')) client.pymongo_test.command('dbstats') uri = ('mongodb://user:pass' '@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1' '&replicaSet=%s' % (host, port, self.replica_set_name)) client = MongoClient(uri) client.pymongo_test.command('dbstats') db = client.get_database( 'pymongo_test', read_preference=ReadPreference.SECONDARY) db.command('dbstats') def tearDown(self): client_context.rs_or_standalone_client.pymongo_test.remove_user('user') class TestAuthURIOptions(unittest.TestCase): @client_context.require_auth def setUp(self): client = MongoClient(host, port) response = client.admin.command('ismaster') self.replica_set_name = str(response.get('setName', '')) client_context.client.admin.add_user('admin', 'pass', roles=['userAdminAnyDatabase', 'dbAdminAnyDatabase', 'readWriteAnyDatabase', 'clusterAdmin']) client.admin.authenticate('admin', 'pass') client.pymongo_test.add_user('user', 'pass', roles=['userAdmin', 'readWrite']) if self.replica_set_name: # GLE requires authentication. client.admin.authenticate('admin', 'pass') # Make sure the admin user is replicated after calling add_user # above. This avoids a race in the replica set tests below. client.admin.command('getLastError', w=len(response['hosts'])) self.client = client def tearDown(self): self.client.admin.authenticate('admin', 'pass') self.client.pymongo_test.remove_user('user') self.client.admin.remove_user('admin') self.client.pymongo_test.logout() self.client.admin.logout() self.client = None def test_uri_options(self): # Test default to admin client = MongoClient('mongodb://admin:pass@%s:%d' % (host, port)) self.assertTrue(client.admin.command('dbstats')) if self.replica_set_name: uri = ('mongodb://admin:pass' '@%s:%d/?replicaSet=%s' % (host, port, self.replica_set_name)) client = MongoClient(uri) self.assertTrue(client.admin.command('dbstats')) db = client.get_database( 'admin', read_preference=ReadPreference.SECONDARY) self.assertTrue(db.command('dbstats')) # Test explicit database uri = 'mongodb://user:pass@%s:%d/pymongo_test' % (host, port) client = MongoClient(uri) self.assertRaises(OperationFailure, client.admin.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) if self.replica_set_name: uri = ('mongodb://user:pass@%s:%d' '/pymongo_test?replicaSet=%s' % (host, port, self.replica_set_name)) client = MongoClient(uri) self.assertRaises(OperationFailure, client.admin.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) db = client.get_database( 'pymongo_test', read_preference=ReadPreference.SECONDARY) self.assertTrue(db.command('dbstats')) # Test authSource uri = ('mongodb://user:pass@%s:%d' '/pymongo_test2?authSource=pymongo_test' % (host, port)) client = MongoClient(uri) self.assertRaises(OperationFailure, client.pymongo_test2.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) if self.replica_set_name: uri = ('mongodb://user:pass@%s:%d/pymongo_test2?replicaSet=' '%s;authSource=pymongo_test' % (host, port, self.replica_set_name)) client = MongoClient(uri) self.assertRaises(OperationFailure, client.pymongo_test2.command, 'dbstats') self.assertTrue(client.pymongo_test.command('dbstats')) db = client.get_database( 'pymongo_test', read_preference=ReadPreference.SECONDARY) self.assertTrue(db.command('dbstats')) class TestDelegatedAuth(unittest.TestCase): @client_context.require_auth @client_context.require_version_max(2, 5, 3) @client_context.require_version_min(2, 4, 0) def setUp(self): self.client = client_context.rs_or_standalone_client def tearDown(self): self.client.pymongo_test.remove_user('user') self.client.pymongo_test2.remove_user('user') self.client.pymongo_test2.foo.drop() def test_delegated_auth(self): self.client.pymongo_test2.foo.drop() self.client.pymongo_test2.foo.insert_one({}) # User definition with no roles in pymongo_test. self.client.pymongo_test.add_user('user', 'pass', roles=[]) # Delegate auth to pymongo_test. self.client.pymongo_test2.add_user('user', userSource='pymongo_test', roles=['read']) auth_c = MongoClient(host, port) self.assertRaises(OperationFailure, auth_c.pymongo_test2.foo.find_one) # Auth must occur on the db where the user is defined. self.assertRaises(OperationFailure, auth_c.pymongo_test2.authenticate, 'user', 'pass') # Auth directly self.assertTrue(auth_c.pymongo_test.authenticate('user', 'pass')) self.assertTrue(auth_c.pymongo_test2.foo.find_one()) auth_c.pymongo_test.logout() self.assertRaises(OperationFailure, auth_c.pymongo_test2.foo.find_one) # Auth using source self.assertTrue(auth_c.pymongo_test2.authenticate( 'user', 'pass', source='pymongo_test')) self.assertTrue(auth_c.pymongo_test2.foo.find_one()) # Must logout from the db authenticate was called on. auth_c.pymongo_test2.logout() self.assertRaises(OperationFailure, auth_c.pymongo_test2.foo.find_one) if __name__ == "__main__": unittest.main() pymongo-3.2/test/high_availability/0000755000175000017500000000000012631423130021377 5ustar behackettbehackett00000000000000pymongo-3.2/test/high_availability/ha_tools.py0000644000175000017500000003473612630145074023605 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for testing high availability in PyMongo.""" import os import random import shutil import signal import socket import subprocess import sys import time from stat import S_IRUSR import pymongo import pymongo.errors from pymongo.read_preferences import ReadPreference from test.utils import connected home = os.environ.get('HOME') default_dbpath = os.path.join(home, 'data', 'pymongo_high_availability') dbpath = os.environ.get('DBPATH', default_dbpath) default_logpath = os.path.join(home, 'log', 'pymongo_high_availability') logpath = os.path.expanduser(os.environ.get('LOGPATH', default_logpath)) hostname = os.environ.get('HOSTNAME', 'localhost') port = int(os.environ.get('DBPORT', 27017)) mongod = os.path.expanduser(os.environ.get('MONGOD', 'mongod')) mongos = os.path.expanduser(os.environ.get('MONGOS', 'mongos')) replica_set_name = os.environ.get('SETNAME', 'repl0') ha_tools_debug = bool(os.environ.get('HA_TOOLS_DEBUG')) nodes = {} routers = {} cur_port = port key_file = None try: from subprocess import DEVNULL # Python 3. except ImportError: DEVNULL = open(os.devnull, 'wb') def kill_members(members, sig, hosts=nodes): for member in sorted(members): try: if ha_tools_debug: print('killing %s' % (member,)) proc = hosts[member]['proc'] if 'java' in sys.platform: # _process is a wrapped java.lang.UNIXProcess. proc._process.destroy() # Not sure if cygwin makes sense here... elif sys.platform in ('win32', 'cygwin'): os.kill(proc.pid, signal.CTRL_C_EVENT) else: os.kill(proc.pid, sig) except OSError: if ha_tools_debug: print('%s already dead?' % (member,)) def kill_all_members(): kill_members(nodes.keys(), 2, nodes) kill_members(routers.keys(), 2, routers) def wait_for(proc, port_num): trys = 0 while proc.poll() is None and trys < 160: trys += 1 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: try: s.connect((hostname, port_num)) return True except (IOError, socket.error): time.sleep(0.25) finally: s.close() kill_all_members() return False def start_subprocess(cmd): """Run cmd (a list of strings) and return a Popen instance.""" return subprocess.Popen(cmd, stdout=DEVNULL, stderr=DEVNULL) def start_replica_set(members, auth=False, fresh=True): global cur_port global key_file if fresh: if os.path.exists(dbpath): try: shutil.rmtree(dbpath) except OSError: pass try: os.makedirs(dbpath) except OSError as exc: print(exc) print("\tWhile creating %s" % (dbpath,)) if auth: key_file = os.path.join(dbpath, 'key.txt') if not os.path.exists(key_file): with open(key_file, 'w') as f: f.write("my super secret system password") os.chmod(key_file, S_IRUSR) for i in range(len(members)): host = '%s:%d' % (hostname, cur_port) members[i].update({'_id': i, 'host': host}) path = os.path.join(dbpath, 'db' + str(i)) if not os.path.exists(path): os.makedirs(path) member_logpath = os.path.join(logpath, 'db' + str(i) + '.log') if not os.path.exists(os.path.dirname(member_logpath)): os.makedirs(os.path.dirname(member_logpath)) cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--replSet', replica_set_name, '--nojournal', '--oplogSize', '64', '--logappend', '--logpath', member_logpath] if auth: cmd += ['--keyFile', key_file] if ha_tools_debug: print('starting %s' % (' '.join(cmd),)) proc = start_subprocess(cmd) nodes[host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) cur_port += 1 if not res: return None config = {'_id': replica_set_name, 'members': members} primary = members[0]['host'] c = pymongo.MongoClient(primary) try: if ha_tools_debug: print('rs.initiate(%s)' % (config,)) c.admin.command('replSetInitiate', config) except pymongo.errors.OperationFailure as exc: # Already initialized from a previous run? if ha_tools_debug: print(exc) expected_arbiters = 0 for member in members: if member.get('arbiterOnly'): expected_arbiters += 1 expected_secondaries = len(members) - expected_arbiters - 1 # Wait a minute for replica set to come up. patience = 1 for i in range(int(patience * 60 / 2)): time.sleep(2) try: if (get_primary() and len(get_secondaries()) == expected_secondaries and len(get_arbiters()) == expected_arbiters): break except pymongo.errors.ConnectionFailure: # Keep waiting pass if ha_tools_debug: print('waiting for RS %s' % (i,)) else: kill_all_members() raise Exception( "Replica set still not initalized after %s minutes" % patience) return primary, replica_set_name def create_sharded_cluster(num_routers=3): global cur_port if not os.path.exists(logpath): os.makedirs(logpath) # Start a config server configdb_host = '%s:%d' % (hostname, cur_port) path = os.path.join(dbpath, 'configdb') if not os.path.exists(path): os.makedirs(path) configdb_logpath = os.path.join(logpath, 'configdb.log') cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--nojournal', '--logappend', '--logpath', configdb_logpath] proc = start_subprocess(cmd) nodes[configdb_host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) if not res: return None # ...and a shard server cur_port = cur_port + 1 shard_host = '%s:%d' % (hostname, cur_port) path = os.path.join(dbpath, 'shard1') if not os.path.exists(path): os.makedirs(path) db_logpath = os.path.join(logpath, 'shard1.log') cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--nojournal', '--logappend', '--logpath', db_logpath] proc = start_subprocess(cmd) nodes[shard_host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) if not res: return None # ...and a few mongos instances cur_port = cur_port + 1 for i in range(num_routers): cur_port = cur_port + i host = '%s:%d' % (hostname, cur_port) mongos_logpath = os.path.join(logpath, 'mongos' + str(i) + '.log') cmd = [mongos, '--port', str(cur_port), '--logappend', '--logpath', mongos_logpath, '--configdb', configdb_host] proc = start_subprocess(cmd) routers[host] = {'proc': proc, 'cmd': cmd} res = wait_for(proc, cur_port) if not res: return None # Add the shard client = pymongo.MongoClient(host) try: client.admin.command({'addshard': shard_host}) except pymongo.errors.OperationFailure: # Already configured. pass return get_mongos_seed_list() # Connect to a random member def get_client(): # Attempt a direct connection to each node until one succeeds. Using a # non-PRIMARY read preference allows us to use the node even if it's a # secondary. for i, node in enumerate(nodes.keys()): try: return connected( pymongo.MongoClient( node, read_preference=ReadPreference.PRIMARY_PREFERRED)) except pymongo.errors.ConnectionFailure: if i == len(nodes) - 1: raise def get_mongos_seed_list(): members = routers.keys() return ','.join(members) def kill_mongos(host): kill_members([host], 2, hosts=routers) return host def restart_mongos(host): restart_members([host], True) def get_members_in_state(state): status = get_client().admin.command('replSetGetStatus') members = status['members'] return [k['name'] for k in members if k['state'] == state] def get_primary(): try: primaries = get_members_in_state(1) assert len(primaries) <= 1 if primaries: return primaries[0] except (pymongo.errors.ConnectionFailure, pymongo.errors.OperationFailure): pass return None def wait_for_primary(): for _ in range(30): time.sleep(1) if get_primary(): break else: raise AssertionError("Primary didn't come back up") def get_random_secondary(): secondaries = get_members_in_state(2) if len(secondaries): return random.choice(secondaries) return None def get_secondaries(): return get_members_in_state(2) def get_arbiters(): return get_members_in_state(7) def get_recovering(): return get_members_in_state(3) def get_passives(): return get_client().admin.command('ismaster').get('passives', []) def get_hosts(): return get_client().admin.command('ismaster').get('hosts', []) def get_hidden_members(): # Both 'hidden' and 'slaveDelay' secondaries = get_secondaries() readers = get_hosts() + get_passives() for member in readers: try: secondaries.remove(member) except: # Skip primary pass return secondaries def get_tags(member): config = get_client().local.system.replset.find_one() for m in config['members']: if m['host'] == member: return m.get('tags', {}) raise Exception('member %s not in config' % repr(member)) def kill_primary(sig=2): primary = get_primary() kill_members([primary], sig) return primary def kill_secondary(sig=2): secondary = get_random_secondary() kill_members([secondary], sig) return secondary def kill_all_secondaries(sig=2): secondaries = get_secondaries() kill_members(secondaries, sig) return secondaries # TODO: refactor w/ start_replica_set def add_member(auth=False): global cur_port host = '%s:%d' % (hostname, cur_port) primary = get_primary() assert primary c = pymongo.MongoClient(primary) config = c.local.system.replset.find_one() _id = max([member['_id'] for member in config['members']]) + 1 member = {'_id': _id, 'host': host} path = os.path.join(dbpath, 'db' + str(_id)) if os.path.exists(path): shutil.rmtree(path) os.makedirs(path) member_logpath = os.path.join(logpath, 'db' + str(_id) + '.log') if not os.path.exists(os.path.dirname(member_logpath)): os.makedirs(os.path.dirname(member_logpath)) cmd = [mongod, '--dbpath', path, '--port', str(cur_port), '--replSet', replica_set_name, '--nojournal', '--oplogSize', '64', '--logappend', '--logpath', member_logpath] if auth: cmd += ['--keyFile', key_file] if ha_tools_debug: print('starting %s' % ' '.join(cmd)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) nodes[host] = {'proc': proc, 'cmd': cmd, 'dbpath': path} res = wait_for(proc, cur_port) cur_port += 1 config['members'].append(member) config['version'] += 1 if ha_tools_debug: print({'replSetReconfig': config}) response = c.admin.command({'replSetReconfig': config}) if ha_tools_debug: print(response) if not res: return None return host def stepdown_primary(): primary = get_primary() if primary: if ha_tools_debug: print('stepping down primary: %s' % (primary,)) c = pymongo.MongoClient(primary) for _ in range(10): try: c.admin.command('replSetStepDown', 20) except pymongo.errors.OperationFailure as exc: if ha_tools_debug: print('Code %s from replSetStepDown: %s' % (exc.code, exc)) print('Trying again in one second....') time.sleep(1) except pymongo.errors.ConnectionFailure as exc: # replSetStepDown causes mongod to close all connections. if ha_tools_debug: print('Exception from replSetStepDown: %s' % exc) # Seems to have succeeded. break else: raise AssertionError("Couldn't complete replSetStepDown") if ha_tools_debug: print('\tcalled replSetStepDown') elif ha_tools_debug: print('stepdown_primary() found no primary') def set_maintenance(member, value): """Put a member into RECOVERING state if value is True, else normal state. """ c = pymongo.MongoClient(member) c.admin.command('replSetMaintenance', value) start = time.time() while value != (member in get_recovering()): assert (time.time() - start) <= 10, ( "Member %s never switched state" % member) time.sleep(0.25) def restart_members(members, router=False): restarted = [] for member in members: if router: cmd = routers[member]['cmd'] else: cmd = nodes[member]['cmd'] lockfile_path = os.path.join(nodes[member]['dbpath'], 'mongod.lock') if os.path.exists(lockfile_path): os.remove(lockfile_path) proc = start_subprocess(cmd) if router: routers[member]['proc'] = proc else: nodes[member]['proc'] = proc res = wait_for(proc, int(member.split(':')[1])) if res: restarted.append(member) return restarted pymongo-3.2/test/high_availability/test_ha.py0000644000175000017500000011353312630145074023415 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test replica set operations and failures.""" # These test methods exuberantly violate the "one assert per test" rule, because # each method requires running setUp, which takes about 30 seconds to bring up # a replica set. Thus each method asserts everything we want to assert for a # given replica-set configuration. import time import ha_tools from pymongo import common from pymongo.common import partition_node from pymongo.errors import (AutoReconnect, OperationFailure, ConnectionFailure, InvalidOperation, WTimeoutError) from pymongo.mongo_client import MongoClient from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.write_concern import WriteConcern from test import unittest, utils, client_knobs from test.utils import one, wait_until, connected # To make the code terser, copy modes into module scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST def partition_nodes(nodes): """Translate from ['host:port', ...] to [(host, port), ...]""" return [partition_node(node) for node in nodes] class HATestCase(unittest.TestCase): """A test case for connections to replica sets or mongos.""" # Override default 10-second interval for faster testing... heartbeat_frequency = 0.5 # ... or disable it by setting "enable_heartbeat" to False. enable_heartbeat = True # Override this to speed up connection-failure tests. server_selection_timeout = common.SERVER_SELECTION_TIMEOUT def setUp(self): if self.enable_heartbeat: heartbeat_frequency = self.heartbeat_frequency else: # Disable periodic monitoring. heartbeat_frequency = 1e6 self.knobs = client_knobs(heartbeat_frequency=heartbeat_frequency) self.knobs.enable() def tearDown(self): ha_tools.kill_all_members() ha_tools.nodes.clear() ha_tools.routers.clear() time.sleep(1) # Let members really die. self.knobs.disable() class TestDirectConnection(HATestCase): def setUp(self): super(TestDirectConnection, self).setUp() members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_secondary_connection(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: len(self.c.secondaries), "discover secondary") # Wait for replication... w = len(self.c.secondaries) + 1 db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, ]: client = MongoClient( primary_host, primary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: primary_host == client.host, "connect to primary") self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient( secondary_host, secondary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: secondary_host == client.host, "connect to secondary") self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient( arbiter_host, arbiter_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: arbiter_host == client.host, "connect to arbiter") self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into arbiter client %s should' 'have raised exception' % (client,)) class TestPassiveAndHidden(HATestCase): def setUp(self): super(TestPassiveAndHidden, self).setUp() members = [{}, {'priority': 0}, {'arbiterOnly': True}, {'priority': 0, 'hidden': True}, {'priority': 0, 'slaveDelay': 5} ] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_passive_and_hidden(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) passives = ha_tools.get_passives() passives = partition_nodes(passives) self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: utils.assertReadFromAll(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) time.sleep(2 * self.heartbeat_frequency) utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED) class TestMonitorRemovesRecoveringMember(HATestCase): # Members in STARTUP2 or RECOVERING states are shown in the primary's # isMaster response, but aren't secondaries and shouldn't be read from. # Verify that if a secondary goes into RECOVERING mode, the Monitor removes # it from the set of readers. def setUp(self): super(TestMonitorRemovesRecoveringMember, self).setUp() members = [{}, {'priority': 0}, {'priority': 0}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_monitor_removes_recovering_member(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = partition_nodes(secondaries) utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) time.sleep(2 * self.heartbeat_frequency) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member utils.assertReadFrom(self, self.c, partition_node(secondary), mode) class TestTriggeredRefresh(HATestCase): # Verify that if a secondary goes into RECOVERING mode or if the primary # changes, the next exception triggers an immediate refresh. enable_heartbeat = False def setUp(self): super(TestTriggeredRefresh, self).setUp() members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = [ MongoClient( self.seed, replicaSet=self.name, read_preference=SECONDARY, serverSelectionTimeoutMS=self.server_selection_timeout) for _ in xrange(2)] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: wait_until( lambda: c.primary == partition_node(primary), 'connect to the primary') wait_until( lambda: one(c.secondaries) == partition_node(secondary), 'connect to the secondary') ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one) self.assertRaises(AutoReconnect, self.c_count.test.test.count) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled time.sleep(1) self.assertFalse(self.c_find_one.secondaries) self.assertEqual(partition_node(primary), self.c_find_one.primary) self.assertFalse(self.c_count.secondaries) self.assertEqual(partition_node(primary), self.c_count.primary) def test_stepdown_triggers_refresh(self): c_find_one = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) c_count = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) # We've started the primary and one secondary wait_until(lambda: len(c_find_one.secondaries), "discover secondary") wait_until(lambda: len(c_count.secondaries), "discover secondary") ha_tools.stepdown_primary() # Trigger a refresh, both with a cursor and a command. self.assertRaises(AutoReconnect, c_find_one.test.test.find_one) self.assertRaises(AutoReconnect, c_count.test.command, 'count') # Both clients detect the stepdown *AND* re-check the server # immediately, they don't just mark it Unknown. Wait for the # immediate refresh to complete - we're not waiting for the # periodic refresh, which has been disabled wait_until(lambda: len(c_find_one.secondaries) == 2, "detect two secondaries") wait_until(lambda: len(c_count.secondaries) == 2, "detect two secondaries") class TestHealthMonitor(HATestCase): def setUp(self): super(TestHealthMonitor, self).setUp() res = ha_tools.start_replica_set([{}, {}, {}]) self.seed, self.name = res def test_primary_failure(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") old_primary = c.primary old_secondaries = c.secondaries killed = ha_tools.kill_primary() self.assertTrue(bool(len(killed))) wait_until(lambda: c.primary and c.primary != old_primary, "discover new primary", timeout=30) wait_until(lambda: c.secondaries != old_secondaries, "discover new secondaries", timeout=30) def test_secondary_failure(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") primary = c.primary old_secondaries = c.secondaries killed = ha_tools.kill_secondary() time.sleep(2 * self.heartbeat_frequency) self.assertTrue(bool(len(killed))) self.assertEqual(primary, c.primary) wait_until(lambda: c.secondaries != old_secondaries, "discover new secondaries", timeout=30) old_secondaries = c.secondaries ha_tools.restart_members([killed]) self.assertEqual(primary, c.primary) wait_until(lambda: c.secondaries != old_secondaries, "discover new secondaries", timeout=30) def test_primary_stepdown(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") ha_tools.stepdown_primary() # Wait for new primary. wait_until(lambda: (ha_tools.get_primary() and c.primary == partition_node(ha_tools.get_primary())), "discover new primary", timeout=30) wait_until(lambda: len(c.secondaries) == 2, "discover new secondaries", timeout=30) class TestWritesWithFailover(HATestCase): enable_heartbeat = False def setUp(self): super(TestWritesWithFailover, self).setUp() res = ha_tools.start_replica_set([{}, {}, {}]) self.seed, self.name = res def test_writes_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") primary = c.primary w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) self.assertEqual('bar', db.test.find_one()['foo']) killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) # Wait past pool's check interval, so it throws an error from # get_socket(). time.sleep(1) # Verify that we only raise AutoReconnect, not some other error, # while we wait for new primary. for _ in xrange(10000): try: db.test.insert_one({'bar': 'baz'}) # No error, found primary. break except AutoReconnect: time.sleep(.01) else: self.fail("Couldn't connect to new primary") # Found new primary. self.assertTrue(c.primary) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar']) class TestReadWithFailover(HATestCase): def setUp(self): super(TestReadWithFailover, self).setUp() res = ha_tools.start_replica_set([{}, {}, {}]) self.seed, self.name = res def test_read_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def iter_cursor(cursor): for _ in cursor: pass return True w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) # Force replication db.test.insert_many([{'foo': i} for i in xrange(10)]) self.assertEqual(10, db.test.count()) db.read_preference = SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) next(cursor) self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor.address in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved) class TestReadPreference(HATestCase): # Speed up assertReadFrom() when no server is suitable. server_selection_timeout = 0.001 def setUp(self): super(TestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) self.w = len(self.c.secondaries) + 1 self.db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=self.w)) self.db.test.delete_many({}) self.db.test.insert_many([{'foo': i} for i in xrange(10)]) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): ServerDescription._host_to_round_trip_time[host] = ping_time_seconds def clear_ping_times(self): ServerDescription._host_to_round_trip_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) ha_tools.wait_for_primary() ha_tools.kill_members([unpartition_node(secondary)], 2) time.sleep(5) ha_tools.wait_for_primary() time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times() class TestReplicaSetAuth(HATestCase): def setUp(self): super(TestReplicaSetAuth, self).setUp() members = [ {}, {'priority': 0}, {'priority': 0}, ] res = ha_tools.start_replica_set(members, auth=True) self.c = MongoClient( res[0], replicaSet=res[1], serverSelectionTimeoutMS=self.server_selection_timeout) # Add an admin user to enable auth self.c.admin.add_user('admin', 'adminpass') self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout() def test_auth_during_failover(self): self.assertTrue(self.db.authenticate('user', 'userpass')) db = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=3, wtimeout=3000)) self.assertTrue(db.foo.insert_one({'foo': 'bar'})) self.db.logout() self.assertRaises(OperationFailure, self.db.foo.find_one) primary = self.c.primary ha_tools.kill_members(['%s:%d' % primary], 2) # Let monitor notice primary's gone time.sleep(2 * self.heartbeat_frequency) self.assertFalse(primary == self.c.primary) # Make sure we can still authenticate self.assertTrue(self.db.authenticate('user', 'userpass')) # And still query. self.db.read_preference = PRIMARY_PREFERRED self.assertEqual('bar', self.db.foo.find_one()['foo']) class TestAlive(HATestCase): def setUp(self): super(TestAlive, self).setUp() members = [{}, {}] self.seed, self.name = ha_tools.start_replica_set(members) def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = connected( MongoClient( primary, serverSelectionTimeoutMS=self.server_selection_timeout)), secondary_cx = connected( MongoClient( secondary, serverSelectionTimeoutMS=self.server_selection_timeout)) rsc = connected( MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout)) self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) class TestMongosLoadBalancing(HATestCase): def setUp(self): super(TestMongosLoadBalancing, self).setUp() seed_list = ha_tools.create_sharded_cluster() self.assertIsNotNone(seed_list) self.dbname = 'pymongo_mongos_ha' self.client = MongoClient( seed_list, serverSelectionTimeoutMS=self.server_selection_timeout) self.client.drop_database(self.dbname) def test_mongos_load_balancing(self): wait_until(lambda: len(ha_tools.routers) == len(self.client.nodes), 'discover all mongoses') # Can't access "address" when load balancing. with self.assertRaises(InvalidOperation): self.client.address coll = self.client[self.dbname].test coll.insert_one({'foo': 'bar'}) live_routers = list(ha_tools.routers) ha_tools.kill_mongos(live_routers.pop()) while live_routers: try: self.assertEqual(1, coll.count()) except ConnectionFailure: # If first attempt happened to select the dead mongos. self.assertEqual(1, coll.count()) wait_until(lambda: len(live_routers) == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.kill_mongos(live_routers.pop()) # Make sure the last one's really dead. time.sleep(1) # I'm alone. self.assertRaises(ConnectionFailure, coll.count) wait_until(lambda: 0 == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.restart_mongos(one(ha_tools.routers)) # Find new mongos self.assertEqual(1, coll.count()) class TestLastErrorDefaults(HATestCase): def setUp(self): super(TestLastErrorDefaults, self).setUp() members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) def test_get_last_error_defaults(self): replset = self.c.local.system.replset.find_one() settings = replset.get('settings', {}) # This should cause a WTimeoutError for every write command settings['getLastErrorDefaults'] = { 'w': 3, 'wtimeout': 1 } replset['settings'] = settings replset['version'] = replset.get("version", 1) + 1 self.c.admin.command("replSetReconfig", replset) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.insert_one, {'_id': 0}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.update_one, {'_id': 0}, {"$set": {"a": 10}}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.delete_one, {'_id': 0}) class TestShipOfTheseus(HATestCase): # If all of a replica set's members are replaced with new ones, is it still # the same replica set, or a different one? def setUp(self): super(TestShipOfTheseus, self).setUp() res = ha_tools.start_replica_set([{}, {}]) self.seed, self.name = res def test_ship_of_theseus(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=len(c.secondaries) + 1)) db.test.insert_one({}) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [] for i in range(3): new_hosts.append(ha_tools.add_member()) # RS closes all connections after reconfig. for j in xrange(30): try: if ha_tools.get_primary(): break except (ConnectionFailure, OperationFailure): pass time.sleep(1) else: self.fail("Couldn't recover from reconfig") # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break time.sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) time.sleep(5) wait_until(lambda: (ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2), "fail over", timeout=30) time.sleep(2 * self.heartbeat_frequency) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Wait for members to figure out they're secondaries. wait_until(lambda: len(ha_tools.get_secondaries()) == 2, "detect two secondaries", timeout=30) # Should be able to reconnect to set again. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) class TestLastError(HATestCase): # A "not master" error from Database.error() should refresh the server. enable_heartbeat = False def setUp(self): super(TestLastError, self).setUp() res = ha_tools.start_replica_set([{}, {}]) self.seed, self.name = res def test_last_error(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: c.secondaries, "discover secondary") ha_tools.stepdown_primary() db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=0)) db.test.insert_one({}) response = db.error() self.assertTrue('err' in response and 'not master' in response['err']) wait_until(lambda: len(c.secondaries) == 2, "discover two secondaries") if __name__ == '__main__': unittest.main() pymongo-3.2/test/pymongo_mocks.py0000644000175000017500000001551412630145074021201 0ustar behackettbehackett00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for mocking parts of PyMongo to test other parts.""" import contextlib from functools import partial import weakref from pymongo import common from pymongo import MongoClient from pymongo.errors import AutoReconnect, NetworkTimeout from pymongo.ismaster import IsMaster from pymongo.monitor import Monitor from pymongo.pool import Pool, PoolOptions from pymongo.server_description import ServerDescription from test import host as default_host, port as default_port class MockPool(Pool): def __init__(self, client, pair, *args, **kwargs): # MockPool gets a 'client' arg, regular pools don't. Weakref it to # avoid cycle with __del__, causing ResourceWarnings in Python 3.3. self.client = weakref.proxy(client) self.mock_host, self.mock_port = pair # Actually connect to the default server. Pool.__init__(self, (default_host, default_port), PoolOptions(connect_timeout=20)) @contextlib.contextmanager def get_socket(self, all_credentials, checkout=False): client = self.client host_and_port = '%s:%s' % (self.mock_host, self.mock_port) if host_and_port in client.mock_down_hosts: raise AutoReconnect('mock error') assert host_and_port in ( client.mock_standalones + client.mock_members + client.mock_mongoses), "bad host: %s" % host_and_port with Pool.get_socket(self, all_credentials) as sock_info: sock_info.mock_host = self.mock_host sock_info.mock_port = self.mock_port yield sock_info class MockMonitor(Monitor): def __init__( self, client, server_description, topology, pool, topology_settings): # MockMonitor gets a 'client' arg, regular monitors don't. self.client = client Monitor.__init__( self, server_description, topology, pool, topology_settings) def _check_once(self): address = self._server_description.address response, rtt = self.client.mock_is_master('%s:%d' % address) return ServerDescription(address, IsMaster(response), rtt) class MockClient(MongoClient): def __init__( self, standalones, members, mongoses, ismaster_hosts=None, *args, **kwargs): """A MongoClient connected to the default server, with a mock topology. standalones, members, mongoses determine the configuration of the topology. They are formatted like ['a:1', 'b:2']. ismaster_hosts provides an alternative host list for the server's mocked ismaster response; see test_connect_with_internal_ips. """ self.mock_standalones = standalones[:] self.mock_members = members[:] if self.mock_members: self.mock_primary = self.mock_members[0] else: self.mock_primary = None if ismaster_hosts is not None: self.mock_ismaster_hosts = ismaster_hosts else: self.mock_ismaster_hosts = members[:] self.mock_mongoses = mongoses[:] # Hosts that should raise socket errors. self.mock_down_hosts = [] # Hostname -> (min wire version, max wire version) self.mock_wire_versions = {} # Hostname -> max write batch size self.mock_max_write_batch_sizes = {} # Hostname -> round trip time self.mock_rtts = {} kwargs['_pool_class'] = partial(MockPool, self) kwargs['_monitor_class'] = partial(MockMonitor, self) super(MockClient, self).__init__(*args, **kwargs) def kill_host(self, host): """Host is like 'a:1'.""" self.mock_down_hosts.append(host) def revive_host(self, host): """Host is like 'a:1'.""" self.mock_down_hosts.remove(host) def set_wire_version_range(self, host, min_version, max_version): self.mock_wire_versions[host] = (min_version, max_version) def set_max_write_batch_size(self, host, size): self.mock_max_write_batch_sizes[host] = size def mock_is_master(self, host): """Return mock ismaster response (a dict) and round trip time.""" min_wire_version, max_wire_version = self.mock_wire_versions.get( host, (common.MIN_WIRE_VERSION, common.MAX_WIRE_VERSION)) max_write_batch_size = self.mock_max_write_batch_sizes.get( host, common.MAX_WRITE_BATCH_SIZE) rtt = self.mock_rtts.get(host, 0) # host is like 'a:1'. if host in self.mock_down_hosts: raise NetworkTimeout('mock timeout') elif host in self.mock_standalones: response = { 'ok': 1, 'ismaster': True, 'minWireVersion': min_wire_version, 'maxWireVersion': max_wire_version, 'maxWriteBatchSize': max_write_batch_size} elif host in self.mock_members: ismaster = (host == self.mock_primary) # Simulate a replica set member. response = { 'ok': 1, 'ismaster': ismaster, 'secondary': not ismaster, 'setName': 'rs', 'hosts': self.mock_ismaster_hosts, 'minWireVersion': min_wire_version, 'maxWireVersion': max_wire_version, 'maxWriteBatchSize': max_write_batch_size} if self.mock_primary: response['primary'] = self.mock_primary elif host in self.mock_mongoses: response = { 'ok': 1, 'ismaster': True, 'minWireVersion': min_wire_version, 'maxWireVersion': max_wire_version, 'msg': 'isdbgrid', 'maxWriteBatchSize': max_write_batch_size} else: # In test_internal_ips(), we try to connect to a host listed # in ismaster['hosts'] but not publicly accessible. raise AutoReconnect('Unknown host: %s' % host) return response, rtt def _process_kill_cursors_queue(self): # Avoid the background thread causing races, e.g. a surprising # reconnect while we're trying to test a disconnected client. pass pymongo-3.2/test/test_threads.py0000644000175000017500000001460412630145074021005 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test that pymongo is thread safe.""" import threading from test import (client_context, db_user, db_pwd, IntegrationTest, unittest) from test.utils import rs_or_single_client_noauth from test.utils import frequent_thread_switches, joinall from pymongo.errors import OperationFailure @client_context.require_connection def setUpModule(): pass class AutoAuthenticateThreads(threading.Thread): def __init__(self, collection, num): threading.Thread.__init__(self) self.coll = collection self.num = num self.success = False self.setDaemon(True) def run(self): for i in range(self.num): self.coll.insert_one({'num': i}) self.coll.find_one({'num': i}) self.success = True class SaveAndFind(threading.Thread): def __init__(self, collection): threading.Thread.__init__(self) self.collection = collection self.setDaemon(True) self.passed = False def run(self): sum = 0 for document in self.collection.find(): sum += document["x"] assert sum == 499500, "sum was %d not 499500" % sum self.passed = True class Insert(threading.Thread): def __init__(self, collection, n, expect_exception): threading.Thread.__init__(self) self.collection = collection self.n = n self.expect_exception = expect_exception self.setDaemon(True) def run(self): for _ in range(self.n): error = True try: self.collection.insert_one({"test": "insert"}) error = False except: if not self.expect_exception: raise if self.expect_exception: assert error class Update(threading.Thread): def __init__(self, collection, n, expect_exception): threading.Thread.__init__(self) self.collection = collection self.n = n self.expect_exception = expect_exception self.setDaemon(True) def run(self): for _ in range(self.n): error = True try: self.collection.update_one({"test": "unique"}, {"$set": {"test": "update"}}) error = False except: if not self.expect_exception: raise if self.expect_exception: assert error class Disconnect(threading.Thread): def __init__(self, client, n): threading.Thread.__init__(self) self.client = client self.n = n self.passed = False def run(self): for _ in range(self.n): self.client.close() self.passed = True class TestThreads(IntegrationTest): def setUp(self): self.db = client_context.rs_or_standalone_client.pymongo_test def test_threading(self): self.db.drop_collection("test") self.db.test.insert_many([{"x": i} for i in range(1000)]) threads = [] for i in range(10): t = SaveAndFind(self.db.test) t.start() threads.append(t) joinall(threads) def test_safe_insert(self): self.db.drop_collection("test1") self.db.test1.insert_one({"test": "insert"}) self.db.drop_collection("test2") self.db.test2.insert_one({"test": "insert"}) self.db.test2.create_index("test", unique=True) self.db.test2.find_one() okay = Insert(self.db.test1, 2000, False) error = Insert(self.db.test2, 2000, True) error.start() okay.start() error.join() okay.join() def test_safe_update(self): self.db.drop_collection("test1") self.db.test1.insert_one({"test": "update"}) self.db.test1.insert_one({"test": "unique"}) self.db.drop_collection("test2") self.db.test2.insert_one({"test": "update"}) self.db.test2.insert_one({"test": "unique"}) self.db.test2.create_index("test", unique=True) self.db.test2.find_one() okay = Update(self.db.test1, 2000, False) error = Update(self.db.test2, 2000, True) error.start() okay.start() error.join() okay.join() def test_client_disconnect(self): self.db.drop_collection("test") self.db.test.insert_many([{"x": i} for i in range(1000)]) # Start 10 threads that execute a query, and 10 threads that call # client.close() 10 times in a row. threads = [SaveAndFind(self.db.test) for _ in range(10)] threads.extend(Disconnect(self.db.client, 10) for _ in range(10)) with frequent_thread_switches(): # Frequent thread switches hurt performance badly enough to # prevent reconnection within 5 seconds, especially in Python 2 # on a Windows build slave. for t in threads: t.start() for t in threads: t.join(30) for t in threads: self.assertTrue(t.passed) class TestThreadsAuth(IntegrationTest): @classmethod @client_context.require_auth def setUpClass(cls): super(TestThreadsAuth, cls).setUpClass() def test_auto_auth_login(self): client = rs_or_single_client_noauth() self.assertRaises(OperationFailure, client.auth_test.test.find_one) # Admin auth client.admin.authenticate(db_user, db_pwd) nthreads = 10 threads = [] for _ in range(nthreads): t = AutoAuthenticateThreads(client.auth_test.test, 10) t.start() threads.append(t) joinall(threads) for t in threads: self.assertTrue(t.success) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_collection.py0000644000175000017500000025150112630145074021505 0ustar behackettbehackett00000000000000# -*- coding: utf-8 -*- # Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the collection module.""" import re import sys import threading from collections import defaultdict sys.path[0:0] = [""] import bson from bson.raw_bson import RawBSONDocument from bson.regex import Regex from bson.code import Code from bson.codec_options import CodecOptions from bson.objectid import ObjectId from bson.py3compat import u, itervalues from bson.son import SON from pymongo import (ASCENDING, DESCENDING, GEO2D, GEOHAYSTACK, GEOSPHERE, HASHED, TEXT) from pymongo import MongoClient, monitoring from pymongo.bulk import BulkWriteError from pymongo.collection import Collection, ReturnDocument from pymongo.command_cursor import CommandCursor from pymongo.cursor import CursorType from pymongo.errors import (DocumentTooLarge, DuplicateKeyError, InvalidDocument, InvalidName, InvalidOperation, OperationFailure, WriteConcernError) from pymongo.message import _COMMAND_OVERHEAD, _gen_find_command from pymongo.operations import * from pymongo.read_preferences import ReadPreference from pymongo.results import (InsertOneResult, InsertManyResult, UpdateResult, DeleteResult) from pymongo.write_concern import WriteConcern from test.test_client import IntegrationTest from test.utils import (is_mongos, enable_text_search, get_pool, rs_or_single_client, single_client, wait_until, EventListener) from test import client_context, host, port, unittest class TestCollectionNoConnect(unittest.TestCase): @classmethod def setUpClass(cls): client = MongoClient(host, port, connect=False) cls.db = client.pymongo_test def test_collection(self): self.assertRaises(TypeError, Collection, self.db, 5) def make_col(base, name): return base[name] self.assertRaises(InvalidName, make_col, self.db, "") self.assertRaises(InvalidName, make_col, self.db, "te$t") self.assertRaises(InvalidName, make_col, self.db, ".test") self.assertRaises(InvalidName, make_col, self.db, "test.") self.assertRaises(InvalidName, make_col, self.db, "tes..t") self.assertRaises(InvalidName, make_col, self.db.test, "") self.assertRaises(InvalidName, make_col, self.db.test, "te$t") self.assertRaises(InvalidName, make_col, self.db.test, ".test") self.assertRaises(InvalidName, make_col, self.db.test, "test.") self.assertRaises(InvalidName, make_col, self.db.test, "tes..t") self.assertRaises(InvalidName, make_col, self.db.test, "tes\x00t") self.assertTrue(isinstance(self.db.test, Collection)) self.assertEqual(self.db.test, self.db["test"]) self.assertEqual(self.db.test, Collection(self.db, "test")) self.assertEqual(self.db.test.mike, self.db["test.mike"]) self.assertEqual(self.db.test["mike"], self.db["test.mike"]) def test_getattr(self): coll = self.db.test self.assertTrue(isinstance(coll['_does_not_exist'], Collection)) with self.assertRaises(AttributeError) as context: coll._does_not_exist # Message should be: # "AttributeError: Collection has no attribute '_does_not_exist'. To # access the test._does_not_exist collection, use # database['test._does_not_exist']." self.assertIn("has no attribute '_does_not_exist'", str(context.exception)) def test_iteration(self): self.assertRaises(TypeError, next, self.db) class TestCollection(IntegrationTest): @classmethod def setUpClass(cls): super(TestCollection, cls).setUpClass() cls.w = client_context.w @classmethod def tearDownClass(cls): cls.db.drop_collection("test_large_limit") def test_drop_nonexistent_collection(self): self.db.drop_collection('test') self.assertFalse('test' in self.db.collection_names()) # No exception self.db.drop_collection('test') @client_context.require_version_min(2, 6) def test_create_indexes(self): db = self.db self.assertRaises(TypeError, db.test.create_indexes, 'foo') self.assertRaises(TypeError, db.test.create_indexes, ['foo']) self.assertRaises(TypeError, IndexModel, 5) self.assertRaises(ValueError, IndexModel, []) db.test.drop_indexes() db.test.insert_one({}) self.assertEqual(len(db.test.index_information()), 1) db.test.create_indexes([IndexModel("hello")]) db.test.create_indexes([IndexModel([("hello", DESCENDING), ("world", ASCENDING)])]) # Tuple instead of list. db.test.create_indexes([IndexModel((("world", ASCENDING),))]) self.assertEqual(len(db.test.index_information()), 4) db.test.drop_indexes() names = db.test.create_indexes([IndexModel([("hello", DESCENDING), ("world", ASCENDING)], name="hello_world")]) self.assertEqual(names, ["hello_world"]) db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) db.test.create_indexes([IndexModel("hello")]) self.assertTrue("hello_1" in db.test.index_information()) db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) names = db.test.create_indexes([IndexModel([("hello", DESCENDING), ("world", ASCENDING)]), IndexModel("hello")]) info = db.test.index_information() for name in names: self.assertTrue(name in info) db.test.drop() db.test.insert_one({'a': 1}) db.test.insert_one({'a': 1}) self.assertRaises( DuplicateKeyError, db.test.create_indexes, [IndexModel('a', unique=True)]) def test_create_index(self): db = self.db self.assertRaises(TypeError, db.test.create_index, 5) self.assertRaises(TypeError, db.test.create_index, {"hello": 1}) self.assertRaises(ValueError, db.test.create_index, []) db.test.drop_indexes() db.test.insert_one({}) self.assertEqual(len(db.test.index_information()), 1) db.test.create_index("hello") db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)]) # Tuple instead of list. db.test.create_index((("world", ASCENDING),)) self.assertEqual(len(db.test.index_information()), 4) db.test.drop_indexes() ix = db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)], name="hello_world") self.assertEqual(ix, "hello_world") db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) db.test.create_index("hello") self.assertTrue("hello_1" in db.test.index_information()) db.test.drop_indexes() self.assertEqual(len(db.test.index_information()), 1) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)]) self.assertTrue("hello_-1_world_1" in db.test.index_information()) db.test.drop() db.test.insert_one({'a': 1}) db.test.insert_one({'a': 1}) self.assertRaises( DuplicateKeyError, db.test.create_index, 'a', unique=True) def test_drop_index(self): db = self.db db.test.drop_indexes() db.test.create_index("hello") name = db.test.create_index("goodbye") self.assertEqual(len(db.test.index_information()), 3) self.assertEqual(name, "goodbye_1") db.test.drop_index(name) # Drop it again. with self.assertRaises(OperationFailure): db.test.drop_index(name) self.assertEqual(len(db.test.index_information()), 2) self.assertTrue("hello_1" in db.test.index_information()) db.test.drop_indexes() db.test.create_index("hello") name = db.test.create_index("goodbye") self.assertEqual(len(db.test.index_information()), 3) self.assertEqual(name, "goodbye_1") db.test.drop_index([("goodbye", ASCENDING)]) self.assertEqual(len(db.test.index_information()), 2) self.assertTrue("hello_1" in db.test.index_information()) def test_reindex(self): db = self.db db.drop_collection("test") db.test.insert_one({"foo": "bar", "who": "what", "when": "how"}) db.test.create_index("foo") db.test.create_index("who") db.test.create_index("when") info = db.test.index_information() def check_result(result): self.assertEqual(4, result['nIndexes']) indexes = result['indexes'] names = [idx['name'] for idx in indexes] for name in names: self.assertTrue(name in info) for key in info: self.assertTrue(key in names) reindexed = db.test.reindex() if 'raw' in reindexed: # mongos for result in itervalues(reindexed['raw']): check_result(result) else: check_result(reindexed) def test_list_indexes(self): db = self.db db.test.drop() db.test.insert_one({}) # create collection def map_indexes(indexes): return dict([(index["name"], index) for index in indexes]) indexes = list(db.test.list_indexes()) self.assertEqual(len(indexes), 1) self.assertTrue("_id_" in map_indexes(indexes)) db.test.create_index("hello") indexes = list(db.test.list_indexes()) self.assertEqual(len(indexes), 2) self.assertEqual(map_indexes(indexes)["hello_1"]["key"], SON([("hello", ASCENDING)])) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)], unique=True) indexes = list(db.test.list_indexes()) self.assertEqual(len(indexes), 3) index_map = map_indexes(indexes) self.assertEqual(index_map["hello_-1_world_1"]["key"], SON([("hello", DESCENDING), ("world", ASCENDING)])) self.assertEqual(True, index_map["hello_-1_world_1"]["unique"]) def test_index_info(self): db = self.db db.test.drop() db.test.insert_one({}) # create collection self.assertEqual(len(db.test.index_information()), 1) self.assertTrue("_id_" in db.test.index_information()) db.test.create_index("hello") self.assertEqual(len(db.test.index_information()), 2) self.assertEqual(db.test.index_information()["hello_1"]["key"], [("hello", ASCENDING)]) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)], unique=True) self.assertEqual(db.test.index_information()["hello_1"]["key"], [("hello", ASCENDING)]) self.assertEqual(len(db.test.index_information()), 3) self.assertEqual([("hello", DESCENDING), ("world", ASCENDING)], db.test.index_information()["hello_-1_world_1"]["key"] ) self.assertEqual( True, db.test.index_information()["hello_-1_world_1"]["unique"]) def test_index_geo2d(self): db = self.db db.test.drop_indexes() self.assertEqual('loc_2d', db.test.create_index([("loc", GEO2D)])) index_info = db.test.index_information()['loc_2d'] self.assertEqual([('loc', '2d')], index_info['key']) @client_context.require_no_mongos def test_index_haystack(self): db = self.db db.test.drop() _id = db.test.insert_one({ "pos": {"long": 34.2, "lat": 33.3}, "type": "restaurant" }).inserted_id db.test.insert_one({ "pos": {"long": 34.2, "lat": 37.3}, "type": "restaurant" }) db.test.insert_one({ "pos": {"long": 59.1, "lat": 87.2}, "type": "office" }) db.test.create_index( [("pos", GEOHAYSTACK), ("type", ASCENDING)], bucketSize=1 ) results = db.command(SON([ ("geoSearch", "test"), ("near", [33, 33]), ("maxDistance", 6), ("search", {"type": "restaurant"}), ("limit", 30), ]))['results'] self.assertEqual(2, len(results)) self.assertEqual({ "_id": _id, "pos": {"long": 34.2, "lat": 33.3}, "type": "restaurant" }, results[0]) @client_context.require_version_min(2, 3, 2) @client_context.require_no_mongos def test_index_text(self): enable_text_search(self.client) db = self.db db.test.drop_indexes() self.assertEqual("t_text", db.test.create_index([("t", TEXT)])) index_info = db.test.index_information()["t_text"] self.assertTrue("weights" in index_info) if client_context.version.at_least(2, 5, 5): db.test.insert_many([ {'t': 'spam eggs and spam'}, {'t': 'spam'}, {'t': 'egg sausage and bacon'}]) # MongoDB 2.6 text search. Create 'score' field in projection. cursor = db.test.find( {'$text': {'$search': 'spam'}}, {'score': {'$meta': 'textScore'}}) # Sort by 'score' field. cursor.sort([('score', {'$meta': 'textScore'})]) results = list(cursor) self.assertTrue(results[0]['score'] >= results[1]['score']) db.test.drop_indexes() @client_context.require_version_min(2, 3, 2) def test_index_2dsphere(self): db = self.db db.test.drop_indexes() self.assertEqual("geo_2dsphere", db.test.create_index([("geo", GEOSPHERE)])) for dummy, info in db.test.index_information().items(): field, idx_type = info['key'][0] if field == 'geo' and idx_type == '2dsphere': break else: self.fail("2dsphere index not found.") poly = {"type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]} query = {"geo": {"$within": {"$geometry": poly}}} # This query will error without a 2dsphere index. db.test.find(query) db.test.drop_indexes() @client_context.require_version_min(2, 3, 2) def test_index_hashed(self): db = self.db db.test.drop_indexes() self.assertEqual("a_hashed", db.test.create_index([("a", HASHED)])) for dummy, info in db.test.index_information().items(): field, idx_type = info['key'][0] if field == 'a' and idx_type == 'hashed': break else: self.fail("hashed index not found.") db.test.drop_indexes() def test_index_sparse(self): db = self.db db.test.drop_indexes() db.test.create_index([('key', ASCENDING)], sparse=True) self.assertTrue(db.test.index_information()['key_1']['sparse']) def test_index_background(self): db = self.db db.test.drop_indexes() db.test.create_index([('keya', ASCENDING)]) db.test.create_index([('keyb', ASCENDING)], background=False) db.test.create_index([('keyc', ASCENDING)], background=True) self.assertFalse('background' in db.test.index_information()['keya_1']) self.assertFalse(db.test.index_information()['keyb_1']['background']) self.assertTrue(db.test.index_information()['keyc_1']['background']) def _drop_dups_setup(self, db): db.drop_collection('test') db.test.insert_one({'i': 1}) db.test.insert_one({'i': 2}) db.test.insert_one({'i': 2}) # duplicate db.test.insert_one({'i': 3}) @client_context.require_version_max(2, 6) def test_index_drop_dups(self): # Try dropping duplicates db = self.db self._drop_dups_setup(db) # No error, just drop the duplicate db.test.create_index([('i', ASCENDING)], unique=True, dropDups=True) # Duplicate was dropped self.assertEqual(3, db.test.count()) # Index was created, plus the index on _id self.assertEqual(2, len(db.test.index_information())) def test_index_dont_drop_dups(self): # Try *not* dropping duplicates db = self.db self._drop_dups_setup(db) # There's a duplicate def test_create(): db.test.create_index( [('i', ASCENDING)], unique=True, dropDups=False ) self.assertRaises(DuplicateKeyError, test_create) # Duplicate wasn't dropped self.assertEqual(4, db.test.count()) # Index wasn't created, only the default index on _id self.assertEqual(1, len(db.test.index_information())) # Get the plan dynamically because the explain format will change. def get_plan_stage(self, root, stage): if root.get('stage') == stage: return root elif "inputStage" in root: return self.get_plan_stage(root['inputStage'], stage) elif "inputStages" in root: for i in root['inputStages']: stage = self.get_plan_stage(i, stage) if stage: return stage elif "shards" in root: for i in root['shards']: stage = self.get_plan_stage(i['winningPlan'], stage) if stage: return stage return {} @client_context.require_version_min(3, 1, 9, -1) def test_index_filter(self): db = self.db db.drop_collection("test") # Test bad filter spec on create. self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression=5) self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression={"x": {"$asdasd": 3}}) self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression={"$and": 5}) self.assertRaises(OperationFailure, db.test.create_index, "x", partialFilterExpression={ "$and": [{"$and": [{"x": {"$lt": 2}}, {"x": {"$gt": 0}}]}, {"x": {"$exists": True}}]}) self.assertEqual("x_1", db.test.create_index( [('x', ASCENDING)], partialFilterExpression={"a": {"$lte": 1.5}})) db.test.insert_one({"x": 5, "a": 2}) db.test.insert_one({"x": 6, "a": 1}) # Operations that use the partial index. explain = db.test.find({"x": 6, "a": 1}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'IXSCAN') self.assertEqual("x_1", stage.get('indexName')) self.assertTrue(stage.get('isPartial')) explain = db.test.find({"x": {"$gt": 1}, "a": 1}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'IXSCAN') self.assertEqual("x_1", stage.get('indexName')) self.assertTrue(stage.get('isPartial')) explain = db.test.find({"x": 6, "a": {"$lte": 1}}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'IXSCAN') self.assertEqual("x_1", stage.get('indexName')) self.assertTrue(stage.get('isPartial')) # Operations that do not use the partial index. explain = db.test.find({"x": 6, "a": {"$lte": 1.6}}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'COLLSCAN') self.assertNotEqual({}, stage) explain = db.test.find({"x": 6}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'COLLSCAN') self.assertNotEqual({}, stage) # Test drop_indexes. db.test.drop_index("x_1") explain = db.test.find({"x": 6, "a": 1}).explain() stage = self.get_plan_stage(explain['queryPlanner']['winningPlan'], 'COLLSCAN') self.assertNotEqual({}, stage) def test_field_selection(self): db = self.db db.drop_collection("test") doc = {"a": 1, "b": 5, "c": {"d": 5, "e": 10}} db.test.insert_one(doc) # Test field inclusion doc = next(db.test.find({}, ["_id"])) self.assertEqual(list(doc), ["_id"]) doc = next(db.test.find({}, ["a"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "a"]) doc = next(db.test.find({}, ["b"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "b"]) doc = next(db.test.find({}, ["c"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "c"]) doc = next(db.test.find({}, ["a"])) self.assertEqual(doc["a"], 1) doc = next(db.test.find({}, ["b"])) self.assertEqual(doc["b"], 5) doc = next(db.test.find({}, ["c"])) self.assertEqual(doc["c"], {"d": 5, "e": 10}) # Test inclusion of fields with dots doc = next(db.test.find({}, ["c.d"])) self.assertEqual(doc["c"], {"d": 5}) doc = next(db.test.find({}, ["c.e"])) self.assertEqual(doc["c"], {"e": 10}) doc = next(db.test.find({}, ["b", "c.e"])) self.assertEqual(doc["c"], {"e": 10}) doc = next(db.test.find({}, ["b", "c.e"])) l = list(doc) l.sort() self.assertEqual(l, ["_id", "b", "c"]) doc = next(db.test.find({}, ["b", "c.e"])) self.assertEqual(doc["b"], 5) # Test field exclusion doc = next(db.test.find({}, {"a": False, "b": 0})) l = list(doc) l.sort() self.assertEqual(l, ["_id", "c"]) doc = next(db.test.find({}, {"_id": False})) l = list(doc) self.assertFalse("_id" in l) def test_options(self): db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=4096) result = db.test.options() # mongos 2.2.x adds an $auth field when auth is enabled. result.pop('$auth', None) self.assertEqual(result, {"capped": True, 'size': 4096}) db.drop_collection("test") def test_insert_one(self): db = self.db db.test.drop() document = {"_id": 1000} result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertTrue(isinstance(result.inserted_id, int)) self.assertEqual(document["_id"], result.inserted_id) self.assertTrue(result.acknowledged) self.assertIsNotNone(db.test.find_one({"_id": document["_id"]})) self.assertEqual(1, db.test.count()) document = {"foo": "bar"} result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertTrue(isinstance(result.inserted_id, ObjectId)) self.assertEqual(document["_id"], result.inserted_id) self.assertTrue(result.acknowledged) self.assertIsNotNone(db.test.find_one({"_id": document["_id"]})) self.assertEqual(2, db.test.count()) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertTrue(isinstance(result.inserted_id, ObjectId)) self.assertEqual(document["_id"], result.inserted_id) self.assertFalse(result.acknowledged) # The insert failed duplicate key... wait_until(lambda: 2 == db.test.count(), 'forcing duplicate key error') document = RawBSONDocument( bson.BSON.encode({'_id': ObjectId(), 'foo': 'bar'})) result = db.test.insert_one(document) self.assertTrue(isinstance(result, InsertOneResult)) self.assertEqual(result.inserted_id, None) def test_insert_many(self): db = self.db db.test.drop() docs = [{} for _ in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(isinstance(result.inserted_ids, list)) self.assertEqual(5, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, ObjectId)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({'_id': _id})) self.assertTrue(result.acknowledged) docs = [{"_id": i} for i in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(isinstance(result.inserted_ids, list)) self.assertEqual(5, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, int)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({"_id": _id})) self.assertTrue(result.acknowledged) docs = [RawBSONDocument(bson.BSON.encode({"_id": i + 5})) for i in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(isinstance(result.inserted_ids, list)) self.assertEqual([], result.inserted_ids) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) docs = [{} for _ in range(5)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertFalse(result.acknowledged) self.assertEqual(20, db.test.count()) def test_delete_one(self): self.db.test.drop() self.db.test.insert_one({"x": 1}) self.db.test.insert_one({"y": 1}) self.db.test.insert_one({"z": 1}) result = self.db.test.delete_one({"x": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertEqual(1, result.deleted_count) self.assertTrue(result.acknowledged) self.assertEqual(2, self.db.test.count()) result = self.db.test.delete_one({"y": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertEqual(1, result.deleted_count) self.assertTrue(result.acknowledged) self.assertEqual(1, self.db.test.count()) db = self.db.client.get_database(self.db.name, write_concern=WriteConcern(w=0)) result = db.test.delete_one({"z": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertRaises(InvalidOperation, lambda: result.deleted_count) self.assertFalse(result.acknowledged) wait_until(lambda: 0 == db.test.count(), 'delete 1 documents') def test_delete_many(self): self.db.test.drop() self.db.test.insert_one({"x": 1}) self.db.test.insert_one({"x": 1}) self.db.test.insert_one({"y": 1}) self.db.test.insert_one({"y": 1}) result = self.db.test.delete_many({"x": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertEqual(2, result.deleted_count) self.assertTrue(result.acknowledged) self.assertEqual(0, self.db.test.count({"x": 1})) db = self.db.client.get_database(self.db.name, write_concern=WriteConcern(w=0)) result = db.test.delete_many({"y": 1}) self.assertTrue(isinstance(result, DeleteResult)) self.assertRaises(InvalidOperation, lambda: result.deleted_count) self.assertFalse(result.acknowledged) wait_until(lambda: 0 == db.test.count(), 'delete 2 documents') def test_command_document_too_large(self): large = '*' * (self.client.max_bson_size + _COMMAND_OVERHEAD) coll = self.db.test self.assertRaises( DocumentTooLarge, coll.insert_one, {'data': large}) # update_one and update_many are the same self.assertRaises( DocumentTooLarge, coll.replace_one, {}, {'data': large}) self.assertRaises( DocumentTooLarge, coll.delete_one, {'data': large}) @client_context.require_version_min(3, 1, 9, -1) @client_context.require_no_auth def test_insert_bypass_document_validation(self): db = self.db db.test.drop() db.create_collection("test", validator={"a": {"$exists": True}}) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) # Test insert_one self.assertRaises(OperationFailure, db.test.insert_one, {"_id": 1, "x": 100}) result = db.test.insert_one({"_id": 1, "x": 100}, bypass_document_validation=True) self.assertTrue(isinstance(result, InsertOneResult)) self.assertEqual(1, result.inserted_id) result = db.test.insert_one({"_id":2, "a":0}) self.assertTrue(isinstance(result, InsertOneResult)) self.assertEqual(2, result.inserted_id) self.assertRaises(OperationFailure, db_w0.test.insert_one, {"x": 1}, bypass_document_validation=True) # Test insert_many docs = [{"_id": i, "x": 100 - i} for i in range(3, 100)] self.assertRaises(OperationFailure, db.test.insert_many, docs) result = db.test.insert_many(docs, bypass_document_validation=True) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(97, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, int)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({"x": doc["x"]})) self.assertTrue(result.acknowledged) docs = [{"_id": i, "a": 200 - i} for i in range(100, 200)] result = db.test.insert_many(docs) self.assertTrue(isinstance(result, InsertManyResult)) self.assertTrue(97, len(result.inserted_ids)) for doc in docs: _id = doc["_id"] self.assertTrue(isinstance(_id, int)) self.assertTrue(_id in result.inserted_ids) self.assertEqual(1, db.test.count({"a": doc["a"]})) self.assertTrue(result.acknowledged) self.assertRaises(OperationFailure, db_w0.test.insert_many, [{"x": 1}, {"x": 2}], bypass_document_validation=True) @client_context.require_version_min(3, 1, 9, -1) @client_context.require_no_auth def test_replace_bypass_document_validation(self): db = self.db db.test.drop() db.create_collection("test", validator={"a": {"$exists": True}}) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) # Test replace_one db.test.insert_one({"a": 101}) self.assertRaises(OperationFailure, db.test.replace_one, {"a": 101}, {"y": 1}) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(1, db.test.count({"a": 101})) db.test.replace_one({"a": 101}, {"y": 1}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"a": 101})) self.assertEqual(1, db.test.count({"y": 1})) db.test.replace_one({"y": 1}, {"a": 102}) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(0, db.test.count({"a": 101})) self.assertEqual(1, db.test.count({"a": 102})) db.test.insert_one({"y": 1}, bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.replace_one, {"y": 1}, {"x": 101}) self.assertEqual(0, db.test.count({"x": 101})) self.assertEqual(1, db.test.count({"y": 1})) db.test.replace_one({"y": 1}, {"x": 101}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(1, db.test.count({"x": 101})) db.test.replace_one({"x": 101}, {"a": 103}, bypass_document_validation=False) self.assertEqual(0, db.test.count({"x": 101})) self.assertEqual(1, db.test.count({"a": 103})) self.assertRaises(OperationFailure, db_w0.test.replace_one, {"y": 1}, {"x": 1}, bypass_document_validation=True) @client_context.require_version_min(3, 1, 9, -1) @client_context.require_no_auth def test_update_bypass_document_validation(self): db = self.db db.test.drop() db.test.insert_one({"z": 5}) db.command(SON([("collMod", "test"), ("validator", {"z": {"$gte": 0}})])) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) # Test update_one self.assertRaises(OperationFailure, db.test.update_one, {"z": 5}, {"$inc": {"z": -10}}) self.assertEqual(0, db.test.count({"z": -5})) self.assertEqual(1, db.test.count({"z": 5})) db.test.update_one({"z": 5}, {"$inc": {"z": -10}}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"z": 5})) self.assertEqual(1, db.test.count({"z": -5})) db.test.update_one({"z": -5}, {"$inc": {"z": 6}}, bypass_document_validation=False) self.assertEqual(1, db.test.count({"z": 1})) self.assertEqual(0, db.test.count({"z": -5})) db.test.insert_one({"z": -10}, bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.update_one, {"z": -10}, {"$inc": {"z": 1}}) self.assertEqual(0, db.test.count({"z": -9})) self.assertEqual(1, db.test.count({"z": -10})) db.test.update_one({"z": -10}, {"$inc": {"z": 1}}, bypass_document_validation=True) self.assertEqual(1, db.test.count({"z": -9})) self.assertEqual(0, db.test.count({"z": -10})) db.test.update_one({"z": -9}, {"$inc": {"z": 9}}, bypass_document_validation=False) self.assertEqual(0, db.test.count({"z": -9})) self.assertEqual(1, db.test.count({"z": 0})) self.assertRaises(OperationFailure, db_w0.test.update_one, {"y": 1}, {"$inc": {"x": 1}}, bypass_document_validation=True) # Test update_many db.test.insert_many([{"z": i} for i in range(3, 101)]) db.test.insert_one({"y": 0}, bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.update_many, {}, {"$inc": {"z": -100}}) self.assertEqual(100, db.test.count({"z": {"$gte": 0}})) self.assertEqual(0, db.test.count({"z": {"$lt": 0}})) self.assertEqual(0, db.test.count({"y": 0, "z": -100})) db.test.update_many({"z": {"$gte": 0}}, {"$inc": {"z": -100}}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"z": {"$gt": 0}})) self.assertEqual(100, db.test.count({"z": {"$lte": 0}})) db.test.update_many({"z": {"$gt": -50}}, {"$inc": {"z": 100}}, bypass_document_validation=False) self.assertEqual(50, db.test.count({"z": {"$gt": 0}})) self.assertEqual(50, db.test.count({"z": {"$lt": 0}})) db.test.insert_many([{"z": -i} for i in range(50)], bypass_document_validation=True) self.assertRaises(OperationFailure, db.test.update_many, {}, {"$inc": {"z": 1}}) self.assertEqual(100, db.test.count({"z": {"$lte": 0}})) self.assertEqual(50, db.test.count({"z": {"$gt": 1}})) db.test.update_many({"z": {"$gte": 0}}, {"$inc": {"z": -100}}, bypass_document_validation=True) self.assertEqual(0, db.test.count({"z": {"$gt": 0}})) self.assertEqual(150, db.test.count({"z": {"$lte": 0}})) db.test.update_many({"z": {"$lte": 0}}, {"$inc": {"z": 100}}, bypass_document_validation=False) self.assertEqual(150, db.test.count({"z": {"$gte": 0}})) self.assertEqual(0, db.test.count({"z": {"$lt": 0}})) self.assertRaises(OperationFailure, db_w0.test.update_many, {"y": 1}, {"$inc": {"x": 1}}, bypass_document_validation=True) @client_context.require_version_min(3, 1, 9, -1) @client_context.require_no_auth def test_bypass_document_validation_bulk_write(self): db = self.db db.test.drop() db.create_collection("test", validator={"a": {"$gte": 0}}) db_w0 = self.db.client.get_database( self.db.name, write_concern=WriteConcern(w=0)) ops = [InsertOne({"a": -10}), InsertOne({"a": -11}), InsertOne({"a": -12}), UpdateOne({"a": {"$lte": -10}}, {"$inc": {"a": 1}}), UpdateMany({"a": {"$lte": -10}}, {"$inc": {"a": 1}}), ReplaceOne({"a": {"$lte": -10}}, {"a": -1})] db.test.bulk_write(ops, bypass_document_validation=True) self.assertEqual(3, db.test.count()) self.assertEqual(1, db.test.count({"a": -11})) self.assertEqual(1, db.test.count({"a": -1})) self.assertEqual(1, db.test.count({"a": -9})) # Assert that the operations would fail without bypass_doc_val for op in ops: self.assertRaises(BulkWriteError, db.test.bulk_write, [op]) self.assertRaises(OperationFailure, db_w0.test.bulk_write, ops, bypass_document_validation=True) def test_find_by_default_dct(self): db = self.db db.test.insert_one({'foo': 'bar'}) dct = defaultdict(dict, [('foo', 'bar')]) self.assertIsNotNone(db.test.find_one(dct)) self.assertEqual(dct, defaultdict(dict, [('foo', 'bar')])) def test_find_w_fields(self): db = self.db db.test.delete_many({}) db.test.insert_one({"x": 1, "mike": "awesome", "extra thing": "abcdefghijklmnopqrstuvwxyz"}) self.assertEqual(1, db.test.count()) doc = next(db.test.find({})) self.assertTrue("x" in doc) doc = next(db.test.find({})) self.assertTrue("mike" in doc) doc = next(db.test.find({})) self.assertTrue("extra thing" in doc) doc = next(db.test.find({}, ["x", "mike"])) self.assertTrue("x" in doc) doc = next(db.test.find({}, ["x", "mike"])) self.assertTrue("mike" in doc) doc = next(db.test.find({}, ["x", "mike"])) self.assertFalse("extra thing" in doc) doc = next(db.test.find({}, ["mike"])) self.assertFalse("x" in doc) doc = next(db.test.find({}, ["mike"])) self.assertTrue("mike" in doc) doc = next(db.test.find({}, ["mike"])) self.assertFalse("extra thing" in doc) def test_fields_specifier_as_dict(self): db = self.db db.test.delete_many({}) db.test.insert_one({"x": [1, 2, 3], "mike": "awesome"}) self.assertEqual([1, 2, 3], db.test.find_one()["x"]) self.assertEqual([2, 3], db.test.find_one( projection={"x": {"$slice": -2}})["x"]) self.assertTrue("x" not in db.test.find_one(projection={"x": 0})) self.assertTrue("mike" in db.test.find_one(projection={"x": 0})) def test_find_w_regex(self): db = self.db db.test.delete_many({}) db.test.insert_one({"x": "hello_world"}) db.test.insert_one({"x": "hello_mike"}) db.test.insert_one({"x": "hello_mikey"}) db.test.insert_one({"x": "hello_test"}) self.assertEqual(db.test.find().count(), 4) self.assertEqual(db.test.find({"x": re.compile("^hello.*")}).count(), 4) self.assertEqual(db.test.find({"x": re.compile("ello")}).count(), 4) self.assertEqual(db.test.find({"x": re.compile("^hello$")}).count(), 0) self.assertEqual(db.test.find({"x": re.compile("^hello_mi.*$")}).count(), 2) def test_id_can_be_anything(self): db = self.db db.test.delete_many({}) auto_id = {"hello": "world"} db.test.insert_one(auto_id) self.assertTrue(isinstance(auto_id["_id"], ObjectId)) numeric = {"_id": 240, "hello": "world"} db.test.insert_one(numeric) self.assertEqual(numeric["_id"], 240) obj = {"_id": numeric, "hello": "world"} db.test.insert_one(obj) self.assertEqual(obj["_id"], numeric) for x in db.test.find(): self.assertEqual(x["hello"], u("world")) self.assertTrue("_id" in x) def test_invalid_key_names(self): db = self.db db.test.drop() db.test.insert_one({"hello": "world"}) db.test.insert_one({"hello": {"hello": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {"$hello": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {"$hello": "world"}}) db.test.insert_one({"he$llo": "world"}) db.test.insert_one({"hello": {"hello$": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {".hello": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {".hello": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello.": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {"hello.": "world"}}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hel.lo": "world"}) self.assertRaises(InvalidDocument, db.test.insert_one, {"hello": {"hel.lo": "world"}}) def test_unique_index(self): db = self.db db.drop_collection("test") db.test.create_index("hello") # No error. db.test.insert_one({"hello": "world"}) db.test.insert_one({"hello": "world"}) db.drop_collection("test") db.test.create_index("hello", unique=True) with self.assertRaises(DuplicateKeyError): db.test.insert_one({"hello": "world"}) db.test.insert_one({"hello": "world"}) def test_duplicate_key_error(self): db = self.db db.drop_collection("test") db.test.create_index("x", unique=True) db.test.insert_one({"_id": 1, "x": 1}) with self.assertRaises(DuplicateKeyError) as context: db.test.insert_one({"x": 1}) self.assertIsNotNone(context.exception.details) with self.assertRaises(DuplicateKeyError) as context: db.test.insert_one({"x": 1}) self.assertIsNotNone(context.exception.details) self.assertEqual(1, db.test.count()) def test_wtimeout(self): # Ensure setting wtimeout doesn't disable write concern altogether. # See SERVER-12596. collection = self.db.test collection.drop() collection.insert_one({'_id': 1}) coll = collection.with_options( write_concern=WriteConcern(w=1, wtimeout=1000)) self.assertRaises(DuplicateKeyError, coll.insert_one, {'_id': 1}) coll = collection.with_options( write_concern=WriteConcern(wtimeout=1000)) self.assertRaises(DuplicateKeyError, coll.insert_one, {'_id': 1}) def test_error_code(self): try: self.db.test.update_many({}, {"$thismodifierdoesntexist": 1}) except OperationFailure as exc: self.assertTrue(exc.code in (9, 10147, 16840, 17009)) # Just check that we set the error document. Fields # vary by MongoDB version. self.assertTrue(exc.details is not None) else: self.fail("OperationFailure was not raised") def test_index_on_subfield(self): db = self.db db.drop_collection("test") db.test.insert_one({"hello": {"a": 4, "b": 5}}) db.test.insert_one({"hello": {"a": 7, "b": 2}}) db.test.insert_one({"hello": {"a": 4, "b": 10}}) db.drop_collection("test") db.test.create_index("hello.a", unique=True) db.test.insert_one({"hello": {"a": 4, "b": 5}}) db.test.insert_one({"hello": {"a": 7, "b": 2}}) self.assertRaises(DuplicateKeyError, db.test.insert_one, {"hello": {"a": 4, "b": 10}}) def test_replace_one(self): db = self.db db.drop_collection("test") self.assertRaises(ValueError, lambda: db.test.replace_one({}, {"$set": {"x": 1}})) id1 = db.test.insert_one({"x": 1}).inserted_id result = db.test.replace_one({"x": 1}, {"y": 1}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"y": 1})) self.assertEqual(0, db.test.count({"x": 1})) self.assertEqual(db.test.find_one(id1)["y"], 1) replacement = RawBSONDocument(bson.BSON.encode({"_id": id1, "z": 1})) result = db.test.replace_one({"y": 1}, replacement, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"z": 1})) self.assertEqual(0, db.test.count({"y": 1})) self.assertEqual(db.test.find_one(id1)["z"], 1) result = db.test.replace_one({"x": 2}, {"y": 2}, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(0, result.matched_count) self.assertTrue(result.modified_count in (None, 0)) self.assertTrue(isinstance(result.upserted_id, ObjectId)) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"y": 2})) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.replace_one({"x": 0}, {"y": 0}) self.assertTrue(isinstance(result, UpdateResult)) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.upserted_id) self.assertFalse(result.acknowledged) def test_update_one(self): db = self.db db.drop_collection("test") self.assertRaises(ValueError, lambda: db.test.update_one({}, {"x": 1})) id1 = db.test.insert_one({"x": 5}).inserted_id result = db.test.update_one({}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(db.test.find_one(id1)["x"], 6) id2 = db.test.insert_one({"x": 1}).inserted_id result = db.test.update_one({"x": 6}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(db.test.find_one(id1)["x"], 7) self.assertEqual(db.test.find_one(id2)["x"], 1) result = db.test.update_one({"x": 2}, {"$set": {"y": 1}}, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(0, result.matched_count) self.assertTrue(result.modified_count in (None, 0)) self.assertTrue(isinstance(result.upserted_id, ObjectId)) self.assertTrue(result.acknowledged) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.update_one({"x": 0}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.upserted_id) self.assertFalse(result.acknowledged) def test_update_many(self): db = self.db db.drop_collection("test") self.assertRaises(ValueError, lambda: db.test.update_many({}, {"x": 1})) db.test.insert_one({"x": 4, "y": 3}) db.test.insert_one({"x": 5, "y": 5}) db.test.insert_one({"x": 4, "y": 4}) result = db.test.update_many({"x": 4}, {"$set": {"y": 5}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(2, result.matched_count) self.assertTrue(result.modified_count in (None, 2)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(3, db.test.count({"y": 5})) result = db.test.update_many({"x": 5}, {"$set": {"y": 6}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(1, result.matched_count) self.assertTrue(result.modified_count in (None, 1)) self.assertIsNone(result.upserted_id) self.assertTrue(result.acknowledged) self.assertEqual(1, db.test.count({"y": 6})) result = db.test.update_many({"x": 2}, {"$set": {"y": 1}}, True) self.assertTrue(isinstance(result, UpdateResult)) self.assertEqual(0, result.matched_count) self.assertTrue(result.modified_count in (None, 0)) self.assertTrue(isinstance(result.upserted_id, ObjectId)) self.assertTrue(result.acknowledged) db = db.client.get_database(db.name, write_concern=WriteConcern(w=0)) result = db.test.update_many({"x": 0}, {"$inc": {"x": 1}}) self.assertTrue(isinstance(result, UpdateResult)) self.assertRaises(InvalidOperation, lambda: result.matched_count) self.assertRaises(InvalidOperation, lambda: result.modified_count) self.assertRaises(InvalidOperation, lambda: result.upserted_id) self.assertFalse(result.acknowledged) def test_update_with_invalid_keys(self): self.db.drop_collection("test") self.assertTrue(self.db.test.insert_one({"hello": "world"})) doc = self.db.test.find_one() doc['a.b'] = 'c' expected = InvalidDocument if client_context.version.at_least(2, 5, 4, -1): expected = OperationFailure # Replace self.assertRaises(expected, self.db.test.replace_one, {"hello": "world"}, doc) # Upsert self.assertRaises(expected, self.db.test.replace_one, {"foo": "bar"}, doc, upsert=True) # Check that the last two ops didn't actually modify anything self.assertTrue('a.b' not in self.db.test.find_one()) # Modify shouldn't check keys... self.assertTrue(self.db.test.update_one({"hello": "world"}, {"$set": {"foo.bar": "baz"}}, upsert=True)) # I know this seems like testing the server but I'd like to be notified # by CI if the server's behavior changes here. doc = SON([("$set", {"foo.bar": "bim"}), ("hello", "world")]) self.assertRaises(OperationFailure, self.db.test.update_one, {"hello": "world"}, doc, upsert=True) # This is going to cause keys to be checked and raise InvalidDocument. # That's OK assuming the server's behavior in the previous assert # doesn't change. If the behavior changes checking the first key for # '$' in update won't be good enough anymore. doc = SON([("hello", "world"), ("$set", {"foo.bar": "bim"})]) self.assertRaises(expected, self.db.test.replace_one, {"hello": "world"}, doc, upsert=True) # Replace with empty document self.assertNotEqual(0, self.db.test.replace_one( {"hello": "world"}, {}).matched_count) def test_acknowledged_delete(self): db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=1000) db.test.insert_one({"x": 1}) self.assertEqual(1, db.test.count()) # Can't remove from capped collection. self.assertRaises(OperationFailure, db.test.delete_one, {"x": 1}) db.drop_collection("test") db.test.insert_one({"x": 1}) db.test.insert_one({"x": 1}) self.assertEqual(2, db.test.delete_many({}).deleted_count) self.assertEqual(0, db.test.delete_many({}).deleted_count) def test_manual_last_error(self): coll = self.db.get_collection("test", write_concern=WriteConcern(w=0)) coll.insert_one({"x": 1}) self.db.command("getlasterror", w=1, wtimeout=1) def test_count(self): db = self.db db.drop_collection("test") self.assertEqual(db.test.count(), 0) db.test.insert_many([{}, {}]) self.assertEqual(db.test.count(), 2) db.test.insert_many([{'foo': 'bar'}, {'foo': 'baz'}]) self.assertEqual(db.test.find({'foo': 'bar'}).count(), 1) self.assertEqual(db.test.count({'foo': 'bar'}), 1) self.assertEqual(db.test.find({'foo': re.compile(r'ba.*')}).count(), 2) self.assertEqual( db.test.count({'foo': re.compile(r'ba.*')}), 2) def test_aggregate(self): db = self.db db.drop_collection("test") db.test.insert_one({'foo': [1, 2]}) self.assertRaises(TypeError, db.test.aggregate, "wow") pipeline = {"$project": {"_id": False, "foo": True}} result = db.test.aggregate([pipeline], useCursor=False) self.assertTrue(isinstance(result, CommandCursor)) self.assertEqual([{'foo': [1, 2]}], list(result)) def test_aggregate_raw_bson(self): db = self.db db.drop_collection("test") db.test.insert_one({'foo': [1, 2]}) self.assertRaises(TypeError, db.test.aggregate, "wow") pipeline = {"$project": {"_id": False, "foo": True}} result = db.get_collection( 'test', codec_options=CodecOptions(document_class=RawBSONDocument) ).aggregate([pipeline], useCursor=False) self.assertTrue(isinstance(result, CommandCursor)) first_result = next(result) self.assertIsInstance(first_result, RawBSONDocument) self.assertEqual([1, 2], list(first_result['foo'])) @client_context.require_version_min(2, 5, 1) def test_aggregation_cursor_validation(self): db = self.db projection = {'$project': {'_id': '$_id'}} cursor = db.test.aggregate([projection], cursor={}) self.assertTrue(isinstance(cursor, CommandCursor)) cursor = db.test.aggregate([projection], useCursor=True) self.assertTrue(isinstance(cursor, CommandCursor)) @client_context.require_version_min(2, 5, 1) def test_aggregation_cursor(self): db = self.db if client_context.replica_set_name: # Test that getMore messages are sent to the right server. db = self.client.get_database( db.name, read_preference=ReadPreference.SECONDARY, write_concern=WriteConcern(w=self.w)) for collection_size in (10, 1000): db.drop_collection("test") db.test.insert_many([{'_id': i} for i in range(collection_size)]) expected_sum = sum(range(collection_size)) # Use batchSize to ensure multiple getMore messages cursor = db.test.aggregate( [{'$project': {'_id': '$_id'}}], batchSize=5) self.assertEqual( expected_sum, sum(doc['_id'] for doc in cursor)) # Test that batchSize is handled properly. cursor = db.test.aggregate([], batchSize=5) self.assertEqual(5, len(cursor._CommandCursor__data)) # Force a getMore cursor._CommandCursor__data.clear() next(cursor) # startingFrom for a command cursor doesn't include the initial batch # returned by the command. self.assertEqual(5, cursor._CommandCursor__retrieved) # batchSize - 1 self.assertEqual(4, len(cursor._CommandCursor__data)) # Exhaust the cursor. There shouldn't be any errors. for doc in cursor: pass @client_context.require_version_min(2, 5, 1) def test_aggregation_cursor_alive(self): self.db.test.delete_many({}) self.db.test.insert_many([{} for _ in range(3)]) self.addCleanup(self.db.test.delete_many, {}) cursor = self.db.test.aggregate(pipeline=[], cursor={'batchSize': 2}) n = 0 while True: cursor.next() n += 1 if 3 == n: self.assertFalse(cursor.alive) break self.assertTrue(cursor.alive) @client_context.require_version_min(2, 5, 5) @client_context.require_no_mongos def test_parallel_scan(self): db = self.db db.drop_collection("test") if client_context.replica_set_name: # Test that getMore messages are sent to the right server. db = self.client.get_database( db.name, read_preference=ReadPreference.SECONDARY, write_concern=WriteConcern(w=self.w)) coll = db.test coll.insert_many([{'_id': i} for i in range(8000)]) docs = [] threads = [threading.Thread(target=docs.extend, args=(cursor,)) for cursor in coll.parallel_scan(3)] for t in threads: t.start() for t in threads: t.join() self.assertEqual( set(range(8000)), set(doc['_id'] for doc in docs)) def test_group(self): db = self.db db.drop_collection("test") self.assertEqual([], db.test.group([], {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) db.test.insert_many([{"a": 2}, {"b": 5}, {"a": 1}]) self.assertEqual([{"count": 3}], db.test.group([], {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) self.assertEqual([{"count": 1}], db.test.group([], {"a": {"$gt": 1}}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) db.test.insert_one({"a": 2, "b": 3}) self.assertEqual([{"a": 2, "count": 2}, {"a": None, "count": 1}, {"a": 1, "count": 1}], db.test.group(["a"], {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) # modifying finalize self.assertEqual([{"a": 2, "count": 3}, {"a": None, "count": 2}, {"a": 1, "count": 2}], db.test.group(["a"], {}, {"count": 0}, "function (obj, prev) " "{ prev.count++; }", "function (obj) { obj.count++; }")) # returning finalize self.assertEqual([2, 1, 1], db.test.group(["a"], {}, {"count": 0}, "function (obj, prev) " "{ prev.count++; }", "function (obj) { return obj.count; }")) # keyf self.assertEqual([2, 2], db.test.group("function (obj) { if (obj.a == 2) " "{ return {a: true} }; " "return {b: true}; }", {}, {"count": 0}, "function (obj, prev) " "{ prev.count++; }", "function (obj) { return obj.count; }")) # no key self.assertEqual([{"count": 4}], db.test.group(None, {}, {"count": 0}, "function (obj, prev) { prev.count++; }" )) self.assertRaises(OperationFailure, db.test.group, [], {}, {}, "5 ++ 5") def test_group_with_scope(self): db = self.db db.drop_collection("test") db.test.insert_many([{"a": 1}, {"b": 1}]) reduce_function = "function (obj, prev) { prev.count += inc_value; }" self.assertEqual(2, db.test.group([], {}, {"count": 0}, Code(reduce_function, {"inc_value": 1}))[0]['count']) self.assertEqual(4, db.test.group([], {}, {"count": 0}, Code(reduce_function, {"inc_value": 2}))[0]['count']) self.assertEqual(1, db.test.group([], {}, {"count": 0}, Code(reduce_function, {"inc_value": 0.5}))[0]['count']) self.assertEqual(2, db.test.group( [], {}, {"count": 0}, Code(reduce_function, {"inc_value": 1}))[0]['count']) self.assertEqual(4, db.test.group( [], {}, {"count": 0}, Code(reduce_function, {"inc_value": 2}))[0]['count']) self.assertEqual(1, db.test.group( [], {}, {"count": 0}, Code(reduce_function, {"inc_value": 0.5}))[0]['count']) def test_large_limit(self): db = self.db db.drop_collection("test_large_limit") db.test_large_limit.create_index([('x', 1)]) my_str = "mongomongo" * 1000 for i in range(2000): doc = {"x": i, "y": my_str} db.test_large_limit.insert_one(doc) i = 0 y = 0 for doc in db.test_large_limit.find(limit=1900).sort([('x', 1)]): i += 1 y += doc["x"] self.assertEqual(1900, i) self.assertEqual((1900 * 1899) / 2, y) def test_find_kwargs(self): db = self.db db.drop_collection("test") for i in range(10): db.test.insert_one({"x": i}) self.assertEqual(10, db.test.count()) total = 0 for x in db.test.find({}, skip=4, limit=2): total += x["x"] self.assertEqual(9, total) def test_rename(self): db = self.db db.drop_collection("test") db.drop_collection("foo") self.assertRaises(TypeError, db.test.rename, 5) self.assertRaises(InvalidName, db.test.rename, "") self.assertRaises(InvalidName, db.test.rename, "te$t") self.assertRaises(InvalidName, db.test.rename, ".test") self.assertRaises(InvalidName, db.test.rename, "test.") self.assertRaises(InvalidName, db.test.rename, "tes..t") self.assertEqual(0, db.test.count()) self.assertEqual(0, db.foo.count()) for i in range(10): db.test.insert_one({"x": i}) self.assertEqual(10, db.test.count()) db.test.rename("foo") self.assertEqual(0, db.test.count()) self.assertEqual(10, db.foo.count()) x = 0 for doc in db.foo.find(): self.assertEqual(x, doc["x"]) x += 1 db.test.insert_one({}) self.assertRaises(OperationFailure, db.foo.rename, "test") db.foo.rename("test", dropTarget=True) def test_find_one(self): db = self.db db.drop_collection("test") _id = db.test.insert_one({"hello": "world", "foo": "bar"}).inserted_id self.assertEqual("world", db.test.find_one()["hello"]) self.assertEqual(db.test.find_one(_id), db.test.find_one()) self.assertEqual(db.test.find_one(None), db.test.find_one()) self.assertEqual(db.test.find_one({}), db.test.find_one()) self.assertEqual(db.test.find_one({"hello": "world"}), db.test.find_one()) self.assertTrue("hello" in db.test.find_one(projection=["hello"])) self.assertTrue("hello" not in db.test.find_one(projection=["foo"])) self.assertEqual(["_id"], list(db.test.find_one(projection=[]))) self.assertEqual(None, db.test.find_one({"hello": "foo"})) self.assertEqual(None, db.test.find_one(ObjectId())) def test_find_one_non_objectid(self): db = self.db db.drop_collection("test") db.test.insert_one({"_id": 5}) self.assertTrue(db.test.find_one(5)) self.assertFalse(db.test.find_one(6)) def test_find_one_with_find_args(self): db = self.db db.drop_collection("test") db.test.insert_many([{"x": i} for i in range(1, 4)]) self.assertEqual(1, db.test.find_one()["x"]) self.assertEqual(2, db.test.find_one(skip=1, limit=2)["x"]) def test_find_with_sort(self): db = self.db db.drop_collection("test") db.test.insert_many([{"x": 2}, {"x": 1}, {"x": 3}]) self.assertEqual(2, db.test.find_one()["x"]) self.assertEqual(1, db.test.find_one(sort=[("x", 1)])["x"]) self.assertEqual(3, db.test.find_one(sort=[("x", -1)])["x"]) def to_list(things): return [thing["x"] for thing in things] self.assertEqual([2, 1, 3], to_list(db.test.find())) self.assertEqual([1, 2, 3], to_list(db.test.find(sort=[("x", 1)]))) self.assertEqual([3, 2, 1], to_list(db.test.find(sort=[("x", -1)]))) self.assertRaises(TypeError, db.test.find, sort=5) self.assertRaises(TypeError, db.test.find, sort="hello") self.assertRaises(ValueError, db.test.find, sort=["hello", 1]) # TODO doesn't actually test functionality, just that it doesn't blow up def test_cursor_timeout(self): list(self.db.test.find(no_cursor_timeout=True)) list(self.db.test.find(no_cursor_timeout=False)) def test_exhaust(self): if is_mongos(self.db.client): self.assertRaises(InvalidOperation, self.db.test.find, cursor_type=CursorType.EXHAUST) return # Limit is incompatible with exhaust. self.assertRaises(InvalidOperation, self.db.test.find, cursor_type=CursorType.EXHAUST, limit=5) cur = self.db.test.find(cursor_type=CursorType.EXHAUST) self.assertRaises(InvalidOperation, cur.limit, 5) cur = self.db.test.find(limit=5) self.assertRaises(InvalidOperation, cur.add_option, 64) cur = self.db.test.find() cur.add_option(64) self.assertRaises(InvalidOperation, cur.limit, 5) self.db.drop_collection("test") # Insert enough documents to require more than one batch self.db.test.insert_many([{'i': i} for i in range(150)]) client = rs_or_single_client(maxPoolSize=1) socks = get_pool(client).sockets # Make sure the socket is returned after exhaustion. cur = client[self.db.name].test.find(cursor_type=CursorType.EXHAUST) next(cur) self.assertEqual(0, len(socks)) for _ in cur: pass self.assertEqual(1, len(socks)) # Same as previous but don't call next() for _ in client[self.db.name].test.find(cursor_type=CursorType.EXHAUST): pass self.assertEqual(1, len(socks)) # If the Cursor instance is discarded before being # completely iterated we have to close and # discard the socket. cur = client[self.db.name].test.find(cursor_type=CursorType.EXHAUST) next(cur) self.assertEqual(0, len(socks)) if sys.platform.startswith('java') or 'PyPy' in sys.version: # Don't wait for GC or use gc.collect(), it's unreliable. cur.close() cur = None # The socket should be discarded. self.assertEqual(0, len(socks)) def test_distinct(self): self.db.drop_collection("test") test = self.db.test test.insert_many([{"a": 1}, {"a": 2}, {"a": 2}, {"a": 2}, {"a": 3}]) distinct = test.distinct("a") distinct.sort() self.assertEqual([1, 2, 3], distinct) distinct = test.find({'a': {'$gt': 1}}).distinct("a") distinct.sort() self.assertEqual([2, 3], distinct) distinct = test.distinct('a', {'a': {'$gt': 1}}) distinct.sort() self.assertEqual([2, 3], distinct) self.db.drop_collection("test") test.insert_one({"a": {"b": "a"}, "c": 12}) test.insert_one({"a": {"b": "b"}, "c": 12}) test.insert_one({"a": {"b": "c"}, "c": 12}) test.insert_one({"a": {"b": "c"}, "c": 12}) distinct = test.distinct("a.b") distinct.sort() self.assertEqual(["a", "b", "c"], distinct) def test_query_on_query_field(self): self.db.drop_collection("test") self.db.test.insert_one({"query": "foo"}) self.db.test.insert_one({"bar": "foo"}) self.assertEqual(1, self.db.test.find({"query": {"$ne": None}}).count()) self.assertEqual(1, len(list(self.db.test.find({"query": {"$ne": None}}))) ) def test_min_query(self): self.db.drop_collection("test") self.db.test.insert_many([{"x": 1}, {"x": 2}]) self.db.test.create_index("x") self.assertEqual(1, len(list(self.db.test.find({"$min": {"x": 2}, "$query": {}})))) self.assertEqual(2, self.db.test.find({"$min": {"x": 2}, "$query": {}})[0]["x"]) def test_numerous_inserts(self): # Ensure we don't exceed server's 1000-document batch size limit. self.db.test.drop() n_docs = 2100 self.db.test.insert_many([{} for _ in range(n_docs)]) self.assertEqual(n_docs, self.db.test.count()) self.db.test.drop() def test_map_reduce(self): db = self.db db.drop_collection("test") db.test.insert_one({"id": 1, "tags": ["dog", "cat"]}) db.test.insert_one({"id": 2, "tags": ["cat"]}) db.test.insert_one({"id": 3, "tags": ["mouse", "cat", "dog"]}) db.test.insert_one({"id": 4, "tags": []}) map = Code("function () {" " this.tags.forEach(function(z) {" " emit(z, 1);" " });" "}") reduce = Code("function (key, values) {" " var total = 0;" " for (var i = 0; i < values.length; i++) {" " total += values[i];" " }" " return total;" "}") result = db.test.map_reduce(map, reduce, out='mrunittests') self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(2, result.find_one({"_id": "dog"})["value"]) self.assertEqual(1, result.find_one({"_id": "mouse"})["value"]) db.test.insert_one({"id": 5, "tags": ["hampster"]}) result = db.test.map_reduce(map, reduce, out='mrunittests') self.assertEqual(1, result.find_one({"_id": "hampster"})["value"]) db.test.delete_one({"id": 5}) result = db.test.map_reduce(map, reduce, out={'merge': 'mrunittests'}) self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(1, result.find_one({"_id": "hampster"})["value"]) result = db.test.map_reduce(map, reduce, out={'reduce': 'mrunittests'}) self.assertEqual(6, result.find_one({"_id": "cat"})["value"]) self.assertEqual(4, result.find_one({"_id": "dog"})["value"]) self.assertEqual(2, result.find_one({"_id": "mouse"})["value"]) self.assertEqual(1, result.find_one({"_id": "hampster"})["value"]) result = db.test.map_reduce( map, reduce, out={'replace': 'mrunittests'} ) self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(2, result.find_one({"_id": "dog"})["value"]) self.assertEqual(1, result.find_one({"_id": "mouse"})["value"]) result = db.test.map_reduce(map, reduce, out=SON([('replace', 'mrunittests'), ('db', 'mrtestdb') ])) self.assertEqual(3, result.find_one({"_id": "cat"})["value"]) self.assertEqual(2, result.find_one({"_id": "dog"})["value"]) self.assertEqual(1, result.find_one({"_id": "mouse"})["value"]) self.client.drop_database('mrtestdb') full_result = db.test.map_reduce(map, reduce, out='mrunittests', full_response=True) self.assertEqual(6, full_result["counts"]["emit"]) result = db.test.map_reduce(map, reduce, out='mrunittests', limit=2) self.assertEqual(2, result.find_one({"_id": "cat"})["value"]) self.assertEqual(1, result.find_one({"_id": "dog"})["value"]) self.assertEqual(None, result.find_one({"_id": "mouse"})) result = db.test.map_reduce(map, reduce, out={'inline': 1}) self.assertTrue(isinstance(result, dict)) self.assertTrue('results' in result) self.assertTrue(result['results'][1]["_id"] in ("cat", "dog", "mouse")) result = db.test.inline_map_reduce(map, reduce) self.assertTrue(isinstance(result, list)) self.assertEqual(3, len(result)) self.assertTrue(result[1]["_id"] in ("cat", "dog", "mouse")) full_result = db.test.inline_map_reduce(map, reduce, full_response=True) self.assertEqual(6, full_result["counts"]["emit"]) def test_messages_with_unicode_collection_names(self): db = self.db db[u("Employés")].insert_one({"x": 1}) db[u("Employés")].replace_one({"x": 1}, {"x": 2}) db[u("Employés")].delete_many({}) db[u("Employés")].find_one() list(db[u("Employés")].find()) def test_drop_indexes_non_existent(self): self.db.drop_collection("test") self.db.test.drop_indexes() # This is really a bson test but easier to just reproduce it here... # (Shame on me) def test_bad_encode(self): c = self.db.test c.drop() self.assertRaises(InvalidDocument, c.insert_one, {"x": c}) class BadGetAttr(dict): def __getattr__(self, name): pass bad = BadGetAttr([('foo', 'bar')]) c.insert_one({'bad': bad}) self.assertEqual('bar', c.find_one()['bad']['foo']) def test_find_one_and(self): c = self.db.test c.drop() c.insert_one({'_id': 1, 'i': 1}) self.assertEqual({'_id': 1, 'i': 1}, c.find_one_and_update({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 3}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, return_document=ReturnDocument.AFTER)) self.assertEqual({'_id': 1, 'i': 3}, c.find_one_and_delete({'_id': 1})) self.assertEqual(None, c.find_one({'_id': 1})) self.assertEqual(None, c.find_one_and_update({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 1}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, return_document=ReturnDocument.AFTER, upsert=True)) self.assertEqual({'_id': 1, 'i': 2}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, return_document=ReturnDocument.AFTER)) self.assertEqual({'_id': 1, 'i': 3}, c.find_one_and_replace( {'_id': 1}, {'i': 3, 'j': 1}, projection=['i'], return_document=ReturnDocument.AFTER)) self.assertEqual({'i': 4}, c.find_one_and_update( {'_id': 1}, {'$inc': {'i': 1}}, projection={'i': 1, '_id': 0}, return_document=ReturnDocument.AFTER)) c.drop() for j in range(5): c.insert_one({'j': j, 'i': 0}) sort = [('j', DESCENDING)] self.assertEqual(4, c.find_one_and_update({}, {'$inc': {'i': 1}}, sort=sort)['j']) def test_find_one_and_write_concern(self): listener = EventListener() saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([]) db = single_client(event_listeners=[listener])[self.db.name] # non-default WriteConcern. c_w0 = db.get_collection( 'test', write_concern=WriteConcern(w=0)) # default WriteConcern. c_default = db.get_collection('test', write_concern=WriteConcern()) results = listener.results # Authenticate the client and throw out auth commands from the listener. db.command('ismaster') results.clear() try: if client_context.version.at_least(3, 1, 9, -1): c_w0.find_and_modify( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() c_w0.find_one_and_update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() c_w0.find_one_and_replace({'_id': 1}, {'foo': 'bar'}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() c_w0.find_one_and_delete({'_id': 1}) self.assertEqual( {'w': 0}, results['started'][0].command['writeConcern']) results.clear() # Test write concern errors. if client_context.is_rs: c_wc_error = db.get_collection( 'test', write_concern=WriteConcern( w=len(client_context.nodes) + 1)) self.assertRaises( WriteConcernError, c_wc_error.find_and_modify, {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_update, {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_replace, {'w': 0}, results['started'][0].command['writeConcern']) self.assertRaises( WriteConcernError, c_wc_error.find_one_and_delete, {'w': 0}, results['started'][0].command['writeConcern']) results.clear() else: c_w0.find_and_modify( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_w0.find_one_and_update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_w0.find_one_and_replace({'_id': 1}, {'foo': 'bar'}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_w0.find_one_and_delete({'_id': 1}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_and_modify({'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_one_and_update({'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_one_and_replace({'_id': 1}, {'foo': 'bar'}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() c_default.find_one_and_delete({'_id': 1}) self.assertNotIn('writeConcern', results['started'][0].command) results.clear() finally: monitoring._LISTENERS = saved_listeners def test_find_with_nested(self): c = self.db.test c.drop() c.insert_many([{'i': i} for i in range(5)]) # [0, 1, 2, 3, 4] self.assertEqual( [2], [i['i'] for i in c.find({ '$and': [ { # This clause gives us [1,2,4] '$or': [ {'i': {'$lte': 2}}, {'i': {'$gt': 3}}, ], }, { # This clause gives us [2,3] '$or': [ {'i': 2}, {'i': 3}, ] }, ] })] ) self.assertEqual( [0, 1, 2], [i['i'] for i in c.find({ '$or': [ { # This clause gives us [2] '$and': [ {'i': {'$gte': 2}}, {'i': {'$lt': 3}}, ], }, { # This clause gives us [0,1] '$and': [ {'i': {'$gt': -100}}, {'i': {'$lt': 2}}, ] }, ] })] ) def test_find_regex(self): c = self.db.test c.drop() c.insert_one({'r': re.compile('.*')}) self.assertTrue(isinstance(c.find_one()['r'], Regex)) for doc in c.find(): self.assertTrue(isinstance(doc['r'], Regex)) def test_find_command_generation(self): cmd = _gen_find_command( 'coll', {'$query': {'foo': 1}, '$dumb': 2}, None, 0, 0, 0, None) self.assertEqual( cmd.to_dict(), SON([('find', 'coll'), ('$dumb', 2), ('filter', {'foo': 1})]).to_dict()) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_pooling.py0000644000175000017500000003447512630145074021032 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test built in connection-pooling with threads.""" import gc import random import socket import sys import threading import time from pymongo import MongoClient from pymongo.errors import (AutoReconnect, ConnectionFailure, DuplicateKeyError, ExceededMaxWaiters) sys.path[0:0] = [""] from pymongo.network import socket_closed from pymongo.pool import Pool, PoolOptions from test import host, port, SkipTest, unittest, client_context from test.utils import (get_pool, joinall, delay, one, rs_or_single_client) @client_context.require_connection def setUpModule(): pass N = 10 DB = "pymongo-pooling-tests" def gc_collect_until_done(threads, timeout=60): start = time.time() running = list(threads) while running: assert (time.time() - start) < timeout, "Threads timed out" for t in running: t.join(0.1) if not t.isAlive(): running.remove(t) gc.collect() class MongoThread(threading.Thread): """A thread that uses a MongoClient.""" def __init__(self, client): super(MongoThread, self).__init__() self.daemon = True # Don't hang whole test if thread hangs. self.client = client self.db = self.client[DB] self.passed = False def run(self): self.run_mongo_thread() self.passed = True def run_mongo_thread(self): raise NotImplementedError class InsertOneAndFind(MongoThread): def run_mongo_thread(self): for _ in range(N): rand = random.randint(0, N) _id = self.db.sf.insert_one({"x": rand}).inserted_id assert rand == self.db.sf.find_one(_id)["x"] class Unique(MongoThread): def run_mongo_thread(self): for _ in range(N): self.db.unique.insert_one({}) # no error class NonUnique(MongoThread): def run_mongo_thread(self): for _ in range(N): try: self.db.unique.insert_one({"_id": "jesse"}) except DuplicateKeyError: pass else: raise AssertionError("Should have raised DuplicateKeyError") class Disconnect(MongoThread): def run_mongo_thread(self): for _ in range(N): self.client.close() class SocketGetter(MongoThread): """Utility for TestPooling. Checks out a socket and holds it forever. Used in test_no_wait_queue_timeout, test_wait_queue_multiple, and test_no_wait_queue_multiple. """ def __init__(self, client, pool): super(SocketGetter, self).__init__(client) self.state = 'init' self.pool = pool self.sock = None def run_mongo_thread(self): self.state = 'get_socket' # Pass 'checkout' so we can hold the socket. with self.pool.get_socket({}, checkout=True) as sock: self.sock = sock self.state = 'sock' def __del__(self): if self.sock: self.sock.close() def run_cases(client, cases): threads = [] n_runs = 5 for case in cases: for i in range(n_runs): t = case(client) t.start() threads.append(t) for t in threads: t.join() for t in threads: assert t.passed, "%s.run() threw an exception" % repr(t) class _TestPoolingBase(unittest.TestCase): """Base class for all connection-pool tests.""" def setUp(self): self.c = rs_or_single_client() db = self.c[DB] db.unique.drop() db.test.drop() db.unique.insert_one({"_id": "jesse"}) db.test.insert_many([{} for _ in range(10)]) def create_pool(self, pair=(host, port), *args, **kwargs): return Pool(pair, PoolOptions(*args, **kwargs)) class TestPooling(_TestPoolingBase): def test_max_pool_size_validation(self): self.assertRaises( ValueError, MongoClient, host=host, port=port, maxPoolSize=-1) self.assertRaises( ValueError, MongoClient, host=host, port=port, maxPoolSize='foo') c = MongoClient(host=host, port=port, maxPoolSize=100) self.assertEqual(c.max_pool_size, 100) def test_no_disconnect(self): run_cases(self.c, [NonUnique, Unique, InsertOneAndFind]) def test_disconnect(self): run_cases(self.c, [InsertOneAndFind, Disconnect, Unique]) def test_pool_reuses_open_socket(self): # Test Pool's _check_closed() method doesn't close a healthy socket. cx_pool = self.create_pool(max_pool_size=10) cx_pool._check_interval_seconds = 0 # Always check. with cx_pool.get_socket({}) as sock_info: pass with cx_pool.get_socket({}) as new_sock_info: self.assertEqual(sock_info, new_sock_info) self.assertEqual(1, len(cx_pool.sockets)) def test_get_socket_and_exception(self): # get_socket() returns socket after a non-network error. cx_pool = self.create_pool(max_pool_size=1, wait_queue_timeout=1) with self.assertRaises(ZeroDivisionError): with cx_pool.get_socket({}) as sock_info: 1 / 0 # Socket was returned, not closed. with cx_pool.get_socket({}) as new_sock_info: self.assertEqual(sock_info, new_sock_info) self.assertEqual(1, len(cx_pool.sockets)) def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. cx_pool = self.create_pool() with cx_pool.get_socket({}) as sock_info: # Use SocketInfo's API to close the socket. sock_info.close() self.assertEqual(0, len(cx_pool.sockets)) def test_pool_removes_dead_socket(self): # Test that Pool removes dead socket and the socket doesn't return # itself PYTHON-344 cx_pool = self.create_pool(max_pool_size=1, wait_queue_timeout=1) cx_pool._check_interval_seconds = 0 # Always check. with cx_pool.get_socket({}) as sock_info: # Simulate a closed socket without telling the SocketInfo it's # closed. sock_info.sock.close() self.assertTrue(socket_closed(sock_info.sock)) with cx_pool.get_socket({}) as new_sock_info: self.assertEqual(0, len(cx_pool.sockets)) self.assertNotEqual(sock_info, new_sock_info) self.assertEqual(1, len(cx_pool.sockets)) # Semaphore was released. with cx_pool.get_socket({}): pass def test_socket_closed(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((host, port)) self.assertFalse(socket_closed(s)) s.close() self.assertTrue(socket_closed(s)) def test_return_socket_after_reset(self): pool = self.create_pool() with pool.get_socket({}) as sock: pool.reset() self.assertTrue(sock.closed) self.assertEqual(0, len(pool.sockets)) def test_pool_check(self): # Test that Pool recovers from two connection failures in a row. # This exercises code at the end of Pool._check(). cx_pool = self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) cx_pool._check_interval_seconds = 0 # Always check. with cx_pool.get_socket({}) as sock_info: # Simulate a closed socket without telling the SocketInfo it's # closed. sock_info.sock.close() # Swap pool's address with a bad one. address, cx_pool.address = cx_pool.address, ('foo.com', 1234) with self.assertRaises(AutoReconnect): with cx_pool.get_socket({}): pass # Back to normal, semaphore was correctly released. cx_pool.address = address with cx_pool.get_socket({}, checkout=True) as sock_info: pass sock_info.close() def test_wait_queue_timeout(self): wait_queue_timeout = 2 # Seconds pool = self.create_pool( max_pool_size=1, wait_queue_timeout=wait_queue_timeout) with pool.get_socket({}) as sock_info: start = time.time() with self.assertRaises(ConnectionFailure): with pool.get_socket({}): pass duration = time.time() - start self.assertTrue( abs(wait_queue_timeout - duration) < 1, "Waited %.2f seconds for a socket, expected %f" % ( duration, wait_queue_timeout)) sock_info.close() def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. pool = self.create_pool(max_pool_size=1) # Reach max_size. with pool.get_socket({}) as s1: t = SocketGetter(self.c, pool) t.start() while t.state != 'get_socket': time.sleep(0.1) time.sleep(1) self.assertEqual(t.state, 'get_socket') while t.state != 'sock': time.sleep(0.1) self.assertEqual(t.state, 'sock') self.assertEqual(t.sock, s1) s1.close() def test_wait_queue_multiple(self): wait_queue_multiple = 3 pool = self.create_pool( max_pool_size=2, wait_queue_multiple=wait_queue_multiple) # Reach max_size sockets. with pool.get_socket({}): with pool.get_socket({}): # Reach max_size * wait_queue_multiple waiters. threads = [] for _ in range(6): t = SocketGetter(self.c, pool) t.start() threads.append(t) time.sleep(1) for t in threads: self.assertEqual(t.state, 'get_socket') with self.assertRaises(ExceededMaxWaiters): with pool.get_socket({}): pass def test_no_wait_queue_multiple(self): pool = self.create_pool(max_pool_size=2) socks = [] for _ in range(2): # Pass 'checkout' so we can hold the socket. with pool.get_socket({}, checkout=True) as sock: socks.append(sock) threads = [] for _ in range(30): t = SocketGetter(self.c, pool) t.start() threads.append(t) time.sleep(1) for t in threads: self.assertEqual(t.state, 'get_socket') for socket_info in socks: socket_info.close() class TestPoolMaxSize(_TestPoolingBase): def test_max_pool_size(self): max_pool_size = 4 c = rs_or_single_client(maxPoolSize=max_pool_size) collection = c[DB].test # Need one document. collection.drop() collection.insert_one({}) # nthreads had better be much larger than max_pool_size to ensure that # max_pool_size sockets are actually required at some point in this # test's execution. cx_pool = get_pool(c) nthreads = 10 threads = [] lock = threading.Lock() self.n_passed = 0 def f(): for _ in range(5): collection.find_one({'$where': delay(0.1)}) assert len(cx_pool.sockets) <= max_pool_size with lock: self.n_passed += 1 for i in range(nthreads): t = threading.Thread(target=f) threads.append(t) t.start() joinall(threads) self.assertEqual(nthreads, self.n_passed) self.assertTrue(len(cx_pool.sockets) > 1) self.assertEqual(max_pool_size, cx_pool._socket_semaphore.counter) def test_max_pool_size_none(self): c = rs_or_single_client(maxPoolSize=None) collection = c[DB].test # Need one document. collection.drop() collection.insert_one({}) cx_pool = get_pool(c) nthreads = 10 threads = [] lock = threading.Lock() self.n_passed = 0 def f(): for _ in range(5): collection.find_one({'$where': delay(0.1)}) with lock: self.n_passed += 1 for i in range(nthreads): t = threading.Thread(target=f) threads.append(t) t.start() joinall(threads) self.assertEqual(nthreads, self.n_passed) self.assertTrue(len(cx_pool.sockets) > 1) def test_max_pool_size_zero(self): with self.assertRaises(ValueError): rs_or_single_client(maxPoolSize=0) def test_max_pool_size_with_connection_failure(self): # The pool acquires its semaphore before attempting to connect; ensure # it releases the semaphore on connection failure. test_pool = Pool( ('example.com', 27017), PoolOptions( max_pool_size=1, connect_timeout=1, socket_timeout=1, wait_queue_timeout=1)) # First call to get_socket fails; if pool doesn't release its semaphore # then the second call raises "ConnectionFailure: Timed out waiting for # socket from pool" instead of AutoReconnect. for i in range(2): with self.assertRaises(AutoReconnect) as context: with test_pool.get_socket({}, checkout=True): pass # Testing for AutoReconnect instead of ConnectionFailure, above, # is sufficient right *now* to catch a semaphore leak. But that # seems error-prone, so check the message too. self.assertNotIn('waiting for socket from pool', str(context.exception)) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_cursor.py0000644000175000017500000013157612630145074020700 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the cursor module.""" import copy import itertools import random import re import sys sys.path[0:0] = [""] from bson.code import Code from bson.py3compat import u, PY3 from bson.son import SON from pymongo import (MongoClient, monitoring, ASCENDING, DESCENDING, ALL, OFF) from pymongo.command_cursor import CommandCursor from pymongo.cursor import CursorType from pymongo.cursor_manager import CursorManager from pymongo.errors import (InvalidOperation, OperationFailure, ExecutionTimeout) from test import (client_context, SkipTest, unittest, host, port, IntegrationTest) from test.utils import server_started_with_auth, single_client, EventListener if PY3: long = int class TestCursorNoConnect(unittest.TestCase): @classmethod def setUpClass(cls): client = MongoClient(host, port, connect=False) cls.db = client.test def test_deepcopy_cursor_littered_with_regexes(self): cursor = self.db.test.find({ "x": re.compile("^hmmm.*"), "y": [re.compile("^hmm.*")], "z": {"a": [re.compile("^hm.*")]}, re.compile("^key.*"): {"a": [re.compile("^hm.*")]}}) cursor2 = copy.deepcopy(cursor) self.assertEqual(cursor._Cursor__spec, cursor2._Cursor__spec) def test_add_remove_option(self): cursor = self.db.test.find() self.assertEqual(0, cursor._Cursor__query_flags) cursor.add_option(2) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE) self.assertEqual(2, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.add_option(32) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(34, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.add_option(128) cursor2 = self.db.test.find( cursor_type=CursorType.TAILABLE_AWAIT).add_option(128) self.assertEqual(162, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertEqual(162, cursor._Cursor__query_flags) cursor.add_option(128) self.assertEqual(162, cursor._Cursor__query_flags) cursor.remove_option(128) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(34, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(32) cursor2 = self.db.test.find(cursor_type=CursorType.TAILABLE) self.assertEqual(2, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertEqual(2, cursor._Cursor__query_flags) cursor.remove_option(32) self.assertEqual(2, cursor._Cursor__query_flags) # Timeout cursor = self.db.test.find(no_cursor_timeout=True) self.assertEqual(16, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(16) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(16) self.assertEqual(0, cursor._Cursor__query_flags) # Tailable / Await data cursor = self.db.test.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(34, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(34) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(32) self.assertEqual(2, cursor._Cursor__query_flags) # Partial cursor = self.db.test.find(allow_partial_results=True) self.assertEqual(128, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(128) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) cursor.remove_option(128) self.assertEqual(0, cursor._Cursor__query_flags) # Exhaust - which mongos doesn't support if not self.db.client.is_mongos: cursor = self.db.test.find(cursor_type=CursorType.EXHAUST) self.assertEqual(64, cursor._Cursor__query_flags) cursor2 = self.db.test.find().add_option(64) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertTrue(cursor._Cursor__exhaust) cursor.remove_option(64) self.assertEqual(0, cursor._Cursor__query_flags) self.assertFalse(cursor._Cursor__exhaust) class TestCursor(IntegrationTest): @client_context.require_version_min(2, 5, 3, -1) def test_max_time_ms(self): db = self.db db.pymongo_test.drop() coll = db.pymongo_test self.assertRaises(TypeError, coll.find().max_time_ms, 'foo') coll.insert_one({"amalia": 1}) coll.insert_one({"amalia": 2}) coll.find().max_time_ms(None) coll.find().max_time_ms(long(1)) cursor = coll.find().max_time_ms(999) self.assertEqual(999, cursor._Cursor__max_time_ms) cursor = coll.find().max_time_ms(10).max_time_ms(1000) self.assertEqual(1000, cursor._Cursor__max_time_ms) cursor = coll.find().max_time_ms(999) c2 = cursor.clone() self.assertEqual(999, c2._Cursor__max_time_ms) self.assertTrue("$maxTimeMS" in cursor._Cursor__query_spec()) self.assertTrue("$maxTimeMS" in c2._Cursor__query_spec()) self.assertTrue(coll.find_one(max_time_ms=1000)) client = self.client if "enableTestCommands=1" in client_context.cmd_line['argv']: # Cursor parses server timeout error in response to initial query. client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: cursor = coll.find().max_time_ms(1) try: next(cursor) except ExecutionTimeout: pass else: self.fail("ExecutionTimeout not raised") self.assertRaises(ExecutionTimeout, coll.find_one, max_time_ms=1) finally: client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") @client_context.require_version_min(3, 1, 9, -1) def test_max_await_time_ms(self): db = self.db db.pymongo_test.drop() coll = db.create_collection("pymongo_test", capped=True, size=4096) self.assertRaises(TypeError, coll.find().max_await_time_ms, 'foo') coll.insert_one({"amalia": 1}) coll.insert_one({"amalia": 2}) coll.find().max_await_time_ms(None) coll.find().max_await_time_ms(long(1)) # When cursor is not tailable_await cursor = coll.find() self.assertEqual(None, cursor._Cursor__max_await_time_ms) cursor = coll.find().max_await_time_ms(99) self.assertEqual(None, cursor._Cursor__max_await_time_ms) # If cursor is tailable_await and timeout is unset cursor = coll.find(cursor_type=CursorType.TAILABLE_AWAIT) self.assertEqual(None, cursor._Cursor__max_await_time_ms) # If cursor is tailable_await and timeout is set cursor = coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms(99) self.assertEqual(99, cursor._Cursor__max_await_time_ms) cursor = coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms( 10).max_await_time_ms(90) self.assertEqual(90, cursor._Cursor__max_await_time_ms) listener = EventListener() saved_listeners = monitoring._LISTENERS monitoring._LISTENERS = monitoring._Listeners([]) coll = single_client( event_listeners=[listener])[self.db.name].pymongo_test results = listener.results try: # Tailable_await defaults. list(coll.find(cursor_type=CursorType.TAILABLE_AWAIT)) # find self.assertFalse('maxTimeMS' in results['started'][0].command) # getMore self.assertFalse('maxTimeMS' in results['started'][1].command) results.clear() # Tailable_await with max_await_time_ms set. list(coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_await_time_ms(99)) # find self.assertFalse('maxTimeMS' in results['started'][0].command) # getMore self.assertTrue('maxTimeMS' in results['started'][1].command) self.assertEqual(99, results['started'][1].command['maxTimeMS']) results.clear() # Tailable_await with max_time_ms list(coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_time_ms(1)) # find self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(1, results['started'][0].command['maxTimeMS']) # getMore self.assertFalse('maxTimeMS' in results['started'][1].command) results.clear() # Tailable_await with both max_time_ms and max_await_time_ms list(coll.find( cursor_type=CursorType.TAILABLE_AWAIT).max_time_ms( 1).max_await_time_ms(99)) # find self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(1, results['started'][0].command['maxTimeMS']) # getMore self.assertTrue('maxTimeMS' in results['started'][1].command) self.assertEqual(99, results['started'][1].command['maxTimeMS']) results.clear() # Non tailable_await with max_await_time_ms list(coll.find(batch_size=1).max_await_time_ms(99)) # find self.assertFalse('maxTimeMS' in results['started'][0].command) # getMore self.assertFalse('maxTimeMS' in results['started'][1].command) results.clear() # Non tailable_await with max_time_ms list(coll.find(batch_size=1).max_time_ms(99)) # find self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(99, results['started'][0].command['maxTimeMS']) # getMore self.assertFalse('maxTimeMS' in results['started'][1].command) # Non tailable_await with both max_time_ms and max_await_time_ms list(coll.find(batch_size=1).max_time_ms(99).max_await_time_ms(88)) # find self.assertTrue('maxTimeMS' in results['started'][0].command) self.assertEqual(99, results['started'][0].command['maxTimeMS']) # getMore self.assertFalse('maxTimeMS' in results['started'][1].command) finally: monitoring._LISTENERS = saved_listeners @client_context.require_version_min(2, 5, 3, -1) @client_context.require_test_commands def test_max_time_ms_getmore(self): # Test that Cursor handles server timeout error in response to getmore. coll = self.db.pymongo_test coll.insert_many([{} for _ in range(200)]) cursor = coll.find().max_time_ms(100) # Send initial query before turning on failpoint. next(cursor) self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") try: try: # Iterate up to first getmore. list(cursor) except ExecutionTimeout: pass else: self.fail("ExecutionTimeout not raised") finally: self.client.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") def test_explain(self): a = self.db.test.find() a.explain() for _ in a: break b = a.explain() # "cursor" pre MongoDB 2.7.6, "executionStats" post self.assertTrue("cursor" in b or "executionStats" in b) def test_hint(self): db = self.db self.assertRaises(TypeError, db.test.find().hint, 5.5) db.test.drop() db.test.insert_many([{"num": i, "foo": i} for i in range(100)]) self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}) .hint([("num", ASCENDING)]).explain) self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}) .hint([("foo", ASCENDING)]).explain) spec = [("num", DESCENDING)] index = db.test.create_index(spec) first = next(db.test.find()) self.assertEqual(0, first.get('num')) first = next(db.test.find().hint(spec)) self.assertEqual(99, first.get('num')) self.assertRaises(OperationFailure, db.test.find({"num": 17, "foo": 17}) .hint([("foo", ASCENDING)]).explain) a = db.test.find({"num": 17}) a.hint(spec) for _ in a: break self.assertRaises(InvalidOperation, a.hint, spec) def test_hint_by_name(self): db = self.db db.test.drop() db.test.insert_many([{"i": i} for i in range(100)]) db.test.create_index([('i', DESCENDING)], name='fooindex') first = next(db.test.find()) self.assertEqual(0, first.get('i')) first = next(db.test.find().hint('fooindex')) self.assertEqual(99, first.get('i')) def test_limit(self): db = self.db self.assertRaises(TypeError, db.test.find().limit, None) self.assertRaises(TypeError, db.test.find().limit, "hello") self.assertRaises(TypeError, db.test.find().limit, 5.5) self.assertTrue(db.test.find().limit(long(5))) db.test.drop() db.test.insert_many([{"x": i} for i in range(100)]) count = 0 for _ in db.test.find(): count += 1 self.assertEqual(count, 100) count = 0 for _ in db.test.find().limit(20): count += 1 self.assertEqual(count, 20) count = 0 for _ in db.test.find().limit(99): count += 1 self.assertEqual(count, 99) count = 0 for _ in db.test.find().limit(1): count += 1 self.assertEqual(count, 1) count = 0 for _ in db.test.find().limit(0): count += 1 self.assertEqual(count, 100) count = 0 for _ in db.test.find().limit(0).limit(50).limit(10): count += 1 self.assertEqual(count, 10) a = db.test.find() a.limit(10) for _ in a: break self.assertRaises(InvalidOperation, a.limit, 5) def test_max(self): db = self.db db.test.drop() db.test.create_index([("j", ASCENDING)]) db.test.insert_many([{"j": j, "k": j} for j in range(10)]) cursor = db.test.find().max([("j", 3)]) self.assertEqual(len(list(cursor)), 3) # Tuple. cursor = db.test.find().max((("j", 3), )) self.assertEqual(len(list(cursor)), 3) # Compound index. db.test.create_index([("j", ASCENDING), ("k", ASCENDING)]) cursor = db.test.find().max([("j", 3), ("k", 3)]) self.assertEqual(len(list(cursor)), 3) # Wrong order. cursor = db.test.find().max([("k", 3), ("j", 3)]) self.assertRaises(OperationFailure, list, cursor) # No such index. cursor = db.test.find().max([("k", 3)]) self.assertRaises(OperationFailure, list, cursor) self.assertRaises(TypeError, db.test.find().max, 10) self.assertRaises(TypeError, db.test.find().max, {"j": 10}) def test_min(self): db = self.db db.test.drop() db.test.create_index([("j", ASCENDING)]) db.test.insert_many([{"j": j, "k": j} for j in range(10)]) cursor = db.test.find().min([("j", 3)]) self.assertEqual(len(list(cursor)), 7) # Tuple. cursor = db.test.find().min((("j", 3), )) self.assertEqual(len(list(cursor)), 7) # Compound index. db.test.create_index([("j", ASCENDING), ("k", ASCENDING)]) cursor = db.test.find().min([("j", 3), ("k", 3)]) self.assertEqual(len(list(cursor)), 7) # Wrong order. cursor = db.test.find().min([("k", 3), ("j", 3)]) self.assertRaises(OperationFailure, list, cursor) # No such index. cursor = db.test.find().min([("k", 3)]) self.assertRaises(OperationFailure, list, cursor) self.assertRaises(TypeError, db.test.find().min, 10) self.assertRaises(TypeError, db.test.find().min, {"j": 10}) def test_batch_size(self): db = self.db db.test.drop() db.test.insert_many([{"x": x} for x in range(200)]) self.assertRaises(TypeError, db.test.find().batch_size, None) self.assertRaises(TypeError, db.test.find().batch_size, "hello") self.assertRaises(TypeError, db.test.find().batch_size, 5.5) self.assertRaises(ValueError, db.test.find().batch_size, -1) self.assertTrue(db.test.find().batch_size(long(5))) a = db.test.find() for _ in a: break self.assertRaises(InvalidOperation, a.batch_size, 5) def cursor_count(cursor, expected_count): count = 0 for _ in cursor: count += 1 self.assertEqual(expected_count, count) cursor_count(db.test.find().batch_size(0), 200) cursor_count(db.test.find().batch_size(1), 200) cursor_count(db.test.find().batch_size(2), 200) cursor_count(db.test.find().batch_size(5), 200) cursor_count(db.test.find().batch_size(100), 200) cursor_count(db.test.find().batch_size(500), 200) cursor_count(db.test.find().batch_size(0).limit(1), 1) cursor_count(db.test.find().batch_size(1).limit(1), 1) cursor_count(db.test.find().batch_size(2).limit(1), 1) cursor_count(db.test.find().batch_size(5).limit(1), 1) cursor_count(db.test.find().batch_size(100).limit(1), 1) cursor_count(db.test.find().batch_size(500).limit(1), 1) cursor_count(db.test.find().batch_size(0).limit(10), 10) cursor_count(db.test.find().batch_size(1).limit(10), 10) cursor_count(db.test.find().batch_size(2).limit(10), 10) cursor_count(db.test.find().batch_size(5).limit(10), 10) cursor_count(db.test.find().batch_size(100).limit(10), 10) cursor_count(db.test.find().batch_size(500).limit(10), 10) def test_limit_and_batch_size(self): db = self.db db.test.drop() db.test.insert_many([{"x": x} for x in range(500)]) curs = db.test.find().limit(0).batch_size(10) next(curs) self.assertEqual(10, curs._Cursor__retrieved) curs = db.test.find(limit=0, batch_size=10) next(curs) self.assertEqual(10, curs._Cursor__retrieved) curs = db.test.find().limit(-2).batch_size(0) next(curs) self.assertEqual(2, curs._Cursor__retrieved) curs = db.test.find(limit=-2, batch_size=0) next(curs) self.assertEqual(2, curs._Cursor__retrieved) curs = db.test.find().limit(-4).batch_size(5) next(curs) self.assertEqual(4, curs._Cursor__retrieved) curs = db.test.find(limit=-4, batch_size=5) next(curs) self.assertEqual(4, curs._Cursor__retrieved) curs = db.test.find().limit(50).batch_size(500) next(curs) self.assertEqual(50, curs._Cursor__retrieved) curs = db.test.find(limit=50, batch_size=500) next(curs) self.assertEqual(50, curs._Cursor__retrieved) curs = db.test.find().batch_size(500) next(curs) self.assertEqual(500, curs._Cursor__retrieved) curs = db.test.find(batch_size=500) next(curs) self.assertEqual(500, curs._Cursor__retrieved) curs = db.test.find().limit(50) next(curs) self.assertEqual(50, curs._Cursor__retrieved) curs = db.test.find(limit=50) next(curs) self.assertEqual(50, curs._Cursor__retrieved) # these two might be shaky, as the default # is set by the server. as of 2.0.0-rc0, 101 # or 1MB (whichever is smaller) is default # for queries without ntoreturn curs = db.test.find() next(curs) self.assertEqual(101, curs._Cursor__retrieved) curs = db.test.find().limit(0).batch_size(0) next(curs) self.assertEqual(101, curs._Cursor__retrieved) curs = db.test.find(limit=0, batch_size=0) next(curs) self.assertEqual(101, curs._Cursor__retrieved) def test_skip(self): db = self.db self.assertRaises(TypeError, db.test.find().skip, None) self.assertRaises(TypeError, db.test.find().skip, "hello") self.assertRaises(TypeError, db.test.find().skip, 5.5) self.assertRaises(ValueError, db.test.find().skip, -5) self.assertTrue(db.test.find().skip(long(5))) db.drop_collection("test") db.test.insert_many([{"x": i} for i in range(100)]) for i in db.test.find(): self.assertEqual(i["x"], 0) break for i in db.test.find().skip(20): self.assertEqual(i["x"], 20) break for i in db.test.find().skip(99): self.assertEqual(i["x"], 99) break for i in db.test.find().skip(1): self.assertEqual(i["x"], 1) break for i in db.test.find().skip(0): self.assertEqual(i["x"], 0) break for i in db.test.find().skip(0).skip(50).skip(10): self.assertEqual(i["x"], 10) break for i in db.test.find().skip(1000): self.fail() a = db.test.find() a.skip(10) for _ in a: break self.assertRaises(InvalidOperation, a.skip, 5) def test_sort(self): db = self.db self.assertRaises(TypeError, db.test.find().sort, 5) self.assertRaises(ValueError, db.test.find().sort, []) self.assertRaises(TypeError, db.test.find().sort, [], ASCENDING) self.assertRaises(TypeError, db.test.find().sort, [("hello", DESCENDING)], DESCENDING) db.test.drop() unsort = list(range(10)) random.shuffle(unsort) db.test.insert_many([{"x": i} for i in unsort]) asc = [i["x"] for i in db.test.find().sort("x", ASCENDING)] self.assertEqual(asc, list(range(10))) asc = [i["x"] for i in db.test.find().sort("x")] self.assertEqual(asc, list(range(10))) asc = [i["x"] for i in db.test.find().sort([("x", ASCENDING)])] self.assertEqual(asc, list(range(10))) expect = list(reversed(range(10))) desc = [i["x"] for i in db.test.find().sort("x", DESCENDING)] self.assertEqual(desc, expect) desc = [i["x"] for i in db.test.find().sort([("x", DESCENDING)])] self.assertEqual(desc, expect) desc = [i["x"] for i in db.test.find().sort("x", ASCENDING).sort("x", DESCENDING)] self.assertEqual(desc, expect) expected = [(1, 5), (2, 5), (0, 3), (7, 3), (9, 2), (2, 1), (3, 1)] shuffled = list(expected) random.shuffle(shuffled) db.test.drop() for (a, b) in shuffled: db.test.insert_one({"a": a, "b": b}) result = [(i["a"], i["b"]) for i in db.test.find().sort([("b", DESCENDING), ("a", ASCENDING)])] self.assertEqual(result, expected) a = db.test.find() a.sort("x", ASCENDING) for _ in a: break self.assertRaises(InvalidOperation, a.sort, "x", ASCENDING) def test_count(self): db = self.db db.test.drop() self.assertEqual(0, db.test.find().count()) db.test.insert_many([{"x": i} for i in range(10)]) self.assertEqual(10, db.test.find().count()) self.assertTrue(isinstance(db.test.find().count(), int)) self.assertEqual(10, db.test.find().limit(5).count()) self.assertEqual(10, db.test.find().skip(5).count()) self.assertEqual(1, db.test.find({"x": 1}).count()) self.assertEqual(5, db.test.find({"x": {"$lt": 5}}).count()) a = db.test.find() b = a.count() for _ in a: break self.assertEqual(b, a.count()) self.assertEqual(0, db.test.acollectionthatdoesntexist.find().count()) def test_count_with_hint(self): collection = self.db.test collection.drop() collection.insert_many([{'i': 1}, {'i': 2}]) self.assertEqual(2, collection.find().count()) collection.create_index([('i', 1)]) self.assertEqual(1, collection.find({'i': 1}).hint("_id_").count()) self.assertEqual(2, collection.find().hint("_id_").count()) if client_context.version.at_least(2, 6, 0): # Count supports hint self.assertRaises(OperationFailure, collection.find({'i': 1}).hint("BAD HINT").count) else: # Hint is ignored self.assertEqual( 1, collection.find({'i': 1}).hint("BAD HINT").count()) # Create a sparse index which should have no entries. collection.create_index([('x', 1)], sparse=True) if client_context.version.at_least(2, 6, 0): # Count supports hint self.assertEqual(0, collection.find({'i': 1}).hint("x_1").count()) self.assertEqual( 0, collection.find({'i': 1}).hint([("x", 1)]).count()) else: # Hint is ignored self.assertEqual(1, collection.find({'i': 1}).hint("x_1").count()) self.assertEqual( 1, collection.find({'i': 1}).hint([("x", 1)]).count()) self.assertEqual(2, collection.find().hint("x_1").count()) self.assertEqual(2, collection.find().hint([("x", 1)]).count()) def test_where(self): db = self.db db.test.drop() a = db.test.find() self.assertRaises(TypeError, a.where, 5) self.assertRaises(TypeError, a.where, None) self.assertRaises(TypeError, a.where, {}) db.test.insert_many([{"x": i} for i in range(10)]) self.assertEqual(3, len(list(db.test.find().where('this.x < 3')))) self.assertEqual(3, len(list(db.test.find().where(Code('this.x < 3'))))) self.assertEqual(3, len(list(db.test.find().where(Code('this.x < i', {"i": 3}))))) self.assertEqual(10, len(list(db.test.find()))) self.assertEqual(3, db.test.find().where('this.x < 3').count()) self.assertEqual(10, db.test.find().count()) self.assertEqual(3, db.test.find().where(u('this.x < 3')).count()) self.assertEqual([0, 1, 2], [a["x"] for a in db.test.find().where('this.x < 3')]) self.assertEqual([], [a["x"] for a in db.test.find({"x": 5}).where('this.x < 3')]) self.assertEqual([5], [a["x"] for a in db.test.find({"x": 5}).where('this.x > 3')]) cursor = db.test.find().where('this.x < 3').where('this.x > 7') self.assertEqual([8, 9], [a["x"] for a in cursor]) a = db.test.find() b = a.where('this.x > 3') for _ in a: break self.assertRaises(InvalidOperation, a.where, 'this.x < 3') def test_rewind(self): self.db.test.insert_many([{"x": i} for i in range(1, 4)]) cursor = self.db.test.find().limit(2) count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) count = 0 for _ in cursor: count += 1 self.assertEqual(0, count) cursor.rewind() count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) cursor.rewind() count = 0 for _ in cursor: break cursor.rewind() for _ in cursor: count += 1 self.assertEqual(2, count) self.assertEqual(cursor, cursor.rewind()) def test_clone(self): self.db.test.insert_many([{"x": i} for i in range(1, 4)]) cursor = self.db.test.find().limit(2) count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) count = 0 for _ in cursor: count += 1 self.assertEqual(0, count) cursor = cursor.clone() cursor2 = cursor.clone() count = 0 for _ in cursor: count += 1 self.assertEqual(2, count) for _ in cursor2: count += 1 self.assertEqual(4, count) cursor.rewind() count = 0 for _ in cursor: break cursor = cursor.clone() for _ in cursor: count += 1 self.assertEqual(2, count) self.assertNotEqual(cursor, cursor.clone()) # Just test attributes cursor = self.db.test.find({"x": re.compile("^hello.*")}, skip=1, no_cursor_timeout=True, cursor_type=CursorType.TAILABLE_AWAIT, allow_partial_results=True, manipulate=False, projection={'_id': False}).limit(2) cursor.min([('a', 1)]).max([('b', 3)]) cursor.add_option(128) cursor.comment('hi!') cursor2 = cursor.clone() self.assertEqual(cursor._Cursor__skip, cursor2._Cursor__skip) self.assertEqual(cursor._Cursor__limit, cursor2._Cursor__limit) self.assertEqual(type(cursor._Cursor__codec_options), type(cursor2._Cursor__codec_options)) self.assertEqual(cursor._Cursor__manipulate, cursor2._Cursor__manipulate) self.assertEqual(cursor._Cursor__query_flags, cursor2._Cursor__query_flags) self.assertEqual(cursor._Cursor__comment, cursor2._Cursor__comment) self.assertEqual(cursor._Cursor__min, cursor2._Cursor__min) self.assertEqual(cursor._Cursor__max, cursor2._Cursor__max) # Shallow copies can so can mutate cursor2 = copy.copy(cursor) cursor2._Cursor__projection['cursor2'] = False self.assertTrue('cursor2' in cursor._Cursor__projection) # Deepcopies and shouldn't mutate cursor3 = copy.deepcopy(cursor) cursor3._Cursor__projection['cursor3'] = False self.assertFalse('cursor3' in cursor._Cursor__projection) cursor4 = cursor.clone() cursor4._Cursor__projection['cursor4'] = False self.assertFalse('cursor4' in cursor._Cursor__projection) # Test memo when deepcopying queries query = {"hello": "world"} query["reflexive"] = query cursor = self.db.test.find(query) cursor2 = copy.deepcopy(cursor) self.assertNotEqual(id(cursor._Cursor__spec), id(cursor2._Cursor__spec)) self.assertEqual(id(cursor2._Cursor__spec['reflexive']), id(cursor2._Cursor__spec)) self.assertEqual(len(cursor2._Cursor__spec), 2) # Ensure hints are cloned as the correct type cursor = self.db.test.find().hint([('z', 1), ("a", 1)]) cursor2 = copy.deepcopy(cursor) self.assertTrue(isinstance(cursor2._Cursor__hint, SON)) self.assertEqual(cursor._Cursor__hint, cursor2._Cursor__hint) def test_count_with_fields(self): self.db.test.drop() self.db.test.insert_one({"x": 1}) self.assertEqual(1, self.db.test.find({}, ["a"]).count()) def test_bad_getitem(self): self.assertRaises(TypeError, lambda x: self.db.test.find()[x], "hello") self.assertRaises(TypeError, lambda x: self.db.test.find()[x], 5.5) self.assertRaises(TypeError, lambda x: self.db.test.find()[x], None) def test_getitem_slice_index(self): self.db.drop_collection("test") self.db.test.insert_many([{"i": i} for i in range(100)]) count = itertools.count self.assertRaises(IndexError, lambda: self.db.test.find()[-1:]) self.assertRaises(IndexError, lambda: self.db.test.find()[1:2:2]) for a, b in zip(count(0), self.db.test.find()): self.assertEqual(a, b['i']) self.assertEqual(100, len(list(self.db.test.find()[0:]))) for a, b in zip(count(0), self.db.test.find()[0:]): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find()[20:]))) for a, b in zip(count(20), self.db.test.find()[20:]): self.assertEqual(a, b['i']) for a, b in zip(count(99), self.db.test.find()[99:]): self.assertEqual(a, b['i']) for i in self.db.test.find()[1000:]: self.fail() self.assertEqual(5, len(list(self.db.test.find()[20:25]))) self.assertEqual(5, len(list( self.db.test.find()[long(20):long(25)]))) for a, b in zip(count(20), self.db.test.find()[20:25]): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find()[40:45][20:]))) for a, b in zip(count(20), self.db.test.find()[40:45][20:]): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find()[40:45].limit(0).skip(20)) ) ) for a, b in zip(count(20), self.db.test.find()[40:45].limit(0).skip(20)): self.assertEqual(a, b['i']) self.assertEqual(80, len(list(self.db.test.find().limit(10).skip(40)[20:])) ) for a, b in zip(count(20), self.db.test.find().limit(10).skip(40)[20:]): self.assertEqual(a, b['i']) self.assertEqual(1, len(list(self.db.test.find()[:1]))) self.assertEqual(5, len(list(self.db.test.find()[:5]))) self.assertEqual(1, len(list(self.db.test.find()[99:100]))) self.assertEqual(1, len(list(self.db.test.find()[99:1000]))) self.assertEqual(0, len(list(self.db.test.find()[10:10]))) self.assertEqual(0, len(list(self.db.test.find()[:0]))) self.assertEqual(80, len(list(self.db.test.find()[10:10].limit(0).skip(20)) ) ) self.assertRaises(IndexError, lambda: self.db.test.find()[10:8]) def test_getitem_numeric_index(self): self.db.drop_collection("test") self.db.test.insert_many([{"i": i} for i in range(100)]) self.assertEqual(0, self.db.test.find()[0]['i']) self.assertEqual(50, self.db.test.find()[50]['i']) self.assertEqual(50, self.db.test.find().skip(50)[0]['i']) self.assertEqual(50, self.db.test.find().skip(49)[1]['i']) self.assertEqual(50, self.db.test.find()[long(50)]['i']) self.assertEqual(99, self.db.test.find()[99]['i']) self.assertRaises(IndexError, lambda x: self.db.test.find()[x], -1) self.assertRaises(IndexError, lambda x: self.db.test.find()[x], 100) self.assertRaises(IndexError, lambda x: self.db.test.find().skip(50)[x], 50) def test_count_with_limit_and_skip(self): self.assertRaises(TypeError, self.db.test.find().count, "foo") def check_len(cursor, length): self.assertEqual(len(list(cursor)), cursor.count(True)) self.assertEqual(length, cursor.count(True)) self.db.drop_collection("test") self.db.test.insert_many([{"i": i} for i in range(100)]) check_len(self.db.test.find(), 100) check_len(self.db.test.find().limit(10), 10) check_len(self.db.test.find().limit(110), 100) check_len(self.db.test.find().skip(10), 90) check_len(self.db.test.find().skip(110), 0) check_len(self.db.test.find().limit(10).skip(10), 10) check_len(self.db.test.find()[10:20], 10) check_len(self.db.test.find().limit(10).skip(95), 5) check_len(self.db.test.find()[95:105], 5) def test_len(self): self.assertRaises(TypeError, len, self.db.test.find()) def test_properties(self): self.assertEqual(self.db.test, self.db.test.find().collection) def set_coll(): self.db.test.find().collection = "hello" self.assertRaises(AttributeError, set_coll) def test_get_more(self): db = self.db db.drop_collection("test") db.test.insert_many([{'i': i} for i in range(10)]) self.assertEqual(10, len(list(db.test.find().batch_size(5)))) def test_tailable(self): db = self.db db.drop_collection("test") db.create_collection("test", capped=True, size=1000, max=3) self.addCleanup(db.drop_collection, "test") cursor = db.test.find(cursor_type=CursorType.TAILABLE) db.test.insert_one({"x": 1}) count = 0 for doc in cursor: count += 1 self.assertEqual(1, doc["x"]) self.assertEqual(1, count) db.test.insert_one({"x": 2}) count = 0 for doc in cursor: count += 1 self.assertEqual(2, doc["x"]) self.assertEqual(1, count) db.test.insert_one({"x": 3}) count = 0 for doc in cursor: count += 1 self.assertEqual(3, doc["x"]) self.assertEqual(1, count) # Capped rollover - the collection can never # have more than 3 documents. Just make sure # this doesn't raise... db.test.insert_many([{"x": i} for i in range(4, 7)]) self.assertEqual(0, len(list(cursor))) # and that the cursor doesn't think it's still alive. self.assertFalse(cursor.alive) self.assertEqual(3, db.test.count()) def test_distinct(self): self.db.drop_collection("test") self.db.test.insert_many( [{"a": 1}, {"a": 2}, {"a": 2}, {"a": 2}, {"a": 3}]) distinct = self.db.test.find({"a": {"$lt": 3}}).distinct("a") distinct.sort() self.assertEqual([1, 2], distinct) self.db.drop_collection("test") self.db.test.insert_one({"a": {"b": "a"}, "c": 12}) self.db.test.insert_one({"a": {"b": "b"}, "c": 8}) self.db.test.insert_one({"a": {"b": "c"}, "c": 12}) self.db.test.insert_one({"a": {"b": "c"}, "c": 8}) distinct = self.db.test.find({"c": 8}).distinct("a.b") distinct.sort() self.assertEqual(["b", "c"], distinct) def test_max_scan(self): self.db.drop_collection("test") self.db.test.insert_many([{} for _ in range(100)]) self.assertEqual(100, len(list(self.db.test.find()))) self.assertEqual(50, len(list(self.db.test.find().max_scan(50)))) self.assertEqual(50, len(list(self.db.test.find() .max_scan(90).max_scan(50)))) def test_with_statement(self): self.db.drop_collection("test") self.db.test.insert_many([{} for _ in range(100)]) c1 = self.db.test.find() with self.db.test.find() as c2: self.assertTrue(c2.alive) self.assertFalse(c2.alive) with self.db.test.find() as c2: self.assertEqual(100, len(list(c2))) self.assertFalse(c2.alive) self.assertTrue(c1.alive) @client_context.require_no_mongos def test_comment(self): if server_started_with_auth(self.db.client): raise SkipTest("SERVER-4754 - This test uses profiling.") # MongoDB 3.1.5 changed the ns for commands. regex = {'$regex': 'pymongo_test.(\$cmd|test)'} if client_context.version.at_least(3, 1, 8, -1): query_key = "query.comment" else: query_key = "query.$comment" self.client.drop_database(self.db) self.db.set_profiling_level(ALL) try: list(self.db.test.find().comment('foo')) op = self.db.system.profile.find({'ns': 'pymongo_test.test', 'op': 'query', query_key: 'foo'}) self.assertEqual(op.count(), 1) self.db.test.find().comment('foo').count() op = self.db.system.profile.find({'ns': regex, 'op': 'command', 'command.count': 'test', 'command.$comment': 'foo'}) self.assertEqual(op.count(), 1) self.db.test.find().comment('foo').distinct('type') op = self.db.system.profile.find({'ns': regex, 'op': 'command', 'command.distinct': 'test', 'command.$comment': 'foo'}) self.assertEqual(op.count(), 1) finally: self.db.set_profiling_level(OFF) self.db.system.profile.drop() self.db.test.insert_many([{}, {}]) cursor = self.db.test.find() next(cursor) self.assertRaises(InvalidOperation, cursor.comment, 'hello') def test_cursor_transfer(self): # This is just a test, don't try this at home... client = client_context.rs_or_standalone_client db = client.pymongo_test db.test.delete_many({}) db.test.insert_many([{'_id': i} for i in range(200)]) class CManager(CursorManager): def __init__(self, client): super(CManager, self).__init__(client) def close(self, dummy, dummy2): # Do absolutely nothing... pass client.set_cursor_manager(CManager) self.addCleanup(client.set_cursor_manager, CursorManager) docs = [] cursor = db.test.find().batch_size(10) docs.append(next(cursor)) cursor.close() docs.extend(cursor) self.assertEqual(len(docs), 10) cmd_cursor = {'id': cursor.cursor_id, 'firstBatch': []} ccursor = CommandCursor(cursor.collection, cmd_cursor, cursor.address, retrieved=cursor.retrieved) docs.extend(ccursor) self.assertEqual(len(docs), 200) def test_modifiers(self): cur = self.db.test.find() self.assertTrue('$query' not in cur._Cursor__query_spec()) cur = self.db.test.find().comment("testing").max_time_ms(500) self.assertTrue('$query' in cur._Cursor__query_spec()) self.assertEqual(cur._Cursor__query_spec()["$comment"], "testing") self.assertEqual(cur._Cursor__query_spec()["$maxTimeMS"], 500) cur = self.db.test.find( modifiers={"$maxTimeMS": 500, "$comment": "testing"}) self.assertTrue('$query' in cur._Cursor__query_spec()) self.assertEqual(cur._Cursor__query_spec()["$comment"], "testing") self.assertEqual(cur._Cursor__query_spec()["$maxTimeMS"], 500) def test_alive(self): self.db.test.delete_many({}) self.db.test.insert_many([{} for _ in range(3)]) self.addCleanup(self.db.test.delete_many, {}) cursor = self.db.test.find().batch_size(2) n = 0 while True: cursor.next() n += 1 if 3 == n: self.assertFalse(cursor.alive) break self.assertTrue(cursor.alive) if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_legacy_api.py0000644000175000017500000012452212630145074021451 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test various legacy / deprecated API features.""" import itertools import sys import threading import time import warnings sys.path[0:0] = [""] from bson.codec_options import CodecOptions from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import u from bson.son import SON from pymongo import ASCENDING, DESCENDING from pymongo.errors import (ConfigurationError, DocumentTooLarge, DuplicateKeyError, InvalidDocument, InvalidOperation, OperationFailure, WTimeoutError) from pymongo.son_manipulator import (AutoReference, NamespaceInjector, ObjectIdShuffler, SONManipulator) from pymongo.write_concern import WriteConcern from test import client_context, qcheck, unittest from test.test_client import IntegrationTest from test.utils import (joinall, oid_generated_on_client, rs_or_single_client, wait_until) class TestDeprecations(IntegrationTest): @classmethod def setUpClass(cls): super(TestDeprecations, cls).setUpClass() cls.warn_context = warnings.catch_warnings() cls.warn_context.__enter__() warnings.simplefilter("error", DeprecationWarning) @classmethod def tearDownClass(cls): cls.warn_context.__exit__() cls.warn_context = None def test_save_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.save({})) def test_insert_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.insert({})) def test_update_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.update({}, {})) def test_remove_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.remove({})) def test_find_and_modify_deprecation(self): self.assertRaises( DeprecationWarning, lambda: self.db.test.find_and_modify({'i': 5}, {})) def test_add_son_manipulator_deprecation(self): db = self.client.pymongo_test self.assertRaises(DeprecationWarning, lambda: db.add_son_manipulator(AutoReference(db))) def test_ensure_index_deprecation(self): try: self.assertRaises( DeprecationWarning, lambda: self.db.test.ensure_index('i')) finally: self.db.test.drop() class TestLegacy(IntegrationTest): @classmethod def setUpClass(cls): super(TestLegacy, cls).setUpClass() cls.w = client_context.w cls.warn_context = warnings.catch_warnings() cls.warn_context.__enter__() warnings.simplefilter("ignore", DeprecationWarning) @classmethod def tearDownClass(cls): cls.warn_context.__exit__() cls.warn_context = None def test_insert_find_one(self): # Tests legacy insert. db = self.db db.test.drop() self.assertEqual(0, len(list(db.test.find()))) doc = {"hello": u("world")} _id = db.test.insert(doc) self.assertEqual(1, len(list(db.test.find()))) self.assertEqual(doc, db.test.find_one()) self.assertEqual(doc["_id"], _id) self.assertTrue(isinstance(_id, ObjectId)) doc_class = dict # Work around http://bugs.jython.org/issue1728 if (sys.platform.startswith('java') and sys.version_info[:3] >= (2, 5, 2)): doc_class = SON db = self.client.get_database( db.name, codec_options=CodecOptions(document_class=doc_class)) def remove_insert_find_one(doc): db.test.remove({}) db.test.insert(doc) # SON equality is order sensitive. return db.test.find_one() == doc.to_dict() qcheck.check_unittest(self, remove_insert_find_one, qcheck.gen_mongo_dict(3)) def test_generator_insert(self): # Only legacy insert currently supports insert from a generator. db = self.db db.test.remove({}) self.assertEqual(db.test.find().count(), 0) db.test.insert(({'a': i} for i in range(5)), manipulate=False) self.assertEqual(5, db.test.count()) db.test.remove({}) db.test.insert(({'a': i} for i in range(5)), manipulate=True) self.assertEqual(5, db.test.count()) db.test.remove({}) def test_insert_multiple(self): # Tests legacy insert. db = self.db db.drop_collection("test") doc1 = {"hello": u("world")} doc2 = {"hello": u("mike")} self.assertEqual(db.test.find().count(), 0) ids = db.test.insert([doc1, doc2]) self.assertEqual(db.test.find().count(), 2) self.assertEqual(doc1, db.test.find_one({"hello": u("world")})) self.assertEqual(doc2, db.test.find_one({"hello": u("mike")})) self.assertEqual(2, len(ids)) self.assertEqual(doc1["_id"], ids[0]) self.assertEqual(doc2["_id"], ids[1]) ids = db.test.insert([{"hello": 1}]) self.assertTrue(isinstance(ids, list)) self.assertEqual(1, len(ids)) self.assertRaises(InvalidOperation, db.test.insert, []) # Generator that raises StopIteration on first call to next(). self.assertRaises(InvalidOperation, db.test.insert, (i for i in [])) def test_insert_multiple_with_duplicate(self): # Tests legacy insert. db = self.db db.drop_collection("test_insert_multiple_with_duplicate") collection = db.test_insert_multiple_with_duplicate collection.ensure_index([('i', ASCENDING)], unique=True) # No error collection.insert([{'i': i} for i in range(5, 10)], w=0) wait_until(lambda: 5 == collection.count(), 'insert 5 documents') collection.remove() # No error collection.insert([{'i': 1}] * 2, w=0) wait_until(lambda: 1 == collection.count(), 'insert 1 document') self.assertRaises( DuplicateKeyError, lambda: collection.insert([{'i': 2}] * 2), ) db.drop_collection("test_insert_multiple_with_duplicate") db = self.client.get_database( db.name, write_concern=WriteConcern(w=0)) collection = db.test_insert_multiple_with_duplicate collection.ensure_index([('i', ASCENDING)], unique=True) # No error. collection.insert([{'i': 1}] * 2) wait_until(lambda: 1 == collection.count(), 'insert 1 document') # Implied acknowledged. self.assertRaises( DuplicateKeyError, lambda: collection.insert([{'i': 2}] * 2, fsync=True), ) # Explicit acknowledged. self.assertRaises( DuplicateKeyError, lambda: collection.insert([{'i': 2}] * 2, w=1)) db.drop_collection("test_insert_multiple_with_duplicate") def test_insert_iterables(self): # Tests legacy insert. db = self.db self.assertRaises(TypeError, db.test.insert, 4) self.assertRaises(TypeError, db.test.insert, None) self.assertRaises(TypeError, db.test.insert, True) db.drop_collection("test") self.assertEqual(db.test.find().count(), 0) db.test.insert(({"hello": u("world")}, {"hello": u("world")})) self.assertEqual(db.test.find().count(), 2) db.drop_collection("test") self.assertEqual(db.test.find().count(), 0) db.test.insert(map(lambda x: {"hello": "world"}, itertools.repeat(None, 10))) self.assertEqual(db.test.find().count(), 10) def test_insert_manipulate_false(self): # Test three aspects of legacy insert with manipulate=False: # 1. The return value is None or [None] as appropriate. # 2. _id is not set on the passed-in document object. # 3. _id is not sent to server. collection = self.db.test_insert_manipulate_false collection.drop() oid = ObjectId() doc = {'a': oid} # The return value is None. self.assertTrue(collection.insert(doc, manipulate=False) is None) # insert() shouldn't set _id on the passed-in document object. self.assertEqual({'a': oid}, doc) server_doc = collection.find_one() # _id is not sent to server, so it's generated server-side. self.assertFalse(oid_generated_on_client(server_doc['_id'])) # Bulk insert. The return value is a list of None. self.assertEqual([None], collection.insert([{}], manipulate=False)) ids = collection.insert([{}, {}], manipulate=False) self.assertEqual([None, None], ids) collection.drop() def test_continue_on_error(self): # Tests legacy insert. db = self.db db.drop_collection("test_continue_on_error") collection = db.test_continue_on_error oid = collection.insert({"one": 1}) self.assertEqual(1, collection.count()) docs = [] docs.append({"_id": oid, "two": 2}) # Duplicate _id. docs.append({"three": 3}) docs.append({"four": 4}) docs.append({"five": 5}) with self.assertRaises(DuplicateKeyError): collection.insert(docs, manipulate=False) self.assertEqual(1, collection.count()) with self.assertRaises(DuplicateKeyError): collection.insert(docs, manipulate=False, continue_on_error=True) self.assertEqual(4, collection.count()) db.drop_collection("test_continue_on_error") oid = collection.insert({"_id": oid, "one": 1}, w=0) wait_until(lambda: 1 == collection.count(), 'insert 1 document') docs[0].pop("_id") docs[2]["_id"] = oid with self.assertRaises(DuplicateKeyError): collection.insert(docs, manipulate=False) self.assertEqual(3, collection.count()) collection.insert(docs, manipulate=False, continue_on_error=True, w=0) wait_until(lambda: 6 == collection.count(), 'insert 3 documents') def test_acknowledged_insert(self): # Tests legacy insert. db = self.db db.drop_collection("test_acknowledged_insert") collection = db.test_acknowledged_insert a = {"hello": "world"} collection.insert(a) collection.insert(a, w=0) self.assertRaises(OperationFailure, collection.insert, a) def test_insert_adds_id(self): # Tests legacy insert. doc = {"hello": "world"} self.db.test.insert(doc) self.assertTrue("_id" in doc) docs = [{"hello": "world"}, {"hello": "world"}] self.db.test.insert(docs) for doc in docs: self.assertTrue("_id" in doc) def test_insert_large_batch(self): # Tests legacy insert. db = self.client.test_insert_large_batch self.addCleanup(self.client.drop_database, 'test_insert_large_batch') max_bson_size = self.client.max_bson_size if client_context.version.at_least(2, 5, 4, -1): # Write commands are limited to 16MB + 16k per batch big_string = 'x' * int(max_bson_size / 2) else: big_string = 'x' * (max_bson_size - 100) # Batch insert that requires 2 batches. successful_insert = [{'x': big_string}, {'x': big_string}, {'x': big_string}, {'x': big_string}] db.collection_0.insert(successful_insert, w=1) self.assertEqual(4, db.collection_0.count()) # Test that inserts fail after first error. insert_second_fails = [{'_id': 'id0', 'x': big_string}, {'_id': 'id0', 'x': big_string}, {'_id': 'id1', 'x': big_string}, {'_id': 'id2', 'x': big_string}] with self.assertRaises(DuplicateKeyError): db.collection_1.insert(insert_second_fails) self.assertEqual(1, db.collection_1.count()) # 2 batches, 2nd insert fails, don't continue on error. self.assertTrue(db.collection_2.insert(insert_second_fails, w=0)) wait_until(lambda: 1 == db.collection_2.count(), 'insert 1 document', timeout=60) # 2 batches, ids of docs 0 and 1 are dupes, ids of docs 2 and 3 are # dupes. Acknowledged, continue on error. insert_two_failures = [{'_id': 'id0', 'x': big_string}, {'_id': 'id0', 'x': big_string}, {'_id': 'id1', 'x': big_string}, {'_id': 'id1', 'x': big_string}] with self.assertRaises(OperationFailure) as context: db.collection_3.insert(insert_two_failures, continue_on_error=True, w=1) self.assertIn('id1', str(context.exception)) # Only the first and third documents should be inserted. self.assertEqual(2, db.collection_3.count()) # 2 batches, 2 errors, unacknowledged, continue on error. db.collection_4.insert(insert_two_failures, continue_on_error=True, w=0) # Only the first and third documents are inserted. wait_until(lambda: 2 == db.collection_4.count(), 'insert 2 documents', timeout=60) def test_bad_dbref(self): # Requires the legacy API to test. c = self.db.test c.drop() # Incomplete DBRefs. self.assertRaises( InvalidDocument, c.insert_one, {'ref': {'$ref': 'collection'}}) self.assertRaises( InvalidDocument, c.insert_one, {'ref': {'$id': ObjectId()}}) ref_only = {'ref': {'$ref': 'collection'}} id_only = {'ref': {'$id': ObjectId()}} # Starting with MongoDB 2.5.2 this is no longer possible # from insert, update, or findAndModify. if not client_context.version.at_least(2, 5, 2): # Force insert of ref without $id. c.insert(ref_only, check_keys=False) self.assertEqual(DBRef('collection', id=None), c.find_one()['ref']) c.drop() # DBRef without $ref is decoded as normal subdocument. c.insert(id_only, check_keys=False) self.assertEqual(id_only, c.find_one()) def test_update(self): # Tests legacy update. db = self.db db.drop_collection("test") id1 = db.test.save({"x": 5}) db.test.update({}, {"$inc": {"x": 1}}) self.assertEqual(db.test.find_one(id1)["x"], 6) id2 = db.test.save({"x": 1}) db.test.update({"x": 6}, {"$inc": {"x": 1}}) self.assertEqual(db.test.find_one(id1)["x"], 7) self.assertEqual(db.test.find_one(id2)["x"], 1) def test_update_manipulate(self): # Tests legacy update. db = self.db db.drop_collection("test") db.test.insert({'_id': 1}) db.test.update({'_id': 1}, {'a': 1}, manipulate=True) self.assertEqual( {'_id': 1, 'a': 1}, db.test.find_one()) class AddField(SONManipulator): def transform_incoming(self, son, dummy): son['field'] = 'value' return son db.add_son_manipulator(AddField()) db.test.update({'_id': 1}, {'a': 2}, manipulate=False) self.assertEqual( {'_id': 1, 'a': 2}, db.test.find_one()) db.test.update({'_id': 1}, {'a': 3}, manipulate=True) self.assertEqual( {'_id': 1, 'a': 3, 'field': 'value'}, db.test.find_one()) def test_update_nmodified(self): # Tests legacy update. db = self.db db.drop_collection("test") ismaster = self.client.admin.command('ismaster') used_write_commands = (ismaster.get("maxWireVersion", 0) > 1) db.test.insert({'_id': 1}) result = db.test.update({'_id': 1}, {'$set': {'x': 1}}) if used_write_commands: self.assertEqual(1, result['nModified']) else: self.assertFalse('nModified' in result) # x is already 1. result = db.test.update({'_id': 1}, {'$set': {'x': 1}}) if used_write_commands: self.assertEqual(0, result['nModified']) else: self.assertFalse('nModified' in result) def test_multi_update(self): # Tests legacy update. db = self.db db.drop_collection("test") db.test.save({"x": 4, "y": 3}) db.test.save({"x": 5, "y": 5}) db.test.save({"x": 4, "y": 4}) db.test.update({"x": 4}, {"$set": {"y": 5}}, multi=True) self.assertEqual(3, db.test.count()) for doc in db.test.find(): self.assertEqual(5, doc["y"]) self.assertEqual(2, db.test.update({"x": 4}, {"$set": {"y": 6}}, multi=True)["n"]) def test_upsert(self): # Tests legacy update. db = self.db db.drop_collection("test") db.test.update({"page": "/"}, {"$inc": {"count": 1}}, upsert=True) db.test.update({"page": "/"}, {"$inc": {"count": 1}}, upsert=True) self.assertEqual(1, db.test.count()) self.assertEqual(2, db.test.find_one()["count"]) def test_acknowledged_update(self): # Tests legacy update. db = self.db db.drop_collection("test_acknowledged_update") collection = db.test_acknowledged_update collection.create_index("x", unique=True) collection.insert({"x": 5}) _id = collection.insert({"x": 4}) self.assertEqual( None, collection.update({"_id": _id}, {"$inc": {"x": 1}}, w=0)) self.assertRaises(DuplicateKeyError, collection.update, {"_id": _id}, {"$inc": {"x": 1}}) self.assertEqual(1, collection.update({"_id": _id}, {"$inc": {"x": 2}})["n"]) self.assertEqual(0, collection.update({"_id": "foo"}, {"$inc": {"x": 2}})["n"]) db.drop_collection("test_acknowledged_update") def test_update_backward_compat(self): # MongoDB versions >= 2.6.0 don't return the updatedExisting field # and return upsert _id in an array subdocument. This test should # pass regardless of server version or type (mongod/s). # Tests legacy update. c = self.db.test c.drop() oid = ObjectId() res = c.update({'_id': oid}, {'$set': {'a': 'a'}}, upsert=True) self.assertFalse(res.get('updatedExisting')) self.assertEqual(oid, res.get('upserted')) res = c.update({'_id': oid}, {'$set': {'b': 'b'}}) self.assertTrue(res.get('updatedExisting')) def test_save(self): # Tests legacy save. self.db.drop_collection("test_save") collection = self.db.test_save # Save a doc with autogenerated id _id = collection.save({"hello": "world"}) self.assertEqual(collection.find_one()["_id"], _id) self.assertTrue(isinstance(_id, ObjectId)) # Save a doc with explicit id collection.save({"_id": "explicit_id", "hello": "bar"}) doc = collection.find_one({"_id": "explicit_id"}) self.assertEqual(doc['_id'], 'explicit_id') self.assertEqual(doc['hello'], 'bar') # Save docs with _id field already present (shouldn't create new docs) self.assertEqual(2, collection.count()) collection.save({'_id': _id, 'hello': 'world'}) self.assertEqual(2, collection.count()) collection.save({'_id': 'explicit_id', 'hello': 'baz'}) self.assertEqual(2, collection.count()) self.assertEqual( 'baz', collection.find_one({'_id': 'explicit_id'})['hello'] ) # Acknowledged mode. collection.create_index("hello", unique=True) # No exception, even though we duplicate the first doc's "hello" value collection.save({'_id': 'explicit_id', 'hello': 'world'}, w=0) self.assertRaises( DuplicateKeyError, collection.save, {'_id': 'explicit_id', 'hello': 'world'}) self.db.drop_collection("test") def test_save_with_invalid_key(self): # Tests legacy save. self.db.drop_collection("test") self.assertTrue(self.db.test.insert({"hello": "world"})) doc = self.db.test.find_one() doc['a.b'] = 'c' expected = InvalidDocument if client_context.version.at_least(2, 5, 4, -1): expected = OperationFailure self.assertRaises(expected, self.db.test.save, doc) def test_acknowledged_save(self): # Tests legacy save. db = self.db db.drop_collection("test_acknowledged_save") collection = db.test_acknowledged_save collection.create_index("hello", unique=True) collection.save({"hello": "world"}) collection.save({"hello": "world"}, w=0) self.assertRaises(DuplicateKeyError, collection.save, {"hello": "world"}) db.drop_collection("test_acknowledged_save") def test_save_adds_id(self): # Tests legacy save. doc = {"hello": "jesse"} self.db.test.save(doc) self.assertTrue("_id" in doc) def test_save_returns_id(self): doc = {"hello": "jesse"} _id = self.db.test.save(doc) self.assertTrue(isinstance(_id, ObjectId)) self.assertEqual(_id, doc["_id"]) doc["hi"] = "bernie" _id = self.db.test.save(doc) self.assertTrue(isinstance(_id, ObjectId)) self.assertEqual(_id, doc["_id"]) def test_remove_one(self): # Tests legacy remove. self.db.test.remove() self.assertEqual(0, self.db.test.count()) self.db.test.insert({"x": 1}) self.db.test.insert({"y": 1}) self.db.test.insert({"z": 1}) self.assertEqual(3, self.db.test.count()) self.db.test.remove(multi=False) self.assertEqual(2, self.db.test.count()) self.db.test.remove() self.assertEqual(0, self.db.test.count()) def test_remove_all(self): # Tests legacy remove. self.db.test.remove() self.assertEqual(0, self.db.test.count()) self.db.test.insert({"x": 1}) self.db.test.insert({"y": 1}) self.assertEqual(2, self.db.test.count()) self.db.test.remove() self.assertEqual(0, self.db.test.count()) def test_remove_non_objectid(self): # Tests legacy remove. db = self.db db.drop_collection("test") db.test.insert_one({"_id": 5}) self.assertEqual(1, db.test.count()) db.test.remove(5) self.assertEqual(0, db.test.count()) def test_write_large_document(self): # Tests legacy insert, save, and update. max_size = self.db.client.max_bson_size half_size = int(max_size / 2) self.assertEqual(max_size, 16777216) expected = DocumentTooLarge if client_context.version.at_least(2, 5, 4, -1): # Document too large handled by the server expected = OperationFailure self.assertRaises(expected, self.db.test.insert, {"foo": "x" * max_size}) self.assertRaises(expected, self.db.test.save, {"foo": "x" * max_size}) self.assertRaises(expected, self.db.test.insert, [{"x": 1}, {"foo": "x" * max_size}]) self.db.test.insert([{"foo": "x" * half_size}, {"foo": "x" * half_size}]) self.db.test.insert({"bar": "x"}) # Use w=0 here to test legacy doc size checking in all server versions self.assertRaises(DocumentTooLarge, self.db.test.update, {"bar": "x"}, {"bar": "x" * (max_size - 14)}, w=0) # This will pass with OP_UPDATE or the update command. self.db.test.update({"bar": "x"}, {"bar": "x" * (max_size - 32)}) def test_last_error_options(self): # Tests legacy write methods. self.db.test.save({"x": 1}, w=1, wtimeout=1) self.db.test.insert({"x": 1}, w=1, wtimeout=1) self.db.test.remove({"x": 1}, w=1, wtimeout=1) self.db.test.update({"x": 1}, {"y": 2}, w=1, wtimeout=1) if client_context.replica_set_name: # client_context.w is the number of hosts in the replica set w = client_context.w + 1 # MongoDB 2.8+ raises error code 100, CannotSatisfyWriteConcern, # if w > number of members. Older versions just time out after 1 ms # as if they had enough secondaries but some are lagging. They # return an error with 'wtimeout': True and no code. def wtimeout_err(f, *args, **kwargs): try: f(*args, **kwargs) except WTimeoutError as exc: self.assertIsNotNone(exc.details) except OperationFailure as exc: self.assertIsNotNone(exc.details) self.assertEqual(100, exc.code, "Unexpected error: %r" % exc) else: self.fail("%s should have failed" % f) coll = self.db.test wtimeout_err(coll.save, {"x": 1}, w=w, wtimeout=1) wtimeout_err(coll.insert, {"x": 1}, w=w, wtimeout=1) wtimeout_err(coll.update, {"x": 1}, {"y": 2}, w=w, wtimeout=1) wtimeout_err(coll.remove, {"x": 1}, w=w, wtimeout=1) # can't use fsync and j options together self.assertRaises(ConfigurationError, self.db.test.insert, {"_id": 1}, j=True, fsync=True) def test_find_and_modify(self): c = self.db.test c.drop() c.insert({'_id': 1, 'i': 1}) # Test that we raise DuplicateKeyError when appropriate. # MongoDB doesn't have a code field for DuplicateKeyError # from commands before 2.2. if client_context.version.at_least(2, 2): c.ensure_index('i', unique=True) self.assertRaises(DuplicateKeyError, c.find_and_modify, query={'i': 1, 'j': 1}, update={'$set': {'k': 1}}, upsert=True) c.drop_indexes() # Test correct findAndModify self.assertEqual({'_id': 1, 'i': 1}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 3}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True)) self.assertEqual({'_id': 1, 'i': 3}, c.find_and_modify({'_id': 1}, remove=True)) self.assertEqual(None, c.find_one({'_id': 1})) self.assertEqual(None, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) # The return value changed in 2.1.2. See SERVER-6226. if client_context.version.at_least(2, 1, 2): self.assertEqual(None, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, upsert=True)) else: self.assertEqual({}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, upsert=True)) self.assertEqual({'_id': 1, 'i': 2}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, upsert=True, new=True)) self.assertEqual({'_id': 1, 'i': 2}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, fields=['i'])) self.assertEqual({'_id': 1, 'i': 4}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, fields={'i': 1})) # Test with full_response=True. result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, upsert=True, full_response=True, fields={'i': 1}) self.assertEqual({'_id': 1, 'i': 5}, result["value"]) self.assertEqual(True, result["lastErrorObject"]["updatedExisting"]) result = c.find_and_modify({'_id': 2}, {'$inc': {'i': 1}}, new=True, upsert=True, full_response=True, fields={'i': 1}) self.assertEqual({'_id': 2, 'i': 1}, result["value"]) self.assertEqual(False, result["lastErrorObject"]["updatedExisting"]) class ExtendedDict(dict): pass result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, fields={'i': 1}) self.assertFalse(isinstance(result, ExtendedDict)) c = self.db.get_collection( "test", codec_options=CodecOptions(document_class=ExtendedDict)) result = c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, fields={'i': 1}) self.assertTrue(isinstance(result, ExtendedDict)) def test_find_and_modify_with_sort(self): c = self.db.test c.drop() for j in range(5): c.insert({'j': j, 'i': 0}) sort = {'j': DESCENDING} self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = {'j': ASCENDING} self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = [('j', DESCENDING)] self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = [('j', ASCENDING)] self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = SON([('j', DESCENDING)]) self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = SON([('j', ASCENDING)]) self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) try: from collections import OrderedDict sort = OrderedDict([('j', DESCENDING)]) self.assertEqual(4, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) sort = OrderedDict([('j', ASCENDING)]) self.assertEqual(0, c.find_and_modify({}, {'$inc': {'i': 1}}, sort=sort)['j']) except ImportError: pass # Test that a standard dict with two keys is rejected. sort = {'j': DESCENDING, 'foo': DESCENDING} self.assertRaises(TypeError, c.find_and_modify, {}, {'$inc': {'i': 1}}, sort=sort) def test_find_and_modify_with_manipulator(self): class AddCollectionNameManipulator(SONManipulator): def will_copy(self): return True def transform_incoming(self, son, dummy): copy = SON(son) if 'collection' in copy: del copy['collection'] return copy def transform_outgoing(self, son, collection): copy = SON(son) copy['collection'] = collection.name return copy db = self.client.pymongo_test db.add_son_manipulator(AddCollectionNameManipulator()) c = db.test c.drop() c.insert({'_id': 1, 'i': 1}) # Test correct findAndModify # With manipulators self.assertEqual({'_id': 1, 'i': 1, 'collection': 'test'}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, manipulate=True)) self.assertEqual({'_id': 1, 'i': 3, 'collection': 'test'}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True, manipulate=True)) # With out manipulators self.assertEqual({'_id': 1, 'i': 3}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}})) self.assertEqual({'_id': 1, 'i': 5}, c.find_and_modify({'_id': 1}, {'$inc': {'i': 1}}, new=True)) def test_last_status(self): # Tests many legacy API elements. # We must call getlasterror on same socket as the last operation. db = rs_or_single_client(maxPoolSize=1).pymongo_test collection = db.test_last_status collection.remove({}) collection.save({"i": 1}) collection.update({"i": 1}, {"$set": {"i": 2}}, w=0) self.assertTrue(db.last_status()["updatedExisting"]) collection.update({"i": 1}, {"$set": {"i": 500}}, w=0) self.assertFalse(db.last_status()["updatedExisting"]) def test_auto_ref_and_deref(self): # Legacy API. db = self.client.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.test.a.remove({}) db.test.b.remove({}) db.test.c.remove({}) a = {"hello": u("world")} db.test.a.save(a) b = {"test": a} db.test.b.save(b) c = {"another test": b} db.test.c.save(c) a["hello"] = "mike" db.test.a.save(a) self.assertEqual(db.test.a.find_one(), a) self.assertEqual(db.test.b.find_one()["test"], a) self.assertEqual(db.test.c.find_one()["another test"]["test"], a) self.assertEqual(db.test.b.find_one(), b) self.assertEqual(db.test.c.find_one()["another test"], b) self.assertEqual(db.test.c.find_one(), c) def test_auto_ref_and_deref_list(self): # Legacy API. db = self.client.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.drop_collection("users") db.drop_collection("messages") message_1 = {"title": "foo"} db.messages.save(message_1) message_2 = {"title": "bar"} db.messages.save(message_2) user = {"messages": [message_1, message_2]} db.users.save(user) db.messages.update(message_1, {"title": "buzz"}) self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"]) self.assertEqual("bar", db.users.find_one()["messages"][1]["title"]) def test_object_to_dict_transformer(self): # PYTHON-709: Some users rely on their custom SONManipulators to run # before any other checks, so they can insert non-dict objects and # have them dictified before the _id is inserted or any other # processing. # Tests legacy API elements. class Thing(object): def __init__(self, value): self.value = value class ThingTransformer(SONManipulator): def transform_incoming(self, thing, dummy): return {'value': thing.value} db = self.client.foo db.add_son_manipulator(ThingTransformer()) t = Thing('value') db.test.remove() db.test.insert([t]) out = db.test.find_one() self.assertEqual('value', out.get('value')) def test_son_manipulator_outgoing(self): class Thing(object): def __init__(self, value): self.value = value class ThingTransformer(SONManipulator): def transform_outgoing(self, doc, collection): # We don't want this applied to the command return # value in pymongo.cursor.Cursor. if 'value' in doc: return Thing(doc['value']) return doc db = self.client.foo db.add_son_manipulator(ThingTransformer()) db.test.delete_many({}) db.test.insert_one({'value': 'value'}) out = db.test.find_one() self.assertTrue(isinstance(out, Thing)) self.assertEqual('value', out.value) if client_context.version.at_least(2, 6): out = next(db.test.aggregate([], cursor={})) self.assertTrue(isinstance(out, Thing)) self.assertEqual('value', out.value) def test_son_manipulator_inheritance(self): # Tests legacy API elements. class Thing(object): def __init__(self, value): self.value = value class ThingTransformer(SONManipulator): def transform_incoming(self, thing, dummy): return {'value': thing.value} def transform_outgoing(self, son, dummy): return Thing(son['value']) class Child(ThingTransformer): pass db = self.client.foo db.add_son_manipulator(Child()) t = Thing('value') db.test.remove() db.test.insert([t]) out = db.test.find_one() self.assertTrue(isinstance(out, Thing)) self.assertEqual('value', out.value) def test_disabling_manipulators(self): class IncByTwo(SONManipulator): def transform_outgoing(self, son, collection): if 'foo' in son: son['foo'] += 2 return son db = self.client.pymongo_test db.add_son_manipulator(IncByTwo()) c = db.test c.drop() c.insert({'foo': 0}) self.assertEqual(2, c.find_one()['foo']) self.assertEqual(0, c.find_one(manipulate=False)['foo']) self.assertEqual(2, c.find_one(manipulate=True)['foo']) c.drop() def test_manipulator_properties(self): db = self.client.foo self.assertEqual([], db.incoming_manipulators) self.assertEqual([], db.incoming_copying_manipulators) self.assertEqual([], db.outgoing_manipulators) self.assertEqual([], db.outgoing_copying_manipulators) db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.add_son_manipulator(ObjectIdShuffler()) self.assertEqual(1, len(db.incoming_manipulators)) self.assertEqual(db.incoming_manipulators, ['NamespaceInjector']) self.assertEqual(2, len(db.incoming_copying_manipulators)) for name in db.incoming_copying_manipulators: self.assertTrue(name in ('ObjectIdShuffler', 'AutoReference')) self.assertEqual([], db.outgoing_manipulators) self.assertEqual(['AutoReference'], db.outgoing_copying_manipulators) def test_ensure_index(self): db = self.db self.assertRaises(TypeError, db.test.ensure_index, {"hello": 1}) self.assertRaises(TypeError, db.test.ensure_index, {"hello": 1}, cache_for='foo') db.test.drop_indexes() self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_indexes() self.assertEqual("foo", db.test.ensure_index("goodbye", name="foo")) self.assertEqual(None, db.test.ensure_index("goodbye", name="foo")) db.test.drop_indexes() self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_index("goodbye_1") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.drop_collection("test") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_index("goodbye_1") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) self.assertEqual(None, db.test.ensure_index("goodbye")) db.test.drop_index("goodbye_1") self.assertEqual("goodbye_1", db.test.ensure_index("goodbye", cache_for=1)) time.sleep(1.2) self.assertEqual("goodbye_1", db.test.ensure_index("goodbye")) # Make sure the expiration time is updated. self.assertEqual(None, db.test.ensure_index("goodbye")) # Clean up indexes for later tests db.test.drop_indexes() def test_ensure_unique_index_threaded(self): coll = self.db.test_unique_threaded coll.drop() coll.insert_many([{'foo': i} for i in range(10000)]) class Indexer(threading.Thread): def run(self): try: coll.ensure_index('foo', unique=True) coll.insert_one({'foo': 'bar'}) coll.insert_one({'foo': 'bar'}) except OperationFailure: pass threads = [] for _ in range(10): t = Indexer() t.setDaemon(True) threads.append(t) for i in range(10): threads[i].start() joinall(threads) self.assertEqual(10001, coll.count()) coll.drop() if __name__ == "__main__": unittest.main() pymongo-3.2/test/test_client.py0000644000175000017500000013107512630145074020633 0ustar behackettbehackett00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test the mongo_client module.""" import contextlib import datetime import os import socket import struct import sys import time import traceback import warnings sys.path[0:0] = [""] from bson import BSON from bson.codec_options import CodecOptions from bson.py3compat import thread, u from bson.son import SON from bson.tz_util import utc from pymongo import auth, message from pymongo.cursor import CursorType from pymongo.database import Database from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, InvalidName, OperationFailure, CursorNotFound, NetworkTimeout, InvalidURI) from pymongo.message import _CursorAddress from pymongo.mongo_client import MongoClient from pymongo.pool import SocketInfo from pymongo.read_preferences import ReadPreference from pymongo.server_selectors import (any_server_selector, writable_server_selector) from pymongo.server_type import SERVER_TYPE from pymongo.write_concern import WriteConcern from test import (client_context, client_knobs, host, pair, port, SkipTest, unittest, IntegrationTest, db_pwd, db_user, MockClientTest) from test.pymongo_mocks import MockClient from test.utils import (assertRaisesExactly, delay, remove_all_users, server_is_master_with_slave, get_pool, one, connected, wait_until, rs_or_single_client, rs_or_single_client_noauth, lazy_client_trial, NTHREADS) class ClientUnitTest(unittest.TestCase): """MongoClient tests that don't require a server.""" @classmethod def setUpClass(cls): cls.client = MongoClient(host, port, connect=False, serverSelectionTimeoutMS=100) def test_keyword_arg_defaults(self): client = MongoClient(socketTimeoutMS=None, connectTimeoutMS=20000, waitQueueTimeoutMS=None, waitQueueMultiple=None, socketKeepAlive=False, replicaSet=None, read_preference=ReadPreference.PRIMARY, ssl=False, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=0, # ssl.CERT_NONE ssl_ca_certs=None, connect=False, serverSelectionTimeoutMS=12000) options = client._MongoClient__options pool_opts = options.pool_options self.assertEqual(None, pool_opts.socket_timeout) # socket.Socket.settimeout takes a float in seconds self.assertEqual(20.0, pool_opts.connect_timeout) self.assertEqual(None, pool_opts.wait_queue_timeout) self.assertEqual(None, pool_opts.wait_queue_multiple) self.assertFalse(pool_opts.socket_keepalive) self.assertEqual(None, pool_opts.ssl_context) self.assertEqual(None, options.replica_set_name) self.assertEqual(ReadPreference.PRIMARY, client.read_preference) self.assertAlmostEqual(12, client.server_selection_timeout) def test_types(self): self.assertRaises(TypeError, MongoClient, 1) self.assertRaises(TypeError, MongoClient, 1.14) self.assertRaises(TypeError, MongoClient, "localhost", "27017") self.assertRaises(TypeError, MongoClient, "localhost", 1.14) self.assertRaises(TypeError, MongoClient, "localhost", []) self.assertRaises(ConfigurationError, MongoClient, []) def test_max_pool_size_zero(self): with self.assertRaises(ValueError): MongoClient(maxPoolSize=0) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.client, "") self.assertRaises(InvalidName, make_db, self.client, "te$t") self.assertRaises(InvalidName, make_db, self.client, "te.t") self.assertRaises(InvalidName, make_db, self.client, "te\\t") self.assertRaises(InvalidName, make_db, self.client, "te/t") self.assertRaises(InvalidName, make_db, self.client, "te st") self.assertTrue(isinstance(self.client.test, Database)) self.assertEqual(self.client.test, self.client["test"]) self.assertEqual(self.client.test, Database(self.client, "test")) def test_get_database(self): codec_options = CodecOptions(tz_aware=True) write_concern = WriteConcern(w=2, j=True) db = self.client.get_database( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) self.assertEqual('foo', db.name) self.assertEqual(codec_options, db.codec_options) self.assertEqual(ReadPreference.SECONDARY, db.read_preference) self.assertEqual(write_concern, db.write_concern) def test_getattr(self): self.assertTrue(isinstance(self.client['_does_not_exist'], Database)) with self.assertRaises(AttributeError) as context: self.client._does_not_exist # Message should be: # "AttributeError: MongoClient has no attribute '_does_not_exist'. To # access the _does_not_exist database, use client['_does_not_exist']". self.assertIn("has no attribute '_does_not_exist'", str(context.exception)) def test_iteration(self): def iterate(): [a for a in self.client] self.assertRaises(TypeError, iterate) def test_get_default_database(self): c = MongoClient("mongodb://%s:%d/foo" % (host, port), connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database()) def test_get_default_database_error(self): # URI with no database. c = MongoClient("mongodb://%s:%d/" % (host, port), connect=False) self.assertRaises(ConfigurationError, c.get_default_database) def test_get_default_database_with_authsource(self): # Ensure we distinguish database name from authSource. uri = "mongodb://%s:%d/foo?authSource=src" % (host, port) c = MongoClient(uri, connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database()) class TestClient(IntegrationTest): def test_constants(self): # Set bad defaults. MongoClient.HOST = "somedomainthatdoesntexist.org" MongoClient.PORT = 123456789 with self.assertRaises(AutoReconnect): connected(MongoClient(serverSelectionTimeoutMS=10)) # Override the defaults. No error. connected(MongoClient(host, port)) # Set good defaults. MongoClient.HOST = host MongoClient.PORT = port # No error. connected(MongoClient()) def test_init_disconnected(self): c = rs_or_single_client(connect=False) # is_primary causes client to block until connected self.assertIsInstance(c.is_primary, bool) c = rs_or_single_client(connect=False) self.assertIsInstance(c.is_mongos, bool) c = rs_or_single_client(connect=False) self.assertIsInstance(c.max_pool_size, int) self.assertIsInstance(c.nodes, frozenset) c = rs_or_single_client(connect=False) self.assertEqual(c.codec_options, CodecOptions()) self.assertIsInstance(c.max_bson_size, int) c = rs_or_single_client(connect=False) self.assertFalse(c.primary) self.assertFalse(c.secondaries) c = rs_or_single_client(connect=False) self.assertIsInstance(c.max_write_batch_size, int) if client_context.is_rs: # The primary's host and port are from the replica set config. self.assertIsNotNone(c.address) else: self.assertEqual(c.address, (host, port)) bad_host = "somedomainthatdoesntexist.org" c = MongoClient(bad_host, port, connectTimeoutMS=1, serverSelectionTimeoutMS=10) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) def test_init_disconnected_with_auth(self): uri = "mongodb://user:pass@somedomainthatdoesntexist" c = MongoClient(uri, connectTimeoutMS=1, serverSelectionTimeoutMS=10) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one) def test_equality(self): c = connected(rs_or_single_client()) self.assertEqual(client_context.rs_or_standalone_client, c) # Explicitly test inequality self.assertFalse(client_context.rs_or_standalone_client != c) def test_host_w_port(self): with self.assertRaises(ValueError): connected(MongoClient("%s:1234567" % host, connectTimeoutMS=1, serverSelectionTimeoutMS=10)) def test_repr(self): # Used to test 'eval' below. import bson client = MongoClient( 'mongodb://localhost:27017,localhost:27018/?replicaSet=replset' '&connectTimeoutMS=12345', connect=False, document_class=SON) the_repr = repr(client) self.assertIn('MongoClient(host=', the_repr) self.assertIn( "document_class=bson.son.SON, " "tz_aware=False, " "connect=False, ", the_repr) self.assertIn("connecttimeoutms='12345'", the_repr) self.assertIn("replicaset=", the_repr) self.assertEqual(eval(the_repr), client) @client_context.require_replica_set def test_repr_replica_set(self): self.assertIn("MongoClient(host=[", repr(self.client)) self.assertIn(pair, repr(self.client)) def test_getters(self): self.assertEqual(client_context.client.address, (host, port)) self.assertEqual(client_context.nodes, self.client.nodes) def test_database_names(self): self.client.pymongo_test.test.insert_one({"dummy": u("object")}) self.client.pymongo_test_mike.test.insert_one({"dummy": u("object")}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.client.drop_database, 5) self.assertRaises(TypeError, self.client.drop_database, None) self.client.pymongo_test.test.insert_one({"dummy": u("object")}) self.client.pymongo_test2.test.insert_one({"dummy": u("object")}) dbs = self.client.database_names() self.assertIn("pymongo_test", dbs) self.assertIn("pymongo_test2", dbs) self.client.drop_database("pymongo_test") self.client.drop_database(self.client.pymongo_test2) raise SkipTest("This test often fails due to SERVER-2329") dbs = self.client.database_names() self.assertNotIn("pymongo_test", dbs) self.assertNotIn("pymongo_test2", dbs) def test_close(self): coll = self.client.pymongo_test.bar self.client.close() self.client.close() coll.count() self.client.close() self.client.close() coll.count() def test_bad_uri(self): with self.assertRaises(InvalidURI): MongoClient("http://localhost") @client_context.require_auth def test_auth_from_uri(self): self.client.admin.add_user("admin", "pass", roles=["root"]) self.addCleanup(self.client.admin.remove_user, 'admin') self.addCleanup(remove_all_users, self.client.pymongo_test) self.client.pymongo_test.add_user( "user", "pass", roles=['userAdmin', 'readWrite']) with self.assertRaises(OperationFailure): connected(rs_or_single_client( "mongodb://a:b@%s:%d" % (host, port))) # No error. connected(rs_or_single_client_noauth( "mongodb://admin:pass@%s:%d" % (host, port))) # Wrong database. uri = "mongodb://admin:pass@%s:%d/pymongo_test" % (host, port) with self.assertRaises(OperationFailure): connected(rs_or_single_client(uri)) # No error. connected(rs_or_single_client_noauth( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port))) # Auth with lazy connection. rs_or_single_client( "mongodb://user:pass@%s:%d/pymongo_test" % (host, port), connect=False).pymongo_test.test.find_one() # Wrong password. bad_client = rs_or_single_client( "mongodb://user:wrong@%s:%d/pymongo_test" % (host, port), connect=False) self.assertRaises(OperationFailure, bad_client.pymongo_test.test.find_one) @client_context.require_auth def test_multiple_logins(self): self.client.pymongo_test.add_user('user1', 'pass', roles=['readWrite']) self.client.pymongo_test.add_user('user2', 'pass', roles=['readWrite']) self.addCleanup(remove_all_users, self.client.pymongo_test) client = rs_or_single_client_noauth( "mongodb://user1:pass@%s:%d/pymongo_test" % (host, port)) client.pymongo_test.test.find_one() with self.assertRaises(OperationFailure): # Can't log in to the same database with multiple users. client.pymongo_test.authenticate('user2', 'pass') client.pymongo_test.test.find_one() client.pymongo_test.logout() with self.assertRaises(OperationFailure): client.pymongo_test.test.find_one() client.pymongo_test.authenticate('user2', 'pass') client.pymongo_test.test.find_one() with self.assertRaises(OperationFailure): client.pymongo_test.authenticate('user1', 'pass') client.pymongo_test.test.find_one() @client_context.require_auth def test_lazy_auth_raises_operation_failure(self): lazy_client = rs_or_single_client( "mongodb://user:wrong@%s/pymongo_test" % host, connect=False) assertRaisesExactly( OperationFailure, lazy_client.test.collection.find_one) def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") mongodb_socket = '/tmp/mongodb-27017.sock' encoded_socket = '%2Ftmp%2Fmongodb-27017.sock' if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessible") if client_context.auth_enabled: uri = "mongodb://%s:%s@%s" % (db_user, db_pwd, encoded_socket) else: uri = "mongodb://%s" % encoded_socket # Confirm we can do operations via the socket. client = MongoClient(uri) client.pymongo_test.test.insert_one({"dummy": "object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket. self.assertRaises( ConnectionFailure, connected, MongoClient("mongodb://%2Ftmp%2Fnon-existent.sock", serverSelectionTimeoutMS=100)) def test_fork(self): # Test using a client before and after a fork. if sys.platform == "win32": raise SkipTest("Can't fork on windows") try: import multiprocessing except ImportError: raise SkipTest("No multiprocessing module") db = self.client.pymongo_test # Ensure a socket is opened before the fork. db.test.find_one() def f(pipe): try: kill_cursors_executor = self.client._kill_cursors_executor servers = self.client._topology.select_servers( any_server_selector) # In child, only the thread that called fork() is alive. # The first operation should revive the rest. db.test.find_one() wait_until( lambda: all(s._monitor._executor._thread.is_alive() for s in servers), "restart monitor threads") wait_until(lambda: kill_cursors_executor._thread.is_alive(), "restart kill-cursors executor") except: traceback.print_exc() # Aid debugging. pipe.send(True) parent_pipe, child_pipe = multiprocessing.Pipe() p = multiprocessing.Process(target=f, args=(child_pipe,)) p.start() p.join(10) child_pipe.close() # Pipe will only have data if the child process failed. try: parent_pipe.recv() self.fail() except EOFError: pass def test_document_class(self): c = self.client db = c.pymongo_test db.test.insert_one({"x": 1}) self.assertEqual(dict, c.codec_options.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c = rs_or_single_client(document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.codec_options.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) def test_timeouts(self): client = rs_or_single_client(connectTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.connect_timeout) client = rs_or_single_client(socketTimeoutMS=10500) self.assertEqual(10.5, get_pool(client).opts.socket_timeout) def test_socket_timeout_ms_validation(self): c = rs_or_single_client(socketTimeoutMS=10 * 1000) self.assertEqual(10, get_pool(c).opts.socket_timeout) c = connected(rs_or_single_client(socketTimeoutMS=None)) self.assertEqual(None, get_pool(c).opts.socket_timeout) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS=0) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS=-1) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS=1e10) self.assertRaises(ValueError, rs_or_single_client, socketTimeoutMS='foo') def test_socket_timeout(self): no_timeout = self.client timeout_sec = 1 timeout = rs_or_single_client(socketTimeoutMS=1000 * timeout_sec) no_timeout.pymongo_test.drop_collection("test") no_timeout.pymongo_test.test.insert_one({"x": 1}) # A $where clause that takes a second longer than the timeout where_func = delay(timeout_sec + 1) def get_x(db): doc = next(db.test.find().where(where_func)) return doc["x"] self.assertEqual(1, get_x(no_timeout.pymongo_test)) self.assertRaises(NetworkTimeout, get_x, timeout.pymongo_test) def test_server_selection_timeout(self): client = MongoClient(serverSelectionTimeoutMS=100, connect=False) self.assertAlmostEqual(0.1, client.server_selection_timeout) client = MongoClient(serverSelectionTimeoutMS=0, connect=False) self.assertAlmostEqual(0, client.server_selection_timeout) self.assertRaises(ValueError, MongoClient, serverSelectionTimeoutMS="foo", connect=False) self.assertRaises(ValueError, MongoClient, serverSelectionTimeoutMS=-1, connect=False) self.assertRaises(ConfigurationError, MongoClient, serverSelectionTimeoutMS=None, connect=False) client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=100', connect=False) self.assertAlmostEqual(0.1, client.server_selection_timeout) client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=0', connect=False) self.assertAlmostEqual(0, client.server_selection_timeout) # Test invalid timeout in URI ignored and set to default. client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=-1', connect=False) self.assertAlmostEqual(30, client.server_selection_timeout) client = MongoClient( 'mongodb://localhost/?serverSelectionTimeoutMS=', connect=False) self.assertAlmostEqual(30, client.server_selection_timeout) def test_waitQueueTimeoutMS(self): client = rs_or_single_client(waitQueueTimeoutMS=2000) self.assertEqual(get_pool(client).opts.wait_queue_timeout, 2) def test_waitQueueMultiple(self): client = rs_or_single_client(maxPoolSize=3, waitQueueMultiple=2) pool = get_pool(client) self.assertEqual(pool.opts.wait_queue_multiple, 2) self.assertEqual(pool._socket_semaphore.waiter_semaphore.counter, 6) def test_socketKeepAlive(self): client = rs_or_single_client(socketKeepAlive=True) self.assertTrue(get_pool(client).opts.socket_keepalive) def test_tz_aware(self): self.assertRaises(ValueError, MongoClient, tz_aware='foo') aware = rs_or_single_client(tz_aware=True) naive = self.client aware.pymongo_test.drop_collection("test") now = datetime.datetime.utcnow() aware.pymongo_test.test.insert_one({"x": now}) self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo) self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo) self.assertEqual( aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), naive.pymongo_test.test.find_one()["x"]) @client_context.require_ipv6 def test_ipv6(self): if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%s[::1]:%d" % (auth_str, port) if client_context.is_rs: uri += '/?replicaSet=' + client_context.replica_set_name client = rs_or_single_client_noauth(uri) client.pymongo_test.test.insert_one({"dummy": u("object")}) client.pymongo_test_bernie.test.insert_one({"dummy": u("object")}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) @client_context.require_no_mongos def test_fsync_lock_unlock(self): if (server_is_master_with_slave(client_context.client) and client_context.version.at_least(2, 3, 0)): raise SkipTest('SERVER-7714') self.assertFalse(self.client.is_locked) # async flushing not supported on windows... if sys.platform not in ('cygwin', 'win32'): self.client.fsync(async=True) self.assertFalse(self.client.is_locked) self.client.fsync(lock=True) self.assertTrue(self.client.is_locked) locked = True self.client.unlock() for _ in range(5): locked = self.client.is_locked if not locked: break time.sleep(1) self.assertFalse(locked) def test_contextlib(self): client = rs_or_single_client() client.pymongo_test.drop_collection("test") client.pymongo_test.test.insert_one({"foo": "bar"}) # The socket used for the previous commands has been returned to the # pool self.assertEqual(1, len(get_pool(client).sockets)) with contextlib.closing(client): self.assertEqual("bar", client.pymongo_test.test.find_one()["foo"]) self.assertEqual(1, len(get_pool(client).sockets)) self.assertEqual(0, len(get_pool(client).sockets)) with client as client: self.assertEqual("bar", client.pymongo_test.test.find_one()["foo"]) self.assertEqual(0, len(get_pool(client).sockets)) def test_interrupt_signal(self): if sys.platform.startswith('java'): # We can't figure out how to raise an exception on a thread that's # blocked on a socket, whether that's the main thread or a worker, # without simply killing the whole thread in Jython. This suggests # PYTHON-294 can't actually occur in Jython. raise SkipTest("Can't test interrupts in Jython") # Test fix for PYTHON-294 -- make sure MongoClient closes its # socket if it gets an interrupt while waiting to recv() from it. db = self.client.pymongo_test # A $where clause which takes 1.5 sec to execute where = delay(1.5) # Need exactly 1 document so find() will execute its $where clause once db.drop_collection('foo') db.foo.insert_one({'_id': 1}) def interrupter(): # Raises KeyboardInterrupt in the main thread time.sleep(0.25) thread.interrupt_main() thread.start_new_thread(interrupter, ()) raised = False try: # Will be interrupted by a KeyboardInterrupt. next(db.foo.find({'$where': where})) except KeyboardInterrupt: raised = True # Can't use self.assertRaises() because it doesn't catch system # exceptions self.assertTrue(raised, "Didn't raise expected KeyboardInterrupt") # Raises AssertionError due to PYTHON-294 -- Mongo's response to the # previous find() is still waiting to be read on the socket, so the # request id's don't match. self.assertEqual( {'_id': 1}, next(db.foo.find()) ) def test_operation_failure(self): # Ensure MongoClient doesn't close socket after it gets an error # response to getLastError. PYTHON-395. pool = get_pool(self.client) socket_count = len(pool.sockets) self.assertGreaterEqual(socket_count, 1) old_sock_info = next(iter(pool.sockets)) self.client.pymongo_test.test.drop() self.client.pymongo_test.test.insert_one({'_id': 'foo'}) self.assertRaises( OperationFailure, self.client.pymongo_test.test.insert_one, {'_id': 'foo'}) self.assertEqual(socket_count, len(pool.sockets)) new_sock_info = next(iter(pool.sockets)) self.assertEqual(old_sock_info, new_sock_info) def test_kill_cursors_with_cursoraddress(self): if (client_context.is_mongos and not client_context.version.at_least(2, 4, 7)): # Old mongos sends incorrectly formatted error response when # cursor isn't found, see SERVER-9738. raise SkipTest("Can't test kill_cursors against old mongos") self.collection = self.client.pymongo_test.test self.collection.drop() self.collection.insert_many([{'_id': i} for i in range(200)]) cursor = self.collection.find().batch_size(1) next(cursor) self.client.kill_cursors( [cursor.cursor_id], _CursorAddress(self.client.address, self.collection.full_name)) # Prevent killcursors from reaching the server while a getmore is in # progress -- the server logs "Assertion: 16089:Cannot kill active # cursor." time.sleep(2) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor') def test_kill_cursors_with_tuple(self): if (client_context.is_mongos and not client_context.version.at_least(2, 4, 7)): # Old mongos sends incorrectly formatted error response when # cursor isn't found, see SERVER-9738. raise SkipTest("Can't test kill_cursors against old mongos") self.collection = self.client.pymongo_test.test self.collection.drop() self.collection.insert_many([{'_id': i} for i in range(200)]) cursor = self.collection.find().batch_size(1) next(cursor) self.client.kill_cursors( [cursor.cursor_id], self.client.address) # Prevent killcursors from reaching the server while a getmore is in # progress -- the server logs "Assertion: 16089:Cannot kill active # cursor." time.sleep(2) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor') def test_kill_cursors_with_server_unavailable(self): with client_knobs(kill_cursor_frequency=9999999): client = MongoClient('doesnt exist', connect=False, serverSelectionTimeoutMS=0) # Wait for the first tick of the periodic kill-cursors to pass. time.sleep(1) # Enqueue a kill-cursors message. client.close_cursor(1234, ('doesnt-exist', 27017)) with warnings.catch_warnings(record=True) as user_warnings: client._process_kill_cursors_queue() self.assertIn("couldn't close cursor on ('doesnt-exist', 27017)", str(user_warnings[0].message)) def test_lazy_connect_w0(self): # Ensure that connect-on-demand works when the first operation is # an unacknowledged write. This exercises _writable_max_wire_version(). # Use a separate collection to avoid races where we're still # completing an operation on a collection while the next test begins. client = rs_or_single_client(connect=False, w=0) client.test_lazy_connect_w0.test.insert_one({}) client = rs_or_single_client(connect=False) client.test_lazy_connect_w0.test.update_one({}, {'$set': {'x': 1}}) client = rs_or_single_client(connect=False) client.test_lazy_connect_w0.test.delete_one({}) @client_context.require_no_mongos def test_exhaust_network_error(self): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid semaphore leaks. client = rs_or_single_client(maxPoolSize=1) collection = client.pymongo_test.test pool = get_pool(client) pool._check_interval_seconds = None # Never check. # Ensure a socket. connected(client) # Cause a network error. sock_info = one(pool.sockets) sock_info.sock.close() cursor = collection.find(cursor_type=CursorType.EXHAUST) with self.assertRaises(ConnectionFailure): next(cursor) self.assertTrue(sock_info.closed) # The semaphore was decremented despite the error. self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) @client_context.require_auth def test_auth_network_error(self): # Make sure there's no semaphore leak if we get a network error # when authenticating a new socket with cached credentials. # Get a client with one socket so we detect if it's leaked. c = connected(rs_or_single_client(maxPoolSize=1, waitQueueTimeoutMS=1)) # Simulate an authenticate() call on a different socket. credentials = auth._build_credentials_tuple( 'DEFAULT', 'admin', db_user, db_pwd, {}) c._cache_credentials('test', credentials, connect=False) # Cause a network error on the actual socket. pool = get_pool(c) socket_info = one(pool.sockets) socket_info.sock.close() # SocketInfo.check_auth logs in with the new credential, but gets a # socket.error. Should be reraised as AutoReconnect. self.assertRaises(AutoReconnect, c.test.collection.find_one) # No semaphore leak, the pool is allowed to make a new socket. c.test.collection.find_one() @client_context.require_no_replica_set def test_connect_to_standalone_using_replica_set_name(self): client = MongoClient(pair, replicaSet='anything', serverSelectionTimeoutMS=100) with self.assertRaises(AutoReconnect): client.test.test.find_one() @client_context.require_replica_set def test_stale_getmore(self): # A cursor is created, but its member goes down and is removed from # the topology before the getMore message is sent. Test that # MongoClient._send_message_with_response handles the error. with self.assertRaises(AutoReconnect): client = MongoClient(host, port, connect=False, serverSelectionTimeoutMS=100, replicaSet=client_context.replica_set_name) client._send_message_with_response( operation=message._GetMore('pymongo_test', 'collection', 101, 1234, client.codec_options), address=('not-a-member', 27017)) class TestExhaustCursor(IntegrationTest): """Test that clients properly handle errors from exhaust cursors.""" def setUp(self): super(TestExhaustCursor, self).setUp() if client_context.is_mongos: raise SkipTest("mongos doesn't support exhaust, SERVER-2627") # mongod < 2.2.0 closes exhaust socket on error, so it behaves like # test_exhaust_query_network_error. Here we test that on query error # the client correctly keeps the socket *open* and checks it in. @client_context.require_version_min(2, 2, 0) def test_exhaust_query_server_error(self): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid semaphore leaks. client = connected(rs_or_single_client(maxPoolSize=1)) collection = client.pymongo_test.test pool = get_pool(client) sock_info = one(pool.sockets) # This will cause OperationFailure in all mongo versions since # the value for $orderby must be a document. cursor = collection.find( SON([('$query', {}), ('$orderby', True)]), cursor_type=CursorType.EXHAUST) self.assertRaises(OperationFailure, cursor.next) self.assertFalse(sock_info.closed) # The socket was checked in and the semaphore was decremented. self.assertIn(sock_info, pool.sockets) self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) def test_exhaust_getmore_server_error(self): # When doing a getmore on an exhaust cursor, the socket stays checked # out on success but it's checked in on error to avoid semaphore leaks. client = rs_or_single_client(maxPoolSize=1) collection = client.pymongo_test.test collection.drop() collection.insert_many([{} for _ in range(200)]) self.addCleanup(client_context.client.pymongo_test.test.drop) pool = get_pool(client) pool._check_interval_seconds = None # Never check. sock_info = one(pool.sockets) cursor = collection.find(cursor_type=CursorType.EXHAUST) # Initial query succeeds. cursor.next() # Cause a server error on getmore. def receive_message(operation, request_id): # Discard the actual server response. SocketInfo.receive_message(sock_info, operation, request_id) # responseFlags bit 1 is QueryFailure. msg = struct.pack(' 4, this returns 0. The Unicode * definition of UTF-8 goes up to 4-byte sequences. */ static unsigned char isLegalUTF8(const unsigned char* source, int length) { unsigned char a; const unsigned char* srcptr = source + length; switch (length) { default: return 0; /* Everything else falls through when "true"... */ case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; case 2: if ((a = (*--srcptr)) > 0xBF) return 0; switch (*source) { /* no fall-through in this inner switch */ case 0xE0: if (a < 0xA0) return 0; break; case 0xF0: if (a < 0x90) return 0; break; case 0xF4: if (a > 0x8F) return 0; break; default: if (a < 0x80) return 0; } case 1: if (*source >= 0x80 && *source < 0xC2) return 0; if (*source > 0xF4) return 0; } return 1; } result_t check_string(const unsigned char* string, const int length, const char check_utf8, const char check_null) { int position = 0; /* By default we go character by character. Will be different for checking * UTF-8 */ int sequence_length = 1; if (!check_utf8 && !check_null) { return VALID; } while (position < length) { if (check_null && *(string + position) == 0) { return HAS_NULL; } if (check_utf8) { sequence_length = trailingBytesForUTF8[*(string + position)] + 1; if ((position + sequence_length) > length) { return NOT_UTF_8; } if (!isLegalUTF8(string + position, sequence_length)) { return NOT_UTF_8; } } position += sequence_length; } return VALID; } pymongo-3.2/bson/code.py0000644000175000017500000000510312630145074017202 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing JavaScript code in BSON. """ import collections from bson.py3compat import string_type class Code(str): """BSON's JavaScript code type. Raises :class:`TypeError` if `code` is not an instance of :class:`basestring` (:class:`str` in python 3) or `scope` is not ``None`` or an instance of :class:`dict`. Scope variables can be set by passing a dictionary as the `scope` argument or by using keyword arguments. If a variable is set as a keyword argument it will override any setting for that variable in the `scope` dictionary. :Parameters: - `code`: string containing JavaScript code to be evaluated - `scope` (optional): dictionary representing the scope in which `code` should be evaluated - a mapping from identifiers (as strings) to values - `**kwargs` (optional): scope variables can also be passed as keyword arguments """ _type_marker = 13 def __new__(cls, code, scope=None, **kwargs): if not isinstance(code, string_type): raise TypeError("code must be an " "instance of %s" % (string_type.__name__)) self = str.__new__(cls, code) try: self.__scope = code.scope except AttributeError: self.__scope = {} if scope is not None: if not isinstance(scope, collections.Mapping): raise TypeError("scope must be an instance of dict") self.__scope.update(scope) self.__scope.update(kwargs) return self @property def scope(self): """Scope dictionary for this instance. """ return self.__scope def __repr__(self): return "Code(%s, %r)" % (str.__repr__(self), self.__scope) def __eq__(self, other): if isinstance(other, Code): return (self.__scope, str(self)) == (other.__scope, str(other)) return False __hash__ = None def __ne__(self, other): return not self == other pymongo-3.2/bson/errors.py0000644000175000017500000000220412560753776017623 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by the BSON package.""" class BSONError(Exception): """Base class for all BSON exceptions. """ class InvalidBSON(BSONError): """Raised when trying to create a BSON object from invalid data. """ class InvalidStringData(BSONError): """Raised when trying to encode a string containing non-UTF8 data. """ class InvalidDocument(BSONError): """Raised when trying to create a BSON object from an invalid document. """ class InvalidId(BSONError): """Raised when trying to create an ObjectId from invalid data. """ pymongo-3.2/bson/json_util.py0000644000175000017500000002244612630145074020307 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for using Python's :mod:`json` module with BSON documents. This module provides two helper methods `dumps` and `loads` that wrap the native :mod:`json` methods and provide explicit BSON conversion to and from json. This allows for specialized encoding and decoding of BSON documents into `Mongo Extended JSON `_'s *Strict* mode. This lets you encode / decode BSON documents to JSON even when they use special BSON types. Example usage (serialization): .. doctest:: >>> from bson import Binary, Code >>> from bson.json_util import dumps >>> dumps([{'foo': [1, 2]}, ... {'bar': {'hello': 'world'}}, ... {'code': Code("function x() { return 1; }")}, ... {'bin': Binary("\x01\x02\x03\x04")}]) '[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$code": "function x() { return 1; }", "$scope": {}}}, {"bin": {"$binary": "AQIDBA==", "$type": "00"}}]' Example usage (deserialization): .. doctest:: >>> from bson.json_util import loads >>> loads('[{"foo": [1, 2]}, {"bar": {"hello": "world"}}, {"code": {"$scope": {}, "$code": "function x() { return 1; }"}}, {"bin": {"$type": "00", "$binary": "AQIDBA=="}}]') [{u'foo': [1, 2]}, {u'bar': {u'hello': u'world'}}, {u'code': Code('function x() { return 1; }', {})}, {u'bin': Binary('...', 0)}] Alternatively, you can manually pass the `default` to :func:`json.dumps`. It won't handle :class:`~bson.binary.Binary` and :class:`~bson.code.Code` instances (as they are extended strings you can't provide custom defaults), but it will be faster as there is less recursion. .. versionchanged:: 2.8 The output format for :class:`~bson.timestamp.Timestamp` has changed from '{"t": , "i": }' to '{"$timestamp": {"t": , "i": }}'. This new format will be decoded to an instance of :class:`~bson.timestamp.Timestamp`. The old format will continue to be decoded to a python dict as before. Encoding to the old format is no longer supported as it was never correct and loses type information. Added support for $numberLong and $undefined - new in MongoDB 2.6 - and parsing $date in ISO-8601 format. .. versionchanged:: 2.7 Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef instances. .. versionchanged:: 2.3 Added dumps and loads helpers to automatically handle conversion to and from json and supports :class:`~bson.binary.Binary` and :class:`~bson.code.Code` """ import base64 import calendar import collections import datetime import json import re import uuid from bson import EPOCH_AWARE, RE_TYPE, SON from bson.binary import Binary from bson.code import Code from bson.dbref import DBRef from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey from bson.objectid import ObjectId from bson.regex import Regex from bson.timestamp import Timestamp from bson.tz_util import utc from bson.py3compat import PY3, iteritems, string_type, text_type _RE_OPT_TABLE = { "i": re.I, "l": re.L, "m": re.M, "s": re.S, "u": re.U, "x": re.X, } def dumps(obj, *args, **kwargs): """Helper function that wraps :class:`json.dumps`. Recursive function that handles all BSON types including :class:`~bson.binary.Binary` and :class:`~bson.code.Code`. .. versionchanged:: 2.7 Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef instances. """ return json.dumps(_json_convert(obj), *args, **kwargs) def loads(s, *args, **kwargs): """Helper function that wraps :class:`json.loads`. Automatically passes the object_hook for BSON type conversion. """ kwargs['object_hook'] = lambda dct: object_hook(dct) return json.loads(s, *args, **kwargs) def _json_convert(obj): """Recursive helper method that converts BSON types so they can be converted into json. """ if hasattr(obj, 'iteritems') or hasattr(obj, 'items'): # PY3 support return SON(((k, _json_convert(v)) for k, v in iteritems(obj))) elif hasattr(obj, '__iter__') and not isinstance(obj, (text_type, bytes)): return list((_json_convert(v) for v in obj)) try: return default(obj) except TypeError: return obj def object_hook(dct): if "$oid" in dct: return ObjectId(str(dct["$oid"])) if "$ref" in dct: return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None)) if "$date" in dct: dtm = dct["$date"] # mongoexport 2.6 and newer if isinstance(dtm, string_type): aware = datetime.datetime.strptime( dtm[:23], "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=utc) offset = dtm[23:] if not offset or offset == 'Z': # UTC return aware else: if len(offset) == 5: # Offset from mongoexport is in format (+|-)HHMM secs = (int(offset[1:3]) * 3600 + int(offset[3:]) * 60) elif ':' in offset and len(offset) == 6: # RFC-3339 format (+|-)HH:MM hours, minutes = offset[1:].split(':') secs = (int(hours) * 3600 + int(minutes) * 60) else: # Not RFC-3339 compliant or mongoexport output. raise ValueError("invalid format for offset") if offset[0] == "-": secs *= -1 return aware - datetime.timedelta(seconds=secs) # mongoexport 2.6 and newer, time before the epoch (SERVER-15275) elif isinstance(dtm, collections.Mapping): secs = float(dtm["$numberLong"]) / 1000.0 # mongoexport before 2.6 else: secs = float(dtm) / 1000.0 return EPOCH_AWARE + datetime.timedelta(seconds=secs) if "$regex" in dct: flags = 0 # PyMongo always adds $options but some other tools may not. for opt in dct.get("$options", ""): flags |= _RE_OPT_TABLE.get(opt, 0) return Regex(dct["$regex"], flags) if "$minKey" in dct: return MinKey() if "$maxKey" in dct: return MaxKey() if "$binary" in dct: if isinstance(dct["$type"], int): dct["$type"] = "%02x" % dct["$type"] subtype = int(dct["$type"], 16) if subtype >= 0xffffff80: # Handle mongoexport values subtype = int(dct["$type"][6:], 16) return Binary(base64.b64decode(dct["$binary"].encode()), subtype) if "$code" in dct: return Code(dct["$code"], dct.get("$scope")) if "$uuid" in dct: return uuid.UUID(dct["$uuid"]) if "$undefined" in dct: return None if "$numberLong" in dct: return Int64(dct["$numberLong"]) if "$timestamp" in dct: tsp = dct["$timestamp"] return Timestamp(tsp["t"], tsp["i"]) return dct def default(obj): # We preserve key order when rendering SON, DBRef, etc. as JSON by # returning a SON for those types instead of a dict. if isinstance(obj, ObjectId): return {"$oid": str(obj)} if isinstance(obj, DBRef): return _json_convert(obj.as_doc()) if isinstance(obj, datetime.datetime): # TODO share this code w/ bson.py? if obj.utcoffset() is not None: obj = obj - obj.utcoffset() millis = int(calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) return {"$date": millis} if isinstance(obj, (RE_TYPE, Regex)): flags = "" if obj.flags & re.IGNORECASE: flags += "i" if obj.flags & re.LOCALE: flags += "l" if obj.flags & re.MULTILINE: flags += "m" if obj.flags & re.DOTALL: flags += "s" if obj.flags & re.UNICODE: flags += "u" if obj.flags & re.VERBOSE: flags += "x" if isinstance(obj.pattern, text_type): pattern = obj.pattern else: pattern = obj.pattern.decode('utf-8') return SON([("$regex", pattern), ("$options", flags)]) if isinstance(obj, MinKey): return {"$minKey": 1} if isinstance(obj, MaxKey): return {"$maxKey": 1} if isinstance(obj, Timestamp): return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])} if isinstance(obj, Code): return SON([('$code', str(obj)), ('$scope', obj.scope)]) if isinstance(obj, Binary): return SON([ ('$binary', base64.b64encode(obj).decode()), ('$type', "%02x" % obj.subtype)]) if PY3 and isinstance(obj, bytes): return SON([ ('$binary', base64.b64encode(obj).decode()), ('$type', "00")]) if isinstance(obj, uuid.UUID): return {"$uuid": obj.hex} raise TypeError("%r is not JSON serializable" % obj) pymongo-3.2/bson/objectid.py0000644000175000017500000002207612630145074020063 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with MongoDB `ObjectIds `_. """ import binascii import calendar import datetime import hashlib import os import random import socket import struct import threading import time from bson.errors import InvalidId from bson.py3compat import PY3, bytes_from_hex, string_type, text_type from bson.tz_util import utc def _machine_bytes(): """Get the machine portion of an ObjectId. """ machine_hash = hashlib.md5() if PY3: # gethostname() returns a unicode string in python 3.x # while update() requires a byte string. machine_hash.update(socket.gethostname().encode()) else: # Calling encode() here will fail with non-ascii hostnames machine_hash.update(socket.gethostname()) return machine_hash.digest()[0:3] def _raise_invalid_id(oid): raise InvalidId( "%r is not a valid ObjectId, it must be a 12-byte input" " or a 24-character hex string" % oid) class ObjectId(object): """A MongoDB ObjectId. """ _inc = random.randint(0, 0xFFFFFF) _inc_lock = threading.Lock() _machine_bytes = _machine_bytes() __slots__ = ('__id') _type_marker = 7 def __init__(self, oid=None): """Initialize a new ObjectId. An ObjectId is a 12-byte unique identifier consisting of: - a 4-byte value representing the seconds since the Unix epoch, - a 3-byte machine identifier, - a 2-byte process id, and - a 3-byte counter, starting with a random value. By default, ``ObjectId()`` creates a new unique identifier. The optional parameter `oid` can be an :class:`ObjectId`, or any 12 :class:`bytes` or, in Python 2, any 12-character :class:`str`. For example, the 12 bytes b'foo-bar-quux' do not follow the ObjectId specification but they are acceptable input:: >>> ObjectId(b'foo-bar-quux') ObjectId('666f6f2d6261722d71757578') `oid` can also be a :class:`unicode` or :class:`str` of 24 hex digits:: >>> ObjectId('0123456789ab0123456789ab') ObjectId('0123456789ab0123456789ab') >>> >>> # A u-prefixed unicode literal: >>> ObjectId(u'0123456789ab0123456789ab') ObjectId('0123456789ab0123456789ab') Raises :class:`~bson.errors.InvalidId` if `oid` is not 12 bytes nor 24 hex digits, or :class:`TypeError` if `oid` is not an accepted type. :Parameters: - `oid` (optional): a valid ObjectId. .. mongodoc:: objectids """ if oid is None: self.__generate() elif isinstance(oid, bytes) and len(oid) == 12: self.__id = oid else: self.__validate(oid) @classmethod def from_datetime(cls, generation_time): """Create a dummy ObjectId instance with a specific generation time. This method is useful for doing range queries on a field containing :class:`ObjectId` instances. .. warning:: It is not safe to insert a document containing an ObjectId generated using this method. This method deliberately eliminates the uniqueness guarantee that ObjectIds generally provide. ObjectIds generated with this method should be used exclusively in queries. `generation_time` will be converted to UTC. Naive datetime instances will be treated as though they already contain UTC. An example using this helper to get documents where ``"_id"`` was generated before January 1, 2010 would be: >>> gen_time = datetime.datetime(2010, 1, 1) >>> dummy_id = ObjectId.from_datetime(gen_time) >>> result = collection.find({"_id": {"$lt": dummy_id}}) :Parameters: - `generation_time`: :class:`~datetime.datetime` to be used as the generation time for the resulting ObjectId. """ if generation_time.utcoffset() is not None: generation_time = generation_time - generation_time.utcoffset() timestamp = calendar.timegm(generation_time.timetuple()) oid = struct.pack( ">i", int(timestamp)) + b"\x00\x00\x00\x00\x00\x00\x00\x00" return cls(oid) @classmethod def is_valid(cls, oid): """Checks if a `oid` string is valid or not. :Parameters: - `oid`: the object id to validate .. versionadded:: 2.3 """ if not oid: return False try: ObjectId(oid) return True except (InvalidId, TypeError): return False def __generate(self): """Generate a new value for this ObjectId. """ # 4 bytes current time oid = struct.pack(">i", int(time.time())) # 3 bytes machine oid += ObjectId._machine_bytes # 2 bytes pid oid += struct.pack(">H", os.getpid() % 0xFFFF) # 3 bytes inc with ObjectId._inc_lock: oid += struct.pack(">i", ObjectId._inc)[1:4] ObjectId._inc = (ObjectId._inc + 1) % 0xFFFFFF self.__id = oid def __validate(self, oid): """Validate and use the given id for this ObjectId. Raises TypeError if id is not an instance of (:class:`basestring` (:class:`str` or :class:`bytes` in python 3), ObjectId) and InvalidId if it is not a valid ObjectId. :Parameters: - `oid`: a valid ObjectId """ if isinstance(oid, ObjectId): self.__id = oid.binary # bytes or unicode in python 2, str in python 3 elif isinstance(oid, string_type): if len(oid) == 24: try: self.__id = bytes_from_hex(oid) except (TypeError, ValueError): _raise_invalid_id(oid) else: _raise_invalid_id(oid) else: raise TypeError("id must be an instance of (bytes, %s, ObjectId), " "not %s" % (text_type.__name__, type(oid))) @property def binary(self): """12-byte binary representation of this ObjectId. """ return self.__id @property def generation_time(self): """A :class:`datetime.datetime` instance representing the time of generation for this :class:`ObjectId`. The :class:`datetime.datetime` is timezone aware, and represents the generation time in UTC. It is precise to the second. """ timestamp = struct.unpack(">i", self.__id[0:4])[0] return datetime.datetime.fromtimestamp(timestamp, utc) def __getstate__(self): """return value of object for pickling. needed explicitly because __slots__() defined. """ return self.__id def __setstate__(self, value): """explicit state set from pickling """ # Provide backwards compatability with OIDs # pickled with pymongo-1.9 or older. if isinstance(value, dict): oid = value["_ObjectId__id"] else: oid = value # ObjectIds pickled in python 2.x used `str` for __id. # In python 3.x this has to be converted to `bytes` # by encoding latin-1. if PY3 and isinstance(oid, text_type): self.__id = oid.encode('latin-1') else: self.__id = oid def __str__(self): if PY3: return binascii.hexlify(self.__id).decode() return binascii.hexlify(self.__id) def __repr__(self): return "ObjectId('%s')" % (str(self),) def __eq__(self, other): if isinstance(other, ObjectId): return self.__id == other.binary return NotImplemented def __ne__(self, other): if isinstance(other, ObjectId): return self.__id != other.binary return NotImplemented def __lt__(self, other): if isinstance(other, ObjectId): return self.__id < other.binary return NotImplemented def __le__(self, other): if isinstance(other, ObjectId): return self.__id <= other.binary return NotImplemented def __gt__(self, other): if isinstance(other, ObjectId): return self.__id > other.binary return NotImplemented def __ge__(self, other): if isinstance(other, ObjectId): return self.__id >= other.binary return NotImplemented def __hash__(self): """Get a hash value for this :class:`ObjectId`.""" return hash(self.__id) pymongo-3.2/bson/time64_limits.h0000644000175000017500000000272512507072032020563 0ustar behackettbehackett00000000000000/* Maximum and minimum inputs your system's respective time functions can correctly handle. time64.h will use your system functions if the input falls inside these ranges and corresponding USE_SYSTEM_* constant is defined. */ #ifndef TIME64_LIMITS_H #define TIME64_LIMITS_H /* Max/min for localtime() */ #define SYSTEM_LOCALTIME_MAX 2147483647 #define SYSTEM_LOCALTIME_MIN -2147483647-1 /* Max/min for gmtime() */ #define SYSTEM_GMTIME_MAX 2147483647 #define SYSTEM_GMTIME_MIN -2147483647-1 /* Max/min for mktime() */ static const struct tm SYSTEM_MKTIME_MAX = { 7, 14, 19, 18, 0, 138, 1, 17, 0 #ifdef HAS_TM_TM_GMTOFF ,-28800 #endif #ifdef HAS_TM_TM_ZONE ,"PST" #endif }; static const struct tm SYSTEM_MKTIME_MIN = { 52, 45, 12, 13, 11, 1, 5, 346, 0 #ifdef HAS_TM_TM_GMTOFF ,-28800 #endif #ifdef HAS_TM_TM_ZONE ,"PST" #endif }; /* Max/min for timegm() */ #ifdef HAS_TIMEGM static const struct tm SYSTEM_TIMEGM_MAX = { 7, 14, 3, 19, 0, 138, 2, 18, 0 #ifdef HAS_TM_TM_GMTOFF ,0 #endif #ifdef HAS_TM_TM_ZONE ,"UTC" #endif }; static const struct tm SYSTEM_TIMEGM_MIN = { 52, 45, 20, 13, 11, 1, 5, 346, 0 #ifdef HAS_TM_TM_GMTOFF ,0 #endif #ifdef HAS_TM_TM_ZONE ,"UTC" #endif }; #endif /* HAS_TIMEGM */ #endif /* TIME64_LIMITS_H */ pymongo-3.2/bson/dbref.py0000644000175000017500000001117512630145074017360 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for manipulating DBRefs (references to MongoDB documents).""" from copy import deepcopy from bson.py3compat import iteritems, string_type from bson.son import SON class DBRef(object): """A reference to a document stored in MongoDB. """ # DBRef isn't actually a BSON "type" so this number was arbitrarily chosen. _type_marker = 100 def __init__(self, collection, id, database=None, _extra={}, **kwargs): """Initialize a new :class:`DBRef`. Raises :class:`TypeError` if `collection` or `database` is not an instance of :class:`basestring` (:class:`str` in python 3). `database` is optional and allows references to documents to work across databases. Any additional keyword arguments will create additional fields in the resultant embedded document. :Parameters: - `collection`: name of the collection the document is stored in - `id`: the value of the document's ``"_id"`` field - `database` (optional): name of the database to reference - `**kwargs` (optional): additional keyword arguments will create additional, custom fields .. mongodoc:: dbrefs """ if not isinstance(collection, string_type): raise TypeError("collection must be an " "instance of %s" % string_type.__name__) if database is not None and not isinstance(database, string_type): raise TypeError("database must be an " "instance of %s" % string_type.__name__) self.__collection = collection self.__id = id self.__database = database kwargs.update(_extra) self.__kwargs = kwargs @property def collection(self): """Get the name of this DBRef's collection as unicode. """ return self.__collection @property def id(self): """Get this DBRef's _id. """ return self.__id @property def database(self): """Get the name of this DBRef's database. Returns None if this DBRef doesn't specify a database. """ return self.__database def __getattr__(self, key): try: return self.__kwargs[key] except KeyError: raise AttributeError(key) # Have to provide __setstate__ to avoid # infinite recursion since we override # __getattr__. def __setstate__(self, state): self.__dict__.update(state) def as_doc(self): """Get the SON document representation of this DBRef. Generally not needed by application developers """ doc = SON([("$ref", self.collection), ("$id", self.id)]) if self.database is not None: doc["$db"] = self.database doc.update(self.__kwargs) return doc def __repr__(self): extra = "".join([", %s=%r" % (k, v) for k, v in iteritems(self.__kwargs)]) if self.database is None: return "DBRef(%r, %r%s)" % (self.collection, self.id, extra) return "DBRef(%r, %r, %r%s)" % (self.collection, self.id, self.database, extra) def __eq__(self, other): if isinstance(other, DBRef): us = (self.__database, self.__collection, self.__id, self.__kwargs) them = (other.__database, other.__collection, other.__id, other.__kwargs) return us == them return NotImplemented def __ne__(self, other): return not self == other def __hash__(self): """Get a hash value for this :class:`DBRef`.""" return hash((self.__collection, self.__id, self.__database, tuple(sorted(self.__kwargs.items())))) def __deepcopy__(self, memo): """Support function for `copy.deepcopy()`.""" return DBRef(deepcopy(self.__collection, memo), deepcopy(self.__id, memo), deepcopy(self.__database, memo), deepcopy(self.__kwargs, memo)) pymongo-3.2/bson/timestamp.py0000644000175000017500000000753412630145074020305 0ustar behackettbehackett00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing MongoDB internal Timestamps. """ import calendar import datetime from bson.py3compat import integer_types from bson.tz_util import utc UPPERBOUND = 4294967296 class Timestamp(object): """MongoDB internal timestamps used in the opLog. """ _type_marker = 17 def __init__(self, time, inc): """Create a new :class:`Timestamp`. This class is only for use with the MongoDB opLog. If you need to store a regular timestamp, please use a :class:`~datetime.datetime`. Raises :class:`TypeError` if `time` is not an instance of :class: `int` or :class:`~datetime.datetime`, or `inc` is not an instance of :class:`int`. Raises :class:`ValueError` if `time` or `inc` is not in [0, 2**32). :Parameters: - `time`: time in seconds since epoch UTC, or a naive UTC :class:`~datetime.datetime`, or an aware :class:`~datetime.datetime` - `inc`: the incrementing counter """ if isinstance(time, datetime.datetime): if time.utcoffset() is not None: time = time - time.utcoffset() time = int(calendar.timegm(time.timetuple())) if not isinstance(time, integer_types): raise TypeError("time must be an instance of int") if not isinstance(inc, integer_types): raise TypeError("inc must be an instance of int") if not 0 <= time < UPPERBOUND: raise ValueError("time must be contained in [0, 2**32)") if not 0 <= inc < UPPERBOUND: raise ValueError("inc must be contained in [0, 2**32)") self.__time = time self.__inc = inc @property def time(self): """Get the time portion of this :class:`Timestamp`. """ return self.__time @property def inc(self): """Get the inc portion of this :class:`Timestamp`. """ return self.__inc def __eq__(self, other): if isinstance(other, Timestamp): return (self.__time == other.time and self.__inc == other.inc) else: return NotImplemented def __hash__(self): return hash(self.time) ^ hash(self.inc) def __ne__(self, other): return not self == other def __lt__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) < (other.time, other.inc) return NotImplemented def __le__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) <= (other.time, other.inc) return NotImplemented def __gt__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) > (other.time, other.inc) return NotImplemented def __ge__(self, other): if isinstance(other, Timestamp): return (self.time, self.inc) >= (other.time, other.inc) return NotImplemented def __repr__(self): return "Timestamp(%s, %s)" % (self.__time, self.__inc) def as_datetime(self): """Return a :class:`~datetime.datetime` instance corresponding to the time portion of this :class:`Timestamp`. The returned datetime's timezone is UTC. """ return datetime.datetime.fromtimestamp(self.__time, utc) pymongo-3.2/bson/raw_bson.py0000644000175000017500000000614412630145074020110 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing raw BSON documents. """ import collections from bson import _UNPACK_INT, _iterate_elements from bson.py3compat import iteritems from bson.codec_options import ( CodecOptions, DEFAULT_CODEC_OPTIONS, _RAW_BSON_DOCUMENT_MARKER) class RawBSONDocument(collections.Mapping): """Representation for a MongoDB document that provides access to the raw BSON bytes that compose it. Only when a field is accessed or modified within the document does RawBSONDocument decode its bytes. """ __slots__ = ('__raw', '__inflated_doc', '__codec_options') _type_marker = _RAW_BSON_DOCUMENT_MARKER def __init__(self, bson_bytes, codec_options=DEFAULT_CODEC_OPTIONS): """Create a new :class:`RawBSONDocument`. :Parameters: - `bson_bytes`: the BSON bytes that compose this document - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. """ self.__raw = bson_bytes self.__inflated_doc = None # Always decode documents to their lazy representations. co = codec_options self.__codec_options = CodecOptions( tz_aware=co.tz_aware, document_class=RawBSONDocument, uuid_representation=co.uuid_representation, unicode_decode_error_handler=co.unicode_decode_error_handler, tzinfo=co.tzinfo) @property def raw(self): """The raw BSON bytes composing this document.""" return self.__raw def items(self): """Lazily decode and iterate elements in this document.""" return iteritems(self.__inflated) @property def __inflated(self): if self.__inflated_doc is None: # We already validated the object's size when this document was # created, so no need to do that again. self.__inflated_doc = dict( element for element in _iterate_elements( self.__raw, 4, _UNPACK_INT(self.__raw[:4])[0] - 1, self.__codec_options)) return self.__inflated_doc def __getitem__(self, item): return self.__inflated[item] def __iter__(self): return iter(self.__inflated) def __len__(self): return len(self.__inflated) def __eq__(self, other): if isinstance(other, RawBSONDocument): return self.__raw == other.raw return NotImplemented def __repr__(self): return ("RawBSONDocument(%r, codec_options=%r)" % (self.raw, self.__codec_options)) pymongo-3.2/bson/regex.py0000644000175000017500000001027712630145074017412 0ustar behackettbehackett00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for representing MongoDB regular expressions. """ import re from bson.son import RE_TYPE from bson.py3compat import string_type, text_type def str_flags_to_int(str_flags): flags = 0 if "i" in str_flags: flags |= re.IGNORECASE if "l" in str_flags: flags |= re.LOCALE if "m" in str_flags: flags |= re.MULTILINE if "s" in str_flags: flags |= re.DOTALL if "u" in str_flags: flags |= re.UNICODE if "x" in str_flags: flags |= re.VERBOSE return flags class Regex(object): """BSON regular expression data.""" _type_marker = 11 @classmethod def from_native(cls, regex): """Convert a Python regular expression into a ``Regex`` instance. Note that in Python 3, a regular expression compiled from a :class:`str` has the ``re.UNICODE`` flag set. If it is undesirable to store this flag in a BSON regular expression, unset it first:: >>> pattern = re.compile('.*') >>> regex = Regex.from_native(pattern) >>> regex.flags ^= re.UNICODE >>> db.collection.insert({'pattern': regex}) :Parameters: - `regex`: A regular expression object from ``re.compile()``. .. warning:: Python regular expressions use a different syntax and different set of flags than MongoDB, which uses `PCRE`_. A regular expression retrieved from the server may not compile in Python, or may match a different set of strings in Python than when used in a MongoDB query. .. _PCRE: http://www.pcre.org/ """ if not isinstance(regex, RE_TYPE): raise TypeError( "regex must be a compiled regular expression, not %s" % type(regex)) return Regex(regex.pattern, regex.flags) def __init__(self, pattern, flags=0): """BSON regular expression data. This class is useful to store and retrieve regular expressions that are incompatible with Python's regular expression dialect. :Parameters: - `pattern`: string - `flags`: (optional) an integer bitmask, or a string of flag characters like "im" for IGNORECASE and MULTILINE """ if not isinstance(pattern, (text_type, bytes)): raise TypeError("pattern must be a string, not %s" % type(pattern)) self.pattern = pattern if isinstance(flags, string_type): self.flags = str_flags_to_int(flags) elif isinstance(flags, int): self.flags = flags else: raise TypeError( "flags must be a string or int, not %s" % type(flags)) def __eq__(self, other): if isinstance(other, Regex): return self.pattern == self.pattern and self.flags == other.flags else: return NotImplemented __hash__ = None def __ne__(self, other): return not self == other def __repr__(self): return "Regex(%r, %r)" % (self.pattern, self.flags) def try_compile(self): """Compile this :class:`Regex` as a Python regular expression. .. warning:: Python regular expressions use a different syntax and different set of flags than MongoDB, which uses `PCRE`_. A regular expression retrieved from the server may not compile in Python, or may match a different set of strings in Python than when used in a MongoDB query. :meth:`try_compile()` may raise :exc:`re.error`. .. _PCRE: http://www.pcre.org/ """ return re.compile(self.pattern, self.flags) pymongo-3.2/bson/encoding_helpers.h0000644000175000017500000000154312560753776021423 0ustar behackettbehackett00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ENCODING_HELPERS_H #define ENCODING_HELPERS_H typedef enum { VALID, NOT_UTF_8, HAS_NULL } result_t; result_t check_string(const unsigned char* string, const int length, const char check_utf8, const char check_null); #endif pymongo-3.2/bson/__init__.py0000644000175000017500000010027612630145074020036 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """BSON (Binary JSON) encoding and decoding. """ import calendar import collections import datetime import itertools import re import struct import sys import uuid from codecs import (utf_8_decode as _utf_8_decode, utf_8_encode as _utf_8_encode) from bson.binary import (Binary, OLD_UUID_SUBTYPE, JAVA_LEGACY, CSHARP_LEGACY, UUIDLegacy) from bson.code import Code from bson.codec_options import ( CodecOptions, DEFAULT_CODEC_OPTIONS, _raw_document_class) from bson.dbref import DBRef from bson.errors import (InvalidBSON, InvalidDocument, InvalidStringData) from bson.int64 import Int64 from bson.max_key import MaxKey from bson.min_key import MinKey from bson.objectid import ObjectId from bson.py3compat import (b, PY3, iteritems, text_type, string_type, reraise) from bson.regex import Regex from bson.son import SON, RE_TYPE from bson.timestamp import Timestamp from bson.tz_util import utc try: from bson import _cbson _USE_C = True except ImportError: _USE_C = False EPOCH_AWARE = datetime.datetime.fromtimestamp(0, utc) EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0) BSONNUM = b"\x01" # Floating point BSONSTR = b"\x02" # UTF-8 string BSONOBJ = b"\x03" # Embedded document BSONARR = b"\x04" # Array BSONBIN = b"\x05" # Binary BSONUND = b"\x06" # Undefined BSONOID = b"\x07" # ObjectId BSONBOO = b"\x08" # Boolean BSONDAT = b"\x09" # UTC Datetime BSONNUL = b"\x0A" # Null BSONRGX = b"\x0B" # Regex BSONREF = b"\x0C" # DBRef BSONCOD = b"\x0D" # Javascript code BSONSYM = b"\x0E" # Symbol BSONCWS = b"\x0F" # Javascript code with scope BSONINT = b"\x10" # 32bit int BSONTIM = b"\x11" # Timestamp BSONLON = b"\x12" # 64bit int BSONMIN = b"\xFF" # Min key BSONMAX = b"\x7F" # Max key _UNPACK_FLOAT = struct.Struct("= obj_end: raise InvalidBSON("invalid object length") if _raw_document_class(opts.document_class): return (opts.document_class(data[position:end + 1], opts), position + obj_size) obj = _elements_to_dict(data, position + 4, end, opts) position += obj_size if "$ref" in obj: return (DBRef(obj.pop("$ref"), obj.pop("$id", None), obj.pop("$db", None), obj), position) return obj, position def _get_array(data, position, obj_end, opts): """Decode a BSON array to python list.""" size = _UNPACK_INT(data[position:position + 4])[0] end = position + size - 1 if data[end:end + 1] != b"\x00": raise InvalidBSON("bad eoo") position += 4 end -= 1 result = [] # Avoid doing global and attibute lookups in the loop. append = result.append index = data.index getter = _ELEMENT_GETTER while position < end: element_type = data[position:position + 1] # Just skip the keys. position = index(b'\x00', position) + 1 value, position = getter[element_type](data, position, obj_end, opts) append(value) return result, position + 1 def _get_binary(data, position, dummy, opts): """Decode a BSON binary to bson.binary.Binary or python UUID.""" length, subtype = _UNPACK_LENGTH_SUBTYPE(data[position:position + 5]) position += 5 if subtype == 2: length2 = _UNPACK_INT(data[position:position + 4])[0] position += 4 if length2 != length - 4: raise InvalidBSON("invalid binary (st 2) - lengths don't match!") length = length2 end = position + length if subtype in (3, 4): # Java Legacy uuid_representation = opts.uuid_representation if uuid_representation == JAVA_LEGACY: java = data[position:end] value = uuid.UUID(bytes=java[0:8][::-1] + java[8:16][::-1]) # C# legacy elif uuid_representation == CSHARP_LEGACY: value = uuid.UUID(bytes_le=data[position:end]) # Python else: value = uuid.UUID(bytes=data[position:end]) return value, end # Python3 special case. Decode subtype 0 to 'bytes'. if PY3 and subtype == 0: value = data[position:end] else: value = Binary(data[position:end], subtype) return value, end def _get_oid(data, position, dummy0, dummy1): """Decode a BSON ObjectId to bson.objectid.ObjectId.""" end = position + 12 return ObjectId(data[position:end]), end def _get_boolean(data, position, dummy0, dummy1): """Decode a BSON true/false to python True/False.""" end = position + 1 return data[position:end] == b"\x01", end def _get_date(data, position, dummy, opts): """Decode a BSON datetime to python datetime.datetime.""" end = position + 8 millis = _UNPACK_LONG(data[position:end])[0] diff = ((millis % 1000) + 1000) % 1000 seconds = (millis - diff) / 1000 micros = diff * 1000 if opts.tz_aware: dt = EPOCH_AWARE + datetime.timedelta( seconds=seconds, microseconds=micros) if opts.tzinfo: dt = dt.astimezone(opts.tzinfo) else: dt = EPOCH_NAIVE + datetime.timedelta( seconds=seconds, microseconds=micros) return dt, end def _get_code(data, position, obj_end, opts): """Decode a BSON code to bson.code.Code.""" code, position = _get_string(data, position, obj_end, opts) return Code(code), position def _get_code_w_scope(data, position, obj_end, opts): """Decode a BSON code_w_scope to bson.code.Code.""" code, position = _get_string(data, position + 4, obj_end, opts) scope, position = _get_object(data, position, obj_end, opts) return Code(code, scope), position def _get_regex(data, position, dummy0, opts): """Decode a BSON regex to bson.regex.Regex or a python pattern object.""" pattern, position = _get_c_string(data, position, opts) bson_flags, position = _get_c_string(data, position, opts) bson_re = Regex(pattern, bson_flags) return bson_re, position def _get_ref(data, position, obj_end, opts): """Decode (deprecated) BSON DBPointer to bson.dbref.DBRef.""" collection, position = _get_string(data, position, obj_end, opts) oid, position = _get_oid(data, position, obj_end, opts) return DBRef(collection, oid), position def _get_timestamp(data, position, dummy0, dummy1): """Decode a BSON timestamp to bson.timestamp.Timestamp.""" end = position + 8 inc, timestamp = _UNPACK_TIMESTAMP(data[position:end]) return Timestamp(timestamp, inc), end def _get_int64(data, position, dummy0, dummy1): """Decode a BSON int64 to bson.int64.Int64.""" end = position + 8 return Int64(_UNPACK_LONG(data[position:end])[0]), end # Each decoder function's signature is: # - data: bytes # - position: int, beginning of object in 'data' to decode # - obj_end: int, end of object to decode in 'data' if variable-length type # - opts: a CodecOptions _ELEMENT_GETTER = { BSONNUM: _get_float, BSONSTR: _get_string, BSONOBJ: _get_object, BSONARR: _get_array, BSONBIN: _get_binary, BSONUND: lambda w, x, y, z: (None, x), # Deprecated undefined BSONOID: _get_oid, BSONBOO: _get_boolean, BSONDAT: _get_date, BSONNUL: lambda w, x, y, z: (None, x), BSONRGX: _get_regex, BSONREF: _get_ref, # Deprecated DBPointer BSONCOD: _get_code, BSONSYM: _get_string, # Deprecated symbol BSONCWS: _get_code_w_scope, BSONINT: _get_int, BSONTIM: _get_timestamp, BSONLON: _get_int64, BSONMIN: lambda w, x, y, z: (MinKey(), x), BSONMAX: lambda w, x, y, z: (MaxKey(), x)} def _element_to_dict(data, position, obj_end, opts): """Decode a single key, value pair.""" element_type = data[position:position + 1] position += 1 element_name, position = _get_c_string(data, position, opts) value, position = _ELEMENT_GETTER[element_type](data, position, obj_end, opts) return element_name, value, position if _USE_C: _element_to_dict = _cbson._element_to_dict def _iterate_elements(data, position, obj_end, opts): end = obj_end - 1 while position < end: (key, value, position) = _element_to_dict(data, position, obj_end, opts) yield key, value def _elements_to_dict(data, position, obj_end, opts): """Decode a BSON document.""" result = opts.document_class() for key, value in _iterate_elements(data, position, obj_end, opts): result[key] = value return result def _bson_to_dict(data, opts): """Decode a BSON string to document_class.""" try: obj_size = _UNPACK_INT(data[:4])[0] except struct.error as exc: raise InvalidBSON(str(exc)) if obj_size != len(data): raise InvalidBSON("invalid object size") if data[obj_size - 1:obj_size] != b"\x00": raise InvalidBSON("bad eoo") try: if _raw_document_class(opts.document_class): return opts.document_class(data, opts) return _elements_to_dict(data, 4, obj_size - 1, opts) except InvalidBSON: raise except Exception: # Change exception type to InvalidBSON but preserve traceback. _, exc_value, exc_tb = sys.exc_info() reraise(InvalidBSON, exc_value, exc_tb) if _USE_C: _bson_to_dict = _cbson._bson_to_dict _PACK_FLOAT = struct.Struct(">> import collections # From Python standard library. >>> import bson >>> from bson.codec_options import CodecOptions >>> data = bson.BSON.encode({'a': 1}) >>> decoded_doc = bson.BSON.decode(data) >>> options = CodecOptions(document_class=collections.OrderedDict) >>> decoded_doc = bson.BSON.decode(data, codec_options=options) >>> type(decoded_doc) :Parameters: - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. .. versionchanged:: 3.0 Removed `compile_re` option: PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. Replaced `as_class`, `tz_aware`, and `uuid_subtype` options with `codec_options`. .. versionchanged:: 2.7 Added `compile_re` option. If set to False, PyMongo represented BSON regular expressions as :class:`~bson.regex.Regex` objects instead of attempting to compile BSON regular expressions as Python native regular expressions, thus preventing errors for some incompatible patterns, see `PYTHON-500`_. .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 """ if not isinstance(codec_options, CodecOptions): raise _CODEC_OPTIONS_TYPE_ERROR return _bson_to_dict(self, codec_options) def has_c(): """Is the C extension installed? """ return _USE_C pymongo-3.2/bson/_cbsonmodule.c0000644000175000017500000026441512630145074020550 0ustar behackettbehackett00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains C implementations of some of the functions * needed by the bson module. If possible, these implementations * should be used to speed up BSON encoding and decoding. */ #include "Python.h" #include "datetime.h" #include "buffer.h" #include "time64.h" #include "encoding_helpers.h" #define _CBSON_MODULE #include "_cbsonmodule.h" /* New module state and initialization code. * See the module-initialization-and-state * section in the following doc: * http://docs.python.org/release/3.1.3/howto/cporting.html * which references the following pep: * http://www.python.org/dev/peps/pep-3121/ * */ struct module_state { PyObject* Binary; PyObject* Code; PyObject* ObjectId; PyObject* DBRef; PyObject* Regex; PyObject* UUID; PyObject* Timestamp; PyObject* MinKey; PyObject* MaxKey; PyObject* UTC; PyTypeObject* REType; PyObject* BSONInt64; PyObject* Mapping; PyObject* CodecOptions; }; /* The Py_TYPE macro was introduced in CPython 2.6 */ #ifndef Py_TYPE #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif /* Maximum number of regex flags */ #define FLAGS_SIZE 7 /* Default UUID representation type code. */ #define PYTHON_LEGACY 3 /* Other UUID representations. */ #define STANDARD 4 #define JAVA_LEGACY 5 #define CSHARP_LEGACY 6 #define BSON_MAX_SIZE 2147483647 /* The smallest possible BSON document, i.e. "{}" */ #define BSON_MIN_SIZE 5 /* Get an error class from the bson.errors module. * * Returns a new ref */ static PyObject* _error(char* name) { PyObject* error; PyObject* errors = PyImport_ImportModule("bson.errors"); if (!errors) { return NULL; } error = PyObject_GetAttrString(errors, name); Py_DECREF(errors); return error; } /* Safely downcast from Py_ssize_t to int, setting an * exception and returning -1 on error. */ static int _downcast_and_check(Py_ssize_t size, int extra) { if (size > BSON_MAX_SIZE || ((BSON_MAX_SIZE - extra) < size)) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyErr_SetString(InvalidStringData, "String length must be <= 2147483647"); Py_DECREF(InvalidStringData); } return -1; } return (int)size + extra; } static PyObject* elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options); static int _write_element_to_buffer(PyObject* self, buffer_t buffer, int type_byte, PyObject* value, unsigned char check_keys, const codec_options_t* options); /* Date stuff */ static PyObject* datetime_from_millis(long long millis) { /* To encode a datetime instance like datetime(9999, 12, 31, 23, 59, 59, 999999) * we follow these steps: * 1. Calculate a timestamp in seconds: 253402300799 * 2. Multiply that by 1000: 253402300799000 * 3. Add in microseconds divided by 1000 253402300799999 * * (Note: BSON doesn't support microsecond accuracy, hence the rounding.) * * To decode we could do: * 1. Get seconds: timestamp / 1000: 253402300799 * 2. Get micros: (timestamp % 1000) * 1000: 999000 * Resulting in datetime(9999, 12, 31, 23, 59, 59, 999000) -- the expected result * * Now what if the we encode (1, 1, 1, 1, 1, 1, 111111)? * 1. and 2. gives: -62135593139000 * 3. Gives us: -62135593138889 * * Now decode: * 1. Gives us: -62135593138 * 2. Gives us: -889000 * Resulting in datetime(1, 1, 1, 1, 1, 2, 15888216) -- an invalid result * * If instead to decode we do: * diff = ((millis % 1000) + 1000) % 1000: 111 * seconds = (millis - diff) / 1000: -62135593139 * micros = diff * 1000 111000 * Resulting in datetime(1, 1, 1, 1, 1, 1, 111000) -- the expected result */ int diff = (int)(((millis % 1000) + 1000) % 1000); int microseconds = diff * 1000; Time64_T seconds = (millis - diff) / 1000; struct TM timeinfo; gmtime64_r(&seconds, &timeinfo); return PyDateTime_FromDateAndTime(timeinfo.tm_year + 1900, timeinfo.tm_mon + 1, timeinfo.tm_mday, timeinfo.tm_hour, timeinfo.tm_min, timeinfo.tm_sec, microseconds); } static long long millis_from_datetime(PyObject* datetime) { struct TM timeinfo; long long millis; timeinfo.tm_year = PyDateTime_GET_YEAR(datetime) - 1900; timeinfo.tm_mon = PyDateTime_GET_MONTH(datetime) - 1; timeinfo.tm_mday = PyDateTime_GET_DAY(datetime); timeinfo.tm_hour = PyDateTime_DATE_GET_HOUR(datetime); timeinfo.tm_min = PyDateTime_DATE_GET_MINUTE(datetime); timeinfo.tm_sec = PyDateTime_DATE_GET_SECOND(datetime); millis = timegm64(&timeinfo) * 1000; millis += PyDateTime_DATE_GET_MICROSECOND(datetime) / 1000; return millis; } /* Just make this compatible w/ the old API. */ int buffer_write_bytes(buffer_t buffer, const char* data, int size) { if (buffer_write(buffer, data, size)) { PyErr_NoMemory(); return 0; } return 1; } static int write_unicode(buffer_t buffer, PyObject* py_string) { int size; const char* data; PyObject* encoded = PyUnicode_AsUTF8String(py_string); if (!encoded) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AS_STRING(encoded); #else data = PyString_AS_STRING(encoded); #endif if (!data) goto unicodefail; #if PY_MAJOR_VERSION >= 3 if ((size = _downcast_and_check(PyBytes_GET_SIZE(encoded), 1)) == -1) #else if ((size = _downcast_and_check(PyString_GET_SIZE(encoded), 1)) == -1) #endif goto unicodefail; if (!buffer_write_bytes(buffer, (const char*)&size, 4)) goto unicodefail; if (!buffer_write_bytes(buffer, data, size)) goto unicodefail; Py_DECREF(encoded); return 1; unicodefail: Py_DECREF(encoded); return 0; } /* returns 0 on failure */ static int write_string(buffer_t buffer, PyObject* py_string) { int size; const char* data; #if PY_MAJOR_VERSION >= 3 if (PyUnicode_Check(py_string)){ return write_unicode(buffer, py_string); } data = PyBytes_AsString(py_string); #else data = PyString_AsString(py_string); #endif if (!data) { return 0; } #if PY_MAJOR_VERSION >= 3 if ((size = _downcast_and_check(PyBytes_Size(py_string), 1)) == -1) #else if ((size = _downcast_and_check(PyString_Size(py_string), 1)) == -1) #endif return 0; if (!buffer_write_bytes(buffer, (const char*)&size, 4)) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; } /* * Are we in the main interpreter or a sub-interpreter? * Useful for deciding if we can use cached pure python * types in mod_wsgi. */ static int _in_main_interpreter(void) { static PyInterpreterState* main_interpreter = NULL; PyInterpreterState* interpreter; if (main_interpreter == NULL) { interpreter = PyInterpreterState_Head(); while (PyInterpreterState_Next(interpreter)) interpreter = PyInterpreterState_Next(interpreter); main_interpreter = interpreter; } return (main_interpreter == PyThreadState_Get()->interp); } /* * Get a reference to a pure python type. If we are in the * main interpreter return the cached object, otherwise import * the object we need and return it instead. */ static PyObject* _get_object(PyObject* object, char* module_name, char* object_name) { if (_in_main_interpreter()) { Py_XINCREF(object); return object; } else { PyObject* imported = NULL; PyObject* module = PyImport_ImportModule(module_name); if (!module) return NULL; imported = PyObject_GetAttrString(module, object_name); Py_DECREF(module); return imported; } } /* Load a Python object to cache. * * Returns non-zero on failure. */ static int _load_object(PyObject** object, char* module_name, char* object_name) { PyObject* module; module = PyImport_ImportModule(module_name); if (!module) { return 1; } *object = PyObject_GetAttrString(module, object_name); Py_DECREF(module); return (*object) ? 0 : 2; } /* Load all Python objects to cache. * * Returns non-zero on failure. */ static int _load_python_objects(PyObject* module) { PyObject* empty_string; PyObject* re_compile; PyObject* compiled; struct module_state *state = GETSTATE(module); if (_load_object(&state->Binary, "bson.binary", "Binary") || _load_object(&state->Code, "bson.code", "Code") || _load_object(&state->ObjectId, "bson.objectid", "ObjectId") || _load_object(&state->DBRef, "bson.dbref", "DBRef") || _load_object(&state->Timestamp, "bson.timestamp", "Timestamp") || _load_object(&state->MinKey, "bson.min_key", "MinKey") || _load_object(&state->MaxKey, "bson.max_key", "MaxKey") || _load_object(&state->UTC, "bson.tz_util", "utc") || _load_object(&state->Regex, "bson.regex", "Regex") || _load_object(&state->BSONInt64, "bson.int64", "Int64") || _load_object(&state->UUID, "uuid", "UUID") || _load_object(&state->Mapping, "collections", "Mapping") || _load_object(&state->CodecOptions, "bson.codec_options", "CodecOptions")) { return 1; } /* Reload our REType hack too. */ #if PY_MAJOR_VERSION >= 3 empty_string = PyBytes_FromString(""); #else empty_string = PyString_FromString(""); #endif if (empty_string == NULL) { state->REType = NULL; return 1; } if (_load_object(&re_compile, "re", "compile")) { state->REType = NULL; Py_DECREF(empty_string); return 1; } compiled = PyObject_CallFunction(re_compile, "O", empty_string); if (compiled == NULL) { state->REType = NULL; Py_DECREF(empty_string); return 1; } Py_INCREF(Py_TYPE(compiled)); state->REType = Py_TYPE(compiled); Py_DECREF(empty_string); Py_DECREF(compiled); return 0; } /* * Get the _type_marker from an Object. * * Return the type marker, 0 if there is no marker, or -1 on failure. */ static long _type_marker(PyObject* object) { PyObject* type_marker = NULL; long type = 0; if (PyObject_HasAttrString(object, "_type_marker")) { type_marker = PyObject_GetAttrString(object, "_type_marker"); if (type_marker == NULL) { return -1; } } /* * Python objects with broken __getattr__ implementations could return * arbitrary types for a call to PyObject_GetAttrString. For example * pymongo.database.Database returns a new Collection instance for * __getattr__ calls with names that don't match an existing attribute * or method. In some cases "value" could be a subtype of something * we know how to serialize. Make a best effort to encode these types. */ #if PY_MAJOR_VERSION >= 3 if (type_marker && PyLong_CheckExact(type_marker)) { type = PyLong_AsLong(type_marker); #else if (type_marker && PyInt_CheckExact(type_marker)) { type = PyInt_AsLong(type_marker); #endif Py_DECREF(type_marker); /* * Py(Long|Int)_AsLong returns -1 for error but -1 is a valid value * so we call PyErr_Occurred to differentiate. */ if (type == -1 && PyErr_Occurred()) { return -1; } } else { Py_XDECREF(type_marker); } return type; } /* Fill out a codec_options_t* from a CodecOptions object. Use with the "O&" * format spec in PyArg_ParseTuple. * * Return 1 on success. options->document_class is a new reference. * Return 0 on failure. */ int convert_codec_options(PyObject* options_obj, void* p) { codec_options_t* options = (codec_options_t*)p; long type_marker; options->unicode_decode_error_handler = NULL; if (!PyArg_ParseTuple(options_obj, "ObbzO", &options->document_class, &options->tz_aware, &options->uuid_rep, &options->unicode_decode_error_handler, &options->tzinfo)) { return 0; } type_marker = _type_marker(options->document_class); if (type_marker < 0) return 0; Py_INCREF(options->document_class); Py_INCREF(options->tzinfo); options->options_obj = options_obj; Py_INCREF(options->options_obj); options->is_raw_bson = (101 == type_marker); return 1; } /* Fill out a codec_options_t* with default options. * * Return 1 on success. * Return 0 on failure. */ int default_codec_options(struct module_state* state, codec_options_t* options) { PyObject* codec_options_func = _get_object( state->CodecOptions, "bson.codec_options", "CodecOptions"); PyObject* options_obj = PyObject_CallFunctionObjArgs( codec_options_func, NULL); return convert_codec_options(options_obj, options); } void destroy_codec_options(codec_options_t* options) { Py_CLEAR(options->document_class); Py_CLEAR(options->tzinfo); Py_CLEAR(options->options_obj); } static int write_element_to_buffer(PyObject* self, buffer_t buffer, int type_byte, PyObject* value, unsigned char check_keys, const codec_options_t* options) { int result; if(Py_EnterRecursiveCall(" while encoding an object to BSON ")) return 0; result = _write_element_to_buffer(self, buffer, type_byte, value, check_keys, options); Py_LeaveRecursiveCall(); return result; } static void _fix_java(const char* in, char* out) { int i, j; for (i = 0, j = 7; i < j; i++, j--) { out[i] = in[j]; out[j] = in[i]; } for (i = 8, j = 15; i < j; i++, j--) { out[i] = in[j]; out[j] = in[i]; } } static void _set_cannot_encode(PyObject* value) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyObject* repr = PyObject_Repr(value); if (repr) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromString("Cannot encode object: "); #else PyObject* errmsg = PyString_FromString("Cannot encode object: "); #endif if (errmsg) { #if PY_MAJOR_VERSION >= 3 PyObject* error = PyUnicode_Concat(errmsg, repr); if (error) { PyErr_SetObject(InvalidDocument, error); Py_DECREF(error); } Py_DECREF(errmsg); Py_DECREF(repr); #else PyString_ConcatAndDel(&errmsg, repr); if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } #endif } else { Py_DECREF(repr); } } Py_DECREF(InvalidDocument); } } /* * Encode a builtin Python regular expression or our custom Regex class. * * Sets exception and returns 0 on failure. */ static int _write_regex_to_buffer( buffer_t buffer, int type_byte, PyObject* value) { PyObject* py_flags; PyObject* py_pattern; PyObject* encoded_pattern; long int_flags; char flags[FLAGS_SIZE]; char check_utf8 = 0; const char* pattern_data; int pattern_length, flags_length; result_t status; /* * Both the builtin re type and our Regex class have attributes * "flags" and "pattern". */ py_flags = PyObject_GetAttrString(value, "flags"); if (!py_flags) { return 0; } #if PY_MAJOR_VERSION >= 3 int_flags = PyLong_AsLong(py_flags); #else int_flags = PyInt_AsLong(py_flags); #endif Py_DECREF(py_flags); py_pattern = PyObject_GetAttrString(value, "pattern"); if (!py_pattern) { return 0; } if (PyUnicode_Check(py_pattern)) { encoded_pattern = PyUnicode_AsUTF8String(py_pattern); Py_DECREF(py_pattern); if (!encoded_pattern) { return 0; } } else { encoded_pattern = py_pattern; check_utf8 = 1; } #if PY_MAJOR_VERSION >= 3 if (!(pattern_data = PyBytes_AsString(encoded_pattern))) { Py_DECREF(encoded_pattern); return 0; } if ((pattern_length = _downcast_and_check(PyBytes_Size(encoded_pattern), 0)) == -1) { Py_DECREF(encoded_pattern); return 0; } #else if (!(pattern_data = PyString_AsString(encoded_pattern))) { Py_DECREF(encoded_pattern); return 0; } if ((pattern_length = _downcast_and_check(PyString_Size(encoded_pattern), 0)) == -1) { Py_DECREF(encoded_pattern); return 0; } #endif status = check_string((const unsigned char*)pattern_data, pattern_length, check_utf8, 1); if (status == NOT_UTF_8) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyErr_SetString(InvalidStringData, "regex patterns must be valid UTF-8"); Py_DECREF(InvalidStringData); } Py_DECREF(encoded_pattern); return 0; } else if (status == HAS_NULL) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "regex patterns must not contain the NULL byte"); Py_DECREF(InvalidDocument); } Py_DECREF(encoded_pattern); return 0; } if (!buffer_write_bytes(buffer, pattern_data, pattern_length + 1)) { Py_DECREF(encoded_pattern); return 0; } Py_DECREF(encoded_pattern); flags[0] = 0; if (int_flags & 2) { STRCAT(flags, FLAGS_SIZE, "i"); } if (int_flags & 4) { STRCAT(flags, FLAGS_SIZE, "l"); } if (int_flags & 8) { STRCAT(flags, FLAGS_SIZE, "m"); } if (int_flags & 16) { STRCAT(flags, FLAGS_SIZE, "s"); } if (int_flags & 32) { STRCAT(flags, FLAGS_SIZE, "u"); } if (int_flags & 64) { STRCAT(flags, FLAGS_SIZE, "x"); } flags_length = (int)strlen(flags) + 1; if (!buffer_write_bytes(buffer, flags, flags_length)) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x0B; return 1; } /* TODO our platform better be little-endian w/ 4-byte ints! */ /* Write a single value to the buffer (also write its type_byte, for which * space has already been reserved. * * returns 0 on failure */ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, int type_byte, PyObject* value, unsigned char check_keys, const codec_options_t* options) { struct module_state *state = GETSTATE(self); PyObject* mapping_type; PyObject* uuid_type; /* * Don't use PyObject_IsInstance for our custom types. It causes * problems with python sub interpreters. Our custom types should * have a _type_marker attribute, which we can switch on instead. */ long type = _type_marker(value); if (type < 0) { return 0; } switch (type) { case 5: { /* Binary */ PyObject* subtype_object; long subtype; const char* data; int size; *(buffer_get_buffer(buffer) + type_byte) = 0x05; subtype_object = PyObject_GetAttrString(value, "subtype"); if (!subtype_object) { return 0; } #if PY_MAJOR_VERSION >= 3 subtype = PyLong_AsLong(subtype_object); #else subtype = PyInt_AsLong(subtype_object); #endif if (subtype == -1) { Py_DECREF(subtype_object); return 0; } #if PY_MAJOR_VERSION >= 3 size = _downcast_and_check(PyBytes_Size(value), 0); #else size = _downcast_and_check(PyString_Size(value), 0); #endif if (size == -1) { Py_DECREF(subtype_object); return 0; } Py_DECREF(subtype_object); if (subtype == 2) { #if PY_MAJOR_VERSION >= 3 int other_size = _downcast_and_check(PyBytes_Size(value), 4); #else int other_size = _downcast_and_check(PyString_Size(value), 4); #endif if (other_size == -1) return 0; if (!buffer_write_bytes(buffer, (const char*)&other_size, 4)) { return 0; } if (!buffer_write_bytes(buffer, (const char*)&subtype, 1)) { return 0; } } if (!buffer_write_bytes(buffer, (const char*)&size, 4)) { return 0; } if (subtype != 2) { if (!buffer_write_bytes(buffer, (const char*)&subtype, 1)) { return 0; } } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(value); #else data = PyString_AsString(value); #endif if (!data) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; } case 7: { /* ObjectId */ const char* data; PyObject* pystring = PyObject_GetAttrString(value, "_ObjectId__id"); if (!pystring) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(pystring); #else data = PyString_AsString(pystring); #endif if (!data) { Py_DECREF(pystring); return 0; } if (!buffer_write_bytes(buffer, data, 12)) { Py_DECREF(pystring); return 0; } Py_DECREF(pystring); *(buffer_get_buffer(buffer) + type_byte) = 0x07; return 1; } case 11: { /* Regex */ return _write_regex_to_buffer(buffer, type_byte, value); } case 13: { /* Code */ int start_position, length_location, length; PyObject* scope = PyObject_GetAttrString(value, "scope"); if (!scope) { return 0; } if (!PyDict_Size(scope)) { Py_DECREF(scope); *(buffer_get_buffer(buffer) + type_byte) = 0x0D; return write_string(buffer, value); } *(buffer_get_buffer(buffer) + type_byte) = 0x0F; start_position = buffer_get_position(buffer); /* save space for length */ length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); Py_DECREF(scope); return 0; } if (!write_string(buffer, value)) { Py_DECREF(scope); return 0; } if (!write_dict(self, buffer, scope, 0, options, 0)) { Py_DECREF(scope); return 0; } Py_DECREF(scope); length = buffer_get_position(buffer) - start_position; memcpy(buffer_get_buffer(buffer) + length_location, &length, 4); return 1; } case 17: { /* Timestamp */ PyObject* obj; long i; obj = PyObject_GetAttrString(value, "inc"); if (!obj) { return 0; } #if PY_MAJOR_VERSION >= 3 i = PyLong_AsLong(obj); #else i = PyInt_AsLong(obj); #endif Py_DECREF(obj); if (!buffer_write_bytes(buffer, (const char*)&i, 4)) { return 0; } obj = PyObject_GetAttrString(value, "time"); if (!obj) { return 0; } #if PY_MAJOR_VERSION >= 3 i = PyLong_AsLong(obj); #else i = PyInt_AsLong(obj); #endif Py_DECREF(obj); if (!buffer_write_bytes(buffer, (const char*)&i, 4)) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x11; return 1; } case 18: { /* Int64 */ const long long ll = PyLong_AsLongLong(value); if (PyErr_Occurred()) { /* Overflow */ PyErr_SetString(PyExc_OverflowError, "MongoDB can only handle up to 8-byte ints"); return 0; } if (!buffer_write_bytes(buffer, (const char*)&ll, 8)) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x12; return 1; } case 100: { /* DBRef */ PyObject* as_doc = PyObject_CallMethod(value, "as_doc", NULL); if (!as_doc) { return 0; } if (!write_dict(self, buffer, as_doc, 0, options, 0)) { Py_DECREF(as_doc); return 0; } Py_DECREF(as_doc); *(buffer_get_buffer(buffer) + type_byte) = 0x03; return 1; } case 101: { /* RawBSONDocument */ char* raw_bson_document_bytes; Py_ssize_t raw_bson_document_bytes_len; int raw_bson_document_bytes_len_int; PyObject* raw_bson_document_bytes_obj = PyObject_GetAttrString(value, "raw"); if (!raw_bson_document_bytes_obj) { return 0; } #if PY_MAJOR_VERSION >= 3 if (-1 == PyBytes_AsStringAndSize(raw_bson_document_bytes_obj, &raw_bson_document_bytes, &raw_bson_document_bytes_len)) { #else if (-1 == PyString_AsStringAndSize(raw_bson_document_bytes_obj, &raw_bson_document_bytes, &raw_bson_document_bytes_len)) { #endif Py_DECREF(raw_bson_document_bytes_obj); return 0; } raw_bson_document_bytes_len_int = _downcast_and_check( raw_bson_document_bytes_len, 0); if (-1 == raw_bson_document_bytes_len_int) { Py_DECREF(raw_bson_document_bytes_obj); return 0; } if(!buffer_write_bytes(buffer, raw_bson_document_bytes, raw_bson_document_bytes_len_int)) { Py_DECREF(raw_bson_document_bytes_obj); return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x03; Py_DECREF(raw_bson_document_bytes_obj); return 1; } case 255: { /* MinKey */ *(buffer_get_buffer(buffer) + type_byte) = 0xFF; return 1; } case 127: { /* MaxKey */ *(buffer_get_buffer(buffer) + type_byte) = 0x7F; return 1; } } /* No _type_marker attibute or not one of our types. */ if (PyBool_Check(value)) { #if PY_MAJOR_VERSION >= 3 const long bool = PyLong_AsLong(value); #else const long bool = PyInt_AsLong(value); #endif const char c = bool ? 0x01 : 0x00; *(buffer_get_buffer(buffer) + type_byte) = 0x08; return buffer_write_bytes(buffer, &c, 1); } #if PY_MAJOR_VERSION >= 3 else if (PyLong_Check(value)) { const long long_value = PyLong_AsLong(value); #else else if (PyInt_Check(value)) { const long long_value = PyInt_AsLong(value); #endif const int int_value = (int)long_value; if (PyErr_Occurred() || long_value != int_value) { /* Overflow */ long long long_long_value; PyErr_Clear(); long_long_value = PyLong_AsLongLong(value); if (PyErr_Occurred()) { /* Overflow AGAIN */ PyErr_SetString(PyExc_OverflowError, "MongoDB can only handle up to 8-byte ints"); return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x12; return buffer_write_bytes(buffer, (const char*)&long_long_value, 8); } *(buffer_get_buffer(buffer) + type_byte) = 0x10; return buffer_write_bytes(buffer, (const char*)&int_value, 4); #if PY_MAJOR_VERSION < 3 } else if (PyLong_Check(value)) { const long long long_long_value = PyLong_AsLongLong(value); if (PyErr_Occurred()) { /* Overflow */ PyErr_SetString(PyExc_OverflowError, "MongoDB can only handle up to 8-byte ints"); return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x12; return buffer_write_bytes(buffer, (const char*)&long_long_value, 8); #endif } else if (PyFloat_Check(value)) { const double d = PyFloat_AsDouble(value); *(buffer_get_buffer(buffer) + type_byte) = 0x01; return buffer_write_bytes(buffer, (const char*)&d, 8); } else if (value == Py_None) { *(buffer_get_buffer(buffer) + type_byte) = 0x0A; return 1; } else if (PyDict_Check(value)) { *(buffer_get_buffer(buffer) + type_byte) = 0x03; return write_dict(self, buffer, value, check_keys, options, 0); } else if (PyList_Check(value) || PyTuple_Check(value)) { Py_ssize_t items, i; int start_position, length_location, length; char zero = 0; *(buffer_get_buffer(buffer) + type_byte) = 0x04; start_position = buffer_get_position(buffer); /* save space for length */ length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); return 0; } if ((items = PySequence_Size(value)) > BSON_MAX_SIZE) { PyObject* BSONError = _error("BSONError"); if (BSONError) { PyErr_SetString(BSONError, "Too many items to serialize."); Py_DECREF(BSONError); } return 0; } for(i = 0; i < items; i++) { int list_type_byte = buffer_save_space(buffer, 1); char name[16]; PyObject* item_value; if (list_type_byte == -1) { PyErr_NoMemory(); return 0; } INT2STRING(name, (int)i); if (!buffer_write_bytes(buffer, name, (int)strlen(name) + 1)) { return 0; } if (!(item_value = PySequence_GetItem(value, i))) return 0; if (!write_element_to_buffer(self, buffer, list_type_byte, item_value, check_keys, options)) { Py_DECREF(item_value); return 0; } Py_DECREF(item_value); } /* write null byte and fill in length */ if (!buffer_write_bytes(buffer, &zero, 1)) { return 0; } length = buffer_get_position(buffer) - start_position; memcpy(buffer_get_buffer(buffer) + length_location, &length, 4); return 1; #if PY_MAJOR_VERSION >= 3 /* Python3 special case. Store bytes as BSON binary subtype 0. */ } else if (PyBytes_Check(value)) { int subtype = 0; int size; const char* data = PyBytes_AS_STRING(value); if (!data) return 0; if ((size = _downcast_and_check(PyBytes_GET_SIZE(value), 0)) == -1) return 0; *(buffer_get_buffer(buffer) + type_byte) = 0x05; if (!buffer_write_bytes(buffer, (const char*)&size, 4)) { return 0; } if (!buffer_write_bytes(buffer, (const char*)&subtype, 1)) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; #else /* PyString_Check only works in Python 2.x. */ } else if (PyString_Check(value)) { result_t status; const char* data; int size; if (!(data = PyString_AS_STRING(value))) return 0; if ((size = _downcast_and_check(PyString_GET_SIZE(value), 1)) == -1) return 0; *(buffer_get_buffer(buffer) + type_byte) = 0x02; status = check_string((const unsigned char*)data, size - 1, 1, 0); if (status == NOT_UTF_8) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyObject* repr = PyObject_Repr(value); char* repr_as_cstr = repr ? PyString_AsString(repr) : NULL; if (repr_as_cstr) { PyObject *message = PyString_FromFormat( "strings in documents must be valid UTF-8: %s", repr_as_cstr); if (message) { PyErr_SetObject(InvalidStringData, message); Py_DECREF(message); } } else { /* repr(value) failed, use a generic message. */ PyErr_SetString( InvalidStringData, "strings in documents must be valid UTF-8"); } Py_XDECREF(repr); Py_DECREF(InvalidStringData); } return 0; } if (!buffer_write_bytes(buffer, (const char*)&size, 4)) { return 0; } if (!buffer_write_bytes(buffer, data, size)) { return 0; } return 1; #endif } else if (PyUnicode_Check(value)) { *(buffer_get_buffer(buffer) + type_byte) = 0x02; return write_unicode(buffer, value); } else if (PyDateTime_Check(value)) { long long millis; PyObject* utcoffset = PyObject_CallMethod(value, "utcoffset", NULL); if (utcoffset == NULL) return 0; if (utcoffset != Py_None) { PyObject* result = PyNumber_Subtract(value, utcoffset); Py_DECREF(utcoffset); if (!result) { return 0; } millis = millis_from_datetime(result); Py_DECREF(result); } else { millis = millis_from_datetime(value); } *(buffer_get_buffer(buffer) + type_byte) = 0x09; return buffer_write_bytes(buffer, (const char*)&millis, 8); } else if (PyObject_TypeCheck(value, state->REType)) { return _write_regex_to_buffer(buffer, type_byte, value); } /* * Try Mapping and UUID last since we have to import * them if we're in a sub-interpreter. */ mapping_type = _get_object(state->Mapping, "collections", "Mapping"); if (mapping_type && PyObject_IsInstance(value, mapping_type)) { Py_DECREF(mapping_type); /* PyObject_IsInstance returns -1 on error */ if (PyErr_Occurred()) { return 0; } *(buffer_get_buffer(buffer) + type_byte) = 0x03; return write_dict(self, buffer, value, check_keys, options, 0); } uuid_type = _get_object(state->UUID, "uuid", "UUID"); if (uuid_type && PyObject_IsInstance(value, uuid_type)) { /* Just a special case of Binary above, but * simpler to do as a separate case. */ PyObject* bytes; /* Could be bytes, bytearray, str... */ const char* data; /* UUID is always 16 bytes */ int size = 16; int subtype; Py_DECREF(uuid_type); /* PyObject_IsInstance returns -1 on error */ if (PyErr_Occurred()) { return 0; } if (options->uuid_rep == JAVA_LEGACY || options->uuid_rep == CSHARP_LEGACY) { subtype = 3; } else { subtype = options->uuid_rep; } *(buffer_get_buffer(buffer) + type_byte) = 0x05; if (!buffer_write_bytes(buffer, (const char*)&size, 4)) { return 0; } if (!buffer_write_bytes(buffer, (const char*)&subtype, 1)) { return 0; } if (options->uuid_rep == CSHARP_LEGACY) { /* Legacy C# byte order */ bytes = PyObject_GetAttrString(value, "bytes_le"); } else { bytes = PyObject_GetAttrString(value, "bytes"); } if (!bytes) { return 0; } #if PY_MAJOR_VERSION >= 3 data = PyBytes_AsString(bytes); #else data = PyString_AsString(bytes); #endif if (data == NULL) { Py_DECREF(bytes); return 0; } if (options->uuid_rep == JAVA_LEGACY) { /* Store in legacy java byte order. */ char as_legacy_java[16]; _fix_java(data, as_legacy_java); if (!buffer_write_bytes(buffer, as_legacy_java, size)) { Py_DECREF(bytes); return 0; } } else { if (!buffer_write_bytes(buffer, data, size)) { Py_DECREF(bytes); return 0; } } Py_DECREF(bytes); return 1; } Py_XDECREF(mapping_type); Py_XDECREF(uuid_type); /* We can't determine value's type. Fail. */ _set_cannot_encode(value); return 0; } static int check_key_name(const char* name, int name_length) { if (name_length > 0 && name[0] == '$') { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromFormat( "key '%s' must not start with '$'", name); #else PyObject* errmsg = PyString_FromFormat( "key '%s' must not start with '$'", name); #endif if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } Py_DECREF(InvalidDocument); } return 0; } if (strchr(name, '.')) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromFormat( "key '%s' must not contain '.'", name); #else PyObject* errmsg = PyString_FromFormat( "key '%s' must not contain '.'", name); #endif if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } Py_DECREF(InvalidDocument); } return 0; } return 1; } /* Write a (key, value) pair to the buffer. * * Returns 0 on failure */ int write_pair(PyObject* self, buffer_t buffer, const char* name, int name_length, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char allow_id) { int type_byte; /* Don't write any _id elements unless we're explicitly told to - * _id has to be written first so we do so, but don't bother * deleting it from the dictionary being written. */ if (!allow_id && strcmp(name, "_id") == 0) { return 1; } type_byte = buffer_save_space(buffer, 1); if (type_byte == -1) { PyErr_NoMemory(); return 0; } if (check_keys && !check_key_name(name, name_length)) { return 0; } if (!buffer_write_bytes(buffer, name, name_length + 1)) { return 0; } if (!write_element_to_buffer(self, buffer, type_byte, value, check_keys, options)) { return 0; } return 1; } int decode_and_write_pair(PyObject* self, buffer_t buffer, PyObject* key, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) { PyObject* encoded; const char* data; int size; if (PyUnicode_Check(key)) { encoded = PyUnicode_AsUTF8String(key); if (!encoded) { return 0; } #if PY_MAJOR_VERSION >= 3 if (!(data = PyBytes_AS_STRING(encoded))) { Py_DECREF(encoded); return 0; } if ((size = _downcast_and_check(PyBytes_GET_SIZE(encoded), 1)) == -1) { Py_DECREF(encoded); return 0; } #else if (!(data = PyString_AS_STRING(encoded))) { Py_DECREF(encoded); return 0; } if ((size = _downcast_and_check(PyString_GET_SIZE(encoded), 1)) == -1) { Py_DECREF(encoded); return 0; } #endif if (strlen(data) != (size_t)(size - 1)) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "Key names must not contain the NULL byte"); Py_DECREF(InvalidDocument); } Py_DECREF(encoded); return 0; } #if PY_MAJOR_VERSION < 3 } else if (PyString_Check(key)) { result_t status; encoded = key; Py_INCREF(encoded); if (!(data = PyString_AS_STRING(encoded))) { Py_DECREF(encoded); return 0; } if ((size = _downcast_and_check(PyString_GET_SIZE(encoded), 1)) == -1) { Py_DECREF(encoded); return 0; } status = check_string((const unsigned char*)data, size - 1, 1, 1); if (status == NOT_UTF_8) { PyObject* InvalidStringData = _error("InvalidStringData"); if (InvalidStringData) { PyErr_SetString(InvalidStringData, "strings in documents must be valid UTF-8"); Py_DECREF(InvalidStringData); } Py_DECREF(encoded); return 0; } else if (status == HAS_NULL) { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "Key names must not contain the NULL byte"); Py_DECREF(InvalidDocument); } Py_DECREF(encoded); return 0; } #endif } else { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyObject* repr = PyObject_Repr(key); if (repr) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromString( "documents must have only string keys, key was "); #else PyObject* errmsg = PyString_FromString( "documents must have only string keys, key was "); #endif if (errmsg) { #if PY_MAJOR_VERSION >= 3 PyObject* error = PyUnicode_Concat(errmsg, repr); if (error) { PyErr_SetObject(InvalidDocument, error); Py_DECREF(error); } Py_DECREF(errmsg); Py_DECREF(repr); #else PyString_ConcatAndDel(&errmsg, repr); if (errmsg) { PyErr_SetObject(InvalidDocument, errmsg); Py_DECREF(errmsg); } #endif } else { Py_DECREF(repr); } } Py_DECREF(InvalidDocument); } return 0; } /* If top_level is True, don't allow writing _id here - it was already written. */ if (!write_pair(self, buffer, data, size - 1, value, check_keys, options, !top_level)) { Py_DECREF(encoded); return 0; } Py_DECREF(encoded); return 1; } /* returns 0 on failure */ int write_dict(PyObject* self, buffer_t buffer, PyObject* dict, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) { PyObject* key; PyObject* iter; char zero = 0; int length; int length_location; struct module_state *state = GETSTATE(self); PyObject* mapping_type = _get_object(state->Mapping, "collections", "Mapping"); if (mapping_type) { if (!PyObject_IsInstance(dict, mapping_type)) { PyObject* repr; Py_DECREF(mapping_type); if ((repr = PyObject_Repr(dict))) { #if PY_MAJOR_VERSION >= 3 PyObject* errmsg = PyUnicode_FromString( "encoder expected a mapping type but got: "); if (errmsg) { PyObject* error = PyUnicode_Concat(errmsg, repr); if (error) { PyErr_SetObject(PyExc_TypeError, error); Py_DECREF(error); } Py_DECREF(errmsg); Py_DECREF(repr); } #else PyObject* errmsg = PyString_FromString( "encoder expected a mapping type but got: "); if (errmsg) { PyString_ConcatAndDel(&errmsg, repr); if (errmsg) { PyErr_SetObject(PyExc_TypeError, errmsg); Py_DECREF(errmsg); } } #endif else { Py_DECREF(repr); } } else { PyErr_SetString(PyExc_TypeError, "encoder expected a mapping type"); } return 0; } Py_DECREF(mapping_type); /* PyObject_IsInstance returns -1 on error */ if (PyErr_Occurred()) { return 0; } } length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); return 0; } /* Write _id first if this is a top level doc. */ if (top_level) { /* * If "dict" is a defaultdict we don't want to call * PyMapping_GetItemString on it. That would **create** * an _id where one didn't previously exist (PYTHON-871). */ if (PyDict_Check(dict)) { /* PyDict_GetItemString returns a borrowed reference. */ PyObject* _id = PyDict_GetItemString(dict, "_id"); if (_id) { if (!write_pair(self, buffer, "_id", 3, _id, check_keys, options, 1)) { return 0; } } } else if (PyMapping_HasKeyString(dict, "_id")) { PyObject* _id = PyMapping_GetItemString(dict, "_id"); if (!_id) { return 0; } if (!write_pair(self, buffer, "_id", 3, _id, check_keys, options, 1)) { Py_DECREF(_id); return 0; } /* PyMapping_GetItemString returns a new reference. */ Py_DECREF(_id); } } iter = PyObject_GetIter(dict); if (iter == NULL) { return 0; } while ((key = PyIter_Next(iter)) != NULL) { PyObject* value = PyObject_GetItem(dict, key); if (!value) { PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); Py_DECREF(iter); return 0; } if (!decode_and_write_pair(self, buffer, key, value, check_keys, options, top_level)) { Py_DECREF(key); Py_DECREF(value); Py_DECREF(iter); return 0; } Py_DECREF(key); Py_DECREF(value); } Py_DECREF(iter); if (PyErr_Occurred()) { return 0; } /* write null byte and fill in length */ if (!buffer_write_bytes(buffer, &zero, 1)) { return 0; } length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &length, 4); return 1; } static PyObject* _cbson_dict_to_bson(PyObject* self, PyObject* args) { PyObject* dict; PyObject* result; unsigned char check_keys; unsigned char top_level = 1; codec_options_t options; buffer_t buffer; PyObject* raw_bson_document_bytes_obj; char* raw_bson_document_bytes; Py_ssize_t raw_bson_document_bytes_len; int raw_bson_document_bytes_len_int; long type_marker; if (!PyArg_ParseTuple(args, "ObO&|b", &dict, &check_keys, convert_codec_options, &options, &top_level)) { return NULL; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); return NULL; } /* check for RawBSONDocument */ type_marker = _type_marker(dict); if (type_marker < 0) { destroy_codec_options(&options); return NULL; } else if (101 == type_marker) { raw_bson_document_bytes_obj = PyObject_GetAttrString(dict, "raw"); if (NULL == raw_bson_document_bytes_obj) { destroy_codec_options(&options); buffer_free(buffer); return NULL; } #if PY_MAJOR_VERSION >= 3 if (-1 == PyBytes_AsStringAndSize(raw_bson_document_bytes_obj, &raw_bson_document_bytes, &raw_bson_document_bytes_len)) { #else if (-1 == PyString_AsStringAndSize(raw_bson_document_bytes_obj, &raw_bson_document_bytes, &raw_bson_document_bytes_len)) { #endif Py_DECREF(raw_bson_document_bytes_obj); destroy_codec_options(&options); buffer_free(buffer); return NULL; } raw_bson_document_bytes_len_int = _downcast_and_check(raw_bson_document_bytes_len, 0); if (raw_bson_document_bytes_len_int < 0 || !buffer_write_bytes(buffer, raw_bson_document_bytes, raw_bson_document_bytes_len_int)) { destroy_codec_options(&options); buffer_free(buffer); Py_DECREF(raw_bson_document_bytes_obj); return NULL; } Py_DECREF(raw_bson_document_bytes_obj); } else if (!write_dict(self, buffer, dict, check_keys, &options, top_level)) { destroy_codec_options(&options); buffer_free(buffer); return NULL; } /* objectify buffer */ #if PY_MAJOR_VERSION >= 3 result = Py_BuildValue("y#", buffer_get_buffer(buffer), buffer_get_position(buffer)); #else result = Py_BuildValue("s#", buffer_get_buffer(buffer), buffer_get_position(buffer)); #endif destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* get_value(PyObject* self, const char* buffer, unsigned* position, unsigned char type, unsigned max, const codec_options_t* options) { struct module_state *state = GETSTATE(self); PyObject* value = NULL; switch (type) { case 1: { double d; if (max < 8) { goto invalid; } memcpy(&d, buffer + *position, 8); value = PyFloat_FromDouble(d); *position += 8; break; } case 2: case 14: { unsigned value_length; if (max < 4) { goto invalid; } memcpy(&value_length, buffer + *position, 4); /* Encoded string length + string */ if (!value_length || max < value_length || max < 4 + value_length) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + value_length - 1]) { goto invalid; } value = PyUnicode_DecodeUTF8( buffer + *position, value_length - 1, options->unicode_decode_error_handler); if (!value) { goto invalid; } *position += value_length; break; } case 3: { PyObject* collection; unsigned size; if (max < 4) { goto invalid; } memcpy(&size, buffer + *position, 4); if (size < BSON_MIN_SIZE || max < size) { goto invalid; } /* Check for bad eoo */ if (buffer[*position + size - 1]) { goto invalid; } if (options->is_raw_bson) { value = PyObject_CallFunction( options->document_class, BYTES_FORMAT_STRING "O", buffer + *position, size, options->options_obj); if (!value) { goto invalid; } *position += size; break; } value = elements_to_dict(self, buffer + *position + 4, size - 5, options); if (!value) { goto invalid; } /* Decoding for DBRefs */ if (PyMapping_HasKeyString(value, "$ref")) { /* DBRef */ PyObject* dbref = NULL; PyObject* dbref_type; PyObject* id; PyObject* database; collection = PyMapping_GetItemString(value, "$ref"); /* PyMapping_GetItemString returns NULL to indicate error. */ if (!collection) { goto invalid; } PyMapping_DelItemString(value, "$ref"); if (PyMapping_HasKeyString(value, "$id")) { id = PyMapping_GetItemString(value, "$id"); if (!id) { Py_DECREF(collection); goto invalid; } PyMapping_DelItemString(value, "$id"); } else { id = Py_None; Py_INCREF(id); } if (PyMapping_HasKeyString(value, "$db")) { database = PyMapping_GetItemString(value, "$db"); if (!database) { Py_DECREF(collection); Py_DECREF(id); goto invalid; } PyMapping_DelItemString(value, "$db"); } else { database = Py_None; Py_INCREF(database); } if ((dbref_type = _get_object(state->DBRef, "bson.dbref", "DBRef"))) { dbref = PyObject_CallFunctionObjArgs(dbref_type, collection, id, database, value, NULL); Py_DECREF(dbref_type); } Py_DECREF(value); value = dbref; Py_DECREF(id); Py_DECREF(collection); Py_DECREF(database); } *position += size; break; } case 4: { unsigned size, end; if (max < 4) { goto invalid; } memcpy(&size, buffer + *position, 4); if (size < BSON_MIN_SIZE || max < size) { goto invalid; } end = *position + size - 1; /* Check for bad eoo */ if (buffer[end]) { goto invalid; } *position += 4; value = PyList_New(0); if (!value) { goto invalid; } while (*position < end) { PyObject* to_append; unsigned char bson_type = (unsigned char)buffer[(*position)++]; size_t key_size = strlen(buffer + *position); if (max < key_size) { Py_DECREF(value); goto invalid; } /* just skip the key, they're in order. */ *position += (unsigned)key_size + 1; if (Py_EnterRecursiveCall(" while decoding a list value")) { Py_DECREF(value); goto invalid; } to_append = get_value(self, buffer, position, bson_type, max - (unsigned)key_size, options); Py_LeaveRecursiveCall(); if (!to_append) { Py_DECREF(value); goto invalid; } if (PyList_Append(value, to_append) < 0) { Py_DECREF(value); Py_DECREF(to_append); goto invalid; } Py_DECREF(to_append); } (*position)++; break; } case 5: { PyObject* data; PyObject* st; PyObject* type_to_create; unsigned length; unsigned char subtype; if (max < 5) { goto invalid; } memcpy(&length, buffer + *position, 4); if (max < length) { goto invalid; } subtype = (unsigned char)buffer[*position + 4]; *position += 5; if (subtype == 2 && length < 4) { goto invalid; } #if PY_MAJOR_VERSION >= 3 /* Python3 special case. Decode BSON binary subtype 0 to bytes. */ if (subtype == 0) { value = PyBytes_FromStringAndSize(buffer + *position, length); *position += length; break; } if (subtype == 2) { data = PyBytes_FromStringAndSize(buffer + *position + 4, length - 4); } else { data = PyBytes_FromStringAndSize(buffer + *position, length); } #else if (subtype == 2) { data = PyString_FromStringAndSize(buffer + *position + 4, length - 4); } else { data = PyString_FromStringAndSize(buffer + *position, length); } #endif if (!data) { goto invalid; } /* Encode as UUID, not Binary */ if (subtype == 3 || subtype == 4) { PyObject* kwargs; PyObject* args = PyTuple_New(0); /* UUID should always be 16 bytes */ if (!args || length != 16) { Py_DECREF(data); goto invalid; } kwargs = PyDict_New(); if (!kwargs) { Py_DECREF(data); Py_DECREF(args); goto invalid; } /* * From this point, we hold refs to args, kwargs, and data. * If anything fails, goto uuiderror to clean them up. */ if (options->uuid_rep == CSHARP_LEGACY) { /* Legacy C# byte order */ if ((PyDict_SetItemString(kwargs, "bytes_le", data)) == -1) goto uuiderror; } else { if (options->uuid_rep == JAVA_LEGACY) { /* Convert from legacy java byte order */ char big_endian[16]; _fix_java(buffer + *position, big_endian); /* Free the previously created PyString object */ Py_DECREF(data); #if PY_MAJOR_VERSION >= 3 data = PyBytes_FromStringAndSize(big_endian, length); #else data = PyString_FromStringAndSize(big_endian, length); #endif if (data == NULL) goto uuiderror; } if ((PyDict_SetItemString(kwargs, "bytes", data)) == -1) goto uuiderror; } if ((type_to_create = _get_object(state->UUID, "uuid", "UUID"))) { value = PyObject_Call(type_to_create, args, kwargs); Py_DECREF(type_to_create); } Py_DECREF(args); Py_DECREF(kwargs); Py_DECREF(data); if (!value) { goto invalid; } *position += length; break; uuiderror: Py_DECREF(args); Py_DECREF(kwargs); Py_XDECREF(data); goto invalid; } #if PY_MAJOR_VERSION >= 3 st = PyLong_FromLong(subtype); #else st = PyInt_FromLong(subtype); #endif if (!st) { Py_DECREF(data); goto invalid; } if ((type_to_create = _get_object(state->Binary, "bson.binary", "Binary"))) { value = PyObject_CallFunctionObjArgs(type_to_create, data, st, NULL); Py_DECREF(type_to_create); } Py_DECREF(st); Py_DECREF(data); if (!value) { goto invalid; } *position += length; break; } case 6: case 10: { value = Py_None; Py_INCREF(value); break; } case 7: { PyObject* objectid_type; if (max < 12) { goto invalid; } if ((objectid_type = _get_object(state->ObjectId, "bson.objectid", "ObjectId"))) { #if PY_MAJOR_VERSION >= 3 value = PyObject_CallFunction(objectid_type, "y#", buffer + *position, 12); #else value = PyObject_CallFunction(objectid_type, "s#", buffer + *position, 12); #endif Py_DECREF(objectid_type); } *position += 12; break; } case 8: { value = buffer[(*position)++] ? Py_True : Py_False; Py_INCREF(value); break; } case 9: { PyObject* utc_type; PyObject* naive; PyObject* replace; PyObject* args; PyObject* kwargs; PyObject* astimezone; long long millis; if (max < 8) { goto invalid; } memcpy(&millis, buffer + *position, 8); naive = datetime_from_millis(millis); *position += 8; if (!options->tz_aware) { /* In the naive case, we're done here. */ value = naive; break; } if (!naive) { goto invalid; } replace = PyObject_GetAttrString(naive, "replace"); Py_DECREF(naive); if (!replace) { goto invalid; } args = PyTuple_New(0); if (!args) { Py_DECREF(replace); goto invalid; } kwargs = PyDict_New(); if (!kwargs) { Py_DECREF(replace); Py_DECREF(args); goto invalid; } utc_type = _get_object(state->UTC, "bson.tz_util", "utc"); if (!utc_type || PyDict_SetItemString(kwargs, "tzinfo", utc_type) == -1) { Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); Py_XDECREF(utc_type); goto invalid; } Py_XDECREF(utc_type); value = PyObject_Call(replace, args, kwargs); if (!value) { Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); goto invalid; } /* convert to local time */ if (options->tzinfo != Py_None) { astimezone = PyObject_GetAttrString(value, "astimezone"); if (!astimezone) { Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); goto invalid; } value = PyObject_CallFunctionObjArgs(astimezone, options->tzinfo, NULL); Py_DECREF(astimezone); } Py_DECREF(replace); Py_DECREF(args); Py_DECREF(kwargs); break; } case 11: { PyObject* regex_class; PyObject* pattern; int flags; size_t flags_length, i; size_t pattern_length = strlen(buffer + *position); if (pattern_length > BSON_MAX_SIZE || max < pattern_length) { goto invalid; } pattern = PyUnicode_DecodeUTF8( buffer + *position, pattern_length, options->unicode_decode_error_handler); if (!pattern) { goto invalid; } *position += (unsigned)pattern_length + 1; flags_length = strlen(buffer + *position); if (flags_length > BSON_MAX_SIZE || (BSON_MAX_SIZE - pattern_length) < flags_length) { Py_DECREF(pattern); goto invalid; } if (max < pattern_length + flags_length) { Py_DECREF(pattern); goto invalid; } flags = 0; for (i = 0; i < flags_length; i++) { if (buffer[*position + i] == 'i') { flags |= 2; } else if (buffer[*position + i] == 'l') { flags |= 4; } else if (buffer[*position + i] == 'm') { flags |= 8; } else if (buffer[*position + i] == 's') { flags |= 16; } else if (buffer[*position + i] == 'u') { flags |= 32; } else if (buffer[*position + i] == 'x') { flags |= 64; } } *position += (unsigned)flags_length + 1; regex_class = _get_object(state->Regex, "bson.regex", "Regex"); if (regex_class) { value = PyObject_CallFunction(regex_class, "Oi", pattern, flags); Py_DECREF(regex_class); } Py_DECREF(pattern); break; } case 12: { unsigned coll_length; PyObject* collection; PyObject* id = NULL; PyObject* objectid_type; PyObject* dbref_type; if (max < 4) { goto invalid; } memcpy(&coll_length, buffer + *position, 4); /* Encoded string length + string + 12 byte ObjectId */ if (!coll_length || max < coll_length || max < 4 + coll_length + 12) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + coll_length - 1]) { goto invalid; } collection = PyUnicode_DecodeUTF8( buffer + *position, coll_length - 1, options->unicode_decode_error_handler); if (!collection) { goto invalid; } *position += coll_length; if ((objectid_type = _get_object(state->ObjectId, "bson.objectid", "ObjectId"))) { #if PY_MAJOR_VERSION >= 3 id = PyObject_CallFunction(objectid_type, "y#", buffer + *position, 12); #else id = PyObject_CallFunction(objectid_type, "s#", buffer + *position, 12); #endif Py_DECREF(objectid_type); } if (!id) { Py_DECREF(collection); goto invalid; } *position += 12; if ((dbref_type = _get_object(state->DBRef, "bson.dbref", "DBRef"))) { value = PyObject_CallFunctionObjArgs(dbref_type, collection, id, NULL); Py_DECREF(dbref_type); } Py_DECREF(collection); Py_DECREF(id); break; } case 13: { PyObject* code; PyObject* code_type; unsigned value_length; if (max < 4) { goto invalid; } memcpy(&value_length, buffer + *position, 4); /* Encoded string length + string */ if (!value_length || max < value_length || max < 4 + value_length) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + value_length - 1]) { goto invalid; } code = PyUnicode_DecodeUTF8( buffer + *position, value_length - 1, options->unicode_decode_error_handler); if (!code) { goto invalid; } *position += value_length; if ((code_type = _get_object(state->Code, "bson.code", "Code"))) { value = PyObject_CallFunctionObjArgs(code_type, code, NULL, NULL); Py_DECREF(code_type); } Py_DECREF(code); break; } case 15: { unsigned c_w_s_size; unsigned code_size; unsigned scope_size; PyObject* code; PyObject* scope; PyObject* code_type; if (max < 8) { goto invalid; } memcpy(&c_w_s_size, buffer + *position, 4); *position += 4; if (max < c_w_s_size) { goto invalid; } memcpy(&code_size, buffer + *position, 4); /* code_w_scope length + code length + code + scope length */ if (!code_size || max < code_size || max < 4 + 4 + code_size + 4) { goto invalid; } *position += 4; /* Strings must end in \0 */ if (buffer[*position + code_size - 1]) { goto invalid; } code = PyUnicode_DecodeUTF8( buffer + *position, code_size - 1, options->unicode_decode_error_handler); if (!code) { goto invalid; } *position += code_size; memcpy(&scope_size, buffer + *position, 4); if (scope_size < BSON_MIN_SIZE) { Py_DECREF(code); goto invalid; } /* code length + code + scope length + scope */ if ((4 + code_size + 4 + scope_size) != c_w_s_size) { Py_DECREF(code); goto invalid; } /* Check for bad eoo */ if (buffer[*position + scope_size - 1]) { goto invalid; } scope = elements_to_dict(self, buffer + *position + 4, scope_size - 5, options); if (!scope) { Py_DECREF(code); goto invalid; } *position += scope_size; if ((code_type = _get_object(state->Code, "bson.code", "Code"))) { value = PyObject_CallFunctionObjArgs(code_type, code, scope, NULL); Py_DECREF(code_type); } Py_DECREF(code); Py_DECREF(scope); break; } case 16: { int i; if (max < 4) { goto invalid; } memcpy(&i, buffer + *position, 4); #if PY_MAJOR_VERSION >= 3 value = PyLong_FromLong(i); #else value = PyInt_FromLong(i); #endif if (!value) { goto invalid; } *position += 4; break; } case 17: { unsigned int time, inc; PyObject* timestamp_type; if (max < 8) { goto invalid; } memcpy(&inc, buffer + *position, 4); memcpy(&time, buffer + *position + 4, 4); if ((timestamp_type = _get_object(state->Timestamp, "bson.timestamp", "Timestamp"))) { value = PyObject_CallFunction(timestamp_type, "II", time, inc); Py_DECREF(timestamp_type); } *position += 8; break; } case 18: { long long ll; PyObject* bson_int64_type = _get_object(state->BSONInt64, "bson.int64", "Int64"); if (!bson_int64_type) goto invalid; if (max < 8) { Py_DECREF(bson_int64_type); goto invalid; } memcpy(&ll, buffer + *position, 8); value = PyObject_CallFunction(bson_int64_type, "L", ll); *position += 8; Py_DECREF(bson_int64_type); break; } case 255: { PyObject* minkey_type = _get_object(state->MinKey, "bson.min_key", "MinKey"); if (!minkey_type) goto invalid; value = PyObject_CallFunctionObjArgs(minkey_type, NULL); Py_DECREF(minkey_type); break; } case 127: { PyObject* maxkey_type = _get_object(state->MaxKey, "bson.max_key", "MaxKey"); if (!maxkey_type) goto invalid; value = PyObject_CallFunctionObjArgs(maxkey_type, NULL); Py_DECREF(maxkey_type); break; } default: { PyObject* InvalidDocument = _error("InvalidDocument"); if (InvalidDocument) { PyErr_SetString(InvalidDocument, "no c decoder for this type yet"); Py_DECREF(InvalidDocument); } goto invalid; } } if (value) { return value; } invalid: /* * Wrap any non-InvalidBSON errors in InvalidBSON. */ if (PyErr_Occurred()) { PyObject *etype, *evalue, *etrace; PyObject *InvalidBSON; /* * Calling _error clears the error state, so fetch it first. */ PyErr_Fetch(&etype, &evalue, &etrace); /* Dont reraise anything but PyExc_Exceptions as InvalidBSON. */ if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { if (!PyErr_GivenExceptionMatches(etype, InvalidBSON)) { /* * Raise InvalidBSON(str(e)). */ Py_DECREF(etype); etype = InvalidBSON; if (evalue) { PyObject *msg = PyObject_Str(evalue); Py_DECREF(evalue); evalue = msg; } PyErr_NormalizeException(&etype, &evalue, &etrace); } else { /* * The current exception matches InvalidBSON, so we don't * need this reference after all. */ Py_DECREF(InvalidBSON); } } } /* Steals references to args. */ PyErr_Restore(etype, evalue, etrace); } else { PyObject *InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid length or type code"); Py_DECREF(InvalidBSON); } } return NULL; } /* * Get the next 'name' and 'value' from a document in a string, whose position * is provided. * * Returns the position of the next element in the document, or -1 on error. */ static int _element_to_dict(PyObject* self, const char* string, unsigned position, unsigned max, const codec_options_t* options, PyObject** name, PyObject** value) { unsigned char type = (unsigned char)string[position++]; size_t name_length = strlen(string + position); if (name_length > BSON_MAX_SIZE || position + name_length >= max) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetNone(InvalidBSON); Py_DECREF(InvalidBSON); } return -1; } *name = PyUnicode_DecodeUTF8( string + position, name_length, options->unicode_decode_error_handler); if (!*name) { /* If NULL is returned then wrap the UnicodeDecodeError in an InvalidBSON error */ PyObject *etype, *evalue, *etrace; PyObject *InvalidBSON; PyErr_Fetch(&etype, &evalue, &etrace); if (PyErr_GivenExceptionMatches(etype, PyExc_Exception)) { InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { Py_DECREF(etype); etype = InvalidBSON; if (evalue) { PyObject *msg = PyObject_Str(evalue); Py_DECREF(evalue); evalue = msg; } PyErr_NormalizeException(&etype, &evalue, &etrace); } } PyErr_Restore(etype, evalue, etrace); return -1; } position += (unsigned)name_length + 1; *value = get_value(self, string, &position, type, max - position, options); if (!*value) { Py_DECREF(name); return -1; } return position; } static PyObject* _cbson_element_to_dict(PyObject* self, PyObject* args) { char* string; PyObject* bson; codec_options_t options; unsigned position; unsigned max; int new_position; PyObject* name; PyObject* value; PyObject* result_tuple; if (!PyArg_ParseTuple(args, "OII|O&", &bson, &position, &max, convert_codec_options, &options)) { return NULL; } if (PyTuple_GET_SIZE(args) < 4) { default_codec_options(GETSTATE(self), &options); } #if PY_MAJOR_VERSION >= 3 if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _element_to_dict must be a bytes object"); #else if (!PyString_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _element_to_dict must be a string"); #endif return NULL; } #if PY_MAJOR_VERSION >= 3 string = PyBytes_AsString(bson); #else string = PyString_AsString(bson); #endif new_position = _element_to_dict(self, string, position, max, &options, &name, &value); if (new_position < 0) { return NULL; } result_tuple = Py_BuildValue("OOi", name, value, new_position); if (!result_tuple) { Py_DECREF(name); Py_DECREF(value); return NULL; } return result_tuple; } static PyObject* _elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options) { int position = 0; PyObject* dict = PyObject_CallObject(options->document_class, NULL); if (!dict) { return NULL; } while (position < max) { PyObject* name; PyObject* value; position = _element_to_dict(self, string, position, max, options, &name, &value); if (position < 0) { Py_DECREF(dict); return NULL; } PyObject_SetItem(dict, name, value); Py_DECREF(name); Py_DECREF(value); } return dict; } static PyObject* elements_to_dict(PyObject* self, const char* string, unsigned max, const codec_options_t* options) { PyObject* result; if (Py_EnterRecursiveCall(" while decoding a BSON document")) return NULL; result = _elements_to_dict(self, string, max, options); Py_LeaveRecursiveCall(); return result; } static PyObject* _cbson_bson_to_dict(PyObject* self, PyObject* args) { int size; Py_ssize_t total_size; const char* string; PyObject* bson; codec_options_t options; PyObject* result; PyObject* options_obj; if (! (PyArg_ParseTuple(args, "OO", &bson, &options_obj) && convert_codec_options(options_obj, &options))) { return NULL; } #if PY_MAJOR_VERSION >= 3 if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _bson_to_dict must be a bytes object"); #else if (!PyString_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _bson_to_dict must be a string"); #endif destroy_codec_options(&options); return NULL; } #if PY_MAJOR_VERSION >= 3 total_size = PyBytes_Size(bson); #else total_size = PyString_Size(bson); #endif if (total_size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "not enough data for a BSON document"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } #if PY_MAJOR_VERSION >= 3 string = PyBytes_AsString(bson); #else string = PyString_AsString(bson); #endif if (!string) { destroy_codec_options(&options); return NULL; } memcpy(&size, string, 4); if (size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid message size"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } if (total_size < size || total_size > BSON_MAX_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "objsize too large"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } if (size != total_size || string[size - 1]) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "bad eoo"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); return NULL; } /* No need to decode fields if using RawBSONDocument */ if (options.is_raw_bson) { return PyObject_CallFunction( options.document_class, BYTES_FORMAT_STRING "O", string, size, options_obj); } result = elements_to_dict(self, string + 4, (unsigned)size - 5, &options); destroy_codec_options(&options); return result; } static PyObject* _cbson_decode_all(PyObject* self, PyObject* args) { int size; Py_ssize_t total_size; const char* string; PyObject* bson; PyObject* dict; PyObject* result; codec_options_t options; PyObject* options_obj; if (!PyArg_ParseTuple(args, "O|O", &bson, &options_obj)) { return NULL; } if (PyTuple_GET_SIZE(args) < 2) { default_codec_options(GETSTATE(self), &options); } else if (!convert_codec_options(options_obj, &options)) { return NULL; } #if PY_MAJOR_VERSION >= 3 if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to decode_all must be a bytes object"); #else if (!PyString_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to decode_all must be a string"); #endif return NULL; } #if PY_MAJOR_VERSION >= 3 total_size = PyBytes_Size(bson); string = PyBytes_AsString(bson); #else total_size = PyString_Size(bson); string = PyString_AsString(bson); #endif if (!string) { return NULL; } if (!(result = PyList_New(0))) { destroy_codec_options(&options); return NULL; } while (total_size > 0) { if (total_size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "not enough data for a BSON document"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } memcpy(&size, string, 4); if (size < BSON_MIN_SIZE) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid message size"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } if (total_size < size) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "objsize too large"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } if (string[size - 1]) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "bad eoo"); Py_DECREF(InvalidBSON); } destroy_codec_options(&options); Py_DECREF(result); return NULL; } /* No need to decode fields if using RawBSONDocument. */ if (options.is_raw_bson) { dict = PyObject_CallFunction( options.document_class, BYTES_FORMAT_STRING "O", string, size, options_obj); } else { dict = elements_to_dict(self, string + 4, (unsigned)size - 5, &options); } if (!dict) { Py_DECREF(result); destroy_codec_options(&options); return NULL; } if (PyList_Append(result, dict) < 0) { Py_DECREF(dict); Py_DECREF(result); destroy_codec_options(&options); return NULL; } Py_DECREF(dict); string += size; total_size -= size; } destroy_codec_options(&options); return result; } static PyMethodDef _CBSONMethods[] = { {"_dict_to_bson", _cbson_dict_to_bson, METH_VARARGS, "convert a dictionary to a string containing its BSON representation."}, {"_bson_to_dict", _cbson_bson_to_dict, METH_VARARGS, "convert a BSON string to a SON object."}, {"decode_all", _cbson_decode_all, METH_VARARGS, "convert binary data to a sequence of documents."}, {"_element_to_dict", _cbson_element_to_dict, METH_VARARGS, "Decode a single key, value pair."}, {NULL, NULL, 0, NULL} }; #if PY_MAJOR_VERSION >= 3 #define INITERROR return NULL static int _cbson_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->Binary); Py_VISIT(GETSTATE(m)->Code); Py_VISIT(GETSTATE(m)->ObjectId); Py_VISIT(GETSTATE(m)->DBRef); Py_VISIT(GETSTATE(m)->Regex); Py_VISIT(GETSTATE(m)->UUID); Py_VISIT(GETSTATE(m)->Timestamp); Py_VISIT(GETSTATE(m)->MinKey); Py_VISIT(GETSTATE(m)->MaxKey); Py_VISIT(GETSTATE(m)->UTC); Py_VISIT(GETSTATE(m)->REType); return 0; } static int _cbson_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->Binary); Py_CLEAR(GETSTATE(m)->Code); Py_CLEAR(GETSTATE(m)->ObjectId); Py_CLEAR(GETSTATE(m)->DBRef); Py_CLEAR(GETSTATE(m)->Regex); Py_CLEAR(GETSTATE(m)->UUID); Py_CLEAR(GETSTATE(m)->Timestamp); Py_CLEAR(GETSTATE(m)->MinKey); Py_CLEAR(GETSTATE(m)->MaxKey); Py_CLEAR(GETSTATE(m)->UTC); Py_CLEAR(GETSTATE(m)->REType); return 0; } static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_cbson", NULL, sizeof(struct module_state), _CBSONMethods, NULL, _cbson_traverse, _cbson_clear, NULL }; PyMODINIT_FUNC PyInit__cbson(void) #else #define INITERROR return PyMODINIT_FUNC init_cbson(void) #endif { PyObject *m; PyObject *c_api_object; static void *_cbson_API[_cbson_API_POINTER_COUNT]; PyDateTime_IMPORT; if (PyDateTimeAPI == NULL) { INITERROR; } /* Export C API */ _cbson_API[_cbson_buffer_write_bytes_INDEX] = (void *) buffer_write_bytes; _cbson_API[_cbson_write_dict_INDEX] = (void *) write_dict; _cbson_API[_cbson_write_pair_INDEX] = (void *) write_pair; _cbson_API[_cbson_decode_and_write_pair_INDEX] = (void *) decode_and_write_pair; _cbson_API[_cbson_convert_codec_options_INDEX] = (void *) convert_codec_options; _cbson_API[_cbson_destroy_codec_options_INDEX] = (void *) destroy_codec_options; #if PY_VERSION_HEX >= 0x03010000 /* PyCapsule is new in python 3.1 */ c_api_object = PyCapsule_New((void *) _cbson_API, "_cbson._C_API", NULL); #else c_api_object = PyCObject_FromVoidPtr((void *) _cbson_API, NULL); #endif if (c_api_object == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&moduledef); #else m = Py_InitModule("_cbson", _CBSONMethods); #endif if (m == NULL) { Py_DECREF(c_api_object); INITERROR; } /* Import several python objects */ if (_load_python_objects(m)) { Py_DECREF(c_api_object); #if PY_MAJOR_VERSION >= 3 Py_DECREF(m); #endif INITERROR; } if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { Py_DECREF(c_api_object); #if PY_MAJOR_VERSION >= 3 Py_DECREF(m); #endif INITERROR; } #if PY_MAJOR_VERSION >= 3 return m; #endif } pymongo-3.2/bson/buffer.h0000644000175000017500000000401412560753776017360 0ustar behackettbehackett00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef BUFFER_H #define BUFFER_H /* Note: if any of these functions return a failure condition then the buffer * has already been freed. */ /* A buffer */ typedef struct buffer* buffer_t; /* A position in the buffer */ typedef int buffer_position; /* Allocate and return a new buffer. * Return NULL on allocation failure. */ buffer_t buffer_new(void); /* Free the memory allocated for `buffer`. * Return non-zero on failure. */ int buffer_free(buffer_t buffer); /* Save `size` bytes from the current position in `buffer` (and grow if needed). * Return offset for writing, or -1 on allocation failure. */ buffer_position buffer_save_space(buffer_t buffer, int size); /* Write `size` bytes from `data` to `buffer` (and grow if needed). * Return non-zero on allocation failure. */ int buffer_write(buffer_t buffer, const char* data, int size); /* Write `size` bytes from `data` to `buffer` at position `position`. * Does not change the internal position of `buffer`. * Return non-zero if buffer isn't large enough for write. */ int buffer_write_at_position(buffer_t buffer, buffer_position position, const char* data, int size); /* Getters for the internals of a buffer_t. * Should try to avoid using these as much as possible * since they break the abstraction. */ buffer_position buffer_get_position(buffer_t buffer); char* buffer_get_buffer(buffer_t buffer); void buffer_update_position(buffer_t buffer, buffer_position new_position); #endif pymongo-3.2/bson/time64.c0000644000175000017500000005144112622440614017177 0ustar behackettbehackett00000000000000/* Copyright (c) 2007-2010 Michael G Schwern This software originally derived from Paul Sheer's pivotal_gmtime_r.c. The MIT License: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* Programmers who have available to them 64-bit time values as a 'long long' type can use localtime64_r() and gmtime64_r() which correctly converts the time even on 32-bit systems. Whether you have 64-bit time values will depend on the operating system. localtime64_r() is a 64-bit equivalent of localtime_r(). gmtime64_r() is a 64-bit equivalent of gmtime_r(). */ #ifdef _MSC_VER #define _CRT_SECURE_NO_WARNINGS #endif #include #include #include #include #include #include #include "time64.h" #include "time64_limits.h" /* Spec says except for stftime() and the _r() functions, these all return static memory. Stabbings! */ static struct TM Static_Return_Date; static const int days_in_month[2][12] = { {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, }; static const int julian_days_by_month[2][12] = { {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}, {0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335}, }; static const int length_of_year[2] = { 365, 366 }; /* Some numbers relating to the gregorian cycle */ static const Year years_in_gregorian_cycle = 400; #define days_in_gregorian_cycle ((365 * 400) + 100 - 4 + 1) static const Time64_T seconds_in_gregorian_cycle = days_in_gregorian_cycle * 60LL * 60LL * 24LL; /* Year range we can trust the time funcitons with */ #define MAX_SAFE_YEAR 2037 #define MIN_SAFE_YEAR 1971 /* 28 year Julian calendar cycle */ #define SOLAR_CYCLE_LENGTH 28 /* Year cycle from MAX_SAFE_YEAR down. */ static const int safe_years_high[SOLAR_CYCLE_LENGTH] = { 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, 2010, 2011, 2012, 2013, 2014, 2015 }; /* Year cycle from MIN_SAFE_YEAR up */ static const int safe_years_low[SOLAR_CYCLE_LENGTH] = { 1996, 1997, 1998, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, }; /* Let's assume people are going to be looking for dates in the future. Let's provide some cheats so you can skip ahead. This has a 4x speed boost when near 2008. */ /* Number of days since epoch on Jan 1st, 2008 GMT */ #define CHEAT_DAYS (1199145600 / 24 / 60 / 60) #define CHEAT_YEARS 108 #define IS_LEAP(n) ((!(((n) + 1900) % 400) || (!(((n) + 1900) % 4) && (((n) + 1900) % 100))) != 0) #define WRAP(a,b,m) ((a) = ((a) < 0 ) ? ((b)--, (a) + (m)) : (a)) #ifdef USE_SYSTEM_LOCALTIME # define SHOULD_USE_SYSTEM_LOCALTIME(a) ( \ (a) <= SYSTEM_LOCALTIME_MAX && \ (a) >= SYSTEM_LOCALTIME_MIN \ ) #else # define SHOULD_USE_SYSTEM_LOCALTIME(a) (0) #endif #ifdef USE_SYSTEM_GMTIME # define SHOULD_USE_SYSTEM_GMTIME(a) ( \ (a) <= SYSTEM_GMTIME_MAX && \ (a) >= SYSTEM_GMTIME_MIN \ ) #else # define SHOULD_USE_SYSTEM_GMTIME(a) (0) #endif /* Multi varadic macros are a C99 thing, alas */ #ifdef TIME_64_DEBUG # define TIME64_TRACE(format) (fprintf(stderr, format)) # define TIME64_TRACE1(format, var1) (fprintf(stderr, format, var1)) # define TIME64_TRACE2(format, var1, var2) (fprintf(stderr, format, var1, var2)) # define TIME64_TRACE3(format, var1, var2, var3) (fprintf(stderr, format, var1, var2, var3)) #else # define TIME64_TRACE(format) ((void)0) # define TIME64_TRACE1(format, var1) ((void)0) # define TIME64_TRACE2(format, var1, var2) ((void)0) # define TIME64_TRACE3(format, var1, var2, var3) ((void)0) #endif static int is_exception_century(Year year) { int is_exception = ((year % 100 == 0) && !(year % 400 == 0)); TIME64_TRACE1("# is_exception_century: %s\n", is_exception ? "yes" : "no"); return(is_exception); } /* Compare two dates. The result is like cmp. Ignores things like gmtoffset and dst */ int cmp_date( const struct TM* left, const struct tm* right ) { if( left->tm_year > right->tm_year ) return 1; else if( left->tm_year < right->tm_year ) return -1; if( left->tm_mon > right->tm_mon ) return 1; else if( left->tm_mon < right->tm_mon ) return -1; if( left->tm_mday > right->tm_mday ) return 1; else if( left->tm_mday < right->tm_mday ) return -1; if( left->tm_hour > right->tm_hour ) return 1; else if( left->tm_hour < right->tm_hour ) return -1; if( left->tm_min > right->tm_min ) return 1; else if( left->tm_min < right->tm_min ) return -1; if( left->tm_sec > right->tm_sec ) return 1; else if( left->tm_sec < right->tm_sec ) return -1; return 0; } /* Check if a date is safely inside a range. The intention is to check if its a few days inside. */ int date_in_safe_range( const struct TM* date, const struct tm* min, const struct tm* max ) { if( cmp_date(date, min) == -1 ) return 0; if( cmp_date(date, max) == 1 ) return 0; return 1; } /* timegm() is not in the C or POSIX spec, but it is such a useful extension I would be remiss in leaving it out. Also I need it for localtime64() */ Time64_T timegm64(const struct TM *date) { Time64_T days = 0; Time64_T seconds = 0; Year year; Year orig_year = (Year)date->tm_year; int cycles = 0; if( orig_year > 100 ) { cycles = (int)((orig_year - 100) / 400); orig_year -= cycles * 400; days += (Time64_T)cycles * days_in_gregorian_cycle; } else if( orig_year < -300 ) { cycles = (int)((orig_year - 100) / 400); orig_year -= cycles * 400; days += (Time64_T)cycles * days_in_gregorian_cycle; } TIME64_TRACE3("# timegm/ cycles: %d, days: %lld, orig_year: %lld\n", cycles, days, orig_year); if( orig_year > 70 ) { year = 70; while( year < orig_year ) { days += length_of_year[IS_LEAP(year)]; year++; } } else if ( orig_year < 70 ) { year = 69; do { days -= length_of_year[IS_LEAP(year)]; year--; } while( year >= orig_year ); } days += julian_days_by_month[IS_LEAP(orig_year)][date->tm_mon]; days += date->tm_mday - 1; seconds = days * 60 * 60 * 24; seconds += date->tm_hour * 60 * 60; seconds += date->tm_min * 60; seconds += date->tm_sec; return(seconds); } #ifndef NDEBUG static int check_tm(struct TM *tm) { /* Don't forget leap seconds */ assert(tm->tm_sec >= 0); assert(tm->tm_sec <= 61); assert(tm->tm_min >= 0); assert(tm->tm_min <= 59); assert(tm->tm_hour >= 0); assert(tm->tm_hour <= 23); assert(tm->tm_mday >= 1); assert(tm->tm_mday <= days_in_month[IS_LEAP(tm->tm_year)][tm->tm_mon]); assert(tm->tm_mon >= 0); assert(tm->tm_mon <= 11); assert(tm->tm_wday >= 0); assert(tm->tm_wday <= 6); assert(tm->tm_yday >= 0); assert(tm->tm_yday <= length_of_year[IS_LEAP(tm->tm_year)]); #ifdef HAS_TM_TM_GMTOFF assert(tm->tm_gmtoff >= -24 * 60 * 60); assert(tm->tm_gmtoff <= 24 * 60 * 60); #endif return 1; } #endif /* The exceptional centuries without leap years cause the cycle to shift by 16 */ static Year cycle_offset(Year year) { const Year start_year = 2000; Year year_diff = year - start_year; Year exceptions; if( year > start_year ) year_diff--; exceptions = year_diff / 100; exceptions -= year_diff / 400; TIME64_TRACE3("# year: %lld, exceptions: %lld, year_diff: %lld\n", year, exceptions, year_diff); return exceptions * 16; } /* For a given year after 2038, pick the latest possible matching year in the 28 year calendar cycle. A matching year... 1) Starts on the same day of the week. 2) Has the same leap year status. This is so the calendars match up. Also the previous year must match. When doing Jan 1st you might wind up on Dec 31st the previous year when doing a -UTC time zone. Finally, the next year must have the same start day of week. This is for Dec 31st with a +UTC time zone. It doesn't need the same leap year status since we only care about January 1st. */ static int safe_year(const Year year) { int safe_year = 0; Year year_cycle; if( year >= MIN_SAFE_YEAR && year <= MAX_SAFE_YEAR ) { return (int)year; } year_cycle = year + cycle_offset(year); /* safe_years_low is off from safe_years_high by 8 years */ if( year < MIN_SAFE_YEAR ) year_cycle -= 8; /* Change non-leap xx00 years to an equivalent */ if( is_exception_century(year) ) year_cycle += 11; /* Also xx01 years, since the previous year will be wrong */ if( is_exception_century(year - 1) ) year_cycle += 17; year_cycle %= SOLAR_CYCLE_LENGTH; if( year_cycle < 0 ) year_cycle = SOLAR_CYCLE_LENGTH + year_cycle; assert( year_cycle >= 0 ); assert( year_cycle < SOLAR_CYCLE_LENGTH ); if( year < MIN_SAFE_YEAR ) safe_year = safe_years_low[year_cycle]; else if( year > MAX_SAFE_YEAR ) safe_year = safe_years_high[year_cycle]; else assert(0); TIME64_TRACE3("# year: %lld, year_cycle: %lld, safe_year: %d\n", year, year_cycle, safe_year); assert(safe_year <= MAX_SAFE_YEAR && safe_year >= MIN_SAFE_YEAR); return safe_year; } void copy_tm_to_TM64(const struct tm *src, struct TM *dest) { if( src == NULL ) { memset(dest, 0, sizeof(*dest)); } else { # ifdef USE_TM64 dest->tm_sec = src->tm_sec; dest->tm_min = src->tm_min; dest->tm_hour = src->tm_hour; dest->tm_mday = src->tm_mday; dest->tm_mon = src->tm_mon; dest->tm_year = (Year)src->tm_year; dest->tm_wday = src->tm_wday; dest->tm_yday = src->tm_yday; dest->tm_isdst = src->tm_isdst; # ifdef HAS_TM_TM_GMTOFF dest->tm_gmtoff = src->tm_gmtoff; # endif # ifdef HAS_TM_TM_ZONE dest->tm_zone = src->tm_zone; # endif # else /* They're the same type */ memcpy(dest, src, sizeof(*dest)); # endif } } void copy_TM64_to_tm(const struct TM *src, struct tm *dest) { if( src == NULL ) { memset(dest, 0, sizeof(*dest)); } else { # ifdef USE_TM64 dest->tm_sec = src->tm_sec; dest->tm_min = src->tm_min; dest->tm_hour = src->tm_hour; dest->tm_mday = src->tm_mday; dest->tm_mon = src->tm_mon; dest->tm_year = (int)src->tm_year; dest->tm_wday = src->tm_wday; dest->tm_yday = src->tm_yday; dest->tm_isdst = src->tm_isdst; # ifdef HAS_TM_TM_GMTOFF dest->tm_gmtoff = src->tm_gmtoff; # endif # ifdef HAS_TM_TM_ZONE dest->tm_zone = src->tm_zone; # endif # else /* They're the same type */ memcpy(dest, src, sizeof(*dest)); # endif } } /* Simulate localtime_r() to the best of our ability */ struct tm * fake_localtime_r(const time_t *time, struct tm *result) { const struct tm *static_result = localtime(time); assert(result != NULL); if( static_result == NULL ) { memset(result, 0, sizeof(*result)); return NULL; } else { memcpy(result, static_result, sizeof(*result)); return result; } } /* Simulate gmtime_r() to the best of our ability */ struct tm * fake_gmtime_r(const time_t *time, struct tm *result) { const struct tm *static_result = gmtime(time); assert(result != NULL); if( static_result == NULL ) { memset(result, 0, sizeof(*result)); return NULL; } else { memcpy(result, static_result, sizeof(*result)); return result; } } static Time64_T seconds_between_years(Year left_year, Year right_year) { int increment = (left_year > right_year) ? 1 : -1; Time64_T seconds = 0; int cycles; if( left_year > 2400 ) { cycles = (int)((left_year - 2400) / 400); left_year -= cycles * 400; seconds += cycles * seconds_in_gregorian_cycle; } else if( left_year < 1600 ) { cycles = (int)((left_year - 1600) / 400); left_year += cycles * 400; seconds += cycles * seconds_in_gregorian_cycle; } while( left_year != right_year ) { seconds += length_of_year[IS_LEAP(right_year - 1900)] * 60 * 60 * 24; right_year += increment; } return seconds * increment; } Time64_T mktime64(const struct TM *input_date) { struct tm safe_date; struct TM date; Time64_T time; Year year = input_date->tm_year + 1900; if( date_in_safe_range(input_date, &SYSTEM_MKTIME_MIN, &SYSTEM_MKTIME_MAX) ) { copy_TM64_to_tm(input_date, &safe_date); return (Time64_T)mktime(&safe_date); } /* Have to make the year safe in date else it won't fit in safe_date */ date = *input_date; date.tm_year = safe_year(year) - 1900; copy_TM64_to_tm(&date, &safe_date); time = (Time64_T)mktime(&safe_date); time += seconds_between_years(year, (Year)(safe_date.tm_year + 1900)); return time; } /* Because I think mktime() is a crappy name */ Time64_T timelocal64(const struct TM *date) { return mktime64(date); } struct TM *gmtime64_r (const Time64_T *in_time, struct TM *p) { int v_tm_sec, v_tm_min, v_tm_hour, v_tm_mon, v_tm_wday; Time64_T v_tm_tday; int leap; Time64_T m; Time64_T time = *in_time; Year year = 70; int cycles = 0; assert(p != NULL); #ifdef USE_SYSTEM_GMTIME /* Use the system gmtime() if time_t is small enough */ if( SHOULD_USE_SYSTEM_GMTIME(*in_time) ) { time_t safe_time = (time_t)*in_time; struct tm safe_date; GMTIME_R(&safe_time, &safe_date); copy_tm_to_TM64(&safe_date, p); assert(check_tm(p)); return p; } #endif #ifdef HAS_TM_TM_GMTOFF p->tm_gmtoff = 0; #endif p->tm_isdst = 0; #ifdef HAS_TM_TM_ZONE p->tm_zone = "UTC"; #endif v_tm_sec = (int)(time % 60); time /= 60; v_tm_min = (int)(time % 60); time /= 60; v_tm_hour = (int)(time % 24); time /= 24; v_tm_tday = time; WRAP (v_tm_sec, v_tm_min, 60); WRAP (v_tm_min, v_tm_hour, 60); WRAP (v_tm_hour, v_tm_tday, 24); v_tm_wday = (int)((v_tm_tday + 4) % 7); if (v_tm_wday < 0) v_tm_wday += 7; m = v_tm_tday; if (m >= CHEAT_DAYS) { year = CHEAT_YEARS; m -= CHEAT_DAYS; } if (m >= 0) { /* Gregorian cycles, this is huge optimization for distant times */ cycles = (int)(m / (Time64_T) days_in_gregorian_cycle); if( cycles ) { m -= (cycles * (Time64_T) days_in_gregorian_cycle); year += (cycles * years_in_gregorian_cycle); } /* Years */ leap = IS_LEAP (year); while (m >= (Time64_T) length_of_year[leap]) { m -= (Time64_T) length_of_year[leap]; year++; leap = IS_LEAP (year); } /* Months */ v_tm_mon = 0; while (m >= (Time64_T) days_in_month[leap][v_tm_mon]) { m -= (Time64_T) days_in_month[leap][v_tm_mon]; v_tm_mon++; } } else { year--; /* Gregorian cycles */ cycles = (int)((m / (Time64_T) days_in_gregorian_cycle) + 1); if( cycles ) { m -= (cycles * (Time64_T) days_in_gregorian_cycle); year += (cycles * years_in_gregorian_cycle); } /* Years */ leap = IS_LEAP (year); while (m < (Time64_T) -length_of_year[leap]) { m += (Time64_T) length_of_year[leap]; year--; leap = IS_LEAP (year); } /* Months */ v_tm_mon = 11; while (m < (Time64_T) -days_in_month[leap][v_tm_mon]) { m += (Time64_T) days_in_month[leap][v_tm_mon]; v_tm_mon--; } m += (Time64_T) days_in_month[leap][v_tm_mon]; } p->tm_year = (int)year; if( p->tm_year != year ) { #ifdef EOVERFLOW errno = EOVERFLOW; #endif return NULL; } /* At this point m is less than a year so casting to an int is safe */ p->tm_mday = (int) m + 1; p->tm_yday = julian_days_by_month[leap][v_tm_mon] + (int)m; p->tm_sec = v_tm_sec; p->tm_min = v_tm_min; p->tm_hour = v_tm_hour; p->tm_mon = v_tm_mon; p->tm_wday = v_tm_wday; assert(check_tm(p)); return p; } struct TM *localtime64_r (const Time64_T *time, struct TM *local_tm) { time_t safe_time; struct tm safe_date; struct TM gm_tm; Year orig_year; int month_diff; assert(local_tm != NULL); #ifdef USE_SYSTEM_LOCALTIME /* Use the system localtime() if time_t is small enough */ if( SHOULD_USE_SYSTEM_LOCALTIME(*time) ) { safe_time = (time_t)*time; TIME64_TRACE1("Using system localtime for %lld\n", *time); LOCALTIME_R(&safe_time, &safe_date); copy_tm_to_TM64(&safe_date, local_tm); assert(check_tm(local_tm)); return local_tm; } #endif if( gmtime64_r(time, &gm_tm) == NULL ) { TIME64_TRACE1("gmtime64_r returned null for %lld\n", *time); return NULL; } orig_year = gm_tm.tm_year; if (gm_tm.tm_year > (2037 - 1900) || gm_tm.tm_year < (1970 - 1900) ) { TIME64_TRACE1("Mapping tm_year %lld to safe_year\n", (Year)gm_tm.tm_year); gm_tm.tm_year = safe_year((Year)(gm_tm.tm_year + 1900)) - 1900; } safe_time = (time_t)timegm64(&gm_tm); if( LOCALTIME_R(&safe_time, &safe_date) == NULL ) { TIME64_TRACE1("localtime_r(%d) returned NULL\n", (int)safe_time); return NULL; } copy_tm_to_TM64(&safe_date, local_tm); local_tm->tm_year = (int)orig_year; if( local_tm->tm_year != orig_year ) { TIME64_TRACE2("tm_year overflow: tm_year %lld, orig_year %lld\n", (Year)local_tm->tm_year, (Year)orig_year); #ifdef EOVERFLOW errno = EOVERFLOW; #endif return NULL; } month_diff = local_tm->tm_mon - gm_tm.tm_mon; /* When localtime is Dec 31st previous year and gmtime is Jan 1st next year. */ if( month_diff == 11 ) { local_tm->tm_year--; } /* When localtime is Jan 1st, next year and gmtime is Dec 31st, previous year. */ if( month_diff == -11 ) { local_tm->tm_year++; } /* GMT is Jan 1st, xx01 year, but localtime is still Dec 31st in a non-leap xx00. There is one point in the cycle we can't account for which the safe xx00 year is a leap year. So we need to correct for Dec 31st comming out as the 366th day of the year. */ if( !IS_LEAP(local_tm->tm_year) && local_tm->tm_yday == 365 ) local_tm->tm_yday--; assert(check_tm(local_tm)); return local_tm; } int valid_tm_wday( const struct TM* date ) { if( 0 <= date->tm_wday && date->tm_wday <= 6 ) return 1; else return 0; } int valid_tm_mon( const struct TM* date ) { if( 0 <= date->tm_mon && date->tm_mon <= 11 ) return 1; else return 0; } /* Non-thread safe versions of the above */ struct TM *localtime64(const Time64_T *time) { #ifdef _MSC_VER _tzset(); #else tzset(); #endif return localtime64_r(time, &Static_Return_Date); } struct TM *gmtime64(const Time64_T *time) { return gmtime64_r(time, &Static_Return_Date); } pymongo-3.2/bson/min_key.py0000644000175000017500000000245412622440614017730 0ustar behackettbehackett00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Representation for the MongoDB internal MinKey type. """ class MinKey(object): """MongoDB internal MinKey type. .. versionchanged:: 2.7 ``MinKey`` now implements comparison operators. """ _type_marker = 255 def __eq__(self, other): return isinstance(other, MinKey) def __hash__(self): return hash(self._type_marker) def __ne__(self, other): return not self == other def __le__(self, dummy): return True def __lt__(self, other): return not isinstance(other, MinKey) def __ge__(self, other): return isinstance(other, MinKey) def __gt__(self, dummy): return False def __repr__(self): return "MinKey()" pymongo-3.2/bson/tz_util.py0000644000175000017500000000275612560753776020015 0ustar behackettbehackett00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Timezone related utilities for BSON.""" from datetime import (timedelta, tzinfo) ZERO = timedelta(0) class FixedOffset(tzinfo): """Fixed offset timezone, in minutes east from UTC. Implementation based from the Python `standard library documentation `_. Defining __getinitargs__ enables pickling / copying. """ def __init__(self, offset, name): if isinstance(offset, timedelta): self.__offset = offset else: self.__offset = timedelta(minutes=offset) self.__name = name def __getinitargs__(self): return self.__offset, self.__name def utcoffset(self, dt): return self.__offset def tzname(self, dt): return self.__name def dst(self, dt): return ZERO utc = FixedOffset(0, "UTC") """Fixed offset timezone representing UTC.""" pymongo-3.2/bson/codec_options.py0000644000175000017500000001256512630145074021132 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for specifying BSON codec options.""" import datetime from collections import MutableMapping, namedtuple from bson.py3compat import string_type from bson.binary import (ALL_UUID_REPRESENTATIONS, PYTHON_LEGACY, UUID_REPRESENTATION_NAMES) _RAW_BSON_DOCUMENT_MARKER = 101 def _raw_document_class(document_class): """Determine if a document_class is a RawBSONDocument class.""" marker = getattr(document_class, '_type_marker', None) return marker == _RAW_BSON_DOCUMENT_MARKER _options_base = namedtuple( 'CodecOptions', ('document_class', 'tz_aware', 'uuid_representation', 'unicode_decode_error_handler', 'tzinfo')) class CodecOptions(_options_base): """Encapsulates BSON options used in CRUD operations. :Parameters: - `document_class`: BSON documents returned in queries will be decoded to an instance of this class. Must be a subclass of :class:`~collections.MutableMapping`. Defaults to :class:`dict`. - `tz_aware`: If ``True``, BSON datetimes will be decoded to timezone aware instances of :class:`~datetime.datetime`. Otherwise they will be naive. Defaults to ``False``. - `uuid_representation`: The BSON representation to use when encoding and decoding instances of :class:`~uuid.UUID`. Defaults to :data:`~bson.binary.PYTHON_LEGACY`. - `unicode_decode_error_handler`: The error handler to use when decoding an invalid BSON string. Valid options include 'strict', 'replace', and 'ignore'. Defaults to 'strict'. - `tzinfo`: A :class:`~datetime.tzinfo` subclass that specifies the timezone to/from which :class:`~datetime.datetime` objects should be encoded/decoded. .. warning:: Care must be taken when changing `unicode_decode_error_handler` from its default value ('strict'). The 'replace' and 'ignore' modes should not be used when documents retrieved from the server will be modified in the client application and stored back to the server. """ def __new__(cls, document_class=dict, tz_aware=False, uuid_representation=PYTHON_LEGACY, unicode_decode_error_handler="strict", tzinfo=None): if not (issubclass(document_class, MutableMapping) or _raw_document_class(document_class)): raise TypeError("document_class must be dict, bson.son.SON, " "bson.raw_bson_document.RawBSONDocument, or a " "sublass of collections.MutableMapping") if not isinstance(tz_aware, bool): raise TypeError("tz_aware must be True or False") if uuid_representation not in ALL_UUID_REPRESENTATIONS: raise ValueError("uuid_representation must be a value " "from bson.binary.ALL_UUID_REPRESENTATIONS") if not isinstance(unicode_decode_error_handler, (string_type, None)): raise ValueError("unicode_decode_error_handler must be a string " "or None") if tzinfo is not None: if not isinstance(tzinfo, datetime.tzinfo): raise TypeError( "tzinfo must be an instance of datetime.tzinfo") if not tz_aware: raise ValueError( "cannot specify tzinfo without also setting tz_aware=True") return tuple.__new__( cls, (document_class, tz_aware, uuid_representation, unicode_decode_error_handler, tzinfo)) def __repr__(self): document_class_repr = ( 'dict' if self.document_class is dict else repr(self.document_class)) uuid_rep_repr = UUID_REPRESENTATION_NAMES.get(self.uuid_representation, self.uuid_representation) return ( 'CodecOptions(document_class=%s, tz_aware=%r, uuid_representation=' '%s, unicode_decode_error_handler=%r, tzinfo=%r)' % (document_class_repr, self.tz_aware, uuid_rep_repr, self.unicode_decode_error_handler, self.tzinfo)) DEFAULT_CODEC_OPTIONS = CodecOptions() def _parse_codec_options(options): """Parse BSON codec options.""" return CodecOptions( document_class=options.get( 'document_class', DEFAULT_CODEC_OPTIONS.document_class), tz_aware=options.get( 'tz_aware', DEFAULT_CODEC_OPTIONS.tz_aware), uuid_representation=options.get( 'uuidrepresentation', DEFAULT_CODEC_OPTIONS.uuid_representation), unicode_decode_error_handler=options.get( 'unicode_decode_error_handler', DEFAULT_CODEC_OPTIONS.unicode_decode_error_handler), tzinfo=options.get('tzinfo', DEFAULT_CODEC_OPTIONS.tzinfo)) pymongo-3.2/bson/max_key.py0000644000175000017500000000245012622440614017726 0ustar behackettbehackett00000000000000# Copyright 2010-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Representation for the MongoDB internal MaxKey type. """ class MaxKey(object): """MongoDB internal MaxKey type. .. versionchanged:: 2.7 ``MaxKey`` now implements comparison operators. """ _type_marker = 127 def __eq__(self, other): return isinstance(other, MaxKey) def __hash__(self): return hash(self._type_marker) def __ne__(self, other): return not self == other def __le__(self, other): return isinstance(other, MaxKey) def __lt__(self, dummy): return False def __ge__(self, dummy): return True def __gt__(self, other): return not isinstance(other, MaxKey) def __repr__(self): return "MaxKey()" pymongo-3.2/bson/son.py0000644000175000017500000002075412630145074017100 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for creating and manipulating SON, the Serialized Ocument Notation. Regular dictionaries can be used instead of SON objects, but not when the order of keys is important. A SON object can be used just like a normal Python dictionary.""" import collections import copy import re from bson.py3compat import iteritems # This sort of sucks, but seems to be as good as it gets... # This is essentially the same as re._pattern_type RE_TYPE = type(re.compile("")) class SON(dict): """SON data. A subclass of dict that maintains ordering of keys and provides a few extra niceties for dealing with SON. SON objects can be converted to and from BSON. The mapping from Python types to BSON types is as follows: ======================================= ============= =================== Python Type BSON Type Supported Direction ======================================= ============= =================== None null both bool boolean both int [#int]_ int32 / int64 py -> bson long int64 py -> bson `bson.int64.Int64` int64 both float number (real) both string string py -> bson unicode string both list array both dict / `SON` object both datetime.datetime [#dt]_ [#dt2]_ date both `bson.regex.Regex` regex both compiled re [#re]_ regex py -> bson `bson.binary.Binary` binary both `bson.objectid.ObjectId` oid both `bson.dbref.DBRef` dbref both None undefined bson -> py unicode code bson -> py `bson.code.Code` code py -> bson unicode symbol bson -> py bytes (Python 3) [#bytes]_ binary both ======================================= ============= =================== Note that to save binary data it must be wrapped as an instance of `bson.binary.Binary`. Otherwise it will be saved as a BSON string and retrieved as unicode. .. [#int] A Python int will be saved as a BSON int32 or BSON int64 depending on its size. A BSON int32 will always decode to a Python int. A BSON int64 will always decode to a :class:`~bson.int64.Int64`. .. [#dt] datetime.datetime instances will be rounded to the nearest millisecond when saved .. [#dt2] all datetime.datetime instances are treated as *naive*. clients should always use UTC. .. [#re] :class:`~bson.regex.Regex` instances and regular expression objects from ``re.compile()`` are both saved as BSON regular expressions. BSON regular expressions are decoded as :class:`~bson.regex.Regex` instances. .. [#bytes] The bytes type from Python 3.x is encoded as BSON binary with subtype 0. In Python 3.x it will be decoded back to bytes. In Python 2.x it will be decoded to an instance of :class:`~bson.binary.Binary` with subtype 0. """ def __init__(self, data=None, **kwargs): self.__keys = [] dict.__init__(self) self.update(data) self.update(kwargs) def __new__(cls, *args, **kwargs): instance = super(SON, cls).__new__(cls, *args, **kwargs) instance.__keys = [] return instance def __repr__(self): result = [] for key in self.__keys: result.append("(%r, %r)" % (key, self[key])) return "SON([%s])" % ", ".join(result) def __setitem__(self, key, value): if key not in self.__keys: self.__keys.append(key) dict.__setitem__(self, key, value) def __delitem__(self, key): self.__keys.remove(key) dict.__delitem__(self, key) def keys(self): return list(self.__keys) def copy(self): other = SON() other.update(self) return other # TODO this is all from UserDict.DictMixin. it could probably be made more # efficient. # second level definitions support higher levels def __iter__(self): for k in self.__keys: yield k def has_key(self, key): return key in self.__keys # third level takes advantage of second level definitions def iteritems(self): for k in self: yield (k, self[k]) def iterkeys(self): return self.__iter__() # fourth level uses definitions from lower levels def itervalues(self): for _, v in self.iteritems(): yield v def values(self): return [v for _, v in self.iteritems()] def items(self): return [(key, self[key]) for key in self] def clear(self): self.__keys = [] super(SON, self).clear() def setdefault(self, key, default=None): try: return self[key] except KeyError: self[key] = default return default def pop(self, key, *args): if len(args) > 1: raise TypeError("pop expected at most 2 arguments, got "\ + repr(1 + len(args))) try: value = self[key] except KeyError: if args: return args[0] raise del self[key] return value def popitem(self): try: k, v = next(self.iteritems()) except StopIteration: raise KeyError('container is empty') del self[k] return (k, v) def update(self, other=None, **kwargs): # Make progressively weaker assumptions about "other" if other is None: pass elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups for k, v in other.iteritems(): self[k] = v elif hasattr(other, 'keys'): for k in other.keys(): self[k] = other[k] else: for k, v in other: self[k] = v if kwargs: self.update(kwargs) def get(self, key, default=None): try: return self[key] except KeyError: return default def __eq__(self, other): """Comparison to another SON is order-sensitive while comparison to a regular dictionary is order-insensitive. """ if isinstance(other, SON): return len(self) == len(other) and self.items() == other.items() return self.to_dict() == other def __ne__(self, other): return not self == other def __len__(self): return len(self.__keys) def to_dict(self): """Convert a SON document to a normal Python dictionary instance. This is trickier than just *dict(...)* because it needs to be recursive. """ def transform_value(value): if isinstance(value, list): return [transform_value(v) for v in value] elif isinstance(value, collections.Mapping): return dict([ (k, transform_value(v)) for k, v in iteritems(value)]) else: return value return transform_value(dict(self)) def __deepcopy__(self, memo): out = SON() val_id = id(self) if val_id in memo: return memo.get(val_id) memo[val_id] = out for k, v in self.iteritems(): if not isinstance(v, RE_TYPE): v = copy.deepcopy(v, memo) out[k] = v return out pymongo-3.2/bson/_cbsonmodule.h0000644000175000017500000001261112630145074020542 0ustar behackettbehackett00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef _CBSONMODULE_H #define _CBSONMODULE_H /* Py_ssize_t was new in python 2.5. See conversion * guidlines in http://www.python.org/dev/peps/pep-0353 * */ #if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX #define PY_SSIZE_T_MIN INT_MIN #endif #if defined(WIN32) || defined(_MSC_VER) /* * This macro is basically an implementation of asprintf for win32 * We print to the provided buffer to get the string value as an int. */ #if defined(_MSC_VER) && (_MSC_VER >= 1400) #define INT2STRING(buffer, i) \ _snprintf_s((buffer), \ _scprintf("%d", (i)) + 1, \ _scprintf("%d", (i)) + 1, \ "%d", \ (i)) #define STRCAT(dest, n, src) strcat_s((dest), (n), (src)) #else #define INT2STRING(buffer, i) \ _snprintf((buffer), \ _scprintf("%d", (i)) + 1, \ "%d", \ (i)) #define STRCAT(dest, n, src) strcat((dest), (src)) #endif #else #define INT2STRING(buffer, i) snprintf((buffer), sizeof((buffer)), "%d", (i)) #define STRCAT(dest, n, src) strcat((dest), (src)) #endif #if PY_MAJOR_VERSION >= 3 #define BYTES_FORMAT_STRING "y#" #else #define BYTES_FORMAT_STRING "s#" #endif typedef struct codec_options_t { PyObject* document_class; unsigned char tz_aware; unsigned char uuid_rep; char* unicode_decode_error_handler; PyObject* tzinfo; PyObject* options_obj; unsigned char is_raw_bson; } codec_options_t; /* C API functions */ #define _cbson_buffer_write_bytes_INDEX 0 #define _cbson_buffer_write_bytes_RETURN int #define _cbson_buffer_write_bytes_PROTO (buffer_t buffer, const char* data, int size) #define _cbson_write_dict_INDEX 1 #define _cbson_write_dict_RETURN int #define _cbson_write_dict_PROTO (PyObject* self, buffer_t buffer, PyObject* dict, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) #define _cbson_write_pair_INDEX 2 #define _cbson_write_pair_RETURN int #define _cbson_write_pair_PROTO (PyObject* self, buffer_t buffer, const char* name, int name_length, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char allow_id) #define _cbson_decode_and_write_pair_INDEX 3 #define _cbson_decode_and_write_pair_RETURN int #define _cbson_decode_and_write_pair_PROTO (PyObject* self, buffer_t buffer, PyObject* key, PyObject* value, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) #define _cbson_convert_codec_options_INDEX 4 #define _cbson_convert_codec_options_RETURN int #define _cbson_convert_codec_options_PROTO (PyObject* options_obj, void* p) #define _cbson_destroy_codec_options_INDEX 5 #define _cbson_destroy_codec_options_RETURN void #define _cbson_destroy_codec_options_PROTO (codec_options_t* options) /* Total number of C API pointers */ #define _cbson_API_POINTER_COUNT 6 #ifdef _CBSON_MODULE /* This section is used when compiling _cbsonmodule */ static _cbson_buffer_write_bytes_RETURN buffer_write_bytes _cbson_buffer_write_bytes_PROTO; static _cbson_write_dict_RETURN write_dict _cbson_write_dict_PROTO; static _cbson_write_pair_RETURN write_pair _cbson_write_pair_PROTO; static _cbson_decode_and_write_pair_RETURN decode_and_write_pair _cbson_decode_and_write_pair_PROTO; static _cbson_convert_codec_options_RETURN convert_codec_options _cbson_convert_codec_options_PROTO; static _cbson_destroy_codec_options_RETURN destroy_codec_options _cbson_destroy_codec_options_PROTO; #else /* This section is used in modules that use _cbsonmodule's API */ static void **_cbson_API; #define buffer_write_bytes (*(_cbson_buffer_write_bytes_RETURN (*)_cbson_buffer_write_bytes_PROTO) _cbson_API[_cbson_buffer_write_bytes_INDEX]) #define write_dict (*(_cbson_write_dict_RETURN (*)_cbson_write_dict_PROTO) _cbson_API[_cbson_write_dict_INDEX]) #define write_pair (*(_cbson_write_pair_RETURN (*)_cbson_write_pair_PROTO) _cbson_API[_cbson_write_pair_INDEX]) #define decode_and_write_pair (*(_cbson_decode_and_write_pair_RETURN (*)_cbson_decode_and_write_pair_PROTO) _cbson_API[_cbson_decode_and_write_pair_INDEX]) #define convert_codec_options (*(_cbson_convert_codec_options_RETURN (*)_cbson_convert_codec_options_PROTO) _cbson_API[_cbson_convert_codec_options_INDEX]) #define destroy_codec_options (*(_cbson_destroy_codec_options_RETURN (*)_cbson_destroy_codec_options_PROTO) _cbson_API[_cbson_destroy_codec_options_INDEX]) #define _cbson_IMPORT _cbson_API = (void **)PyCapsule_Import("_cbson._C_API", 0) #endif #endif // _CBSONMODULE_H pymongo-3.2/bson/binary.py0000644000175000017500000001567712630145074017575 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from uuid import UUID from bson.py3compat import PY3 """Tools for representing BSON binary data. """ BINARY_SUBTYPE = 0 """BSON binary subtype for binary data. This is the default subtype for binary data. """ FUNCTION_SUBTYPE = 1 """BSON binary subtype for functions. """ OLD_BINARY_SUBTYPE = 2 """Old BSON binary subtype for binary data. This is the old default subtype, the current default is :data:`BINARY_SUBTYPE`. """ OLD_UUID_SUBTYPE = 3 """Old BSON binary subtype for a UUID. :class:`uuid.UUID` instances will automatically be encoded by :mod:`bson` using this subtype. .. versionadded:: 2.1 """ UUID_SUBTYPE = 4 """BSON binary subtype for a UUID. This is the new BSON binary subtype for UUIDs. The current default is :data:`OLD_UUID_SUBTYPE` but will change to this in a future release. .. versionchanged:: 2.1 Changed to subtype 4. """ STANDARD = UUID_SUBTYPE """The standard UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary, using RFC-4122 byte order with binary subtype :data:`UUID_SUBTYPE`. .. versionadded:: 3.0 """ PYTHON_LEGACY = OLD_UUID_SUBTYPE """The Python legacy UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary, using RFC-4122 byte order with binary subtype :data:`OLD_UUID_SUBTYPE`. .. versionadded:: 3.0 """ JAVA_LEGACY = 5 """The Java legacy UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary, using the Java driver's legacy byte order with binary subtype :data:`OLD_UUID_SUBTYPE`. .. versionadded:: 2.3 """ CSHARP_LEGACY = 6 """The C#/.net legacy UUID representation. :class:`uuid.UUID` instances will automatically be encoded to and decoded from BSON binary, using the C# driver's legacy byte order and binary subtype :data:`OLD_UUID_SUBTYPE`. .. versionadded:: 2.3 """ ALL_UUID_SUBTYPES = (OLD_UUID_SUBTYPE, UUID_SUBTYPE) ALL_UUID_REPRESENTATIONS = (STANDARD, PYTHON_LEGACY, JAVA_LEGACY, CSHARP_LEGACY) UUID_REPRESENTATION_NAMES = { PYTHON_LEGACY: 'PYTHON_LEGACY', STANDARD: 'STANDARD', JAVA_LEGACY: 'JAVA_LEGACY', CSHARP_LEGACY: 'CSHARP_LEGACY'} MD5_SUBTYPE = 5 """BSON binary subtype for an MD5 hash. """ USER_DEFINED_SUBTYPE = 128 """BSON binary subtype for any user defined structure. """ class Binary(bytes): """Representation of BSON binary data. This is necessary because we want to represent Python strings as the BSON string type. We need to wrap binary data so we can tell the difference between what should be considered binary data and what should be considered a string when we encode to BSON. Raises TypeError if `data` is not an instance of :class:`str` (:class:`bytes` in python 3) or `subtype` is not an instance of :class:`int`. Raises ValueError if `subtype` is not in [0, 256). .. note:: In python 3 instances of Binary with subtype 0 will be decoded directly to :class:`bytes`. :Parameters: - `data`: the binary data to represent - `subtype` (optional): the `binary subtype `_ to use """ _type_marker = 5 def __new__(cls, data, subtype=BINARY_SUBTYPE): if not isinstance(data, bytes): raise TypeError("data must be an instance of bytes") if not isinstance(subtype, int): raise TypeError("subtype must be an instance of int") if subtype >= 256 or subtype < 0: raise ValueError("subtype must be contained in [0, 256)") self = bytes.__new__(cls, data) self.__subtype = subtype return self @property def subtype(self): """Subtype of this binary data. """ return self.__subtype def __getnewargs__(self): # Work around http://bugs.python.org/issue7382 data = super(Binary, self).__getnewargs__()[0] if PY3 and not isinstance(data, bytes): data = data.encode('latin-1') return data, self.__subtype def __eq__(self, other): if isinstance(other, Binary): return ((self.__subtype, bytes(self)) == (other.subtype, bytes(other))) # We don't return NotImplemented here because if we did then # Binary("foo") == "foo" would return True, since Binary is a # subclass of str... return False def __hash__(self): return super(Binary, self).__hash__() ^ hash(self.__subtype) def __ne__(self, other): return not self == other def __repr__(self): return "Binary(%s, %s)" % (bytes.__repr__(self), self.__subtype) class UUIDLegacy(Binary): """UUID wrapper to support working with UUIDs stored as PYTHON_LEGACY. .. doctest:: >>> import uuid >>> from bson.binary import Binary, UUIDLegacy, STANDARD >>> from bson.codec_options import CodecOptions >>> my_uuid = uuid.uuid4() >>> coll = db.get_collection('test', ... CodecOptions(uuid_representation=STANDARD)) >>> coll.insert_one({'uuid': Binary(my_uuid.bytes, 3)}).inserted_id ObjectId('...') >>> coll.find({'uuid': my_uuid}).count() 0 >>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count() 1 >>> coll.find({'uuid': UUIDLegacy(my_uuid)})[0]['uuid'] UUID('...') >>> >>> # Convert from subtype 3 to subtype 4 >>> doc = coll.find_one({'uuid': UUIDLegacy(my_uuid)}) >>> coll.replace_one({"_id": doc["_id"]}, doc).matched_count 1 >>> coll.find({'uuid': UUIDLegacy(my_uuid)}).count() 0 >>> coll.find({'uuid': {'$in': [UUIDLegacy(my_uuid), my_uuid]}}).count() 1 >>> coll.find_one({'uuid': my_uuid})['uuid'] UUID('...') Raises TypeError if `obj` is not an instance of :class:`~uuid.UUID`. :Parameters: - `obj`: An instance of :class:`~uuid.UUID`. """ def __new__(cls, obj): if not isinstance(obj, UUID): raise TypeError("obj must be an instance of uuid.UUID") self = Binary.__new__(cls, obj.bytes, OLD_UUID_SUBTYPE) self.__uuid = obj return self def __getnewargs__(self): # Support copy and deepcopy return (self.__uuid,) @property def uuid(self): """UUID instance wrapped by this UUIDLegacy instance. """ return self.__uuid def __repr__(self): return "UUIDLegacy('%s')" % self.__uuid pymongo-3.2/bson/time64.h0000644000175000017500000000275412507072032017204 0ustar behackettbehackett00000000000000#ifndef TIME64_H # define TIME64_H #include #include "time64_config.h" /* Set our custom types */ typedef INT_64_T Int64; typedef Int64 Time64_T; typedef Int64 Year; /* A copy of the tm struct but with a 64 bit year */ struct TM64 { int tm_sec; int tm_min; int tm_hour; int tm_mday; int tm_mon; Year tm_year; int tm_wday; int tm_yday; int tm_isdst; #ifdef HAS_TM_TM_GMTOFF long tm_gmtoff; #endif #ifdef HAS_TM_TM_ZONE char *tm_zone; #endif }; /* Decide which tm struct to use */ #ifdef USE_TM64 #define TM TM64 #else #define TM tm #endif /* Declare public functions */ struct TM *gmtime64_r (const Time64_T *, struct TM *); struct TM *localtime64_r (const Time64_T *, struct TM *); struct TM *gmtime64 (const Time64_T *); struct TM *localtime64 (const Time64_T *); Time64_T timegm64 (const struct TM *); Time64_T mktime64 (const struct TM *); Time64_T timelocal64 (const struct TM *); /* Not everyone has gm/localtime_r(), provide a replacement */ #ifdef HAS_LOCALTIME_R # define LOCALTIME_R(clock, result) localtime_r(clock, result) #else # define LOCALTIME_R(clock, result) fake_localtime_r(clock, result) #endif #ifdef HAS_GMTIME_R # define GMTIME_R(clock, result) gmtime_r(clock, result) #else # define GMTIME_R(clock, result) fake_gmtime_r(clock, result) #endif #endif pymongo-3.2/bson/buffer.c0000644000175000017500000000770612560753776017366 0ustar behackettbehackett00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "buffer.h" #define INITIAL_BUFFER_SIZE 256 struct buffer { char* buffer; int size; int position; }; /* Allocate and return a new buffer. * Return NULL on allocation failure. */ buffer_t buffer_new(void) { buffer_t buffer; buffer = (buffer_t)malloc(sizeof(struct buffer)); if (buffer == NULL) { return NULL; } buffer->size = INITIAL_BUFFER_SIZE; buffer->position = 0; buffer->buffer = (char*)malloc(sizeof(char) * INITIAL_BUFFER_SIZE); if (buffer->buffer == NULL) { free(buffer); return NULL; } return buffer; } /* Free the memory allocated for `buffer`. * Return non-zero on failure. */ int buffer_free(buffer_t buffer) { if (buffer == NULL) { return 1; } free(buffer->buffer); free(buffer); return 0; } /* Grow `buffer` to at least `min_length`. * Return non-zero on allocation failure. */ static int buffer_grow(buffer_t buffer, int min_length) { int old_size = 0; int size = buffer->size; char* old_buffer = buffer->buffer; if (size >= min_length) { return 0; } while (size < min_length) { old_size = size; size *= 2; if (size <= old_size) { /* Size did not increase. Could be an overflow * or size < 1. Just go with min_length. */ size = min_length; } } buffer->buffer = (char*)realloc(buffer->buffer, sizeof(char) * size); if (buffer->buffer == NULL) { free(old_buffer); free(buffer); return 1; } buffer->size = size; return 0; } /* Assure that `buffer` has at least `size` free bytes (and grow if needed). * Return non-zero on allocation failure. */ static int buffer_assure_space(buffer_t buffer, int size) { if (buffer->position + size <= buffer->size) { return 0; } return buffer_grow(buffer, buffer->position + size); } /* Save `size` bytes from the current position in `buffer` (and grow if needed). * Return offset for writing, or -1 on allocation failure. */ buffer_position buffer_save_space(buffer_t buffer, int size) { int position = buffer->position; if (buffer_assure_space(buffer, size) != 0) { return -1; } buffer->position += size; return position; } /* Write `size` bytes from `data` to `buffer` (and grow if needed). * Return non-zero on allocation failure. */ int buffer_write(buffer_t buffer, const char* data, int size) { if (buffer_assure_space(buffer, size) != 0) { return 1; } memcpy(buffer->buffer + buffer->position, data, size); buffer->position += size; return 0; } /* Write `size` bytes from `data` to `buffer` at position `position`. * Does not change the internal position of `buffer`. * Return non-zero if buffer isn't large enough for write. */ int buffer_write_at_position(buffer_t buffer, buffer_position position, const char* data, int size) { if (position + size > buffer->size) { buffer_free(buffer); return 1; } memcpy(buffer->buffer + position, data, size); return 0; } int buffer_get_position(buffer_t buffer) { return buffer->position; } char* buffer_get_buffer(buffer_t buffer) { return buffer->buffer; } void buffer_update_position(buffer_t buffer, buffer_position new_position) { buffer->position = new_position; } pymongo-3.2/bson/time64_config.h0000644000175000017500000000322212507072032020520 0ustar behackettbehackett00000000000000/* Configuration ------------- Define as appropriate for your system. Sensible defaults provided. */ #ifndef TIME64_CONFIG_H # define TIME64_CONFIG_H /* Debugging TIME_64_DEBUG Define if you want debugging messages */ /* #define TIME_64_DEBUG */ /* INT_64_T A 64 bit integer type to use to store time and others. Must be defined. */ #define INT_64_T long long /* USE_TM64 Should we use a 64 bit safe replacement for tm? This will let you go past year 2 billion but the struct will be incompatible with tm. Conversion functions will be provided. */ /* #define USE_TM64 */ /* Availability of system functions. HAS_GMTIME_R Define if your system has gmtime_r() HAS_LOCALTIME_R Define if your system has localtime_r() HAS_TIMEGM Define if your system has timegm(), a GNU extension. */ #if !defined(WIN32) && !defined(_MSC_VER) #define HAS_GMTIME_R #define HAS_LOCALTIME_R #endif /* #define HAS_TIMEGM */ /* Details of non-standard tm struct elements. HAS_TM_TM_GMTOFF True if your tm struct has a "tm_gmtoff" element. A BSD extension. HAS_TM_TM_ZONE True if your tm struct has a "tm_zone" element. A BSD extension. */ /* #define HAS_TM_TM_GMTOFF */ /* #define HAS_TM_TM_ZONE */ /* USE_SYSTEM_LOCALTIME USE_SYSTEM_GMTIME USE_SYSTEM_MKTIME USE_SYSTEM_TIMEGM Should we use the system functions if the time is inside their range? Your system localtime() is probably more accurate, but our gmtime() is fast and safe. */ #define USE_SYSTEM_LOCALTIME /* #define USE_SYSTEM_GMTIME */ #define USE_SYSTEM_MKTIME /* #define USE_SYSTEM_TIMEGM */ #endif /* TIME64_CONFIG_H */ pymongo-3.2/bson/int64.py0000644000175000017500000000204012630145074017231 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A BSON wrapper for long (int in python3)""" from bson.py3compat import PY3 if PY3: long = int class Int64(long): """Representation of the BSON int64 type. This is necessary because every integral number is an :class:`int` in Python 3. Small integral numbers are encoded to BSON int32 by default, but Int64 numbers will always be encoded to BSON int64. :Parameters: - `value`: the numeric value to represent """ _type_marker = 18 pymongo-3.2/bson/py3compat.py0000644000175000017500000000471412630145074020216 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Utility functions and definitions for python3 compatibility.""" import sys PY3 = sys.version_info[0] == 3 if PY3: import codecs import _thread as thread from io import BytesIO as StringIO MAXSIZE = sys.maxsize imap = map def b(s): # BSON and socket operations deal in binary data. In # python 3 that means instances of `bytes`. In python # 2.6 and 2.7 you can create an alias for `bytes` using # the b prefix (e.g. b'foo'). # See http://python3porting.com/problems.html#nicer-solutions return codecs.latin_1_encode(s)[0] def u(s): # PY3 strings may already be treated as unicode literals return s def bytes_from_hex(h): return bytes.fromhex(h) def iteritems(d): return iter(d.items()) def itervalues(d): return iter(d.values()) def reraise(exctype, value, trace=None): raise exctype(str(value)).with_traceback(trace) def _unicode(s): return s text_type = str string_type = str integer_types = int else: import thread from itertools import imap try: from cStringIO import StringIO except ImportError: from StringIO import StringIO MAXSIZE = sys.maxint def b(s): # See comments above. In python 2.x b('foo') is just 'foo'. return s def u(s): """Replacement for unicode literal prefix.""" return unicode(s.replace('\\', '\\\\'), 'unicode_escape') def bytes_from_hex(h): return h.decode('hex') def iteritems(d): return d.iteritems() def itervalues(d): return d.itervalues() # "raise x, y, z" raises SyntaxError in Python 3 exec("""def reraise(exctype, value, trace=None): raise exctype, str(value), trace """) _unicode = unicode string_type = basestring text_type = unicode integer_types = (int, long) pymongo-3.2/setup.cfg0000644000175000017500000000007312631423130016570 0ustar behackettbehackett00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 pymongo-3.2/README.rst0000644000175000017500000001432112630145074016446 0ustar behackettbehackett00000000000000======= PyMongo ======= :Info: See `the mongo site `_ for more information. See `github `_ for the latest source. :Author: Mike Dirolf :Maintainer: Bernie Hackett About ===== The PyMongo distribution contains tools for interacting with MongoDB database from Python. The ``bson`` package is an implementation of the `BSON format `_ for Python. The ``pymongo`` package is a native Python driver for MongoDB. The ``gridfs`` package is a `gridfs `_ implementation on top of ``pymongo``. Support / Feedback ================== For issues with, questions about, or feedback for PyMongo, please look into our `support channels `_. Please do not email any of the PyMongo developers directly with issues or questions - you're more likely to get an answer on the `mongodb-user `_ list on Google Groups. Bugs / Feature Requests ======================= Think you’ve found a bug? Want to see a new feature in PyMongo? Please open a case in our issue management tool, JIRA: - `Create an account and login `_. - Navigate to `the PYTHON project `_. - Click **Create Issue** - Please provide as much information as possible about the issue type and how to reproduce it. Bug reports in JIRA for all driver projects (i.e. PYTHON, CSHARP, JAVA) and the Core Server (i.e. SERVER) project are **public**. How To Ask For Help ------------------- Please include all of the following information when opening an issue: - Detailed steps to reproduce the problem, including full traceback, if possible. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used, with patch level:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The operating system and version (e.g. Windows 7, OSX 10.8, ...) - Web framework or asynchronous network library used, if any, with version (e.g. Django 1.7, mod_wsgi 4.3.0, gevent 1.0.1, Tornado 4.0.2, ...) Security Vulnerabilities ------------------------ If you’ve identified a security vulnerability in a driver or any other MongoDB project, please report it according to the `instructions here `_. Installation ============ If you have `setuptools `_ installed you should be able to do **easy_install pymongo** to install PyMongo. Otherwise you can download the project source and do **python setup.py install** to install. Do **not** install the "bson" package. PyMongo comes with its own bson package; doing "easy_install bson" installs a third-party package that is incompatible with PyMongo. Dependencies ============ The PyMongo distribution is supported and tested on Python 2.x (where x >= 6) and Python 3.x (where x >= 2). PyMongo versions before 3.0 also support Python 2.4, 2.5, and 3.1. Optional packages: - `backports.pbkdf2 `_, improves authentication performance with SCRAM-SHA-1, the default authentication mechanism for MongoDB 3.0+. It especially improves performance on Python older than 2.7.8, or on Python 3 before Python 3.4. - `pykerberos `_ is required for the GSSAPI authentication mechanism. - `monotonic `_ adds support for a monotonic clock, which improves reliability in environments where clock adjustments are frequent. Not needed in Python 3.3+. - `wincertstore `_ adds support for verifying server SSL certificates using Windows provided CA certificates on older versions of python. Not needed or used with versions of Python 2 beginning with 2.7.9, or versions of Python 3 beginning with 3.4.0. - `certifi `_ adds support for using the Mozilla CA bundle with SSL to verify server certificates. Not needed or used with versions of Python 2 beginning with 2.7.9 on any OS, versions of Python 3 beginning with Python 3.4.0 on Windows, or versions of Python 3 beginning with Python 3.2.0 on operating systems other than Windows. Additional dependencies are: - (to generate documentation) sphinx_ - (to run the tests under Python 2.6) unittest2_ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: pycon >>> import pymongo >>> client = pymongo.MongoClient("localhost", 27017) >>> db = client.test >>> db.name u'test' >>> db.my_collection Collection(Database(MongoClient('localhost', 27017), u'test'), u'my_collection') >>> db.my_collection.insert_one({"x": 10}).inserted_id ObjectId('4aba15ebe23f6b53b0000000') >>> db.my_collection.insert_one({"x": 8}).inserted_id ObjectId('4aba160ee23f6b543e000000') >>> db.my_collection.insert_one({"x": 11}).inserted_id ObjectId('4aba160ee23f6b543e000002') >>> db.my_collection.find_one() {u'x': 10, u'_id': ObjectId('4aba15ebe23f6b53b0000000')} >>> for item in db.my_collection.find(): ... print(item["x"]) ... 10 8 11 >>> db.my_collection.create_index("x") u'x_1' >>> for item in db.my_collection.find().sort("x", pymongo.ASCENDING): ... print(item["x"]) ... 8 10 11 >>> [item["x"] for item in db.my_collection.find().limit(2).skip(1)] [8, 11] Documentation ============= You will need sphinx_ installed to generate the documentation. Documentation can be generated by running **python setup.py doc**. Generated documentation can be found in the *doc/build/html/* directory. Testing ======= The easiest way to run the tests is to run **python setup.py test** in the root of the distribution. Note that you will need unittest2_ to run the tests under Python 2.6. To verify that PyMongo works with Gevent's monkey-patching:: $ python green_framework_test.py gevent Or with Eventlet's:: $ python green_framework_test.py eventlet .. _sphinx: http://sphinx.pocoo.org/ .. _unittest2: https://pypi.python.org/pypi/unittest2 pymongo-3.2/pymongo/0000755000175000017500000000000012631423130016437 5ustar behackettbehackett00000000000000pymongo-3.2/pymongo/server_description.py0000644000175000017500000001035612630145074022736 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Represent one server in the topology.""" from pymongo.server_type import SERVER_TYPE from pymongo.ismaster import IsMaster class ServerDescription(object): """Immutable representation of one server. :Parameters: - `address`: A (host, port) pair - `ismaster`: Optional IsMaster instance - `round_trip_time`: Optional float - `error`: Optional, the last error attempting to connect to the server """ __slots__ = ( '_address', '_server_type', '_all_hosts', '_tags', '_replica_set_name', '_primary', '_max_bson_size', '_max_message_size', '_max_write_batch_size', '_min_wire_version', '_max_wire_version', '_round_trip_time', '_me', '_is_writable', '_is_readable', '_error', '_election_id') def __init__( self, address, ismaster=None, round_trip_time=None, error=None): self._address = address if not ismaster: ismaster = IsMaster({}) self._server_type = ismaster.server_type self._all_hosts = ismaster.all_hosts self._tags = ismaster.tags self._replica_set_name = ismaster.replica_set_name self._primary = ismaster.primary self._max_bson_size = ismaster.max_bson_size self._max_message_size = ismaster.max_message_size self._max_write_batch_size = ismaster.max_write_batch_size self._min_wire_version = ismaster.min_wire_version self._max_wire_version = ismaster.max_wire_version self._election_id = ismaster.election_id self._is_writable = ismaster.is_writable self._is_readable = ismaster.is_readable self._round_trip_time = round_trip_time self._me = ismaster.me self._error = error @property def address(self): return self._address @property def server_type(self): return self._server_type @property def all_hosts(self): """List of hosts, passives, and arbiters known to this server.""" return self._all_hosts @property def tags(self): return self._tags @property def replica_set_name(self): """Replica set name or None.""" return self._replica_set_name @property def primary(self): """This server's opinion about who the primary is, or None.""" return self._primary @property def max_bson_size(self): return self._max_bson_size @property def max_message_size(self): return self._max_message_size @property def max_write_batch_size(self): return self._max_write_batch_size @property def min_wire_version(self): return self._min_wire_version @property def max_wire_version(self): return self._max_wire_version @property def election_id(self): return self._election_id @property def me(self): return self._me @property def round_trip_time(self): """The current average latency or None.""" # This override is for unittesting only! if self._address in self._host_to_round_trip_time: return self._host_to_round_trip_time[self._address] return self._round_trip_time @property def error(self): """The last error attempting to connect to the server, or None.""" return self._error @property def is_writable(self): return self._is_writable @property def is_readable(self): return self._is_readable @property def is_server_type_known(self): return self.server_type != SERVER_TYPE.Unknown # For unittesting only. Use under no circumstances! _host_to_round_trip_time = {} pymongo-3.2/pymongo/server_type.py0000644000175000017500000000156212630145074021373 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Type codes for MongoDB servers.""" from collections import namedtuple SERVER_TYPE = namedtuple('ServerType', ['Unknown', 'Mongos', 'RSPrimary', 'RSSecondary', 'RSArbiter', 'RSOther', 'RSGhost', 'Standalone'])(*range(8)) pymongo-3.2/pymongo/bulk.py0000644000175000017500000005623612630145074017771 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """The bulk write operations interface. .. versionadded:: 2.7 """ from bson.objectid import ObjectId from bson.py3compat import u from bson.raw_bson import RawBSONDocument from bson.son import SON from pymongo.common import (validate_is_mapping, validate_is_document_type, validate_ok_for_replace, validate_ok_for_update) from pymongo.errors import (BulkWriteError, DocumentTooLarge, InvalidOperation, OperationFailure) from pymongo.message import (_INSERT, _UPDATE, _DELETE, _do_batched_write_command, _randint, _BulkWriteContext) from pymongo.write_concern import WriteConcern _DELETE_ALL = 0 _DELETE_ONE = 1 # For backwards compatibility. See MongoDB src/mongo/base/error_codes.err _BAD_VALUE = 2 _UNKNOWN_ERROR = 8 _WRITE_CONCERN_ERROR = 64 _COMMANDS = ('insert', 'update', 'delete') # These string literals are used when we create fake server return # documents client side. We use unicode literals in python 2.x to # match the actual return values from the server. _UID = u("_id") _UCODE = u("code") _UERRMSG = u("errmsg") _UINDEX = u("index") _UOP = u("op") class _Run(object): """Represents a batch of write operations. """ def __init__(self, op_type): """Initialize a new Run object. """ self.op_type = op_type self.index_map = [] self.ops = [] def index(self, idx): """Get the original index of an operation in this run. :Parameters: - `idx`: The Run index that maps to the original index. """ return self.index_map[idx] def add(self, original_index, operation): """Add an operation to this Run instance. :Parameters: - `original_index`: The original index of this operation within a larger bulk operation. - `operation`: The operation document. """ self.index_map.append(original_index) self.ops.append(operation) def _make_error(index, code, errmsg, operation): """Create and return an error document. """ return { _UINDEX: index, _UCODE: code, _UERRMSG: errmsg, _UOP: operation } def _merge_legacy(run, full_result, result, index): """Merge a result from a legacy opcode into the full results. """ affected = result.get('n', 0) errmsg = result.get("errmsg", result.get("err", "")) if errmsg: # wtimeout is not considered a hard failure in # MongoDB 2.6 so don't treat it like one here. if result.get("wtimeout"): error_doc = {'errmsg': errmsg, 'code': _WRITE_CONCERN_ERROR} full_result['writeConcernErrors'].append(error_doc) else: code = result.get("code", _UNKNOWN_ERROR) error = _make_error(run.index(index), code, errmsg, run.ops[index]) if "errInfo" in result: error["errInfo"] = result["errInfo"] full_result["writeErrors"].append(error) return if run.op_type == _INSERT: full_result['nInserted'] += 1 elif run.op_type == _UPDATE: if "upserted" in result: doc = {_UINDEX: run.index(index), _UID: result["upserted"]} full_result["upserted"].append(doc) full_result['nUpserted'] += affected # Versions of MongoDB before 2.6 don't return the _id for an # upsert if _id is not an ObjectId. elif result.get("updatedExisting") is False and affected == 1: op = run.ops[index] # If _id is in both the update document *and* the query spec # the update document _id takes precedence. _id = op['u'].get('_id', op['q'].get('_id')) doc = {_UINDEX: run.index(index), _UID: _id} full_result["upserted"].append(doc) full_result['nUpserted'] += affected else: full_result['nMatched'] += affected elif run.op_type == _DELETE: full_result['nRemoved'] += affected def _merge_command(run, full_result, results): """Merge a group of results from write commands into the full result. """ for offset, result in results: affected = result.get("n", 0) if run.op_type == _INSERT: full_result["nInserted"] += affected elif run.op_type == _DELETE: full_result["nRemoved"] += affected elif run.op_type == _UPDATE: upserted = result.get("upserted") if upserted: if isinstance(upserted, list): n_upserted = len(upserted) for doc in upserted: doc["index"] = run.index(doc["index"] + offset) full_result["upserted"].extend(upserted) else: n_upserted = 1 index = run.index(offset) doc = {_UINDEX: index, _UID: upserted} full_result["upserted"].append(doc) full_result["nUpserted"] += n_upserted full_result["nMatched"] += (affected - n_upserted) else: full_result["nMatched"] += affected n_modified = result.get("nModified") # SERVER-13001 - in a mixed sharded cluster a call to # update could return nModified (>= 2.6) or not (<= 2.4). # If any call does not return nModified we can't report # a valid final count so omit the field completely. if n_modified is not None and "nModified" in full_result: full_result["nModified"] += n_modified else: full_result.pop("nModified", None) write_errors = result.get("writeErrors") if write_errors: for doc in write_errors: # Leave the server response intact for APM. replacement = doc.copy() idx = doc["index"] + offset replacement["index"] = run.index(idx) # Add the failed operation to the error document. replacement[_UOP] = run.ops[idx] full_result["writeErrors"].append(replacement) wc_error = result.get("writeConcernError") if wc_error: full_result["writeConcernErrors"].append(wc_error) class _Bulk(object): """The private guts of the bulk write API. """ def __init__(self, collection, ordered, bypass_document_validation): """Initialize a _Bulk instance. """ self.collection = collection self.ordered = ordered self.ops = [] self.name = "%s.%s" % (collection.database.name, collection.name) self.namespace = collection.database.name + '.$cmd' self.executed = False self.bypass_doc_val = bypass_document_validation def add_insert(self, document): """Add an insert document to the list of ops. """ validate_is_document_type("document", document) # Generate ObjectId client side. if not (isinstance(document, RawBSONDocument) or '_id' in document): document['_id'] = ObjectId() self.ops.append((_INSERT, document)) def add_update(self, selector, update, multi=False, upsert=False): """Create an update document and add it to the list of ops. """ validate_ok_for_update(update) cmd = SON([('q', selector), ('u', update), ('multi', multi), ('upsert', upsert)]) self.ops.append((_UPDATE, cmd)) def add_replace(self, selector, replacement, upsert=False): """Create a replace document and add it to the list of ops. """ validate_ok_for_replace(replacement) cmd = SON([('q', selector), ('u', replacement), ('multi', False), ('upsert', upsert)]) self.ops.append((_UPDATE, cmd)) def add_delete(self, selector, limit): """Create a delete document and add it to the list of ops. """ cmd = SON([('q', selector), ('limit', limit)]) self.ops.append((_DELETE, cmd)) def gen_ordered(self): """Generate batches of operations, batched by type of operation, in the order **provided**. """ run = None for idx, (op_type, operation) in enumerate(self.ops): if run is None: run = _Run(op_type) elif run.op_type != op_type: yield run run = _Run(op_type) run.add(idx, operation) yield run def gen_unordered(self): """Generate batches of operations, batched by type of operation, in arbitrary order. """ operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] for idx, (op_type, operation) in enumerate(self.ops): operations[op_type].add(idx, operation) for run in operations: if run.ops: yield run def execute_command(self, sock_info, generator, write_concern): """Execute using write commands. """ # nModified is only reported for write commands, not legacy ops. full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nModified": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() db_name = self.collection.database.name listeners = self.collection.database.client._event_listeners for run in generator: cmd = SON([(_COMMANDS[run.op_type], self.collection.name), ('ordered', self.ordered)]) if write_concern.document: cmd['writeConcern'] = write_concern.document if self.bypass_doc_val and sock_info.max_wire_version >= 4: cmd['bypassDocumentValidation'] = True bwc = _BulkWriteContext(db_name, cmd, sock_info, op_id, listeners) results = _do_batched_write_command( self.namespace, run.op_type, cmd, run.ops, True, self.collection.codec_options, bwc) _merge_command(run, full_result, results) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result['writeErrors']: break if full_result["writeErrors"] or full_result["writeConcernErrors"]: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result def execute_no_results(self, sock_info, generator): """Execute all operations, returning no results (w=0). """ # Cannot have both unacknowledged write and bypass document validation. if self.bypass_doc_val and sock_info.max_wire_version >= 4: raise OperationFailure("Cannot set bypass_document_validation with" " unacknowledged write concern") coll = self.collection # If ordered is True we have to send GLE or use write # commands so we can abort on the first error. write_concern = WriteConcern(w=int(self.ordered)) op_id = _randint() for run in generator: try: if run.op_type == _INSERT: coll._insert( sock_info, run.ops, self.ordered, write_concern=write_concern, op_id=op_id, bypass_doc_val=self.bypass_doc_val) elif run.op_type == _UPDATE: for operation in run.ops: doc = operation['u'] check_keys = True if doc and next(iter(doc)).startswith('$'): check_keys = False coll._update( sock_info, operation['q'], doc, operation['upsert'], check_keys, operation['multi'], write_concern=write_concern, op_id=op_id, ordered=self.ordered, bypass_doc_val=self.bypass_doc_val) else: for operation in run.ops: coll._delete(sock_info, operation['q'], not operation['limit'], write_concern, op_id, self.ordered) except OperationFailure: if self.ordered: break def execute_legacy(self, sock_info, generator, write_concern): """Execute using legacy wire protocol ops. """ coll = self.collection full_result = { "writeErrors": [], "writeConcernErrors": [], "nInserted": 0, "nUpserted": 0, "nMatched": 0, "nRemoved": 0, "upserted": [], } op_id = _randint() stop = False for run in generator: for idx, operation in enumerate(run.ops): try: # To do per-operation reporting we have to do ops one # at a time. That means the performance of bulk insert # will be slower here than calling Collection.insert() if run.op_type == _INSERT: coll._insert(sock_info, operation, self.ordered, write_concern=write_concern, op_id=op_id) result = {} elif run.op_type == _UPDATE: doc = operation['u'] check_keys = True if doc and next(iter(doc)).startswith('$'): check_keys = False result = coll._update(sock_info, operation['q'], doc, operation['upsert'], check_keys, operation['multi'], write_concern=write_concern, op_id=op_id, ordered=self.ordered) else: result = coll._delete(sock_info, operation['q'], not operation['limit'], write_concern, op_id, self.ordered) _merge_legacy(run, full_result, result, idx) except DocumentTooLarge as exc: # MongoDB 2.6 uses error code 2 for "too large". error = _make_error( run.index(idx), _BAD_VALUE, str(exc), operation) full_result['writeErrors'].append(error) if self.ordered: stop = True break except OperationFailure as exc: if not exc.details: # Some error not related to the write operation # (e.g. kerberos failure). Re-raise immediately. raise _merge_legacy(run, full_result, exc.details, idx) # We're supposed to continue if errors are # at the write concern level (e.g. wtimeout) if self.ordered and full_result["writeErrors"]: stop = True break if stop: break if full_result["writeErrors"] or full_result['writeConcernErrors']: if full_result['writeErrors']: full_result['writeErrors'].sort( key=lambda error: error['index']) raise BulkWriteError(full_result) return full_result def execute(self, write_concern): """Execute operations. """ if not self.ops: raise InvalidOperation('No operations to execute') if self.executed: raise InvalidOperation('Bulk operations can ' 'only be executed once.') self.executed = True write_concern = (WriteConcern(**write_concern) if write_concern else self.collection.write_concern) if self.ordered: generator = self.gen_ordered() else: generator = self.gen_unordered() client = self.collection.database.client with client._socket_for_writes() as sock_info: if not write_concern.acknowledged: self.execute_no_results(sock_info, generator) elif sock_info.max_wire_version > 1: return self.execute_command(sock_info, generator, write_concern) else: return self.execute_legacy(sock_info, generator, write_concern) class BulkUpsertOperation(object): """An interface for adding upsert operations. """ __slots__ = ('__selector', '__bulk') def __init__(self, selector, bulk): self.__selector = selector self.__bulk = bulk def update_one(self, update): """Update one document matching the selector. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=False, upsert=True) def update(self, update): """Update all documents matching the selector. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=True, upsert=True) def replace_one(self, replacement): """Replace one entire document matching the selector criteria. :Parameters: - `replacement` (dict): the replacement document """ self.__bulk.add_replace(self.__selector, replacement, upsert=True) class BulkWriteOperation(object): """An interface for adding update or remove operations. """ __slots__ = ('__selector', '__bulk') def __init__(self, selector, bulk): self.__selector = selector self.__bulk = bulk def update_one(self, update): """Update one document matching the selector criteria. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=False) def update(self, update): """Update all documents matching the selector criteria. :Parameters: - `update` (dict): the update operations to apply """ self.__bulk.add_update(self.__selector, update, multi=True) def replace_one(self, replacement): """Replace one entire document matching the selector criteria. :Parameters: - `replacement` (dict): the replacement document """ self.__bulk.add_replace(self.__selector, replacement) def remove_one(self): """Remove a single document matching the selector criteria. """ self.__bulk.add_delete(self.__selector, _DELETE_ONE) def remove(self): """Remove all documents matching the selector criteria. """ self.__bulk.add_delete(self.__selector, _DELETE_ALL) def upsert(self): """Specify that all chained update operations should be upserts. :Returns: - A :class:`BulkUpsertOperation` instance, used to add update operations to this bulk operation. """ return BulkUpsertOperation(self.__selector, self.__bulk) class BulkOperationBuilder(object): """An interface for executing a batch of write operations. """ __slots__ = '__bulk' def __init__(self, collection, ordered=True, bypass_document_validation=False): """Initialize a new BulkOperationBuilder instance. :Parameters: - `collection`: A :class:`~pymongo.collection.Collection` instance. - `ordered` (optional): If ``True`` all operations will be executed serially, in the order provided, and the entire execution will abort on the first error. If ``False`` operations will be executed in arbitrary order (possibly in parallel on the server), reporting any errors that occurred after attempting all operations. Defaults to ``True``. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support """ self.__bulk = _Bulk(collection, ordered, bypass_document_validation) def find(self, selector): """Specify selection criteria for bulk operations. :Parameters: - `selector` (dict): the selection criteria for update and remove operations. :Returns: - A :class:`BulkWriteOperation` instance, used to add update and remove operations to this bulk operation. """ validate_is_mapping("selector", selector) return BulkWriteOperation(selector, self.__bulk) def insert(self, document): """Insert a single document. :Parameters: - `document` (dict): the document to insert .. seealso:: :ref:`writes-and-ids` """ self.__bulk.add_insert(document) def execute(self, write_concern=None): """Execute all provided operations. :Parameters: - write_concern (optional): the write concern for this bulk execution. """ if write_concern is not None: validate_is_mapping("write_concern", write_concern) return self.__bulk.execute(write_concern) pymongo-3.2/pymongo/thread_util.py0000644000175000017500000000756712630145074021343 0ustar behackettbehackett00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for multi-threading support.""" import threading try: from time import monotonic as _time except ImportError: from time import time as _time from pymongo.monotonic import time as _time from pymongo.errors import ExceededMaxWaiters ### Begin backport from CPython 3.2 for timeout support for Semaphore.acquire class Semaphore: # After Tim Peters' semaphore class, but not quite the same (no maximum) def __init__(self, value=1): if value < 0: raise ValueError("semaphore initial value must be >= 0") self._cond = threading.Condition(threading.Lock()) self._value = value def acquire(self, blocking=True, timeout=None): if not blocking and timeout is not None: raise ValueError("can't specify timeout for non-blocking acquire") rc = False endtime = None self._cond.acquire() while self._value == 0: if not blocking: break if timeout is not None: if endtime is None: endtime = _time() + timeout else: timeout = endtime - _time() if timeout <= 0: break self._cond.wait(timeout) else: self._value = self._value - 1 rc = True self._cond.release() return rc __enter__ = acquire def release(self): self._cond.acquire() self._value = self._value + 1 self._cond.notify() self._cond.release() def __exit__(self, t, v, tb): self.release() @property def counter(self): return self._value class BoundedSemaphore(Semaphore): """Semaphore that checks that # releases is <= # acquires""" def __init__(self, value=1): Semaphore.__init__(self, value) self._initial_value = value def release(self): if self._value >= self._initial_value: raise ValueError("Semaphore released too many times") return Semaphore.release(self) ### End backport from CPython 3.2 class DummySemaphore(object): def __init__(self, value=None): pass def acquire(self, blocking=True, timeout=None): return True def release(self): pass class MaxWaitersBoundedSemaphore(object): def __init__(self, semaphore_class, value=1, max_waiters=1): self.waiter_semaphore = semaphore_class(max_waiters) self.semaphore = semaphore_class(value) def acquire(self, blocking=True, timeout=None): if not self.waiter_semaphore.acquire(False): raise ExceededMaxWaiters() try: return self.semaphore.acquire(blocking, timeout) finally: self.waiter_semaphore.release() def __getattr__(self, name): return getattr(self.semaphore, name) class MaxWaitersBoundedSemaphoreThread(MaxWaitersBoundedSemaphore): def __init__(self, value=1, max_waiters=1): MaxWaitersBoundedSemaphore.__init__( self, BoundedSemaphore, value, max_waiters) def create_semaphore(max_size, max_waiters): if max_size is None: return DummySemaphore() else: if max_waiters is None: return BoundedSemaphore(max_size) else: return MaxWaitersBoundedSemaphoreThread(max_size, max_waiters) pymongo-3.2/pymongo/operations.py0000644000175000017500000002141012630145074021201 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Operation class definitions.""" from pymongo.common import validate_boolean, validate_is_mapping from pymongo.helpers import _gen_index_name, _index_document, _index_list class _WriteOp(object): """Private base class for all write operations.""" __slots__ = ("_filter", "_doc", "_upsert") def __init__(self, filter=None, doc=None, upsert=None): if filter is not None: validate_is_mapping("filter", filter) if upsert is not None: validate_boolean("upsert", upsert) self._filter = filter self._doc = doc self._upsert = upsert def __eq__(self, other): if type(other) == type(self): return (other._filter, other._doc, other._upsert) == \ (self._filter, self._doc, self._upsert) return NotImplemented def __ne__(self, other): return not self == other class InsertOne(_WriteOp): """Represents an insert_one operation.""" def __init__(self, document): """Create an InsertOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `document`: The document to insert. If the document is missing an _id field one will be added. """ super(InsertOne, self).__init__(doc=document) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_insert(self._doc) def __repr__(self): return "InsertOne(%r)" % (self._doc,) class DeleteOne(_WriteOp): """Represents a delete_one operation.""" def __init__(self, filter): """Create a DeleteOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the document to delete. """ super(DeleteOne, self).__init__(filter) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_delete(self._filter, 1) def __repr__(self): return "DeleteOne(%r)" % (self._filter,) class DeleteMany(_WriteOp): """Represents a delete_many operation.""" def __init__(self, filter): """Create a DeleteMany instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the documents to delete. """ super(DeleteMany, self).__init__(filter) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_delete(self._filter, 0) def __repr__(self): return "DeleteMany(%r)" % (self._filter,) class ReplaceOne(_WriteOp): """Represents a replace_one operation.""" def __init__(self, filter, replacement, upsert=False): """Create a ReplaceOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The new document. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. """ super(ReplaceOne, self).__init__(filter, replacement, upsert) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_replace(self._filter, self._doc, self._upsert) def __repr__(self): return "ReplaceOne(%r, %r, %r)" % (self._filter, self._doc, self._upsert) class UpdateOne(_WriteOp): """Represents an update_one operation.""" def __init__(self, filter, update, upsert=False): """Represents an update_one operation. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the document to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. """ super(UpdateOne, self).__init__(filter, update, upsert) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_update(self._filter, self._doc, False, self._upsert) def __repr__(self): return "UpdateOne(%r, %r, %r)" % (self._filter, self._doc, self._upsert) class UpdateMany(_WriteOp): """Represents an update_many operation.""" def __init__(self, filter, update, upsert=False): """Create an UpdateMany instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. :Parameters: - `filter`: A query that matches the documents to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. """ super(UpdateMany, self).__init__(filter, update, upsert) def _add_to_bulk(self, bulkobj): """Add this operation to the _Bulk instance `bulkobj`.""" bulkobj.add_update(self._filter, self._doc, True, self._upsert) def __repr__(self): return "UpdateMany(%r, %r, %r)" % (self._filter, self._doc, self._upsert) class IndexModel(object): """Represents an index to create.""" __slots__ = ("__document",) def __init__(self, keys, **kwargs): """Create an Index instance. For use with :meth:`~pymongo.collection.Collection.create_indexes`. Takes either a single key or a list of (key, direction) pairs. The key(s) must be an instance of :class:`basestring` (:class:`str` in python 3), and the direction(s) must be one of (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, :data:`~pymongo.TEXT`). Valid options include, but are not limited to: - `name`: custom name to use for this index - if none is given, a name will be generated. - `unique`: if ``True`` creates a uniqueness constraint on the index. - `background`: if ``True`` this index should be created in the background. - `sparse`: if ``True``, omit from the index any documents that lack the indexed field. - `bucketSize`: for use with geoHaystack indexes. Number of documents to group together within a certain proximity to a given longitude and latitude. - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` index. - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` index. - `expireAfterSeconds`: Used to create an expiring (TTL) collection. MongoDB will automatically delete documents from this collection after seconds. The indexed field must be a UTC datetime or the data will not expire. - `partialFilterExpression`: A document that specifies a filter for a partial index. See the MongoDB documentation for a full list of supported options by server version. .. note:: `partialFilterExpression` requires server version **>= 3.2** :Parameters: - `keys`: a single key or a list of (key, direction) pairs specifying the index to create - `**kwargs` (optional): any additional index creation options (see the above list) should be passed as keyword arguments .. versionchanged:: 3.2 Added partialFilterExpression to support partial indexes. """ keys = _index_list(keys) if "name" not in kwargs: kwargs["name"] = _gen_index_name(keys) kwargs["key"] = _index_document(keys) self.__document = kwargs @property def document(self): """An index document suitable for passing to the createIndexes command. """ return self.__document pymongo-3.2/pymongo/monitor.py0000644000175000017500000001320212630145074020505 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Class to monitor a MongoDB server on a background thread.""" import weakref from bson.codec_options import DEFAULT_CODEC_OPTIONS from pymongo import common, helpers, message, periodic_executor from pymongo.server_type import SERVER_TYPE from pymongo.ismaster import IsMaster from pymongo.monotonic import time as _time from pymongo.read_preferences import MovingAverage from pymongo.server_description import ServerDescription class Monitor(object): def __init__( self, server_description, topology, pool, topology_settings): """Class to monitor a MongoDB server on a background thread. Pass an initial ServerDescription, a Topology, a Pool, and TopologySettings. The Topology is weakly referenced. The Pool must be exclusive to this Monitor. """ self._server_description = server_description self._pool = pool self._settings = topology_settings self._avg_round_trip_time = MovingAverage() # We strongly reference the executor and it weakly references us via # this closure. When the monitor is freed, stop the executor soon. def target(): monitor = self_ref() if monitor is None: return False # Stop the executor. Monitor._run(monitor) return True executor = periodic_executor.PeriodicExecutor( interval=common.HEARTBEAT_FREQUENCY, min_interval=common.MIN_HEARTBEAT_INTERVAL, target=target, name="pymongo_server_monitor_thread") self._executor = executor # Avoid cycles. When self or topology is freed, stop executor soon. self_ref = weakref.ref(self, executor.close) self._topology = weakref.proxy(topology, executor.close) def open(self): """Start monitoring, or restart after a fork. Multiple calls have no effect. """ self._executor.open() def close(self): """Close and stop monitoring. open() restarts the monitor after closing. """ self._executor.close() # Increment the pool_id and maybe close the socket. If the executor # thread has the socket checked out, it will be closed when checked in. self._pool.reset() def join(self, timeout=None): self._executor.join(timeout) def request_check(self): """If the monitor is sleeping, wake and check the server soon.""" self._executor.wake() def _run(self): try: self._server_description = self._check_with_retry() self._topology.on_change(self._server_description) except ReferenceError: # Topology was garbage-collected. self.close() def _check_with_retry(self): """Call ismaster once or twice. Reset server's pool on error. Returns a ServerDescription. """ # According to the spec, if an ismaster call fails we reset the # server's pool. If a server was once connected, change its type # to Unknown only after retrying once. address = self._server_description.address retry = self._server_description.server_type != SERVER_TYPE.Unknown try: return self._check_once() except ReferenceError: raise except Exception as error: self._topology.reset_pool(address) default = ServerDescription(address, error=error) if not retry: self._avg_round_trip_time.reset() # Server type defaults to Unknown. return default # Try a second and final time. If it fails return original error. try: return self._check_once() except ReferenceError: raise except Exception: self._avg_round_trip_time.reset() return default def _check_once(self): """A single attempt to call ismaster. Returns a ServerDescription, or raises an exception. """ with self._pool.get_socket({}) as sock_info: response, round_trip_time = self._check_with_socket(sock_info) self._avg_round_trip_time.add_sample(round_trip_time) sd = ServerDescription( address=self._server_description.address, ismaster=response, round_trip_time=self._avg_round_trip_time.get()) return sd def _check_with_socket(self, sock_info): """Return (IsMaster, round_trip_time). Can raise ConnectionFailure or OperationFailure. """ start = _time() request_id, msg, max_doc_size = message.query( 0, 'admin.$cmd', 0, -1, {'ismaster': 1}, None, DEFAULT_CODEC_OPTIONS) # TODO: use sock_info.command() sock_info.send_message(msg, max_doc_size) raw_response = sock_info.receive_message(1, request_id) result = helpers._unpack_response(raw_response) return IsMaster(result['data'][0]), _time() - start pymongo-3.2/pymongo/uri_parser.py0000644000175000017500000003006012630145074021172 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools to parse and validate a MongoDB URI.""" import warnings from bson.py3compat import PY3, iteritems, string_type if PY3: from urllib.parse import unquote_plus else: from urllib import unquote_plus from pymongo.common import (validate as _validate, get_validated_options) from pymongo.errors import ConfigurationError, InvalidURI SCHEME = 'mongodb://' SCHEME_LEN = len(SCHEME) DEFAULT_PORT = 27017 def _partition(entity, sep): """Python2.4 doesn't have a partition method so we provide our own that mimics str.partition from later releases. Split the string at the first occurrence of sep, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. """ parts = entity.split(sep, 1) if len(parts) == 2: return parts[0], sep, parts[1] else: return entity, '', '' def _rpartition(entity, sep): """Python2.4 doesn't have an rpartition method so we provide our own that mimics str.rpartition from later releases. Split the string at the last occurrence of sep, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing two empty strings, followed by the string itself. """ idx = entity.rfind(sep) if idx == -1: return '', '', entity return entity[:idx], sep, entity[idx + 1:] def parse_userinfo(userinfo): """Validates the format of user information in a MongoDB URI. Reserved characters like ':', '/', '+' and '@' must be escaped following RFC 2396. Returns a 2-tuple containing the unescaped username followed by the unescaped password. :Paramaters: - `userinfo`: A string of the form : .. versionchanged:: 2.2 Now uses `urllib.unquote_plus` so `+` characters must be escaped. """ if '@' in userinfo or userinfo.count(':') > 1: raise InvalidURI("':' or '@' characters in a username or password " "must be escaped according to RFC 2396.") user, _, passwd = _partition(userinfo, ":") # No password is expected with GSSAPI authentication. if not user: raise InvalidURI("The empty string is not valid username.") user = unquote_plus(user) passwd = unquote_plus(passwd) return user, passwd def parse_ipv6_literal_host(entity, default_port): """Validates an IPv6 literal host:port string. Returns a 2-tuple of IPv6 literal followed by port where port is default_port if it wasn't specified in entity. :Parameters: - `entity`: A string that represents an IPv6 literal enclosed in braces (e.g. '[::1]' or '[::1]:27017'). - `default_port`: The port number to use when one wasn't specified in entity. """ if entity.find(']') == -1: raise ValueError("an IPv6 address literal must be " "enclosed in '[' and ']' according " "to RFC 2732.") i = entity.find(']:') if i == -1: return entity[1:-1], default_port return entity[1: i], entity[i + 2:] def parse_host(entity, default_port=DEFAULT_PORT): """Validates a host string Returns a 2-tuple of host followed by port where port is default_port if it wasn't specified in the string. :Parameters: - `entity`: A host or host:port string where host could be a hostname or IP address. - `default_port`: The port number to use when one wasn't specified in entity. """ host = entity port = default_port if entity[0] == '[': host, port = parse_ipv6_literal_host(entity, default_port) elif entity.endswith(".sock"): return entity, default_port elif entity.find(':') != -1: if entity.count(':') > 1: raise ValueError("Reserved characters such as ':' must be " "escaped according RFC 2396. An IPv6 " "address literal must be enclosed in '[' " "and ']' according to RFC 2732.") host, port = host.split(':', 1) if isinstance(port, string_type): if not port.isdigit() or int(port) > 65535 or int(port) <= 0: raise ValueError("Port must be an integer between 0 and 65535: %s" % (port,)) port = int(port) # Normalize hostname to lowercase, since DNS is case-insensitive: # http://tools.ietf.org/html/rfc4343 # This prevents useless rediscovery if "foo.com" is in the seed list but # "FOO.com" is in the ismaster response. return host.lower(), port def validate_options(opts, warn=False): """Validates and normalizes options passed in a MongoDB URI. Returns a new dictionary of validated and normalized options. If warn is False then errors will be thrown for invalid options, otherwise they will be ignored and a warning will be issued. :Parameters: - `opts`: A dict of MongoDB URI options. - `warn` (optional): If ``True`` then warnigns will be logged and invalid options will be ignored. Otherwise invalid options will cause errors. """ if warn: return get_validated_options(opts) else: return dict([_validate(opt, val) for opt, val in iteritems(opts)]) def _parse_options(opts, delim): """Helper method for split_options which creates the options dict. Also handles the creation of a list for the URI tag_sets/ readpreferencetags portion.""" options = {} for opt in opts.split(delim): key, val = opt.split("=") if key.lower() == 'readpreferencetags': options.setdefault('readpreferencetags', []).append(val) else: # str(option) to ensure that a unicode URI results in plain 'str' # option names. 'normalized' is then suitable to be passed as # kwargs in all Python versions. if str(key) in options: warnings.warn("Duplicate URI option %s" % (str(key),)) options[str(key)] = unquote_plus(val) # Special case for deprecated options if "wtimeout" in options: if "wtimeoutMS" in options: options.pop("wtimeout") warnings.warn("Option wtimeout is deprecated, use 'wtimeoutMS'" " instead") return options def split_options(opts, validate=True, warn=False): """Takes the options portion of a MongoDB URI, validates each option and returns the options in a dictionary. :Parameters: - `opt`: A string representing MongoDB URI options. - `validate`: If ``True`` (the default), validate and normalize all options. """ and_idx = opts.find("&") semi_idx = opts.find(";") try: if and_idx >= 0 and semi_idx >= 0: raise InvalidURI("Can not mix '&' and ';' for option separators.") elif and_idx >= 0: options = _parse_options(opts, "&") elif semi_idx >= 0: options = _parse_options(opts, ";") elif opts.find("=") != -1: options = _parse_options(opts, None) else: raise ValueError except ValueError: raise InvalidURI("MongoDB URI options are key=value pairs.") if validate: return validate_options(options, warn) return options def split_hosts(hosts, default_port=DEFAULT_PORT): """Takes a string of the form host1[:port],host2[:port]... and splits it into (host, port) tuples. If [:port] isn't present the default_port is used. Returns a set of 2-tuples containing the host name (or IP) followed by port number. :Parameters: - `hosts`: A string of the form host1[:port],host2[:port],... - `default_port`: The port number to use when one wasn't specified for a host. """ nodes = [] for entity in hosts.split(','): if not entity: raise ConfigurationError("Empty host " "(or extra comma in host list).") port = default_port # Unix socket entities don't have ports if entity.endswith('.sock'): port = None nodes.append(parse_host(entity, port)) return nodes def parse_uri(uri, default_port=DEFAULT_PORT, validate=True, warn=False): """Parse and validate a MongoDB URI. Returns a dict of the form:: { 'nodelist': , 'username': or None, 'password': or None, 'database': or None, 'collection': or None, 'options': } :Parameters: - `uri`: The MongoDB URI to parse. - `default_port`: The port number to use when one wasn't specified for a host in the URI. - `validate`: If ``True`` (the default), validate and normalize all options. - `warn` (optional): When validating, if ``True`` then will warn the user then ignore any invalid options or values. If ``False``, validation will error when options are unsupported or values are invalid. .. versionchanged:: 3.1 ``warn`` added so invalid options can be ignored. """ if not uri.startswith(SCHEME): raise InvalidURI("Invalid URI scheme: URI " "must begin with '%s'" % (SCHEME,)) scheme_free = uri[SCHEME_LEN:] if not scheme_free: raise InvalidURI("Must provide at least one hostname or IP.") user = None passwd = None dbase = None collection = None options = {} # Check for unix domain sockets in the uri if '.sock' in scheme_free: host_part, _, path_part = _rpartition(scheme_free, '/') if not host_part: host_part = path_part path_part = "" if '/' in host_part: raise InvalidURI("Any '/' in a unix domain socket must be" " URL encoded: %s" % host_part) host_part = unquote_plus(host_part) path_part = unquote_plus(path_part) else: host_part, _, path_part = _partition(scheme_free, '/') if not path_part and '?' in host_part: raise InvalidURI("A '/' is required between " "the host list and any options.") if '@' in host_part: userinfo, _, hosts = _rpartition(host_part, '@') user, passwd = parse_userinfo(userinfo) else: hosts = host_part nodes = split_hosts(hosts, default_port=default_port) if path_part: if path_part[0] == '?': opts = path_part[1:] else: dbase, _, opts = _partition(path_part, '?') if '.' in dbase: dbase, collection = dbase.split('.', 1) if opts: options = split_options(opts, validate, warn) if dbase is not None: dbase = unquote_plus(dbase) if collection is not None: collection = unquote_plus(collection) return { 'nodelist': nodes, 'username': user, 'password': passwd, 'database': dbase, 'collection': collection, 'options': options } if __name__ == '__main__': import pprint import sys try: pprint.pprint(parse_uri(sys.argv[1])) except InvalidURI as e: print(e) sys.exit(0) pymongo-3.2/pymongo/server_selectors.py0000644000175000017500000001063612630145074022417 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Criteria to select some ServerDescriptions out of a list.""" from pymongo.server_type import SERVER_TYPE def any_server_selector(server_descriptions): return server_descriptions def readable_server_selector(server_descriptions): return [s for s in server_descriptions if s.is_readable] def writable_server_selector(server_descriptions): return [s for s in server_descriptions if s.is_writable] def secondary_server_selector(server_descriptions): return [s for s in server_descriptions if s.server_type == SERVER_TYPE.RSSecondary] def arbiter_server_selector(server_descriptions): return [s for s in server_descriptions if s.server_type == SERVER_TYPE.RSArbiter] def writable_preferred_server_selector(server_descriptions): """Like PrimaryPreferred but doesn't use tags or latency.""" return ( writable_server_selector(server_descriptions) or secondary_server_selector(server_descriptions)) def single_tag_set_server_selector(tag_set, server_descriptions): """All servers matching one tag set. A tag set is a dict. A server matches if its tags are a superset: A server tagged {'a': '1', 'b': '2'} matches the tag set {'a': '1'}. The empty tag set {} matches any server. The `server_descriptions` passed to this function should have non-readable servers (e.g. RSGhost, RSArbiter, Unknown) filtered out (e.g. by readable_server_selector or secondary_server_selector) first. """ def tags_match(server_tags): for key, value in tag_set.items(): if key not in server_tags or server_tags[key] != value: return False return True return [s for s in server_descriptions if tags_match(s.tags)] def tag_sets_server_selector(tag_sets, server_descriptions): """All servers match a list of tag sets. tag_sets is a list of dicts. The empty tag set {} matches any server, and may be provided at the end of the list as a fallback. So [{'a': 'value'}, {}] expresses a preference for servers tagged {'a': 'value'}, but accepts any server if none matches the first preference. The `server_descriptions` passed to this function should have non-readable servers (e.g. RSGhost, RSArbiter, Unknown) filtered out (e.g. by readable_server_selector or secondary_server_selector) first. """ for tag_set in tag_sets: selected = single_tag_set_server_selector(tag_set, server_descriptions) if selected: return selected return [] def apply_local_threshold(latency_ms, server_descriptions): """All servers with round trip times within latency_ms of the fastest one. No ServerDescription's round_trip_time can be None. The `server_descriptions` passed to this function should have non-readable servers (e.g. RSGhost, RSArbiter, Unknown) filtered out (e.g. by readable_server_selector or secondary_server_selector) first. """ if not server_descriptions: # Avoid ValueError from min() with empty sequence. return [] # round_trip_time is in seconds. if any(s for s in server_descriptions if s.round_trip_time is None): raise ValueError("Not all servers' round trip times are known") fastest = min(s.round_trip_time for s in server_descriptions) return [ s for s in server_descriptions if (s.round_trip_time - fastest) <= latency_ms / 1000.] def secondary_with_tags_server_selector(tag_sets, server_descriptions): """All near-enough secondaries matching the tag sets.""" return tag_sets_server_selector( tag_sets, secondary_server_selector(server_descriptions)) def member_with_tags_server_selector(tag_sets, server_descriptions): """All near-enough members matching the tag sets.""" return tag_sets_server_selector( tag_sets, readable_server_selector(server_descriptions)) pymongo-3.2/pymongo/pool.py0000644000175000017500000005576712630145074020015 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. import contextlib import os import socket import threading from bson import DEFAULT_CODEC_OPTIONS from bson.py3compat import u, itervalues from pymongo import auth, helpers, thread_util from pymongo.errors import (AutoReconnect, ConnectionFailure, ConfigurationError, DocumentTooLarge, NetworkTimeout, NotMasterError, OperationFailure) from pymongo.ismaster import IsMaster from pymongo.monotonic import time as _time from pymongo.network import (command, receive_message, socket_closed) from pymongo.read_concern import DEFAULT_READ_CONCERN from pymongo.read_preferences import ReadPreference from pymongo.server_type import SERVER_TYPE # If the first getaddrinfo call of this interpreter's life is on a thread, # while the main thread holds the import lock, getaddrinfo deadlocks trying # to import the IDNA codec. Import it here, where presumably we're on the # main thread, to avoid the deadlock. See PYTHON-607. u('foo').encode('idna') try: from ssl import match_hostname, CertificateError except ImportError: # These don't require the ssl module from pymongo.ssl_match_hostname import match_hostname, CertificateError def _raise_connection_failure(address, error): """Convert a socket.error to ConnectionFailure and raise it.""" host, port = address # If connecting to a Unix socket, port will be None. if port is not None: msg = '%s:%d: %s' % (host, port, error) else: msg = '%s: %s' % (host, error) if isinstance(error, socket.timeout): raise NetworkTimeout(msg) else: raise AutoReconnect(msg) class PoolOptions(object): __slots__ = ('__max_pool_size', '__connect_timeout', '__socket_timeout', '__wait_queue_timeout', '__wait_queue_multiple', '__ssl_context', '__ssl_match_hostname', '__socket_keepalive', '__event_listeners') def __init__(self, max_pool_size=100, connect_timeout=None, socket_timeout=None, wait_queue_timeout=None, wait_queue_multiple=None, ssl_context=None, ssl_match_hostname=True, socket_keepalive=False, event_listeners=None): self.__max_pool_size = max_pool_size self.__connect_timeout = connect_timeout self.__socket_timeout = socket_timeout self.__wait_queue_timeout = wait_queue_timeout self.__wait_queue_multiple = wait_queue_multiple self.__ssl_context = ssl_context self.__ssl_match_hostname = ssl_match_hostname self.__socket_keepalive = socket_keepalive self.__event_listeners = event_listeners @property def max_pool_size(self): """The maximum number of connections that the pool will open simultaneously. If this is set, operations will block if there are `max_pool_size` outstanding connections. """ return self.__max_pool_size @property def connect_timeout(self): """How long a connection can take to be opened before timing out. """ return self.__connect_timeout @property def socket_timeout(self): """How long a send or receive on a socket can take before timing out. """ return self.__socket_timeout @property def wait_queue_timeout(self): """How long a thread will wait for a socket from the pool if the pool has no free sockets. """ return self.__wait_queue_timeout @property def wait_queue_multiple(self): """Multiplied by max_pool_size to give the number of threads allowed to wait for a socket at one time. """ return self.__wait_queue_multiple @property def ssl_context(self): """An SSLContext instance or None. """ return self.__ssl_context @property def ssl_match_hostname(self): """Call ssl.match_hostname if cert_reqs is not ssl.CERT_NONE. """ return self.__ssl_match_hostname @property def socket_keepalive(self): """Whether to send periodic messages to determine if a connection is closed. """ return self.__socket_keepalive @property def event_listeners(self): """An instance of pymongo.monitoring._EventListeners. """ return self.__event_listeners class SocketInfo(object): """Store a socket with some metadata. :Parameters: - `sock`: a raw socket object - `pool`: a Pool instance - `ismaster`: optional IsMaster instance, response to ismaster on `sock` - `address`: the server's (host, port) """ def __init__(self, sock, pool, ismaster, address): self.sock = sock self.address = address self.authset = set() self.closed = False self.last_checkout = _time() self.is_writable = ismaster.is_writable if ismaster else None self.max_wire_version = ismaster.max_wire_version if ismaster else None self.max_bson_size = ismaster.max_bson_size if ismaster else None self.max_message_size = ismaster.max_message_size if ismaster else None self.max_write_batch_size = ( ismaster.max_write_batch_size if ismaster else None) self.listeners = pool.opts.event_listeners if ismaster: self.is_mongos = ismaster.server_type == SERVER_TYPE.Mongos else: self.is_mongos = None # The pool's pool_id changes with each reset() so we can close sockets # created before the last reset. self.pool_id = pool.pool_id def command(self, dbname, spec, slave_ok=False, read_preference=ReadPreference.PRIMARY, codec_options=DEFAULT_CODEC_OPTIONS, check=True, allowable_errors=None, check_keys=False, read_concern=DEFAULT_READ_CONCERN): """Execute a command or raise ConnectionFailure or OperationFailure. :Parameters: - `dbname`: name of the database on which to run the command - `spec`: a command document as a dict, SON, or mapping object - `slave_ok`: whether to set the SlaveOkay wire protocol bit - `read_preference`: a read preference - `codec_options`: a CodecOptions instance - `check`: raise OperationFailure if there are errors - `allowable_errors`: errors to ignore if `check` is True - `check_keys`: if True, check `spec` for invalid keys - `read_concern`: The read concern for this command. """ if self.max_wire_version < 4 and not read_concern.ok_for_legacy: raise ConfigurationError( 'read concern level of %s is not valid ' 'with a max wire version of %d.' % (read_concern.level, self.max_wire_version)) try: return command(self.sock, dbname, spec, slave_ok, self.is_mongos, read_preference, codec_options, check, allowable_errors, self.address, check_keys, self.listeners, self.max_bson_size, read_concern) except OperationFailure: raise # Catch socket.error, KeyboardInterrupt, etc. and close ourselves. except BaseException as error: self._raise_connection_failure(error) def send_message(self, message, max_doc_size): """Send a raw BSON message or raise ConnectionFailure. If a network exception is raised, the socket is closed. """ if (self.max_bson_size is not None and max_doc_size > self.max_bson_size): raise DocumentTooLarge( "BSON document too large (%d bytes) - the connected server" "supports BSON document sizes up to %d bytes." % (max_doc_size, self.max_bson_size)) try: self.sock.sendall(message) except BaseException as error: self._raise_connection_failure(error) def receive_message(self, operation, request_id): """Receive a raw BSON message or raise ConnectionFailure. If any exception is raised, the socket is closed. """ try: return receive_message(self.sock, operation, request_id) except BaseException as error: self._raise_connection_failure(error) def legacy_write(self, request_id, msg, max_doc_size, with_last_error): """Send OP_INSERT, etc., optionally returning response as a dict. Can raise ConnectionFailure or OperationFailure. :Parameters: - `request_id`: an int. - `msg`: bytes, an OP_INSERT, OP_UPDATE, or OP_DELETE message, perhaps with a getlasterror command appended. - `max_doc_size`: size in bytes of the largest document in `msg`. - `with_last_error`: True if a getlasterror command is appended. """ if not with_last_error and not self.is_writable: # Write won't succeed, bail as if we'd done a getlasterror. raise NotMasterError("not master") self.send_message(msg, max_doc_size) if with_last_error: response = self.receive_message(1, request_id) return helpers._check_gle_response(response) def write_command(self, request_id, msg): """Send "insert" etc. command, returning response as a dict. Can raise ConnectionFailure or OperationFailure. :Parameters: - `request_id`: an int. - `msg`: bytes, the command message. """ self.send_message(msg, 0) response = helpers._unpack_response(self.receive_message(1, request_id)) assert response['number_returned'] == 1 result = response['data'][0] # Raises NotMasterError or OperationFailure. helpers._check_command_response(result) return result def check_auth(self, all_credentials): """Update this socket's authentication. Log in or out to bring this socket's credentials up to date with those provided. Can raise ConnectionFailure or OperationFailure. :Parameters: - `all_credentials`: dict, maps auth source to MongoCredential. """ if all_credentials or self.authset: cached = set(itervalues(all_credentials)) authset = self.authset.copy() # Logout any credentials that no longer exist in the cache. for credentials in authset - cached: auth.logout(credentials.source, self) self.authset.discard(credentials) for credentials in cached - authset: auth.authenticate(credentials, self) self.authset.add(credentials) def authenticate(self, credentials): """Log in to the server and store these credentials in `authset`. Can raise ConnectionFailure or OperationFailure. :Parameters: - `credentials`: A MongoCredential. """ auth.authenticate(credentials, self) self.authset.add(credentials) def close(self): self.closed = True # Avoid exceptions on interpreter shutdown. try: self.sock.close() except: pass def _raise_connection_failure(self, error): # Catch *all* exceptions from socket methods and close the socket. In # regular Python, socket operations only raise socket.error, even if # the underlying cause was a Ctrl-C: a signal raised during socket.recv # is expressed as an EINTR error from poll. See internal_select_ex() in # socketmodule.c. All error codes from poll become socket.error at # first. Eventually in PyEval_EvalFrameEx the interpreter checks for # signals and throws KeyboardInterrupt into the current frame on the # main thread. # # But in Gevent and Eventlet, the polling mechanism (epoll, kqueue, # ...) is called in Python code, which experiences the signal as a # KeyboardInterrupt from the start, rather than as an initial # socket.error, so we catch that, close the socket, and reraise it. self.close() if isinstance(error, socket.error): _raise_connection_failure(self.address, error) else: raise error def __eq__(self, other): return self.sock == other.sock def __ne__(self, other): return not self == other def __hash__(self): return hash(self.sock) def __repr__(self): return "SocketInfo(%s)%s at %s" % ( repr(self.sock), self.closed and " CLOSED" or "", id(self) ) def _create_connection(address, options): """Given (host, port) and PoolOptions, connect and return a socket object. Can raise socket.error. This is a modified version of create_connection from CPython >= 2.6. """ host, port = address # Check if dealing with a unix domain socket if host.endswith('.sock'): if not hasattr(socket, "AF_UNIX"): raise ConnectionFailure("UNIX-sockets are not supported " "on this system") sock = socket.socket(socket.AF_UNIX) try: sock.connect(host) return sock except socket.error: sock.close() raise # Don't try IPv6 if we don't support it. Also skip it if host # is 'localhost' (::1 is fine). Avoids slow connect issues # like PYTHON-356. family = socket.AF_INET if socket.has_ipv6 and host != 'localhost': family = socket.AF_UNSPEC err = None for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): af, socktype, proto, dummy, sa = res sock = socket.socket(af, socktype, proto) try: sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.settimeout(options.connect_timeout) sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, options.socket_keepalive) sock.connect(sa) return sock except socket.error as e: err = e sock.close() if err is not None: raise err else: # This likely means we tried to connect to an IPv6 only # host with an OS/kernel or Python interpreter that doesn't # support IPv6. The test case is Jython2.5.1 which doesn't # support IPv6 at all. raise socket.error('getaddrinfo failed') def _configured_socket(address, options): """Given (host, port) and PoolOptions, return a configured socket. Can raise socket.error, ConnectionFailure, or CertificateError. Sets socket's SSL and timeout options. """ sock = _create_connection(address, options) ssl_context = options.ssl_context if ssl_context is not None: try: sock = ssl_context.wrap_socket(sock) except IOError as exc: sock.close() raise ConnectionFailure("SSL handshake failed: %s" % (str(exc),)) if ssl_context.verify_mode and options.ssl_match_hostname: try: match_hostname(sock.getpeercert(), hostname=address[0]) except CertificateError: sock.close() raise sock.settimeout(options.socket_timeout) return sock # Do *not* explicitly inherit from object or Jython won't call __del__ # http://bugs.jython.org/issue1057 class Pool: def __init__(self, address, options, handshake=True): """ :Parameters: - `address`: a (hostname, port) tuple - `options`: a PoolOptions instance - `handshake`: whether to call ismaster for each new SocketInfo """ # Check a socket's health with socket_closed() every once in a while. # Can override for testing: 0 to always check, None to never check. self._check_interval_seconds = 1 self.sockets = set() self.lock = threading.Lock() # Keep track of resets, so we notice sockets created before the most # recent reset and close them. self.pool_id = 0 self.pid = os.getpid() self.address = address self.opts = options self.handshake = handshake if (self.opts.wait_queue_multiple is None or self.opts.max_pool_size is None): max_waiters = None else: max_waiters = ( self.opts.max_pool_size * self.opts.wait_queue_multiple) self._socket_semaphore = thread_util.create_semaphore( self.opts.max_pool_size, max_waiters) def reset(self): with self.lock: self.pool_id += 1 self.pid = os.getpid() sockets, self.sockets = self.sockets, set() for sock_info in sockets: sock_info.close() def connect(self): """Connect to Mongo and return a new SocketInfo. Can raise ConnectionFailure or CertificateError. Note that the pool does not keep a reference to the socket -- you must call return_socket() when you're done with it. """ sock = None try: sock = _configured_socket(self.address, self.opts) if self.handshake: ismaster = IsMaster(command(sock, 'admin', {'ismaster': 1}, False, False, ReadPreference.PRIMARY, DEFAULT_CODEC_OPTIONS)) else: ismaster = None return SocketInfo(sock, self, ismaster, self.address) except socket.error as error: if sock is not None: sock.close() _raise_connection_failure(self.address, error) @contextlib.contextmanager def get_socket(self, all_credentials, checkout=False): """Get a socket from the pool. Use with a "with" statement. Returns a :class:`SocketInfo` object wrapping a connected :class:`socket.socket`. This method should always be used in a with-statement:: with pool.get_socket(credentials, checkout) as socket_info: socket_info.send_message(msg) data = socket_info.receive_message(op_code, request_id) The socket is logged in or out as needed to match ``all_credentials`` using the correct authentication mechanism for the server's wire protocol version. Can raise ConnectionFailure or OperationFailure. :Parameters: - `all_credentials`: dict, maps auth source to MongoCredential. - `checkout` (optional): keep socket checked out. """ # First get a socket, then attempt authentication. Simplifies # semaphore management in the face of network errors during auth. sock_info = self._get_socket_no_auth() try: sock_info.check_auth(all_credentials) yield sock_info except: # Exception in caller. Decrement semaphore. self.return_socket(sock_info) raise else: if not checkout: self.return_socket(sock_info) def _get_socket_no_auth(self): """Get or create a SocketInfo. Can raise ConnectionFailure.""" # We use the pid here to avoid issues with fork / multiprocessing. # See test.test_client:TestClient.test_fork for an example of # what could go wrong otherwise if self.pid != os.getpid(): self.reset() # Get a free socket or create one. if not self._socket_semaphore.acquire( True, self.opts.wait_queue_timeout): self._raise_wait_queue_timeout() # We've now acquired the semaphore and must release it on error. try: try: # set.pop() isn't atomic in Jython less than 2.7, see # http://bugs.jython.org/issue1854 with self.lock: sock_info, from_pool = self.sockets.pop(), True except KeyError: # Can raise ConnectionFailure or CertificateError. sock_info, from_pool = self.connect(), False if from_pool: # Can raise ConnectionFailure. sock_info = self._check(sock_info) except: self._socket_semaphore.release() raise sock_info.last_checkout = _time() return sock_info def return_socket(self, sock_info): """Return the socket to the pool, or if it's closed discard it.""" if self.pid != os.getpid(): self.reset() else: if sock_info.pool_id != self.pool_id: sock_info.close() elif not sock_info.closed: with self.lock: self.sockets.add(sock_info) self._socket_semaphore.release() def _check(self, sock_info): """This side-effecty function checks if this pool has been reset since the last time this socket was used, or if the socket has been closed by some external network error, and if so, attempts to create a new socket. If this connection attempt fails we reset the pool and reraise the ConnectionFailure. Checking sockets lets us avoid seeing *some* :class:`~pymongo.errors.AutoReconnect` exceptions on server hiccups, etc. We only do this if it's been > 1 second since the last socket checkout, to keep performance reasonable - we can't avoid AutoReconnects completely anyway. """ error = False # How long since socket was last checked out. age = _time() - sock_info.last_checkout if (self._check_interval_seconds is not None and ( 0 == self._check_interval_seconds or age > self._check_interval_seconds)): if socket_closed(sock_info.sock): sock_info.close() error = True if not error: return sock_info else: return self.connect() def _raise_wait_queue_timeout(self): raise ConnectionFailure( 'Timed out waiting for socket from pool with max_size %r and' ' wait_queue_timeout %r' % ( self.opts.max_pool_size, self.opts.wait_queue_timeout)) def __del__(self): # Avoid ResourceWarnings in Python 3 for sock_info in self.sockets: sock_info.close() pymongo-3.2/pymongo/errors.py0000644000175000017500000001425512630145074020343 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Exceptions raised by PyMongo.""" from bson.errors import * try: from ssl import CertificateError except ImportError: from pymongo.ssl_match_hostname import CertificateError class PyMongoError(Exception): """Base class for all PyMongo exceptions.""" class ConnectionFailure(PyMongoError): """Raised when a connection to the database cannot be made or is lost.""" class AutoReconnect(ConnectionFailure): """Raised when a connection to the database is lost and an attempt to auto-reconnect will be made. In order to auto-reconnect you must handle this exception, recognizing that the operation which caused it has not necessarily succeeded. Future operations will attempt to open a new connection to the database (and will continue to raise this exception until the first successful connection is made). Subclass of :exc:`~pymongo.errors.ConnectionFailure`. """ def __init__(self, message='', errors=None): self.errors = self.details = errors or [] ConnectionFailure.__init__(self, message) class NetworkTimeout(AutoReconnect): """An operation on an open connection exceeded socketTimeoutMS. The remaining connections in the pool stay open. In the case of a write operation, you cannot know whether it succeeded or failed. Subclass of :exc:`~pymongo.errors.AutoReconnect`. """ class NotMasterError(AutoReconnect): """The server responded "not master" or "node is recovering". These errors result from a query, write, or command. The operation failed because the client thought it was using the primary but the primary has stepped down, or the client thought it was using a healthy secondary but the secondary is stale and trying to recover. The client launches a refresh operation on a background thread, to update its view of the server as soon as possible after throwing this exception. Subclass of :exc:`~pymongo.errors.AutoReconnect`. """ class ServerSelectionTimeoutError(AutoReconnect): """Thrown when no MongoDB server is available for an operation If there is no suitable server for an operation PyMongo tries for ``serverSelectionTimeoutMS`` (default 30 seconds) to find one, then throws this exception. For example, it is thrown after attempting an operation when PyMongo cannot connect to any server, or if you attempt an insert into a replica set that has no primary and does not elect one within the timeout window, or if you attempt to query with a Read Preference that the replica set cannot satisfy. """ class ConfigurationError(PyMongoError): """Raised when something is incorrectly configured. """ class OperationFailure(PyMongoError): """Raised when a database operation fails. .. versionadded:: 2.7 The :attr:`details` attribute. """ def __init__(self, error, code=None, details=None): self.__code = code self.__details = details PyMongoError.__init__(self, error) @property def code(self): """The error code returned by the server, if any. """ return self.__code @property def details(self): """The complete error document returned by the server. Depending on the error that occurred, the error document may include useful information beyond just the error message. When connected to a mongos the error document may contain one or more subdocuments if errors occurred on multiple shards. """ return self.__details class CursorNotFound(OperationFailure): """Raised while iterating query results if the cursor is invalidated on the server. .. versionadded:: 2.7 """ class ExecutionTimeout(OperationFailure): """Raised when a database operation times out, exceeding the $maxTimeMS set in the query or command option. .. note:: Requires server version **>= 2.6.0** .. versionadded:: 2.7 """ class WriteConcernError(OperationFailure): """Base exception type for errors raised due to write concern. .. versionadded:: 3.0 """ class WriteError(OperationFailure): """Base exception type for errors raised during write operations. .. versionadded:: 3.0 """ class WTimeoutError(WriteConcernError): """Raised when a database operation times out (i.e. wtimeout expires) before replication completes. With newer versions of MongoDB the `details` attribute may include write concern fields like 'n', 'updatedExisting', or 'writtenTo'. .. versionadded:: 2.7 """ class DuplicateKeyError(WriteError): """Raised when an insert or update fails due to a duplicate key error.""" class BulkWriteError(OperationFailure): """Exception class for bulk write errors. .. versionadded:: 2.7 """ def __init__(self, results): OperationFailure.__init__( self, "batch op errors occurred", 65, results) class InvalidOperation(PyMongoError): """Raised when a client attempts to perform an invalid operation.""" class InvalidName(PyMongoError): """Raised when an invalid name is used.""" class CollectionInvalid(PyMongoError): """Raised when collection validation fails.""" class InvalidURI(ConfigurationError): """Raised when trying to parse an invalid mongodb URI.""" class ExceededMaxWaiters(Exception): """Raised when a thread tries to get a connection from a pool and ``maxPoolSize * waitQueueMultiple`` threads are already waiting. .. versionadded:: 2.6 """ pass class DocumentTooLarge(InvalidDocument): """Raised when an encoded document is too large for the connected server. """ pass pymongo-3.2/pymongo/mongo_replica_set_client.py0000644000175000017500000000364312630145074024055 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Deprecated. See :doc:`/examples/high_availability`.""" import warnings from pymongo import mongo_client class MongoReplicaSetClient(mongo_client.MongoClient): """Deprecated alias for :class:`~pymongo.mongo_client.MongoClient`. :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` will be removed in a future version of PyMongo. .. versionchanged:: 3.0 :class:`~pymongo.mongo_client.MongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. The ``refresh`` method is removed from :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`, as are the ``seeds`` and ``hosts`` properties. """ def __init__(self, *args, **kwargs): warnings.warn('MongoReplicaSetClient is deprecated, use MongoClient' ' to connect to a replica set', DeprecationWarning, stacklevel=2) super(MongoReplicaSetClient, self).__init__(*args, **kwargs) def __repr__(self): return "MongoReplicaSetClient(%s)" % (self._repr_helper(),) pymongo-3.2/pymongo/database.py0000644000175000017500000013343212630145074020572 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Database level operations.""" import warnings from bson.code import Code from bson.codec_options import CodecOptions, DEFAULT_CODEC_OPTIONS from bson.dbref import DBRef from bson.objectid import ObjectId from bson.py3compat import iteritems, string_type, _unicode from bson.son import SON from pymongo import auth, common, helpers from pymongo.collection import Collection from pymongo.command_cursor import CommandCursor from pymongo.errors import (CollectionInvalid, ConfigurationError, InvalidName, OperationFailure) from pymongo.helpers import _first_batch from pymongo.read_preferences import ReadPreference from pymongo.son_manipulator import SONManipulator from pymongo.write_concern import WriteConcern def _check_name(name): """Check if a database name is valid. """ if not name: raise InvalidName("database name cannot be the empty string") for invalid_char in [" ", ".", "$", "/", "\\", "\x00"]: if invalid_char in name: raise InvalidName("database names cannot contain the " "character %r" % invalid_char) class Database(common.BaseObject): """A Mongo database. """ def __init__(self, client, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a database by client and name. Raises :class:`TypeError` if `name` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidName` if `name` is not a valid database name. :Parameters: - `client`: A :class:`~pymongo.mongo_client.MongoClient` instance. - `name`: The database name. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) client.codec_options is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) client.read_preference is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) client.write_concern is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) client.read_concern is used. .. mongodoc:: databases .. versionchanged:: 3.2 Added the read_concern option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. :class:`~pymongo.database.Database` no longer returns an instance of :class:`~pymongo.collection.Collection` for attribute names with leading underscores. You must use dict-style lookups instead:: db['__my_collection__'] Not: db.__my_collection__ """ super(Database, self).__init__( codec_options or client.codec_options, read_preference or client.read_preference, write_concern or client.write_concern, read_concern or client.read_concern) if not isinstance(name, string_type): raise TypeError("name must be an instance " "of %s" % (string_type.__name__,)) if name != '$external': _check_name(name) self.__name = _unicode(name) self.__client = client self.__incoming_manipulators = [] self.__incoming_copying_manipulators = [] self.__outgoing_manipulators = [] self.__outgoing_copying_manipulators = [] def add_son_manipulator(self, manipulator): """Add a new son manipulator to this database. **DEPRECATED** - `add_son_manipulator` is deprecated. .. versionchanged:: 3.0 Deprecated add_son_manipulator. """ warnings.warn("add_son_manipulator is deprecated", DeprecationWarning, stacklevel=2) base = SONManipulator() def method_overwritten(instance, method): """Test if this method has been overridden.""" return (getattr( instance, method).__func__ != getattr(base, method).__func__) if manipulator.will_copy(): if method_overwritten(manipulator, "transform_incoming"): self.__incoming_copying_manipulators.insert(0, manipulator) if method_overwritten(manipulator, "transform_outgoing"): self.__outgoing_copying_manipulators.insert(0, manipulator) else: if method_overwritten(manipulator, "transform_incoming"): self.__incoming_manipulators.insert(0, manipulator) if method_overwritten(manipulator, "transform_outgoing"): self.__outgoing_manipulators.insert(0, manipulator) @property def system_js(self): """A :class:`SystemJS` helper for this :class:`Database`. See the documentation for :class:`SystemJS` for more details. """ return SystemJS(self) @property def client(self): """The client instance for this :class:`Database`.""" return self.__client @property def name(self): """The name of this :class:`Database`.""" return self.__name @property def incoming_manipulators(self): """All incoming SON manipulators installed on this instance. .. versionadded:: 2.0 """ return [manipulator.__class__.__name__ for manipulator in self.__incoming_manipulators] @property def incoming_copying_manipulators(self): """All incoming SON copying manipulators installed on this instance. .. versionadded:: 2.0 """ return [manipulator.__class__.__name__ for manipulator in self.__incoming_copying_manipulators] @property def outgoing_manipulators(self): """List all outgoing SON manipulators installed on this instance. .. versionadded:: 2.0 """ return [manipulator.__class__.__name__ for manipulator in self.__outgoing_manipulators] @property def outgoing_copying_manipulators(self): """List all outgoing SON copying manipulators installed on this instance. .. versionadded:: 2.0 """ return [manipulator.__class__.__name__ for manipulator in self.__outgoing_copying_manipulators] def __eq__(self, other): if isinstance(other, Database): return (self.__client == other.client and self.__name == other.name) return NotImplemented def __ne__(self, other): return not self == other def __repr__(self): return "Database(%r, %r)" % (self.__client, self.__name) def __getattr__(self, name): """Get a collection of this database by name. Raises InvalidName if an invalid collection name is used. :Parameters: - `name`: the name of the collection to get """ if name.startswith('_'): raise AttributeError( "Database has no attribute %r. To access the %s" " collection, use database[%r]." % (name, name, name)) return self.__getitem__(name) def __getitem__(self, name): """Get a collection of this database by name. Raises InvalidName if an invalid collection name is used. :Parameters: - `name`: the name of the collection to get """ return Collection(self, name) def get_collection(self, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a :class:`~pymongo.collection.Collection` with the given name and options. Useful for creating a :class:`~pymongo.collection.Collection` with different codec options, read preference, and/or write concern from this :class:`Database`. >>> db.read_preference Primary() >>> coll1 = db.test >>> coll1.read_preference Primary() >>> from pymongo import ReadPreference >>> coll2 = db.get_collection( ... 'test', read_preference=ReadPreference.SECONDARY) >>> coll2.read_preference Secondary(tag_sets=None) :Parameters: - `name`: The name of the collection - a string. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Database` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Database` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Database` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Database` is used. """ return Collection( self, name, False, codec_options, read_preference, write_concern, read_concern) def _collection_default_options(self, name, **kargs): """Get a Collection instance with the default settings.""" wc = (self.write_concern if self.write_concern.acknowledged else WriteConcern()) return self.get_collection( name, codec_options=DEFAULT_CODEC_OPTIONS, read_preference=ReadPreference.PRIMARY, write_concern=wc) def create_collection(self, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None, **kwargs): """Create a new :class:`~pymongo.collection.Collection` in this database. Normally collection creation is automatic. This method should only be used to specify options on creation. :class:`~pymongo.errors.CollectionInvalid` will be raised if the collection already exists. Options should be passed as keyword arguments to this method. Supported options vary with MongoDB release. Some examples include: - "size": desired initial size for the collection (in bytes). For capped collections this size is the max size of the collection. - "capped": if True, this is a capped collection - "max": maximum number of objects if capped (optional) See the MongoDB documentation for a full list of supported options by server version. :Parameters: - `name`: the name of the collection to create - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Database` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Database` is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Database` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Database` is used. - `**kwargs` (optional): additional keyword arguments will be passed as options for the create collection command .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. .. versionchanged:: 2.2 Removed deprecated argument: options """ if name in self.collection_names(): raise CollectionInvalid("collection %s already exists" % name) return Collection(self, name, True, codec_options, read_preference, write_concern, read_concern, **kwargs) def _apply_incoming_manipulators(self, son, collection): """Apply incoming manipulators to `son`.""" for manipulator in self.__incoming_manipulators: son = manipulator.transform_incoming(son, collection) return son def _apply_incoming_copying_manipulators(self, son, collection): """Apply incoming copying manipulators to `son`.""" for manipulator in self.__incoming_copying_manipulators: son = manipulator.transform_incoming(son, collection) return son def _fix_incoming(self, son, collection): """Apply manipulators to an incoming SON object before it gets stored. :Parameters: - `son`: the son object going into the database - `collection`: the collection the son object is being saved in """ son = self._apply_incoming_manipulators(son, collection) son = self._apply_incoming_copying_manipulators(son, collection) return son def _fix_outgoing(self, son, collection): """Apply manipulators to a SON object as it comes out of the database. :Parameters: - `son`: the son object coming out of the database - `collection`: the collection the son object was saved in """ for manipulator in reversed(self.__outgoing_manipulators): son = manipulator.transform_outgoing(son, collection) for manipulator in reversed(self.__outgoing_copying_manipulators): son = manipulator.transform_outgoing(son, collection) return son def _command(self, sock_info, command, slave_ok=False, value=1, check=True, allowable_errors=None, read_preference=ReadPreference.PRIMARY, codec_options=DEFAULT_CODEC_OPTIONS, **kwargs): """Internal command helper.""" if isinstance(command, string_type): command = SON([(command, value)]) command.update(kwargs) return sock_info.command(self.__name, command, slave_ok, read_preference, codec_options, check, allowable_errors) def command(self, command, value=1, check=True, allowable_errors=None, read_preference=ReadPreference.PRIMARY, codec_options=DEFAULT_CODEC_OPTIONS, **kwargs): """Issue a MongoDB command. Send command `command` to the database and return the response. If `command` is an instance of :class:`basestring` (:class:`str` in python 3) then the command {`command`: `value`} will be sent. Otherwise, `command` must be an instance of :class:`dict` and will be sent as is. Any additional keyword arguments will be added to the final command document before it is sent. For example, a command like ``{buildinfo: 1}`` can be sent using: >>> db.command("buildinfo") For a command where the value matters, like ``{collstats: collection_name}`` we can do: >>> db.command("collstats", collection_name) For commands that take additional arguments we can use kwargs. So ``{filemd5: object_id, root: file_root}`` becomes: >>> db.command("filemd5", object_id, root=file_root) :Parameters: - `command`: document representing the command to be issued, or the name of the command (for simple commands only). .. note:: the order of keys in the `command` document is significant (the "verb" must come first), so commands which require multiple keys (e.g. `findandmodify`) should use an instance of :class:`~bson.son.SON` or a string and kwargs instead of a Python `dict`. - `value` (optional): value to use for the command verb when `command` is passed as a string - `check` (optional): check the response for errors, raising :class:`~pymongo.errors.OperationFailure` if there are any - `allowable_errors`: if `check` is ``True``, error messages in this list will be ignored by error-checking - `read_preference`: The read preference for this operation. See :mod:`~pymongo.read_preferences` for options. - `codec_options`: A :class:`~bson.codec_options.CodecOptions` instance. - `**kwargs` (optional): additional keyword arguments will be added to the command document before it is sent .. note:: :meth:`command` does **not** obey :attr:`read_preference` or :attr:`codec_options`. You must use the `read_preference` and `codec_options` parameters instead. .. versionchanged:: 3.0 Removed the `as_class`, `fields`, `uuid_subtype`, `tag_sets`, and `secondary_acceptable_latency_ms` option. Removed `compile_re` option: PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. Added the `codec_options` parameter. .. versionchanged:: 2.7 Added `compile_re` option. If set to False, PyMongo represented BSON regular expressions as :class:`~bson.regex.Regex` objects instead of attempting to compile BSON regular expressions as Python native regular expressions, thus preventing errors for some incompatible patterns, see `PYTHON-500`_. .. versionchanged:: 2.3 Added `tag_sets` and `secondary_acceptable_latency_ms` options. .. versionchanged:: 2.2 Added support for `as_class` - the class you want to use for the resulting documents .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 .. mongodoc:: commands """ client = self.__client with client._socket_for_reads(read_preference) as (sock_info, slave_ok): return self._command(sock_info, command, slave_ok, value, check, allowable_errors, read_preference, codec_options, **kwargs) def _list_collections(self, sock_info, slave_okay, criteria=None): """Internal listCollections helper.""" criteria = criteria or {} cmd = SON([("listCollections", 1), ("cursor", {})]) if criteria: cmd["filter"] = criteria if sock_info.max_wire_version > 2: coll = self["$cmd"] cursor = self._command(sock_info, cmd, slave_okay)["cursor"] return CommandCursor(coll, cursor, sock_info.address) else: coll = self["system.namespaces"] res = _first_batch(sock_info, coll.database.name, coll.name, criteria, 0, slave_okay, CodecOptions(), ReadPreference.PRIMARY, cmd, self.client._event_listeners) data = res["data"] cursor = { "id": res["cursor_id"], "firstBatch": data, "ns": coll.full_name, } # Need to tell the cursor how many docs were in the first batch. return CommandCursor(coll, cursor, sock_info.address, len(data)) def collection_names(self, include_system_collections=True): """Get a list of all the collection names in this database. :Parameters: - `include_system_collections` (optional): if ``False`` list will not include system collections (e.g ``system.indexes``) """ with self.__client._socket_for_reads( ReadPreference.PRIMARY) as (sock_info, slave_okay): wire_version = sock_info.max_wire_version results = self._list_collections(sock_info, slave_okay) # Iterating the cursor to completion may require a socket for getmore. # Ensure we do that outside the "with" block so we don't require more # than one socket at a time. names = [result["name"] for result in results] if wire_version <= 2: # MongoDB 2.4 and older return index namespaces and collection # namespaces prefixed with the database name. names = [n[len(self.__name) + 1:] for n in names if n.startswith(self.__name + ".") and "$" not in n] if not include_system_collections: names = [name for name in names if not name.startswith("system.")] return names def drop_collection(self, name_or_collection): """Drop a collection. :Parameters: - `name_or_collection`: the name of a collection to drop or the collection object itself """ name = name_or_collection if isinstance(name, Collection): name = name.name if not isinstance(name, string_type): raise TypeError("name_or_collection must be an " "instance of %s" % (string_type.__name__,)) self.__client._purge_index(self.__name, name) self.command("drop", _unicode(name), allowable_errors=["ns not found"]) def validate_collection(self, name_or_collection, scandata=False, full=False): """Validate a collection. Returns a dict of validation info. Raises CollectionInvalid if validation fails. With MongoDB < 1.9 the result dict will include a `result` key with a string value that represents the validation results. With MongoDB >= 1.9 the `result` key no longer exists and the results are split into individual fields in the result dict. :Parameters: - `name_or_collection`: A Collection object or the name of a collection to validate. - `scandata`: Do extra checks beyond checking the overall structure of the collection. - `full`: Have the server do a more thorough scan of the collection. Use with `scandata` for a thorough scan of the structure of the collection and the individual documents. Ignored in MongoDB versions before 1.9. """ name = name_or_collection if isinstance(name, Collection): name = name.name if not isinstance(name, string_type): raise TypeError("name_or_collection must be an instance of " "%s or Collection" % (string_type.__name__,)) result = self.command("validate", _unicode(name), scandata=scandata, full=full) valid = True # Pre 1.9 results if "result" in result: info = result["result"] if info.find("exception") != -1 or info.find("corrupt") != -1: raise CollectionInvalid("%s invalid: %s" % (name, info)) # Sharded results elif "raw" in result: for _, res in iteritems(result["raw"]): if "result" in res: info = res["result"] if (info.find("exception") != -1 or info.find("corrupt") != -1): raise CollectionInvalid("%s invalid: " "%s" % (name, info)) elif not res.get("valid", False): valid = False break # Post 1.9 non-sharded results. elif not result.get("valid", False): valid = False if not valid: raise CollectionInvalid("%s invalid: %r" % (name, result)) return result def current_op(self, include_all=False): """Get information on operations currently running. :Parameters: - `include_all` (optional): if ``True`` also list currently idle operations in the result """ cmd = SON([("currentOp", 1), ("$all", include_all)]) with self.__client._socket_for_writes() as sock_info: if sock_info.max_wire_version >= 4: return sock_info.command("admin", cmd) else: spec = {"$all": True} if include_all else {} x = helpers._first_batch(sock_info, "admin", "$cmd.sys.inprog", spec, -1, True, self.codec_options, ReadPreference.PRIMARY, cmd, self.client._event_listeners) return x.get('data', [None])[0] def profiling_level(self): """Get the database's current profiling level. Returns one of (:data:`~pymongo.OFF`, :data:`~pymongo.SLOW_ONLY`, :data:`~pymongo.ALL`). .. mongodoc:: profiling """ result = self.command("profile", -1) assert result["was"] >= 0 and result["was"] <= 2 return result["was"] def set_profiling_level(self, level, slow_ms=None): """Set the database's profiling level. :Parameters: - `level`: Specifies a profiling level, see list of possible values below. - `slow_ms`: Optionally modify the threshold for the profile to consider a query or operation. Even if the profiler is off queries slower than the `slow_ms` level will get written to the logs. Possible `level` values: +----------------------------+------------------------------------+ | Level | Setting | +============================+====================================+ | :data:`~pymongo.OFF` | Off. No profiling. | +----------------------------+------------------------------------+ | :data:`~pymongo.SLOW_ONLY` | On. Only includes slow operations. | +----------------------------+------------------------------------+ | :data:`~pymongo.ALL` | On. Includes all operations. | +----------------------------+------------------------------------+ Raises :class:`ValueError` if level is not one of (:data:`~pymongo.OFF`, :data:`~pymongo.SLOW_ONLY`, :data:`~pymongo.ALL`). .. mongodoc:: profiling """ if not isinstance(level, int) or level < 0 or level > 2: raise ValueError("level must be one of (OFF, SLOW_ONLY, ALL)") if slow_ms is not None and not isinstance(slow_ms, int): raise TypeError("slow_ms must be an integer") if slow_ms is not None: self.command("profile", level, slowms=slow_ms) else: self.command("profile", level) def profiling_info(self): """Returns a list containing current profiling information. .. mongodoc:: profiling """ return list(self["system.profile"].find()) def error(self): """**DEPRECATED**: Get the error if one occurred on the last operation. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("Database.error() is deprecated", DeprecationWarning, stacklevel=2) error = self.command("getlasterror") error_msg = error.get("err", "") if error_msg is None: return None if error_msg.startswith("not master"): # Reset primary server and request check, if another thread isn't # doing so already. primary = self.__client.primary if primary: self.__client._reset_server_and_request_check(primary) return error def last_status(self): """**DEPRECATED**: Get status information from the last operation. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. Returns a SON object with status information. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("last_status() is deprecated", DeprecationWarning, stacklevel=2) return self.command("getlasterror") def previous_error(self): """**DEPRECATED**: Get the most recent error on this database. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. Only returns errors that have occurred since the last call to :meth:`reset_error_history`. Returns None if no such errors have occurred. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("previous_error() is deprecated", DeprecationWarning, stacklevel=2) error = self.command("getpreverror") if error.get("err", 0) is None: return None return error def reset_error_history(self): """**DEPRECATED**: Reset the error history of this database. This method is obsolete: all MongoDB write operations (insert, update, remove, and so on) use the write concern ``w=1`` and report their errors by default. Calls to :meth:`previous_error` will only return errors that have occurred since the most recent call to this method. .. versionchanged:: 2.8 Deprecated. """ warnings.warn("reset_error_history() is deprecated", DeprecationWarning, stacklevel=2) self.command("reseterror") def __iter__(self): return self def __next__(self): raise TypeError("'Database' object is not iterable") next = __next__ def _default_role(self, read_only): """Return the default user role for this database.""" if self.name == "admin": if read_only: return "readAnyDatabase" else: return "root" else: if read_only: return "read" else: return "dbOwner" def _create_or_update_user( self, create, name, password, read_only, **kwargs): """Use a command to create (if create=True) or modify a user. """ opts = {} if read_only or (create and "roles" not in kwargs): warnings.warn("Creating a user with the read_only option " "or without roles is deprecated in MongoDB " ">= 2.6", DeprecationWarning) opts["roles"] = [self._default_role(read_only)] elif read_only: warnings.warn("The read_only option is deprecated in MongoDB " ">= 2.6, use 'roles' instead", DeprecationWarning) if password is not None: # We always salt and hash client side. if "digestPassword" in kwargs: raise ConfigurationError("The digestPassword option is not " "supported via add_user. Please use " "db.command('createUser', ...) " "instead for this option.") opts["pwd"] = auth._password_digest(name, password) opts["digestPassword"] = False # Don't send {} as writeConcern. if self.write_concern.acknowledged and self.write_concern.document: opts["writeConcern"] = self.write_concern.document opts.update(kwargs) if create: command_name = "createUser" else: command_name = "updateUser" self.command(command_name, name, **opts) def _legacy_add_user(self, name, password, read_only, **kwargs): """Uses v1 system to add users, i.e. saving to system.users. """ # Use a Collection with the default codec_options. system_users = self._collection_default_options('system.users') user = system_users.find_one({"user": name}) or {"user": name} if password is not None: user["pwd"] = auth._password_digest(name, password) if read_only is not None: user["readOnly"] = read_only user.update(kwargs) # We don't care what the _id is, only that it has one # for the replace_one call below. user.setdefault("_id", ObjectId()) try: system_users.replace_one({"_id": user["_id"]}, user, True) except OperationFailure as exc: # First admin user add fails gle in MongoDB >= 2.1.2 # See SERVER-4225 for more information. if 'login' in str(exc): pass # First admin user add fails gle from mongos 2.0.x # and 2.2.x. elif (exc.details and 'getlasterror' in exc.details.get('note', '')): pass else: raise def add_user(self, name, password=None, read_only=None, **kwargs): """Create user `name` with password `password`. Add a new user with permissions for this :class:`Database`. .. note:: Will change the password if user `name` already exists. :Parameters: - `name`: the name of the user to create - `password` (optional): the password of the user to create. Can not be used with the ``userSource`` argument. - `read_only` (optional): if ``True`` the user will be read only - `**kwargs` (optional): optional fields for the user document (e.g. ``userSource``, ``otherDBRoles``, or ``roles``). See ``_ for more information. .. note:: The use of optional keyword arguments like ``userSource``, ``otherDBRoles``, or ``roles`` requires MongoDB >= 2.4.0 .. versionchanged:: 2.5 Added kwargs support for optional fields introduced in MongoDB 2.4 .. versionchanged:: 2.2 Added support for read only users """ if not isinstance(name, string_type): raise TypeError("name must be an " "instance of %s" % (string_type.__name__,)) if password is not None: if not isinstance(password, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) if len(password) == 0: raise ValueError("password can't be empty") if read_only is not None: read_only = common.validate_boolean('read_only', read_only) if 'roles' in kwargs: raise ConfigurationError("Can not use " "read_only and roles together") try: uinfo = self.command("usersInfo", name) # Create the user if not found in uinfo, otherwise update one. self._create_or_update_user( (not uinfo["users"]), name, password, read_only, **kwargs) except OperationFailure as exc: # MongoDB >= 2.5.3 requires the use of commands to manage # users. if exc.code in common.COMMAND_NOT_FOUND_CODES: self._legacy_add_user(name, password, read_only, **kwargs) return # Unauthorized. Attempt to create the user in case of # localhost exception. elif exc.code == 13: self._create_or_update_user( True, name, password, read_only, **kwargs) else: raise def remove_user(self, name): """Remove user `name` from this :class:`Database`. User `name` will no longer have permissions to access this :class:`Database`. :Parameters: - `name`: the name of the user to remove """ try: cmd = SON([("dropUser", name)]) # Don't send {} as writeConcern. if self.write_concern.acknowledged and self.write_concern.document: cmd["writeConcern"] = self.write_concern.document self.command(cmd) except OperationFailure as exc: # See comment in add_user try / except above. if exc.code in common.COMMAND_NOT_FOUND_CODES: coll = self._collection_default_options('system.users') coll.delete_one({"user": name}) return raise def authenticate(self, name, password=None, source=None, mechanism='DEFAULT', **kwargs): """Authenticate to use this database. Authentication lasts for the life of the underlying client instance, or until :meth:`logout` is called. Raises :class:`TypeError` if (required) `name`, (optional) `password`, or (optional) `source` is not an instance of :class:`basestring` (:class:`str` in python 3). .. note:: - This method authenticates the current connection, and will also cause all new :class:`~socket.socket` connections in the underlying client instance to be authenticated automatically. - Authenticating more than once on the same database with different credentials is not supported. You must call :meth:`logout` before authenticating with new credentials. - When sharing a client instance between multiple threads, all threads will share the authentication. If you need different authentication profiles for different purposes you must use distinct client instances. :Parameters: - `name`: the name of the user to authenticate. - `password` (optional): the password of the user to authenticate. Not used with GSSAPI or MONGODB-X509 authentication. - `source` (optional): the database to authenticate on. If not specified the current database is used. - `mechanism` (optional): See :data:`~pymongo.auth.MECHANISMS` for options. By default, use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. - `authMechanismProperties` (optional): Used to specify authentication mechanism specific options. To specify the service name for GSSAPI authentication pass authMechanismProperties='SERVICE_NAME:' .. versionadded:: 2.8 Use SCRAM-SHA-1 with MongoDB 3.0 and later. .. versionchanged:: 2.5 Added the `source` and `mechanism` parameters. :meth:`authenticate` now raises a subclass of :class:`~pymongo.errors.PyMongoError` if authentication fails due to invalid credentials or configuration issues. .. mongodoc:: authenticate """ if not isinstance(name, string_type): raise TypeError("name must be an " "instance of %s" % (string_type.__name__,)) if password is not None and not isinstance(password, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) if source is not None and not isinstance(source, string_type): raise TypeError("source must be an " "instance of %s" % (string_type.__name__,)) common.validate_auth_mechanism('mechanism', mechanism) validated_options = {} for option, value in iteritems(kwargs): normalized, val = common.validate_auth_option(option, value) validated_options[normalized] = val credentials = auth._build_credentials_tuple( mechanism, source or self.name, name, password, validated_options) self.client._cache_credentials( self.name, credentials, connect=True) return True def logout(self): """Deauthorize use of this database for this client instance.""" # Sockets will be deauthenticated as they are used. self.client._purge_credentials(self.name) def dereference(self, dbref, **kwargs): """Dereference a :class:`~bson.dbref.DBRef`, getting the document it points to. Raises :class:`TypeError` if `dbref` is not an instance of :class:`~bson.dbref.DBRef`. Returns a document, or ``None`` if the reference does not point to a valid document. Raises :class:`ValueError` if `dbref` has a database specified that is different from the current database. :Parameters: - `dbref`: the reference - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`~pymongo.collection.Collection.find`. """ if not isinstance(dbref, DBRef): raise TypeError("cannot dereference a %s" % type(dbref)) if dbref.database is not None and dbref.database != self.__name: raise ValueError("trying to dereference a DBRef that points to " "another database (%r not %r)" % (dbref.database, self.__name)) return self[dbref.collection].find_one({"_id": dbref.id}, **kwargs) def eval(self, code, *args): """Evaluate a JavaScript expression in MongoDB. Useful if you need to touch a lot of data lightly; in such a scenario the network transfer of the data could be a bottleneck. The `code` argument must be a JavaScript function. Additional positional arguments will be passed to that function when it is run on the server. Raises :class:`TypeError` if `code` is not an instance of :class:`basestring` (:class:`str` in python 3) or `Code`. Raises :class:`~pymongo.errors.OperationFailure` if the eval fails. Returns the result of the evaluation. :Parameters: - `code`: string representation of JavaScript code to be evaluated - `args` (optional): additional positional arguments are passed to the `code` being evaluated .. warning:: the eval command is deprecated in MongoDB 3.0 and will be removed in a future server version. """ if not isinstance(code, Code): code = Code(code) result = self.command("$eval", code, args=args) return result.get("retval", None) def __call__(self, *args, **kwargs): """This is only here so that some API misusages are easier to debug. """ raise TypeError("'Database' object is not callable. If you meant to " "call the '%s' method on a '%s' object it is " "failing because no such method exists." % ( self.__name, self.__client.__class__.__name__)) class SystemJS(object): """Helper class for dealing with stored JavaScript. """ def __init__(self, database): """Get a system js helper for the database `database`. An instance of :class:`SystemJS` can be created with an instance of :class:`Database` through :attr:`Database.system_js`, manual instantiation of this class should not be necessary. :class:`SystemJS` instances allow for easy manipulation and access to server-side JavaScript: .. doctest:: >>> db.system_js.add1 = "function (x) { return x + 1; }" >>> db.system.js.find({"_id": "add1"}).count() 1 >>> db.system_js.add1(5) 6.0 >>> del db.system_js.add1 >>> db.system.js.find({"_id": "add1"}).count() 0 """ if not database.write_concern.acknowledged: database = database.client.get_database( database.name, write_concern=WriteConcern()) # can't just assign it since we've overridden __setattr__ object.__setattr__(self, "_db", database) def __setattr__(self, name, code): self._db.system.js.replace_one( {"_id": name}, {"_id": name, "value": Code(code)}, True) def __setitem__(self, name, code): self.__setattr__(name, code) def __delattr__(self, name): self._db.system.js.delete_one({"_id": name}) def __delitem__(self, name): self.__delattr__(name) def __getattr__(self, name): return lambda *args: self._db.eval(Code("function() { " "return this[name].apply(" "this, arguments); }", scope={'name': name}), *args) def __getitem__(self, name): return self.__getattr__(name) def list(self): """Get a list of the names of the functions stored in this database.""" return [x["_id"] for x in self._db.system.js.find(projection=["_id"])] pymongo-3.2/pymongo/server.py0000644000175000017500000001441612630145074020334 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Communicate with one MongoDB server in a topology.""" import contextlib from datetime import datetime from pymongo.errors import ConfigurationError from pymongo.message import _Query, _convert_exception from pymongo.response import Response, ExhaustResponse from pymongo.server_type import SERVER_TYPE class Server(object): def __init__(self, server_description, pool, monitor): """Represent one MongoDB server.""" self._description = server_description self._pool = pool self._monitor = monitor def open(self): """Start monitoring, or restart after a fork. Multiple calls have no effect. """ self._monitor.open() def reset(self): """Clear the connection pool.""" self.pool.reset() def close(self): """Clear the connection pool and stop the monitor. Reconnect with open(). """ self._monitor.close() self._pool.reset() def request_check(self): """Check the server's state soon.""" self._monitor.request_check() def send_message(self, message, all_credentials): """Send an unacknowledged message to MongoDB. Can raise ConnectionFailure. :Parameters: - `message`: (request_id, data). - `all_credentials`: dict, maps auth source to MongoCredential. """ _, data, max_doc_size = self._split_message(message) with self.get_socket(all_credentials) as sock_info: sock_info.send_message(data, max_doc_size) def send_message_with_response( self, operation, set_slave_okay, all_credentials, listeners, exhaust=False): """Send a message to MongoDB and return a Response object. Can raise ConnectionFailure. :Parameters: - `operation`: A _Query or _GetMore object. - `set_slave_okay`: Pass to operation.get_message. - `all_credentials`: dict, maps auth source to MongoCredential. - `exhaust` (optional): If True, the socket used stays checked out. It is returned along with its Pool in the Response. """ with self.get_socket(all_credentials, exhaust) as sock_info: duration = None publish = listeners.enabled_for_commands if publish: start = datetime.now() use_find_cmd = False if sock_info.max_wire_version >= 4: if not exhaust: use_find_cmd = True elif (isinstance(operation, _Query) and not operation.read_concern.ok_for_legacy): raise ConfigurationError( 'read concern level of %s is not valid ' 'with a max wire version of %d.' % (operation.read_concern.level, sock_info.max_wire_version)) message = operation.get_message( set_slave_okay, sock_info.is_mongos, use_find_cmd) request_id, data, max_doc_size = self._split_message(message) if publish: encoding_duration = datetime.now() - start cmd, dbn = operation.as_command() listeners.publish_command_start( cmd, dbn, request_id, sock_info.address) start = datetime.now() try: sock_info.send_message(data, max_doc_size) response_data = sock_info.receive_message(1, request_id) except Exception as exc: if publish: duration = (datetime.now() - start) + encoding_duration failure = _convert_exception(exc) listeners.publish_command_failure( duration, failure, next(iter(cmd)), request_id, sock_info.address) raise if publish: duration = (datetime.now() - start) + encoding_duration if exhaust: return ExhaustResponse( data=response_data, address=self._description.address, socket_info=sock_info, pool=self._pool, duration=duration, request_id=request_id, from_command=use_find_cmd) else: return Response( data=response_data, address=self._description.address, duration=duration, request_id=request_id, from_command=use_find_cmd) @contextlib.contextmanager def get_socket(self, all_credentials, checkout=False): with self.pool.get_socket(all_credentials, checkout) as sock_info: yield sock_info @property def description(self): return self._description @description.setter def description(self, server_description): assert server_description.address == self._description.address self._description = server_description @property def pool(self): return self._pool def _split_message(self, message): """Return request_id, data, max_doc_size. :Parameters: - `message`: (request_id, data, max_doc_size) or (request_id, data) """ if len(message) == 3: return message else: # get_more and kill_cursors messages don't include BSON documents. request_id, data = message return request_id, data, 0 def __str__(self): d = self._description return '' % ( d.address[0], d.address[1], SERVER_TYPE._fields[d.server_type]) pymongo-3.2/pymongo/read_preferences.py0000644000175000017500000003076012630145074022322 0ustar behackettbehackett00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License", # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for choosing which member of a replica set to read from.""" from collections import Mapping from pymongo.errors import ConfigurationError from pymongo.server_selectors import (member_with_tags_server_selector, secondary_with_tags_server_selector, writable_server_selector) _PRIMARY = 0 _PRIMARY_PREFERRED = 1 _SECONDARY = 2 _SECONDARY_PREFERRED = 3 _NEAREST = 4 _MONGOS_MODES = ( 'primary', 'primaryPreferred', 'secondary', 'secondaryPreferred', 'nearest', ) def _validate_tag_sets(tag_sets): """Validate tag sets for a MongoReplicaSetClient. """ if tag_sets is None: return tag_sets if not isinstance(tag_sets, list): raise TypeError(( "Tag sets %r invalid, must be a list") % (tag_sets,)) if len(tag_sets) == 0: raise ValueError(( "Tag sets %r invalid, must be None or contain at least one set of" " tags") % (tag_sets,)) for tags in tag_sets: if not isinstance(tags, Mapping): raise TypeError( "Tag set %r invalid, must be an instance of dict, " "bson.son.SON or other type that inherits from " "collection.Mapping" % (tags,)) return tag_sets class _ServerMode(object): """Base class for all read preferences. """ __slots__ = ("__mongos_mode", "__mode", "__tag_sets") def __init__(self, mode, tag_sets=None): if mode == _PRIMARY and tag_sets is not None: raise ConfigurationError("Read preference primary " "cannot be combined with tags") self.__mongos_mode = _MONGOS_MODES[mode] self.__mode = mode self.__tag_sets = _validate_tag_sets(tag_sets) @property def name(self): """The name of this read preference. """ return self.__class__.__name__ @property def document(self): """Read preference as a document. """ if self.__tag_sets in (None, [{}]): return {'mode': self.__mongos_mode} return {'mode': self.__mongos_mode, 'tags': self.__tag_sets} @property def mode(self): """The mode of this read preference instance. """ return self.__mode @property def tag_sets(self): """Set ``tag_sets`` to a list of dictionaries like [{'dc': 'ny'}] to read only from members whose ``dc`` tag has the value ``"ny"``. To specify a priority-order for tag sets, provide a list of tag sets: ``[{'dc': 'ny'}, {'dc': 'la'}, {}]``. A final, empty tag set, ``{}``, means "read from any member that matches the mode, ignoring tags." MongoReplicaSetClient tries each set of tags in turn until it finds a set of tags with at least one matching member. .. seealso:: `Data-Center Awareness `_ """ return list(self.__tag_sets) if self.__tag_sets else [{}] def __repr__(self): return "%s(tag_sets=%r)" % ( self.name, self.__tag_sets) def __eq__(self, other): if isinstance(other, _ServerMode): return (self.mode == other.mode and self.tag_sets == other.tag_sets) return NotImplemented def __ne__(self, other): return not self == other def __getstate__(self): """Return value of object for pickling. Needed explicitly because __slots__() defined. """ return {'mode': self.__mode, 'tag_sets': self.__tag_sets} def __setstate__(self, value): """Restore from pickling.""" self.__mode = value['mode'] self.__mongos_mode = _MONGOS_MODES[self.__mode] self.__tag_sets = _validate_tag_sets(value['tag_sets']) class Primary(_ServerMode): """Primary read preference. * When directly connected to one mongod queries are allowed if the server is standalone or a replica set primary. * When connected to a mongos queries are sent to the primary of a shard. * When connected to a replica set queries are sent to the primary of the replica set. """ def __init__(self): super(Primary, self).__init__(_PRIMARY) def __call__(self, server_descriptions): """Return matching ServerDescriptions from a list.""" return writable_server_selector(server_descriptions) def __repr__(self): return "Primary()" def __eq__(self, other): if isinstance(other, _ServerMode): return other.mode == _PRIMARY return NotImplemented class PrimaryPreferred(_ServerMode): """PrimaryPreferred read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are sent to the primary of a shard if available, otherwise a shard secondary. * When connected to a replica set queries are sent to the primary if available, otherwise a secondary. :Parameters: - `tag_sets`: The :attr:`~tag_sets` to use if the primary is not available. """ def __init__(self, tag_sets=None): super(PrimaryPreferred, self).__init__(_PRIMARY_PREFERRED, tag_sets) def __call__(self, server_descriptions): """Return matching ServerDescriptions from a list.""" writable_servers = writable_server_selector(server_descriptions) if writable_servers: return writable_servers else: return secondary_with_tags_server_selector( self.tag_sets, server_descriptions) class Secondary(_ServerMode): """Secondary read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are distributed among shard secondaries. An error is raised if no secondaries are available. * When connected to a replica set queries are distributed among secondaries. An error is raised if no secondaries are available. :Parameters: - `tag_sets`: The :attr:`~tag_sets` to use with this read_preference """ def __init__(self, tag_sets=None): super(Secondary, self).__init__(_SECONDARY, tag_sets) def __call__(self, server_descriptions): """Return matching ServerDescriptions from a list.""" return secondary_with_tags_server_selector( self.tag_sets, server_descriptions) class SecondaryPreferred(_ServerMode): """SecondaryPreferred read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are distributed among shard secondaries, or the shard primary if no secondary is available. * When connected to a replica set queries are distributed among secondaries, or the primary if no secondary is available. :Parameters: - `tag_sets`: The :attr:`~tag_sets` to use with this read_preference """ def __init__(self, tag_sets=None): super(SecondaryPreferred, self).__init__(_SECONDARY_PREFERRED, tag_sets) def __call__(self, server_descriptions): """Return matching ServerDescriptions from a list.""" secondaries = secondary_with_tags_server_selector( self.tag_sets, server_descriptions) if secondaries: return secondaries else: return writable_server_selector(server_descriptions) class Nearest(_ServerMode): """Nearest read preference. * When directly connected to one mongod queries are allowed to standalone servers, to a replica set primary, or to replica set secondaries. * When connected to a mongos queries are distributed among all members of a shard. * When connected to a replica set queries are distributed among all members. :Parameters: - `tag_sets`: The :attr:`~tag_sets` to use with this read_preference """ def __init__(self, tag_sets=None): super(Nearest, self).__init__(_NEAREST, tag_sets) def __call__(self, server_descriptions): """Return matching ServerDescriptions from a list.""" return member_with_tags_server_selector( self.tag_sets or [{}], server_descriptions) _ALL_READ_PREFERENCES = (Primary, PrimaryPreferred, Secondary, SecondaryPreferred, Nearest) def make_read_preference(mode, tag_sets): if mode == _PRIMARY: if tag_sets not in (None, [{}]): raise ConfigurationError("Read preference primary " "cannot be combined with tags") return Primary() return _ALL_READ_PREFERENCES[mode](tag_sets) _MODES = ( 'PRIMARY', 'PRIMARY_PREFERRED', 'SECONDARY', 'SECONDARY_PREFERRED', 'NEAREST', ) class ReadPreference(object): """An enum that defines the read preference modes supported by PyMongo. See :doc:`/examples/high_availability` for code examples. A read preference is used in three cases: :class:`~pymongo.mongo_client.MongoClient` connected to a single mongod: - ``PRIMARY``: Queries are allowed if the server is standalone or a replica set primary. - All other modes allow queries to standalone servers, to a replica set primary, or to replica set secondaries. :class:`~pymongo.mongo_client.MongoClient` initialized with the ``replicaSet`` option: - ``PRIMARY``: Read from the primary. This is the default, and provides the strongest consistency. If no primary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``PRIMARY_PREFERRED``: Read from the primary if available, or if there is none, read from a secondary. - ``SECONDARY``: Read from a secondary. If no secondary is available, raise :class:`~pymongo.errors.AutoReconnect`. - ``SECONDARY_PREFERRED``: Read from a secondary if available, otherwise from the primary. - ``NEAREST``: Read from any member. :class:`~pymongo.mongo_client.MongoClient` connected to a mongos, with a sharded cluster of replica sets: - ``PRIMARY``: Read from the primary of the shard, or raise :class:`~pymongo.errors.OperationFailure` if there is none. This is the default. - ``PRIMARY_PREFERRED``: Read from the primary of the shard, or if there is none, read from a secondary of the shard. - ``SECONDARY``: Read from a secondary of the shard, or raise :class:`~pymongo.errors.OperationFailure` if there is none. - ``SECONDARY_PREFERRED``: Read from a secondary of the shard if available, otherwise from the shard primary. - ``NEAREST``: Read from any shard member. """ PRIMARY = Primary() PRIMARY_PREFERRED = PrimaryPreferred() SECONDARY = Secondary() SECONDARY_PREFERRED = SecondaryPreferred() NEAREST = Nearest() def read_pref_mode_from_name(name): """Get the read preference mode from mongos/uri name. """ return _MONGOS_MODES.index(name) class MovingAverage(object): """Tracks an exponentially-weighted moving average.""" def __init__(self): self.average = None def add_sample(self, sample): if sample < 0: # Likely system time change while waiting for ismaster response # and not using time.monotonic. Ignore it, the next one will # probably be valid. return if self.average is None: self.average = sample else: # The Server Selection Spec requires an exponentially weighted # average with alpha = 0.2. self.average = 0.8 * self.average + 0.2 * sample def get(self): """Get the calculated average, or None if no samples yet.""" return self.average def reset(self): self.average = None pymongo-3.2/pymongo/ssl_context.py0000644000175000017500000000711412630145074021370 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """A fake SSLContext implementation.""" try: import ssl except ImportError: pass class SSLContext(object): """A fake SSLContext. This implements an API similar to ssl.SSLContext from python 3.2 but does not implement methods or properties that would be incompatible with ssl.wrap_socket from python 2.6. You must pass protocol which must be one of the PROTOCOL_* constants defined in the ssl module. ssl.PROTOCOL_SSLv23 is recommended for maximum interoperability. """ __slots__ = ('_cafile', '_certfile', '_keyfile', '_protocol', '_verify_mode') def __init__(self, protocol): self._cafile = None self._certfile = None self._keyfile = None self._protocol = protocol self._verify_mode = ssl.CERT_NONE @property def protocol(self): """The protocol version chosen when constructing the context. This attribute is read-only. """ return self._protocol def __get_verify_mode(self): """Whether to try to verify other peers' certificates and how to behave if verification fails. This attribute must be one of ssl.CERT_NONE, ssl.CERT_OPTIONAL or ssl.CERT_REQUIRED. """ return self._verify_mode def __set_verify_mode(self, value): """Setter for verify_mode.""" self._verify_mode = value verify_mode = property(__get_verify_mode, __set_verify_mode) def load_cert_chain(self, certfile, keyfile=None): """Load a private key and the corresponding certificate. The certfile string must be the path to a single file in PEM format containing the certificate as well as any number of CA certificates needed to establish the certificate's authenticity. The keyfile string, if present, must point to a file containing the private key. Otherwise the private key will be taken from certfile as well. """ self._certfile = certfile self._keyfile = keyfile def load_verify_locations(self, cafile=None, dummy=None): """Load a set of "certification authority"(CA) certificates used to validate other peers' certificates when `~verify_mode` is other than ssl.CERT_NONE. """ self._cafile = cafile def wrap_socket(self, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, dummy=None): """Wrap an existing Python socket sock and return an ssl.SSLSocket object. """ return ssl.wrap_socket(sock, keyfile=self._keyfile, certfile=self._certfile, server_side=server_side, cert_reqs=self._verify_mode, ssl_version=self._protocol, ca_certs=self._cafile, do_handshake_on_connect=do_handshake_on_connect, suppress_ragged_eofs=suppress_ragged_eofs) pymongo-3.2/pymongo/ssl_match_hostname.py0000644000175000017500000000701712571426507022707 0ustar behackettbehackett00000000000000# Backport of the match_hostname logic introduced in python 3.2 # http://hg.python.org/releasing/3.3.5/file/993955b807b3/Lib/ssl.py import re class CertificateError(ValueError): pass def _dnsname_match(dn, hostname, max_wildcards=1): """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 """ pats = [] if not dn: return False parts = dn.split(r'.') leftmost = parts[0] remainder = parts[1:] wildcards = leftmost.count('*') if wildcards > max_wildcards: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a # reasonable choice. raise CertificateError( "too many wildcards in certificate DNS name: " + repr(dn)) # speed up common case w/o wildcards if not wildcards: return dn.lower() == hostname.lower() # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which # the wildcard character comprises a label other than the left-most label. if leftmost == '*': # When '*' is a fragment by itself, it matches a non-empty dotless # fragment. pats.append('[^.]+') elif leftmost.startswith('xn--') or hostname.startswith('xn--'): # RFC 6125, section 6.4.3, subitem 3. # The client SHOULD NOT attempt to match a presented identifier # where the wildcard character is embedded within an A-label or # U-label of an internationalized domain name. pats.append(re.escape(leftmost)) else: # Otherwise, '*' matches any dotless string, e.g. www* pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) # add the remaining fragments, ignore any wildcards for frag in remainder: pats.append(re.escape(frag)) pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) return pat.match(hostname) def match_hostname(cert, hostname): """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 rules are followed, but IP addresses are not accepted for *hostname*. CertificateError is raised on failure. On success, the function returns nothing. """ if not cert: raise ValueError("empty or no certificate") dnsnames = [] san = cert.get('subjectAltName', ()) for key, value in san: if key == 'DNS': if _dnsname_match(value, hostname): return dnsnames.append(value) if not dnsnames: # The subject is only checked when there is no dNSName entry # in subjectAltName for sub in cert.get('subject', ()): for key, value in sub: # XXX according to RFC 2818, the most specific Common Name # must be used. if key == 'commonName': if _dnsname_match(value, hostname): return dnsnames.append(value) if len(dnsnames) > 1: raise CertificateError("hostname %r " "doesn't match either of %s" % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: raise CertificateError("hostname %r " "doesn't match %r" % (hostname, dnsnames[0])) else: raise CertificateError("no appropriate commonName or " "subjectAltName fields were found") pymongo-3.2/pymongo/write_concern.py0000644000175000017500000001053412630145074021664 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with write concerns.""" from bson.py3compat import integer_types, string_type from pymongo.errors import ConfigurationError class WriteConcern(object): """WriteConcern :Parameters: - `w`: (integer or string) Used with replication, write operations will block until they have been replicated to the specified number or tagged set of servers. `w=` always includes the replica set primary (e.g. w=3 means write to the primary and wait until replicated to **two** secondaries). **w=0 disables acknowledgement of write operations and can not be used with other write concern options.** - `wtimeout`: (integer) Used in conjunction with `w`. Specify a value in milliseconds to control how long to wait for write propagation to complete. If replication does not complete in the given timeframe, a timeout exception is raised. - `j`: If ``True`` block until write operations have been committed to the journal. Cannot be used in combination with `fsync`. Prior to MongoDB 2.6 this option was ignored if the server was running without journaling. Starting with MongoDB 2.6 write operations will fail with an exception if this option is used when the server is running without journaling. - `fsync`: If ``True`` and the server is running without journaling, blocks until the server has synced all data files to disk. If the server is running with journaling, this acts the same as the `j` option, blocking until write operations have been committed to the journal. Cannot be used in combination with `j`. """ __slots__ = ("__document", "__acknowledged") def __init__(self, w=None, wtimeout=None, j=None, fsync=None): self.__document = {} self.__acknowledged = True if wtimeout is not None: if not isinstance(wtimeout, integer_types): raise TypeError("wtimeout must be an integer") self.__document["wtimeout"] = wtimeout if j is not None: if not isinstance(j, bool): raise TypeError("j must be True or False") self.__document["j"] = j if fsync is not None: if not isinstance(fsync, bool): raise TypeError("fsync must be True or False") if j and fsync: raise ConfigurationError("Can't set both j " "and fsync at the same time") self.__document["fsync"] = fsync if self.__document and w == 0: raise ConfigurationError("Can not use w value " "of 0 with other options") if w is not None: if isinstance(w, integer_types): self.__acknowledged = w > 0 elif not isinstance(w, string_type): raise TypeError("w must be an integer or string") self.__document["w"] = w @property def document(self): """The document representation of this write concern. .. note:: :class:`WriteConcern` is immutable. Mutating the value of :attr:`document` does not mutate this :class:`WriteConcern`. """ return self.__document.copy() @property def acknowledged(self): """If ``True`` write operations will wait for acknowledgement before returning. """ return self.__acknowledged def __repr__(self): return ("WriteConcern(%s)" % ( ", ".join("%s=%s" % kvt for kvt in self.document.items()),)) def __eq__(self, other): return self.document == other.document def __ne__(self, other): return self.document != other.document pymongo-3.2/pymongo/monotonic.py0000644000175000017500000000204712630145074021030 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Time. Monotonic if possible. """ __all__ = ['time'] try: # Patches standard time module. # From https://pypi.python.org/pypi/Monotime. import monotime except ImportError: pass try: # From https://pypi.python.org/pypi/monotinic. from monotonic import monotonic as time except ImportError: try: # Monotime or Python 3.3+. from time import monotonic as time except ImportError: # Not monotonic. from time import time pymongo-3.2/pymongo/cursor_manager.py0000644000175000017500000000347712630145074022042 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A manager to handle when cursors are killed after they are closed. New cursor managers should be defined as subclasses of CursorManager and can be installed on a client by calling :meth:`~pymongo.mongo_client.MongoClient.set_cursor_manager`. .. versionchanged:: 3.0 Undeprecated. :meth:`~pymongo.cursor_manager.CursorManager.close` now requires an `address` argument. The ``BatchCursorManager`` class is removed. """ import weakref from bson.py3compat import integer_types class CursorManager(object): """The cursor manager base class.""" def __init__(self, client): """Instantiate the manager. :Parameters: - `client`: a MongoClient """ self.__client = weakref.ref(client) def close(self, cursor_id, address): """Kill a cursor. Raises TypeError if cursor_id is not an instance of (int, long). :Parameters: - `cursor_id`: cursor id to close - `address`: the cursor's server's (host, port) pair .. versionchanged:: 3.0 Now requires an `address` argument. """ if not isinstance(cursor_id, integer_types): raise TypeError("cursor_id must be an integer") self.__client().kill_cursors([cursor_id], address) pymongo-3.2/pymongo/message.py0000644000175000017500000006522412630145074020455 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for creating `messages `_ to be sent to MongoDB. .. note:: This module is for internal use and is generally not needed by application developers. """ import datetime import random import struct import bson from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.py3compat import b, StringIO, u from bson.son import SON try: from pymongo import _cmessage _use_c = True except ImportError: _use_c = False from pymongo.errors import DocumentTooLarge, InvalidOperation, OperationFailure from pymongo.read_concern import DEFAULT_READ_CONCERN from pymongo.read_preferences import ReadPreference MAX_INT32 = 2147483647 MIN_INT32 = -2147483648 # Overhead allowed for encoded command documents. _COMMAND_OVERHEAD = 16382 _INSERT = 0 _UPDATE = 1 _DELETE = 2 _EMPTY = b'' _BSONOBJ = b'\x03' _ZERO_8 = b'\x00' _ZERO_16 = b'\x00\x00' _ZERO_32 = b'\x00\x00\x00\x00' _ZERO_64 = b'\x00\x00\x00\x00\x00\x00\x00\x00' _SKIPLIM = b'\x00\x00\x00\x00\xff\xff\xff\xff' _OP_MAP = { _INSERT: b'\x04documents\x00\x00\x00\x00\x00', _UPDATE: b'\x04updates\x00\x00\x00\x00\x00', _DELETE: b'\x04deletes\x00\x00\x00\x00\x00', } _UJOIN = u("%s.%s") def _randint(): """Generate a pseudo random 32 bit integer.""" return random.randint(MIN_INT32, MAX_INT32) def _maybe_add_read_preference(spec, read_preference): """Add $readPreference to spec when appropriate.""" mode = read_preference.mode tag_sets = read_preference.tag_sets # Only add $readPreference if it's something other than primary to avoid # problems with mongos versions that don't support read preferences. Also, # for maximum backwards compatibility, don't add $readPreference for # secondaryPreferred unless tags are in use (setting the slaveOkay bit # has the same effect). if mode and ( mode != ReadPreference.SECONDARY_PREFERRED.mode or tag_sets != [{}]): if "$query" not in spec: spec = SON([("$query", spec)]) spec["$readPreference"] = read_preference.document return spec def _convert_exception(exception): """Convert an Exception into a failure document for publishing.""" return {'errmsg': str(exception), 'errtype': exception.__class__.__name__} def _convert_write_result(operation, command, result): """Convert a legacy write result to write commmand format.""" # Based on _merge_legacy from bulk.py affected = result.get("n", 0) res = {"ok": 1, "n": affected} errmsg = result.get("errmsg", result.get("err", "")) if errmsg: # The write was successful on at least the primary so don't return. if result.get("wtimeout"): res["writeConcernError"] = {"errmsg": errmsg, "code": 64, "errInfo": {"wtimeout": True}} else: # The write failed. error = {"index": 0, "code": result.get("code", 8), "errmsg": errmsg} if "errInfo" in result: error["errInfo"] = result["errInfo"] res["writeErrors"] = [error] return res if operation == "insert": # GLE result for insert is always 0 in most MongoDB versions. res["n"] = len(command['documents']) elif operation == "update": if "upserted" in result: res["upserted"] = [{"index": 0, "_id": result["upserted"]}] # Versions of MongoDB before 2.6 don't return the _id for an # upsert if _id is not an ObjectId. elif result.get("updatedExisting") is False and affected == 1: # If _id is in both the update document *and* the query spec # the update document _id takes precedence. update = command['updates'][0] _id = update["u"].get("_id", update["q"].get("_id")) res["upserted"] = [{"index": 0, "_id": _id}] return res _OPTIONS = SON([ ('tailable', 2), ('oplogReplay', 8), ('noCursorTimeout', 16), ('awaitData', 32), ('allowPartialResults', 128)]) _MODIFIERS = SON([ ('$query', 'filter'), ('$orderby', 'sort'), ('$hint', 'hint'), ('$comment', 'comment'), ('$maxScan', 'maxScan'), ('$maxTimeMS', 'maxTimeMS'), ('$max', 'max'), ('$min', 'min'), ('$returnKey', 'returnKey'), ('$showRecordId', 'showRecordId'), ('$showDiskLoc', 'showRecordId'), # <= MongoDb 3.0 ('$snapshot', 'snapshot')]) def _gen_explain_command( coll, spec, projection, skip, limit, batch_size, options, read_concern): """Generate an explain command document.""" cmd = _gen_find_command( coll, spec, projection, skip, limit, batch_size, options) if read_concern.level: return SON([('explain', cmd), ('readConcern', read_concern.document)]) return SON([('explain', cmd)]) def _gen_find_command(coll, spec, projection, skip, limit, batch_size, options, read_concern=DEFAULT_READ_CONCERN): """Generate a find command document.""" cmd = SON([('find', coll)]) if '$query' in spec: cmd.update([(_MODIFIERS[key], val) if key in _MODIFIERS else (key, val) for key, val in spec.items()]) if '$explain' in cmd: cmd.pop('$explain') if '$readPreference' in cmd: cmd.pop('$readPreference') else: cmd['filter'] = spec if projection: cmd['projection'] = projection if skip: cmd['skip'] = skip if limit: cmd['limit'] = abs(limit) if limit < 0: cmd['singleBatch'] = True if batch_size: cmd['batchSize'] = batch_size if read_concern.level: cmd['readConcern'] = read_concern.document if options: cmd.update([(opt, True) for opt, val in _OPTIONS.items() if options & val]) return cmd def _gen_get_more_command(cursor_id, coll, batch_size, max_await_time_ms): """Generate a getMore command document.""" cmd = SON([('getMore', cursor_id), ('collection', coll)]) if batch_size: cmd['batchSize'] = batch_size if max_await_time_ms is not None: cmd['maxTimeMS'] = max_await_time_ms return cmd class _Query(object): """A query operation.""" __slots__ = ('flags', 'db', 'coll', 'ntoskip', 'ntoreturn', 'spec', 'fields', 'codec_options', 'read_preference', 'limit', 'batch_size', 'name', 'read_concern') def __init__(self, flags, db, coll, ntoskip, ntoreturn, spec, fields, codec_options, read_preference, limit, batch_size, read_concern): self.flags = flags self.db = db self.coll = coll self.ntoskip = ntoskip self.ntoreturn = ntoreturn self.spec = spec self.fields = fields self.codec_options = codec_options self.read_preference = read_preference self.read_concern = read_concern self.limit = limit self.batch_size = batch_size self.name = 'find' def as_command(self): """Return a find command document for this query. Should be called *after* get_message. """ if '$explain' in self.spec: self.name = 'explain' return _gen_explain_command( self.coll, self.spec, self.fields, self.ntoskip, self.limit, self.batch_size, self.flags, self.read_concern), self.db return _gen_find_command( self.coll, self.spec, self.fields, self.ntoskip, self.limit, self.batch_size, self.flags, self.read_concern), self.db def get_message(self, set_slave_ok, is_mongos, use_cmd=False): """Get a query message, possibly setting the slaveOk bit.""" if set_slave_ok: # Set the slaveOk bit. flags = self.flags | 4 else: flags = self.flags ns = _UJOIN % (self.db, self.coll) spec = self.spec ntoreturn = self.ntoreturn if use_cmd: ns = _UJOIN % (self.db, "$cmd") spec = self.as_command()[0] ntoreturn = -1 # All DB commands return 1 document if is_mongos: spec = _maybe_add_read_preference(spec, self.read_preference) return query(flags, ns, self.ntoskip, ntoreturn, spec, self.fields, self.codec_options) class _GetMore(object): """A getmore operation.""" __slots__ = ('db', 'coll', 'ntoreturn', 'cursor_id', 'max_await_time_ms', 'codec_options') name = 'getMore' def __init__(self, db, coll, ntoreturn, cursor_id, codec_options, max_await_time_ms=None): self.db = db self.coll = coll self.ntoreturn = ntoreturn self.cursor_id = cursor_id self.codec_options = codec_options self.max_await_time_ms = max_await_time_ms def as_command(self): """Return a getMore command document for this query.""" return _gen_get_more_command(self.cursor_id, self.coll, self.ntoreturn, self.max_await_time_ms), self.db def get_message(self, dummy0, dummy1, use_cmd=False): """Get a getmore message.""" ns = _UJOIN % (self.db, self.coll) if use_cmd: ns = _UJOIN % (self.db, "$cmd") spec = self.as_command()[0] return query(0, ns, 0, -1, spec, None, self.codec_options) return get_more(ns, self.ntoreturn, self.cursor_id) class _CursorAddress(tuple): """The server address (host, port) of a cursor, with namespace property.""" def __new__(cls, address, namespace): self = tuple.__new__(cls, address) self.__namespace = namespace return self @property def namespace(self): """The namespace this cursor.""" return self.__namespace def __hash__(self): # Two _CursorAddress instances with different namespaces # must not hash the same. return (self + (self.__namespace,)).__hash__() def __eq__(self, other): if isinstance(other, _CursorAddress): return (tuple(self) == tuple(other) and self.namespace == other.namespace) return NotImplemented def __ne__(self, other): return not self == other def __last_error(namespace, args): """Data to send to do a lastError. """ cmd = SON([("getlasterror", 1)]) cmd.update(args) splitns = namespace.split('.', 1) return query(0, splitns[0] + '.$cmd', 0, -1, cmd, None, DEFAULT_CODEC_OPTIONS) def __pack_message(operation, data): """Takes message data and adds a message header based on the operation. Returns the resultant message string. """ request_id = _randint() message = struct.pack(" ctx.max_bson_size) message_length += encoded_length if message_length < ctx.max_message_size and not too_large: data.write(encoded) to_send.append(doc) has_docs = True continue if has_docs: # We have enough data, send this message. try: request_id, msg = _insert_message(data.getvalue(), send_safe) ctx.legacy_write(request_id, msg, 0, send_safe, to_send) # Exception type could be OperationFailure or a subtype # (e.g. DuplicateKeyError) except OperationFailure as exc: # Like it says, continue on error... if continue_on_error: # Store exception details to re-raise after the final batch. last_error = exc # With unacknowledged writes just return at the first error. elif not safe: return # With acknowledged writes raise immediately. else: raise if too_large: _raise_document_too_large( "insert", encoded_length, ctx.max_bson_size) message_length = begin_loc + encoded_length data.seek(begin_loc) data.truncate() data.write(encoded) to_send = [doc] if not has_docs: raise InvalidOperation("cannot do an empty bulk insert") request_id, msg = _insert_message(data.getvalue(), safe) ctx.legacy_write(request_id, msg, 0, safe, to_send) # Re-raise any exception stored due to continue_on_error if last_error is not None: raise last_error if _use_c: _do_batched_insert = _cmessage._do_batched_insert def _do_batched_write_command(namespace, operation, command, docs, check_keys, opts, ctx): """Execute a batch of insert, update, or delete commands. """ max_bson_size = ctx.max_bson_size max_write_batch_size = ctx.max_write_batch_size # Max BSON object size + 16k - 2 bytes for ending NUL bytes. # Server guarantees there is enough room: SERVER-10643. max_cmd_size = max_bson_size + _COMMAND_OVERHEAD ordered = command.get('ordered', True) buf = StringIO() # Save space for message length and request id buf.write(_ZERO_64) # responseTo, opCode buf.write(b"\x00\x00\x00\x00\xd4\x07\x00\x00") # No options buf.write(_ZERO_32) # Namespace as C string buf.write(b(namespace)) buf.write(_ZERO_8) # Skip: 0, Limit: -1 buf.write(_SKIPLIM) # Where to write command document length command_start = buf.tell() buf.write(bson.BSON.encode(command)) # Start of payload buf.seek(-1, 2) # Work around some Jython weirdness. buf.truncate() try: buf.write(_OP_MAP[operation]) except KeyError: raise InvalidOperation('Unknown command') if operation in (_UPDATE, _DELETE): check_keys = False # Where to write list document length list_start = buf.tell() - 4 to_send = [] def send_message(): """Finalize and send the current OP_QUERY message. """ # Close list and command documents buf.write(_ZERO_16) # Write document lengths and request id length = buf.tell() buf.seek(list_start) buf.write(struct.pack('= max_cmd_size enough_documents = (idx >= max_write_batch_size) if enough_data or enough_documents: if not idx: write_op = "insert" if operation == _INSERT else None _raise_document_too_large( write_op, len(value), max_bson_size) result = send_message() results.append((idx_offset, result)) if ordered and "writeErrors" in result: return results # Truncate back to the start of list elements buf.seek(list_start + 4) buf.truncate() idx_offset += idx idx = 0 key = b'0' to_send = [] buf.write(_BSONOBJ) buf.write(key) buf.write(_ZERO_8) buf.write(value) to_send.append(doc) idx += 1 if not has_docs: raise InvalidOperation("cannot do an empty bulk write") results.append((idx_offset, send_message())) return results if _use_c: _do_batched_write_command = _cmessage._do_batched_write_command pymongo-3.2/pymongo/command_cursor.py0000644000175000017500000002061112630145074022033 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """CommandCursor class to iterate over command results.""" import datetime from collections import deque from bson.py3compat import integer_types from pymongo import helpers from pymongo.errors import AutoReconnect, NotMasterError, OperationFailure from pymongo.message import _CursorAddress, _GetMore, _convert_exception class CommandCursor(object): """A cursor / iterator over command cursors. """ def __init__(self, collection, cursor_info, address, retrieved=0): """Create a new command cursor. """ self.__collection = collection self.__id = cursor_info['id'] self.__address = address self.__data = deque(cursor_info['firstBatch']) self.__retrieved = retrieved self.__batch_size = 0 self.__killed = (self.__id == 0) if "ns" in cursor_info: self.__ns = cursor_info["ns"] else: self.__ns = collection.full_name def __del__(self): if self.__id and not self.__killed: self.__die() def __die(self): """Closes this cursor. """ if self.__id and not self.__killed: self.__collection.database.client.close_cursor( self.__id, _CursorAddress(self.__address, self.__ns)) self.__killed = True def close(self): """Explicitly close / kill this cursor. Required for PyPy, Jython and other Python implementations that don't use reference counting garbage collection. """ self.__die() def batch_size(self, batch_size): """Limits the number of documents returned in one batch. Each batch requires a round trip to the server. It can be adjusted to optimize performance and limit data transfer. .. note:: batch_size can not override MongoDB's internal limits on the amount of data it will return to the client in a single batch (i.e if you set batch size to 1,000,000,000, MongoDB will currently only return 4-16MB of results per batch). Raises :exc:`TypeError` if `batch_size` is not an integer. Raises :exc:`ValueError` if `batch_size` is less than ``0``. :Parameters: - `batch_size`: The size of each batch of results requested. """ if not isinstance(batch_size, integer_types): raise TypeError("batch_size must be an integer") if batch_size < 0: raise ValueError("batch_size must be >= 0") self.__batch_size = batch_size == 1 and 2 or batch_size return self def __send_message(self, operation): """Send a getmore message and handle the response. """ client = self.__collection.database.client listeners = client._event_listeners publish = listeners.enabled_for_commands try: response = client._send_message_with_response( operation, address=self.__address) except AutoReconnect: # Don't try to send kill cursors on another socket # or to another server. It can cause a _pinValue # assertion on some server releases if we get here # due to a socket timeout. self.__killed = True raise cmd_duration = response.duration rqst_id = response.request_id from_command = response.from_command if publish: start = datetime.datetime.now() try: doc = helpers._unpack_response(response.data, self.__id, self.__collection.codec_options) if from_command: helpers._check_command_response(doc['data'][0]) except OperationFailure as exc: self.__killed = True if publish: duration = (datetime.datetime.now() - start) + cmd_duration listeners.publish_command_failure( duration, exc.details, "getMore", rqst_id, self.__address) raise except NotMasterError as exc: # Don't send kill cursors to another server after a "not master" # error. It's completely pointless. self.__killed = True if publish: duration = (datetime.datetime.now() - start) + cmd_duration listeners.publish_command_failure( duration, exc.details, "getMore", rqst_id, self.__address) client._reset_server_and_request_check(self.address) raise except Exception as exc: if publish: duration = (datetime.datetime.now() - start) + cmd_duration listeners.publish_command_failure( duration, _convert_exception(exc), "getMore", rqst_id, self.__address) raise if from_command: cursor = doc['data'][0]['cursor'] documents = cursor['nextBatch'] self.__id = cursor['id'] self.__retrieved += len(documents) else: documents = doc["data"] self.__id = doc["cursor_id"] self.__retrieved += doc["number_returned"] if publish: duration = (datetime.datetime.now() - start) + cmd_duration # Must publish in getMore command response format. res = {"cursor": {"id": self.__id, "ns": self.__collection.full_name, "nextBatch": documents}, "ok": 1} listeners.publish_command_success( duration, res, "getMore", rqst_id, self.__address) if self.__id == 0: self.__killed = True self.__data = deque(documents) def _refresh(self): """Refreshes the cursor with more data from the server. Returns the length of self.__data after refresh. Will exit early if self.__data is already non-empty. Raises OperationFailure when the cursor cannot be refreshed due to an error on the query. """ if len(self.__data) or self.__killed: return len(self.__data) if self.__id: # Get More self.__send_message( _GetMore(self.__collection.database.name, self.__collection.name, self.__batch_size, self.__id, self.__collection.codec_options)) else: # Cursor id is zero nothing else to return self.__killed = True return len(self.__data) @property def alive(self): """Does this cursor have the potential to return more data? Even if :attr:`alive` is ``True``, :meth:`next` can raise :exc:`StopIteration`. Best to use a for loop:: for doc in collection.aggregate(pipeline): print(doc) .. note:: :attr:`alive` can be True while iterating a cursor from a failed server. In this case :attr:`alive` will return False after :meth:`next` fails to retrieve the next batch of results from the server. """ return bool(len(self.__data) or (not self.__killed)) @property def cursor_id(self): """Returns the id of the cursor.""" return self.__id @property def address(self): """The (host, port) of the server used, or None. .. versionadded:: 3.0 """ return self.__address def __iter__(self): return self def next(self): """Advance the cursor.""" if len(self.__data) or self._refresh(): coll = self.__collection return coll.database._fix_outgoing(self.__data.popleft(), coll) else: raise StopIteration __next__ = next def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.__die() pymongo-3.2/pymongo/cursor.py0000644000175000017500000012371112630145074020342 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Cursor class to iterate over Mongo query results.""" import copy import datetime from collections import deque from bson import RE_TYPE from bson.code import Code from bson.py3compat import (iteritems, integer_types, string_type) from bson.son import SON from pymongo import helpers from pymongo.common import validate_boolean, validate_is_mapping from pymongo.errors import (AutoReconnect, ConnectionFailure, InvalidOperation, NotMasterError, OperationFailure) from pymongo.message import _CursorAddress, _GetMore, _Query, _convert_exception from pymongo.read_preferences import ReadPreference _QUERY_OPTIONS = { "tailable_cursor": 2, "slave_okay": 4, "oplog_replay": 8, "no_timeout": 16, "await_data": 32, "exhaust": 64, "partial": 128} class CursorType(object): NON_TAILABLE = 0 """The standard cursor type.""" TAILABLE = _QUERY_OPTIONS["tailable_cursor"] """The tailable cursor type. Tailable cursors are only for use with capped collections. They are not closed when the last data is retrieved but are kept open and the cursor location marks the final document position. If more data is received iteration of the cursor will continue from the last document received. """ TAILABLE_AWAIT = TAILABLE | _QUERY_OPTIONS["await_data"] """A tailable cursor with the await option set. Creates a tailable cursor that will wait for a few seconds after returning the full result set so that it can capture and return additional data added during the query. """ EXHAUST = _QUERY_OPTIONS["exhaust"] """An exhaust cursor. MongoDB will stream batched results to the client without waiting for the client to request each batch, reducing latency. """ # This has to be an old style class due to # http://bugs.jython.org/issue1057 class _SocketManager: """Used with exhaust cursors to ensure the socket is returned. """ def __init__(self, sock, pool): self.sock = sock self.pool = pool self.__closed = False def __del__(self): self.close() def close(self): """Return this instance's socket to the connection pool. """ if not self.__closed: self.__closed = True self.pool.return_socket(self.sock) self.sock, self.pool = None, None class Cursor(object): """A cursor / iterator over Mongo query results. """ def __init__(self, collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True): """Create a new cursor. Should not be called directly by application developers - see :meth:`~pymongo.collection.Collection.find` instead. .. mongodoc:: cursors """ self.__id = None spec = filter if spec is None: spec = {} validate_is_mapping("filter", spec) if not isinstance(skip, int): raise TypeError("skip must be an instance of int") if not isinstance(limit, int): raise TypeError("limit must be an instance of int") validate_boolean("no_cursor_timeout", no_cursor_timeout) if cursor_type not in (CursorType.NON_TAILABLE, CursorType.TAILABLE, CursorType.TAILABLE_AWAIT, CursorType.EXHAUST): raise ValueError("not a valid value for cursor_type") validate_boolean("allow_partial_results", allow_partial_results) validate_boolean("oplog_replay", oplog_replay) if modifiers is not None: validate_is_mapping("modifiers", modifiers) if not isinstance(batch_size, integer_types): raise TypeError("batch_size must be an integer") if batch_size < 0: raise ValueError("batch_size must be >= 0") if projection is not None: if not projection: projection = {"_id": 1} projection = helpers._fields_list_to_dict(projection, "projection") self.__collection = collection self.__spec = spec self.__projection = projection self.__skip = skip self.__limit = limit self.__batch_size = batch_size self.__modifiers = modifiers and modifiers.copy() or {} self.__ordering = sort and helpers._index_document(sort) or None self.__max_scan = None self.__explain = False self.__hint = None self.__comment = None self.__max_time_ms = None self.__max_await_time_ms = None self.__max = None self.__min = None self.__manipulate = manipulate # Exhaust cursor support self.__exhaust = False self.__exhaust_mgr = None if cursor_type == CursorType.EXHAUST: if self.__collection.database.client.is_mongos: raise InvalidOperation('Exhaust cursors are ' 'not supported by mongos') if limit: raise InvalidOperation("Can't use limit and exhaust together.") self.__exhaust = True # This is ugly. People want to be able to do cursor[5:5] and # get an empty result set (old behavior was an # exception). It's hard to do that right, though, because the # server uses limit(0) to mean 'no limit'. So we set __empty # in that case and check for it when iterating. We also unset # it anytime we change __limit. self.__empty = False self.__data = deque() self.__address = None self.__retrieved = 0 self.__killed = False self.__codec_options = collection.codec_options self.__read_preference = collection.read_preference self.__read_concern = collection.read_concern self.__query_flags = cursor_type if self.__read_preference != ReadPreference.PRIMARY: self.__query_flags |= _QUERY_OPTIONS["slave_okay"] if no_cursor_timeout: self.__query_flags |= _QUERY_OPTIONS["no_timeout"] if allow_partial_results: self.__query_flags |= _QUERY_OPTIONS["partial"] if oplog_replay: self.__query_flags |= _QUERY_OPTIONS["oplog_replay"] @property def collection(self): """The :class:`~pymongo.collection.Collection` that this :class:`Cursor` is iterating. """ return self.__collection @property def retrieved(self): """The number of documents retrieved so far. """ return self.__retrieved def __del__(self): if self.__id and not self.__killed: self.__die() def rewind(self): """Rewind this cursor to its unevaluated state. Reset this cursor if it has been partially or completely evaluated. Any options that are present on the cursor will remain in effect. Future iterating performed on this cursor will cause new queries to be sent to the server, even if the resultant data has already been retrieved by this cursor. """ self.__data = deque() self.__id = None self.__address = None self.__retrieved = 0 self.__killed = False return self def clone(self): """Get a clone of this cursor. Returns a new Cursor instance with options matching those that have been set on the current instance. The clone will be completely unevaluated, even if the current instance has been partially or completely evaluated. """ return self._clone(True) def _clone(self, deepcopy=True): """Internal clone helper.""" clone = self._clone_base() values_to_clone = ("spec", "projection", "skip", "limit", "max_time_ms", "max_await_time_ms", "comment", "max", "min", "ordering", "explain", "hint", "batch_size", "max_scan", "manipulate", "query_flags", "modifiers") data = dict((k, v) for k, v in iteritems(self.__dict__) if k.startswith('_Cursor__') and k[9:] in values_to_clone) if deepcopy: data = self._deepcopy(data) clone.__dict__.update(data) return clone def _clone_base(self): """Creates an empty Cursor object for information to be copied into. """ return Cursor(self.__collection) def __die(self): """Closes this cursor. """ if self.__id and not self.__killed: if self.__exhaust and self.__exhaust_mgr: # If this is an exhaust cursor and we haven't completely # exhausted the result set we *must* close the socket # to stop the server from sending more data. self.__exhaust_mgr.sock.close() else: self.__collection.database.client.close_cursor( self.__id, _CursorAddress( self.__address, self.__collection.full_name)) if self.__exhaust and self.__exhaust_mgr: self.__exhaust_mgr.close() self.__killed = True def close(self): """Explicitly close / kill this cursor. Required for PyPy, Jython and other Python implementations that don't use reference counting garbage collection. """ self.__die() def __query_spec(self): """Get the spec to use for a query. """ operators = self.__modifiers.copy() if self.__ordering: operators["$orderby"] = self.__ordering if self.__explain: operators["$explain"] = True if self.__hint: operators["$hint"] = self.__hint if self.__comment: operators["$comment"] = self.__comment if self.__max_scan: operators["$maxScan"] = self.__max_scan if self.__max_time_ms is not None: operators["$maxTimeMS"] = self.__max_time_ms if self.__max: operators["$max"] = self.__max if self.__min: operators["$min"] = self.__min if operators: # Make a shallow copy so we can cleanly rewind or clone. spec = self.__spec.copy() # White-listed commands must be wrapped in $query. if "$query" not in spec: # $query has to come first spec = SON([("$query", spec)]) if not isinstance(spec, SON): # Ensure the spec is SON. As order is important this will # ensure its set before merging in any extra operators. spec = SON(spec) spec.update(operators) return spec # Have to wrap with $query if "query" is the first key. # We can't just use $query anytime "query" is a key as # that breaks commands like count and find_and_modify. # Checking spec.keys()[0] covers the case that the spec # was passed as an instance of SON or OrderedDict. elif ("query" in self.__spec and (len(self.__spec) == 1 or next(iter(self.__spec)) == "query")): return SON({"$query": self.__spec}) return self.__spec def __check_okay_to_chain(self): """Check if it is okay to chain more options onto this cursor. """ if self.__retrieved or self.__id is not None: raise InvalidOperation("cannot set options after executing query") def add_option(self, mask): """Set arbitrary query flags using a bitmask. To set the tailable flag: cursor.add_option(2) """ if not isinstance(mask, int): raise TypeError("mask must be an int") self.__check_okay_to_chain() if mask & _QUERY_OPTIONS["exhaust"]: if self.__limit: raise InvalidOperation("Can't use limit and exhaust together.") if self.__collection.database.client.is_mongos: raise InvalidOperation('Exhaust cursors are ' 'not supported by mongos') self.__exhaust = True self.__query_flags |= mask return self def remove_option(self, mask): """Unset arbitrary query flags using a bitmask. To unset the tailable flag: cursor.remove_option(2) """ if not isinstance(mask, int): raise TypeError("mask must be an int") self.__check_okay_to_chain() if mask & _QUERY_OPTIONS["exhaust"]: self.__exhaust = False self.__query_flags &= ~mask return self def limit(self, limit): """Limits the number of results to be returned by this cursor. Raises :exc:`TypeError` if `limit` is not an integer. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. The last `limit` applied to this cursor takes precedence. A limit of ``0`` is equivalent to no limit. :Parameters: - `limit`: the number of results to return .. mongodoc:: limit """ if not isinstance(limit, integer_types): raise TypeError("limit must be an integer") if self.__exhaust: raise InvalidOperation("Can't use limit and exhaust together.") self.__check_okay_to_chain() self.__empty = False self.__limit = limit return self def batch_size(self, batch_size): """Limits the number of documents returned in one batch. Each batch requires a round trip to the server. It can be adjusted to optimize performance and limit data transfer. .. note:: batch_size can not override MongoDB's internal limits on the amount of data it will return to the client in a single batch (i.e if you set batch size to 1,000,000,000, MongoDB will currently only return 4-16MB of results per batch). Raises :exc:`TypeError` if `batch_size` is not an integer. Raises :exc:`ValueError` if `batch_size` is less than ``0``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. The last `batch_size` applied to this cursor takes precedence. :Parameters: - `batch_size`: The size of each batch of results requested. """ if not isinstance(batch_size, integer_types): raise TypeError("batch_size must be an integer") if batch_size < 0: raise ValueError("batch_size must be >= 0") self.__check_okay_to_chain() self.__batch_size = batch_size == 1 and 2 or batch_size return self def skip(self, skip): """Skips the first `skip` results of this cursor. Raises :exc:`TypeError` if `skip` is not an integer. Raises :exc:`ValueError` if `skip` is less than ``0``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. The last `skip` applied to this cursor takes precedence. :Parameters: - `skip`: the number of results to skip """ if not isinstance(skip, integer_types): raise TypeError("skip must be an integer") if skip < 0: raise ValueError("skip must be >= 0") self.__check_okay_to_chain() self.__skip = skip return self def max_time_ms(self, max_time_ms): """Specifies a time limit for a query operation. If the specified time is exceeded, the operation will be aborted and :exc:`~pymongo.errors.ExecutionTimeout` is raised. If `max_time_ms` is ``None`` no limit is applied. Raises :exc:`TypeError` if `max_time_ms` is not an integer or ``None``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. :Parameters: - `max_time_ms`: the time limit after which the operation is aborted """ if (not isinstance(max_time_ms, integer_types) and max_time_ms is not None): raise TypeError("max_time_ms must be an integer or None") self.__check_okay_to_chain() self.__max_time_ms = max_time_ms return self def max_await_time_ms(self, max_await_time_ms): """Specifies a time limit for a getMore operation on a :attr:`~pymongo.cursor.CursorType.TAILABLE_AWAIT` cursor. For all other types of cursor max_await_time_ms is ignored. Raises :exc:`TypeError` if `max_await_time_ms` is not an integer or ``None``. Raises :exc:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. .. note:: `max_await_time_ms` requires server version **>= 3.2** :Parameters: - `max_await_time_ms`: the time limit after which the operation is aborted .. versionadded:: 3.2 """ if (not isinstance(max_await_time_ms, integer_types) and max_await_time_ms is not None): raise TypeError("max_await_time_ms must be an integer or None") self.__check_okay_to_chain() # Ignore max_await_time_ms if not tailable or await_data is False. if self.__query_flags & CursorType.TAILABLE_AWAIT: self.__max_await_time_ms = max_await_time_ms return self def __getitem__(self, index): """Get a single document or a slice of documents from this cursor. Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. To get a single document use an integral index, e.g.:: >>> db.test.find()[50] An :class:`IndexError` will be raised if the index is negative or greater than the amount of documents in this cursor. Any limit previously applied to this cursor will be ignored. To get a slice of documents use a slice index, e.g.:: >>> db.test.find()[20:25] This will return this cursor with a limit of ``5`` and skip of ``20`` applied. Using a slice index will override any prior limits or skips applied to this cursor (including those applied through previous calls to this method). Raises :class:`IndexError` when the slice has a step, a negative start value, or a stop value less than or equal to the start value. :Parameters: - `index`: An integer or slice index to be applied to this cursor """ self.__check_okay_to_chain() self.__empty = False if isinstance(index, slice): if index.step is not None: raise IndexError("Cursor instances do not support slice steps") skip = 0 if index.start is not None: if index.start < 0: raise IndexError("Cursor instances do not support" "negative indices") skip = index.start if index.stop is not None: limit = index.stop - skip if limit < 0: raise IndexError("stop index must be greater than start" "index for slice %r" % index) if limit == 0: self.__empty = True else: limit = 0 self.__skip = skip self.__limit = limit return self if isinstance(index, integer_types): if index < 0: raise IndexError("Cursor instances do not support negative" "indices") clone = self.clone() clone.skip(index + self.__skip) clone.limit(-1) # use a hard limit for doc in clone: return doc raise IndexError("no such item for Cursor instance") raise TypeError("index %r cannot be applied to Cursor " "instances" % index) def max_scan(self, max_scan): """Limit the number of documents to scan when performing the query. Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. Only the last :meth:`max_scan` applied to this cursor has any effect. :Parameters: - `max_scan`: the maximum number of documents to scan """ self.__check_okay_to_chain() self.__max_scan = max_scan return self def max(self, spec): """Adds `max` operator that specifies upper bound for specific index. :Parameters: - `spec`: a list of field, limit pairs specifying the exclusive upper bound for all keys of a specific index in order. .. versionadded:: 2.7 """ if not isinstance(spec, (list, tuple)): raise TypeError("spec must be an instance of list or tuple") self.__check_okay_to_chain() self.__max = SON(spec) return self def min(self, spec): """Adds `min` operator that specifies lower bound for specific index. :Parameters: - `spec`: a list of field, limit pairs specifying the inclusive lower bound for all keys of a specific index in order. .. versionadded:: 2.7 """ if not isinstance(spec, (list, tuple)): raise TypeError("spec must be an instance of list or tuple") self.__check_okay_to_chain() self.__min = SON(spec) return self def sort(self, key_or_list, direction=None): """Sorts this cursor's results. Pass a field name and a direction, either :data:`~pymongo.ASCENDING` or :data:`~pymongo.DESCENDING`:: for doc in collection.find().sort('field', pymongo.ASCENDING): print(doc) To sort by multiple fields, pass a list of (key, direction) pairs:: for doc in collection.find().sort([ ('field1', pymongo.ASCENDING), ('field2', pymongo.DESCENDING)]): print(doc) Beginning with MongoDB version 2.6, text search results can be sorted by relevance:: cursor = db.test.find( {'$text': {'$search': 'some words'}}, {'score': {'$meta': 'textScore'}}) # Sort by 'score' field. cursor.sort([('score', {'$meta': 'textScore'})]) for doc in cursor: print(doc) Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. Only the last :meth:`sort` applied to this cursor has any effect. :Parameters: - `key_or_list`: a single key or a list of (key, direction) pairs specifying the keys to sort on - `direction` (optional): only used if `key_or_list` is a single key, if not given :data:`~pymongo.ASCENDING` is assumed """ self.__check_okay_to_chain() keys = helpers._index_list(key_or_list, direction) self.__ordering = helpers._index_document(keys) return self def count(self, with_limit_and_skip=False): """Get the size of the results set for this query. Returns the number of documents in the results set for this query. Does not take :meth:`limit` and :meth:`skip` into account by default - set `with_limit_and_skip` to ``True`` if that is the desired behavior. Raises :class:`~pymongo.errors.OperationFailure` on a database error. When used with MongoDB >= 2.6, :meth:`~count` uses any :meth:`~hint` applied to the query. In the following example the hint is passed to the count command: collection.find({'field': 'value'}).hint('field_1').count() The :meth:`count` method obeys the :attr:`~pymongo.collection.Collection.read_preference` of the :class:`~pymongo.collection.Collection` instance on which :meth:`~pymongo.collection.Collection.find` was called. :Parameters: - `with_limit_and_skip` (optional): take any :meth:`limit` or :meth:`skip` that has been applied to this cursor into account when getting the count .. note:: The `with_limit_and_skip` parameter requires server version **>= 1.1.4-** .. versionchanged:: 2.8 The :meth:`~count` method now supports :meth:`~hint`. """ validate_boolean("with_limit_and_skip", with_limit_and_skip) cmd = SON([("count", self.__collection.name), ("query", self.__spec)]) if self.__max_time_ms is not None: cmd["maxTimeMS"] = self.__max_time_ms if self.__comment: cmd["$comment"] = self.__comment if self.__hint is not None: cmd["hint"] = self.__hint if with_limit_and_skip: if self.__limit: cmd["limit"] = self.__limit if self.__skip: cmd["skip"] = self.__skip return self.__collection._count(cmd) def distinct(self, key): """Get a list of distinct values for `key` among all documents in the result set of this query. Raises :class:`TypeError` if `key` is not an instance of :class:`basestring` (:class:`str` in python 3). The :meth:`distinct` method obeys the :attr:`~pymongo.collection.Collection.read_preference` of the :class:`~pymongo.collection.Collection` instance on which :meth:`~pymongo.collection.Collection.find` was called. :Parameters: - `key`: name of key for which we want to get the distinct values .. seealso:: :meth:`pymongo.collection.Collection.distinct` """ options = {} if self.__spec: options["query"] = self.__spec if self.__max_time_ms is not None: options['maxTimeMS'] = self.__max_time_ms if self.__comment: options['$comment'] = self.__comment return self.__collection.distinct(key, **options) def explain(self): """Returns an explain plan record for this cursor. .. mongodoc:: explain """ c = self.clone() c.__explain = True # always use a hard limit for explains if c.__limit: c.__limit = -abs(c.__limit) return next(c) def hint(self, index): """Adds a 'hint', telling Mongo the proper index to use for the query. Judicious use of hints can greatly improve query performance. When doing a query on multiple fields (at least one of which is indexed) pass the indexed field as a hint to the query. Hinting will not do anything if the corresponding index does not exist. Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. `index` should be an index as passed to :meth:`~pymongo.collection.Collection.create_index` (e.g. ``[('field', ASCENDING)]``) or the name of the index. If `index` is ``None`` any existing hint for this query is cleared. The last hint applied to this cursor takes precedence over all others. :Parameters: - `index`: index to hint on (as an index specifier) .. versionchanged:: 2.8 The :meth:`~hint` method accepts the name of the index. """ self.__check_okay_to_chain() if index is None: self.__hint = None return self if isinstance(index, string_type): self.__hint = index else: self.__hint = helpers._index_document(index) return self def comment(self, comment): """Adds a 'comment' to the cursor. http://docs.mongodb.org/manual/reference/operator/comment/ :Parameters: - `comment`: A string or document .. versionadded:: 2.7 """ self.__check_okay_to_chain() self.__comment = comment return self def where(self, code): """Adds a $where clause to this query. The `code` argument must be an instance of :class:`basestring` (:class:`str` in python 3) or :class:`~bson.code.Code` containing a JavaScript expression. This expression will be evaluated for each document scanned. Only those documents for which the expression evaluates to *true* will be returned as results. The keyword *this* refers to the object currently being scanned. Raises :class:`TypeError` if `code` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidOperation` if this :class:`Cursor` has already been used. Only the last call to :meth:`where` applied to a :class:`Cursor` has any effect. :Parameters: - `code`: JavaScript expression to use as a filter """ self.__check_okay_to_chain() if not isinstance(code, Code): code = Code(code) self.__spec["$where"] = code return self def __send_message(self, operation): """Send a query or getmore operation and handles the response. If operation is ``None`` this is an exhaust cursor, which reads the next result batch off the exhaust socket instead of sending getMore messages to the server. Can raise ConnectionFailure. """ client = self.__collection.database.client listeners = client._event_listeners publish = listeners.enabled_for_commands from_command = False if operation: kwargs = { "read_preference": self.__read_preference, "exhaust": self.__exhaust, } if self.__address is not None: kwargs["address"] = self.__address try: response = client._send_message_with_response(operation, **kwargs) self.__address = response.address if self.__exhaust: # 'response' is an ExhaustResponse. self.__exhaust_mgr = _SocketManager(response.socket_info, response.pool) cmd_name = operation.name data = response.data cmd_duration = response.duration rqst_id = response.request_id from_command = response.from_command except AutoReconnect: # Don't try to send kill cursors on another socket # or to another server. It can cause a _pinValue # assertion on some server releases if we get here # due to a socket timeout. self.__killed = True raise else: # Exhaust cursor - no getMore message. rqst_id = 0 cmd_name = 'getMore' if publish: # Fake a getMore command. cmd = SON([('getMore', self.__id), ('collection', self.__collection.name)]) if self.__batch_size: cmd['batchSize'] = self.__batch_size if self.__max_time_ms: cmd['maxTimeMS'] = self.__max_time_ms listeners.publish_command_start( cmd, self.__collection.database.name, 0, self.__address) start = datetime.datetime.now() try: data = self.__exhaust_mgr.sock.receive_message(1, None) except Exception as exc: if publish: duration = datetime.datetime.now() - start listeners.publish_command_failure( duration, _convert_exception(exc), cmd_name, rqst_id, self.__address) if isinstance(exc, ConnectionFailure): self.__die() raise if publish: cmd_duration = datetime.datetime.now() - start if publish: start = datetime.datetime.now() try: doc = helpers._unpack_response(response=data, cursor_id=self.__id, codec_options=self.__codec_options) if from_command: helpers._check_command_response(doc['data'][0]) except OperationFailure as exc: self.__killed = True # Make sure exhaust socket is returned immediately, if necessary. self.__die() if publish: duration = (datetime.datetime.now() - start) + cmd_duration listeners.publish_command_failure( duration, exc.details, cmd_name, rqst_id, self.__address) # If this is a tailable cursor the error is likely # due to capped collection roll over. Setting # self.__killed to True ensures Cursor.alive will be # False. No need to re-raise. if self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]: return raise except NotMasterError as exc: # Don't send kill cursors to another server after a "not master" # error. It's completely pointless. self.__killed = True # Make sure exhaust socket is returned immediately, if necessary. self.__die() if publish: duration = (datetime.datetime.now() - start) + cmd_duration listeners.publish_command_failure( duration, exc.details, cmd_name, rqst_id, self.__address) client._reset_server_and_request_check(self.__address) raise except Exception as exc: if publish: duration = (datetime.datetime.now() - start) + cmd_duration listeners.publish_command_failure( duration, _convert_exception(exc), cmd_name, rqst_id, self.__address) raise if publish: duration = (datetime.datetime.now() - start) + cmd_duration # Must publish in find / getMore / explain command response format. if from_command: res = doc['data'][0] elif cmd_name == "explain": res = doc["data"][0] if doc["number_returned"] else {} else: res = {"cursor": {"id": doc["cursor_id"], "ns": self.__collection.full_name}, "ok": 1} if cmd_name == "find": res["cursor"]["firstBatch"] = doc["data"] else: res["cursor"]["nextBatch"] = doc["data"] listeners.publish_command_success( duration, res, cmd_name, rqst_id, self.__address) if from_command and cmd_name != "explain": cursor = doc['data'][0]['cursor'] self.__id = cursor['id'] if cmd_name == 'find': documents = cursor['firstBatch'] else: documents = cursor['nextBatch'] self.__data = deque(documents) self.__retrieved += len(documents) else: self.__id = doc["cursor_id"] self.__data = deque(doc["data"]) self.__retrieved += doc["number_returned"] if self.__id == 0: self.__killed = True if self.__limit and self.__id and self.__limit <= self.__retrieved: self.__die() # Don't wait for garbage collection to call __del__, return the # socket to the pool now. if self.__exhaust and self.__id == 0: self.__exhaust_mgr.close() def _refresh(self): """Refreshes the cursor with more data from Mongo. Returns the length of self.__data after refresh. Will exit early if self.__data is already non-empty. Raises OperationFailure when the cursor cannot be refreshed due to an error on the query. """ if len(self.__data) or self.__killed: return len(self.__data) if self.__id is None: # Query ntoreturn = self.__batch_size if self.__limit: if self.__batch_size: ntoreturn = min(self.__limit, self.__batch_size) else: ntoreturn = self.__limit self.__send_message(_Query(self.__query_flags, self.__collection.database.name, self.__collection.name, self.__skip, ntoreturn, self.__query_spec(), self.__projection, self.__codec_options, self.__read_preference, self.__limit, self.__batch_size, self.__read_concern)) if not self.__id: self.__killed = True elif self.__id: # Get More if self.__limit: limit = self.__limit - self.__retrieved if self.__batch_size: limit = min(limit, self.__batch_size) else: limit = self.__batch_size # Exhaust cursors don't send getMore messages. if self.__exhaust: self.__send_message(None) else: self.__send_message(_GetMore(self.__collection.database.name, self.__collection.name, limit, self.__id, self.__codec_options, self.__max_await_time_ms)) else: # Cursor id is zero nothing else to return self.__killed = True return len(self.__data) @property def alive(self): """Does this cursor have the potential to return more data? This is mostly useful with `tailable cursors `_ since they will stop iterating even though they *may* return more results in the future. With regular cursors, simply use a for loop instead of :attr:`alive`:: for doc in collection.find(): print(doc) .. note:: Even if :attr:`alive` is True, :meth:`next` can raise :exc:`StopIteration`. :attr:`alive` can also be True while iterating a cursor from a failed server. In this case :attr:`alive` will return False after :meth:`next` fails to retrieve the next batch of results from the server. """ return bool(len(self.__data) or (not self.__killed)) @property def cursor_id(self): """Returns the id of the cursor Useful if you need to manage cursor ids and want to handle killing cursors manually using :meth:`~pymongo.mongo_client.MongoClient.kill_cursors` .. versionadded:: 2.2 """ return self.__id @property def address(self): """The (host, port) of the server used, or None. .. versionchanged:: 3.0 Renamed from "conn_id". """ return self.__address def __iter__(self): return self def next(self): """Advance the cursor.""" if self.__empty: raise StopIteration _db = self.__collection.database if len(self.__data) or self._refresh(): if self.__manipulate: return _db._fix_outgoing(self.__data.popleft(), self.__collection) else: return self.__data.popleft() else: raise StopIteration __next__ = next def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.__die() def __copy__(self): """Support function for `copy.copy()`. .. versionadded:: 2.4 """ return self._clone(deepcopy=False) def __deepcopy__(self, memo): """Support function for `copy.deepcopy()`. .. versionadded:: 2.4 """ return self._clone(deepcopy=True) def _deepcopy(self, x, memo=None): """Deepcopy helper for the data dictionary or list. Regular expressions cannot be deep copied but as they are immutable we don't have to copy them when cloning. """ if not hasattr(x, 'items'): y, is_list, iterator = [], True, enumerate(x) else: y, is_list, iterator = {}, False, iteritems(x) if memo is None: memo = {} val_id = id(x) if val_id in memo: return memo.get(val_id) memo[val_id] = y for key, value in iterator: if isinstance(value, (dict, list)) and not isinstance(value, SON): value = self._deepcopy(value, memo) elif not isinstance(value, RE_TYPE): value = copy.deepcopy(value, memo) if is_list: y.append(value) else: if not isinstance(key, RE_TYPE): key = copy.deepcopy(key, memo) y[key] = value return y pymongo-3.2/pymongo/collection.py0000644000175000017500000030412312630145074021156 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Collection level utilities for Mongo.""" import collections import datetime import warnings from bson.code import Code from bson.objectid import ObjectId from bson.py3compat import (_unicode, integer_types, string_type, u) from bson.raw_bson import RawBSONDocument from bson.codec_options import CodecOptions from bson.son import SON from pymongo import (common, helpers, message) from pymongo.bulk import BulkOperationBuilder, _Bulk from pymongo.command_cursor import CommandCursor from pymongo.cursor import Cursor from pymongo.errors import ConfigurationError, InvalidName, OperationFailure from pymongo.helpers import _check_write_command_response from pymongo.operations import _WriteOp, IndexModel from pymongo.read_concern import DEFAULT_READ_CONCERN from pymongo.read_preferences import ReadPreference from pymongo.results import (BulkWriteResult, DeleteResult, InsertOneResult, InsertManyResult, UpdateResult) from pymongo.write_concern import WriteConcern try: from collections import OrderedDict _ORDERED_TYPES = (SON, OrderedDict) except ImportError: _ORDERED_TYPES = (SON,) _NO_OBJ_ERROR = "No matching object found" _UJOIN = u("%s.%s") class ReturnDocument(object): """An enum used with :meth:`~pymongo.collection.Collection.find_one_and_replace` and :meth:`~pymongo.collection.Collection.find_one_and_update`. """ BEFORE = False """Return the original document before it was updated/replaced, or ``None`` if no document matches the query. """ AFTER = True """Return the updated/replaced or inserted document.""" class Collection(common.BaseObject): """A Mongo collection. """ def __init__(self, database, name, create=False, codec_options=None, read_preference=None, write_concern=None, read_concern=None, **kwargs): """Get / create a Mongo collection. Raises :class:`TypeError` if `name` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidName` if `name` is not a valid collection name. Any additional keyword arguments will be used as options passed to the create command. See :meth:`~pymongo.database.Database.create_collection` for valid options. If `create` is ``True`` or additional keyword arguments are present a create command will be sent. Otherwise, a create command will not be sent and the collection will be created implicitly on first use. :Parameters: - `database`: the database to get a collection from - `name`: the name of the collection to get - `create` (optional): if ``True``, force collection creation even without options being set - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) database.codec_options is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) database.read_preference is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) database.write_concern is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) database.read_concern is used. - `**kwargs` (optional): additional keyword arguments will be passed as options for the create collection command .. versionchanged:: 3.2 Added the read_concern option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. Removed the uuid_subtype attribute. :class:`~pymongo.collection.Collection` no longer returns an instance of :class:`~pymongo.collection.Collection` for attribute names with leading underscores. You must use dict-style lookups instead:: collection['__my_collection__'] Not: collection.__my_collection__ .. versionchanged:: 2.2 Removed deprecated argument: options .. versionadded:: 2.1 uuid_subtype attribute .. mongodoc:: collections """ super(Collection, self).__init__( codec_options or database.codec_options, read_preference or database.read_preference, write_concern or database.write_concern, read_concern or database.read_concern) if not isinstance(name, string_type): raise TypeError("name must be an instance " "of %s" % (string_type.__name__,)) if not name or ".." in name: raise InvalidName("collection names cannot be empty") if "$" in name and not (name.startswith("oplog.$main") or name.startswith("$cmd")): raise InvalidName("collection names must not " "contain '$': %r" % name) if name[0] == "." or name[-1] == ".": raise InvalidName("collection names must not start " "or end with '.': %r" % name) if "\x00" in name: raise InvalidName("collection names must not contain the " "null character") self.__database = database self.__name = _unicode(name) self.__full_name = _UJOIN % (self.__database.name, self.__name) if create or kwargs: self.__create(kwargs) def _socket_for_reads(self): return self.__database.client._socket_for_reads(self.read_preference) def _socket_for_primary_reads(self): return self.__database.client._socket_for_reads(ReadPreference.PRIMARY) def _socket_for_writes(self): return self.__database.client._socket_for_writes() def _command(self, sock_info, command, slave_ok=False, read_preference=None, codec_options=None, check=True, allowable_errors=None, read_concern=DEFAULT_READ_CONCERN): """Internal command helper. :Parameters: - `sock_info` - A SocketInfo instance. - `command` - The command itself, as a SON instance. - `slave_ok`: whether to set the SlaveOkay wire protocol bit. - `codec_options` (optional) - An instance of :class:`~bson.codec_options.CodecOptions`. - `check`: raise OperationFailure if there are errors - `allowable_errors`: errors to ignore if `check` is True - `read_concern` (optional) - An instance of :class:`~pymongo.read_concern.ReadConcern`. :Returns: # todo: don't return address (result document, address of server the command was run on) """ return sock_info.command(self.__database.name, command, slave_ok, read_preference or self.read_preference, codec_options or self.codec_options, check, allowable_errors, read_concern=read_concern) def __create(self, options): """Sends a create command with the given options. """ cmd = SON([("create", self.__name)]) if options: if "size" in options: options["size"] = float(options["size"]) cmd.update(options) with self._socket_for_writes() as sock_info: self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY) def __getattr__(self, name): """Get a sub-collection of this collection by name. Raises InvalidName if an invalid collection name is used. :Parameters: - `name`: the name of the collection to get """ if name.startswith('_'): full_name = _UJOIN % (self.__name, name) raise AttributeError( "Collection has no attribute %r. To access the %s" " collection, use database['%s']." % ( name, full_name, full_name)) return self.__getitem__(name) def __getitem__(self, name): return Collection(self.__database, _UJOIN % (self.__name, name)) def __repr__(self): return "Collection(%r, %r)" % (self.__database, self.__name) def __eq__(self, other): if isinstance(other, Collection): return (self.__database == other.database and self.__name == other.name) return NotImplemented def __ne__(self, other): return not self == other @property def full_name(self): """The full name of this :class:`Collection`. The full name is of the form `database_name.collection_name`. """ return self.__full_name @property def name(self): """The name of this :class:`Collection`.""" return self.__name @property def database(self): """The :class:`~pymongo.database.Database` that this :class:`Collection` is a part of. """ return self.__database def with_options( self, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a clone of this collection changing the specified settings. >>> coll1.read_preference Primary() >>> from pymongo import ReadPreference >>> coll2 = coll1.with_options(read_preference=ReadPreference.SECONDARY) >>> coll1.read_preference Primary() >>> coll2.read_preference Secondary(tag_sets=None) :Parameters: - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Collection` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Collection` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Collection` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Collection` is used. """ return Collection(self.__database, self.__name, False, codec_options or self.codec_options, read_preference or self.read_preference, write_concern or self.write_concern, read_concern or self.read_concern) def initialize_unordered_bulk_op(self, bypass_document_validation=False): """Initialize an unordered batch of write operations. Operations will be performed on the server in arbitrary order, possibly in parallel. All operations will be attempted. :Parameters: - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. Returns a :class:`~pymongo.bulk.BulkOperationBuilder` instance. See :ref:`unordered_bulk` for examples. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 2.7 """ return BulkOperationBuilder(self, False, bypass_document_validation) def initialize_ordered_bulk_op(self, bypass_document_validation=False): """Initialize an ordered batch of write operations. Operations will be performed on the server serially, in the order provided. If an error occurs all remaining operations are aborted. :Parameters: - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. Returns a :class:`~pymongo.bulk.BulkOperationBuilder` instance. See :ref:`ordered_bulk` for examples. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 2.7 """ return BulkOperationBuilder(self, True, bypass_document_validation) def bulk_write(self, requests, ordered=True, bypass_document_validation=False): """Send a batch of write operations to the server. Requests are passed as a list of write operation instances ( :class:`~pymongo.operations.InsertOne`, :class:`~pymongo.operations.UpdateOne`, :class:`~pymongo.operations.UpdateMany`, :class:`~pymongo.operations.ReplaceOne`, :class:`~pymongo.operations.DeleteOne`, or :class:`~pymongo.operations.DeleteMany`). >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': ObjectId('54f62e60fba5226811f634ef')} {u'x': 1, u'_id': ObjectId('54f62e60fba5226811f634f0')} >>> # DeleteMany, UpdateOne, and UpdateMany are also available. ... >>> from pymongo import InsertOne, DeleteOne, ReplaceOne >>> requests = [InsertOne({'y': 1}), DeleteOne({'x': 1}), ... ReplaceOne({'w': 1}, {'z': 1}, upsert=True)] >>> result = db.test.bulk_write(requests) >>> result.inserted_count 1 >>> result.deleted_count 1 >>> result.modified_count 0 >>> result.upserted_ids {2: ObjectId('54f62ee28891e756a6e1abd5')} >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': ObjectId('54f62e60fba5226811f634f0')} {u'y': 1, u'_id': ObjectId('54f62ee2fba5226811f634f1')} {u'z': 1, u'_id': ObjectId('54f62ee28891e756a6e1abd5')} :Parameters: - `requests`: A list of write operations (see examples above). - `ordered` (optional): If ``True`` (the default) requests will be performed on the server serially, in the order provided. If an error occurs all remaining operations are aborted. If ``False`` requests will be performed on the server in arbitrary order, possibly in parallel, and all operations will be attempted. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. :Returns: An instance of :class:`~pymongo.results.BulkWriteResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ if not isinstance(requests, list): raise TypeError("requests must be a list") blk = _Bulk(self, ordered, bypass_document_validation) for request in requests: if not isinstance(request, _WriteOp): raise TypeError("%r is not a valid request" % (request,)) request._add_to_bulk(blk) bulk_api_result = blk.execute(self.write_concern.document) if bulk_api_result is not None: return BulkWriteResult(bulk_api_result, True) return BulkWriteResult({}, False) def _legacy_write(self, sock_info, name, cmd, acknowledged, op_id, bypass_doc_val, func, *args): """Internal legacy write helper.""" # Cannot have both unacknowledged write and bypass document validation. if (bypass_doc_val and not acknowledged and sock_info.max_wire_version >= 4): raise OperationFailure("Cannot set bypass_document_validation with" " unacknowledged write concern") listeners = self.database.client._event_listeners publish = listeners.enabled_for_commands if publish: start = datetime.datetime.now() rqst_id, msg, max_size = func(*args) if publish: duration = datetime.datetime.now() - start listeners.publish_command_start( cmd, self.__database.name, rqst_id, sock_info.address, op_id) start = datetime.datetime.now() try: result = sock_info.legacy_write( rqst_id, msg, max_size, acknowledged) except Exception as exc: if publish: dur = (datetime.datetime.now() - start) + duration if isinstance(exc, OperationFailure): details = exc.details # Succeed if GLE was successful and this is a write error. if details.get("ok") and "n" in details: reply = message._convert_write_result( name, cmd, details) listeners.publish_command_success( dur, reply, name, rqst_id, sock_info.address, op_id) raise else: details = message._convert_exception(exc) listeners.publish_command_failure( dur, details, name, rqst_id, sock_info.address, op_id) raise if publish: if result is not None: reply = message._convert_write_result(name, cmd, result) else: # Comply with APM spec. reply = {'ok': 1} duration = (datetime.datetime.now() - start) + duration listeners.publish_command_success( duration, reply, name, rqst_id, sock_info.address, op_id) return result def _insert_one( self, sock_info, doc, ordered, check_keys, manipulate, write_concern, op_id, bypass_doc_val): """Internal helper for inserting a single document.""" if manipulate: doc = self.__database._apply_incoming_manipulators(doc, self) if not isinstance(doc, RawBSONDocument) and '_id' not in doc: doc['_id'] = ObjectId() doc = self.__database._apply_incoming_copying_manipulators(doc, self) concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 command = SON([('insert', self.name), ('ordered', ordered), ('documents', [doc])]) if concern: command['writeConcern'] = concern if sock_info.max_wire_version > 1 and acknowledged: if bypass_doc_val and sock_info.max_wire_version >= 4: command['bypassDocumentValidation'] = True # Insert command. result = sock_info.command(self.__database.name, command, codec_options=self.codec_options, check_keys=check_keys) _check_write_command_response([(0, result)]) else: # Legacy OP_INSERT. self._legacy_write( sock_info, 'insert', command, acknowledged, op_id, bypass_doc_val, message.insert, self.__full_name, [doc], check_keys, acknowledged, concern, False, self.codec_options) if not isinstance(doc, RawBSONDocument): return doc.get('_id') def _insert(self, sock_info, docs, ordered=True, check_keys=True, manipulate=False, write_concern=None, op_id=None, bypass_doc_val=False): """Internal insert helper.""" if isinstance(docs, collections.Mapping): return self._insert_one( sock_info, docs, ordered, check_keys, manipulate, write_concern, op_id, bypass_doc_val) ids = [] if manipulate: def gen(): """Generator that applies SON manipulators to each document and adds _id if necessary. """ _db = self.__database for doc in docs: # Apply user-configured SON manipulators. This order of # operations is required for backwards compatibility, # see PYTHON-709. doc = _db._apply_incoming_manipulators(doc, self) if not (isinstance(doc, RawBSONDocument) or '_id' in doc): doc['_id'] = ObjectId() doc = _db._apply_incoming_copying_manipulators(doc, self) ids.append(doc['_id']) yield doc else: def gen(): """Generator that only tracks existing _ids.""" for doc in docs: # Don't inflate RawBSONDocument by touching fields. if not isinstance(doc, RawBSONDocument): ids.append(doc.get('_id')) yield doc concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 command = SON([('insert', self.name), ('ordered', ordered)]) if concern: command['writeConcern'] = concern if op_id is None: op_id = message._randint() if bypass_doc_val and sock_info.max_wire_version >= 4: command['bypassDocumentValidation'] = True bwc = message._BulkWriteContext( self.database.name, command, sock_info, op_id, self.database.client._event_listeners) if sock_info.max_wire_version > 1 and acknowledged: # Batched insert command. results = message._do_batched_write_command( self.database.name + ".$cmd", message._INSERT, command, gen(), check_keys, self.codec_options, bwc) _check_write_command_response(results) else: # Legacy batched OP_INSERT. message._do_batched_insert(self.__full_name, gen(), check_keys, acknowledged, concern, not ordered, self.codec_options, bwc) return ids def insert_one(self, document, bypass_document_validation=False): """Insert a single document. >>> db.test.count({'x': 1}) 0 >>> result = db.test.insert_one({'x': 1}) >>> result.inserted_id ObjectId('54f112defba522406c9cc208') >>> db.test.find_one({'x': 1}) {u'x': 1, u'_id': ObjectId('54f112defba522406c9cc208')} :Parameters: - `document`: The document to insert. Must be a mutable mapping type. If the document does not have an _id field one will be added automatically. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. :Returns: - An instance of :class:`~pymongo.results.InsertOneResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_document_type("document", document) if not (isinstance(document, RawBSONDocument) or "_id" in document): document["_id"] = ObjectId() with self._socket_for_writes() as sock_info: return InsertOneResult( self._insert(sock_info, document, bypass_doc_val=bypass_document_validation), self.write_concern.acknowledged) def insert_many(self, documents, ordered=True, bypass_document_validation=False): """Insert an iterable of documents. >>> db.test.count() 0 >>> result = db.test.insert_many([{'x': i} for i in range(2)]) >>> result.inserted_ids [ObjectId('54f113fffba522406c9cc20e'), ObjectId('54f113fffba522406c9cc20f')] >>> db.test.count() 2 :Parameters: - `documents`: A iterable of documents to insert. - `ordered` (optional): If ``True`` (the default) documents will be inserted on the server serially, in the order provided. If an error occurs all remaining inserts are aborted. If ``False``, documents will be inserted on the server in arbitrary order, possibly in parallel, and all document inserts will be attempted. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. :Returns: An instance of :class:`~pymongo.results.InsertManyResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ if not isinstance(documents, collections.Iterable) or not documents: raise TypeError("documents must be a non-empty list") inserted_ids = [] def gen(): """A generator that validates documents and handles _ids.""" for document in documents: common.validate_is_document_type("document", document) if not isinstance(document, RawBSONDocument): if "_id" not in document: document["_id"] = ObjectId() inserted_ids.append(document["_id"]) yield (message._INSERT, document) blk = _Bulk(self, ordered, bypass_document_validation) blk.ops = [doc for doc in gen()] blk.execute(self.write_concern.document) return InsertManyResult(inserted_ids, self.write_concern.acknowledged) def _update(self, sock_info, criteria, document, upsert=False, check_keys=True, multi=False, manipulate=False, write_concern=None, op_id=None, ordered=True, bypass_doc_val=False): """Internal update / replace helper.""" common.validate_boolean("upsert", upsert) if manipulate: document = self.__database._fix_incoming(document, self) concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 command = SON([('update', self.name), ('ordered', ordered), ('updates', [SON([('q', criteria), ('u', document), ('multi', multi), ('upsert', upsert)])])]) if concern: command['writeConcern'] = concern if sock_info.max_wire_version > 1 and acknowledged: # Update command. if bypass_doc_val and sock_info.max_wire_version >= 4: command['bypassDocumentValidation'] = True # The command result has to be published for APM unmodified # so we make a shallow copy here before adding updatedExisting. result = sock_info.command(self.__database.name, command, codec_options=self.codec_options).copy() _check_write_command_response([(0, result)]) # Add the updatedExisting field for compatibility. if result.get('n') and 'upserted' not in result: result['updatedExisting'] = True else: result['updatedExisting'] = False # MongoDB >= 2.6.0 returns the upsert _id in an array # element. Break it out for backward compatibility. if 'upserted' in result: result['upserted'] = result['upserted'][0]['_id'] return result else: # Legacy OP_UPDATE. return self._legacy_write( sock_info, 'update', command, acknowledged, op_id, bypass_doc_val, message.update, self.__full_name, upsert, multi, criteria, document, acknowledged, concern, check_keys, self.codec_options) def replace_one(self, filter, replacement, upsert=False, bypass_document_validation=False): """Replace a single document matching the filter. >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': ObjectId('54f4c5befba5220aa4d6dee7')} >>> result = db.test.replace_one({'x': 1}, {'y': 1}) >>> result.matched_count 1 >>> result.modified_count 1 >>> for doc in db.test.find({}): ... print(doc) ... {u'y': 1, u'_id': ObjectId('54f4c5befba5220aa4d6dee7')} The *upsert* option can be used to insert a new document if a matching document does not exist. >>> result = db.test.replace_one({'x': 1}, {'x': 1}, True) >>> result.matched_count 0 >>> result.modified_count 0 >>> result.upserted_id ObjectId('54f11e5c8891e756a6e1abd4') >>> db.test.find_one({'x': 1}) {u'x': 1, u'_id': ObjectId('54f11e5c8891e756a6e1abd4')} :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The new document. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_mapping("filter", filter) common.validate_ok_for_replace(replacement) with self._socket_for_writes() as sock_info: result = self._update(sock_info, filter, replacement, upsert, bypass_doc_val=bypass_document_validation) return UpdateResult(result, self.write_concern.acknowledged) def update_one(self, filter, update, upsert=False, bypass_document_validation=False): """Update a single document matching the filter. >>> for doc in db.test.find(): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> result = db.test.update_one({'x': 1}, {'$inc': {'x': 3}}) >>> result.matched_count 1 >>> result.modified_count 1 >>> for doc in db.test.find(): ... print(doc) ... {u'x': 4, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} :Parameters: - `filter`: A query that matches the document to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_mapping("filter", filter) common.validate_ok_for_update(update) with self._socket_for_writes() as sock_info: result = self._update(sock_info, filter, update, upsert, check_keys=False, bypass_doc_val=bypass_document_validation) return UpdateResult(result, self.write_concern.acknowledged) def update_many(self, filter, update, upsert=False, bypass_document_validation=False): """Update one or more documents that match the filter. >>> for doc in db.test.find(): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> result = db.test.update_many({'x': 1}, {'$inc': {'x': 3}}) >>> result.matched_count 3 >>> result.modified_count 3 >>> for doc in db.test.find(): ... print(doc) ... {u'x': 4, u'_id': 0} {u'x': 4, u'_id': 1} {u'x': 4, u'_id': 2} :Parameters: - `filter`: A query that matches the documents to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 3.2 Added bypass_document_validation support .. versionadded:: 3.0 """ common.validate_is_mapping("filter", filter) common.validate_ok_for_update(update) with self._socket_for_writes() as sock_info: result = self._update(sock_info, filter, update, upsert, check_keys=False, multi=True, bypass_doc_val=bypass_document_validation) return UpdateResult(result, self.write_concern.acknowledged) def drop(self): """Alias for :meth:`~pymongo.database.Database.drop_collection`. The following two calls are equivalent: >>> db.foo.drop() >>> db.drop_collection("foo") """ self.__database.drop_collection(self.__name) def _delete( self, sock_info, criteria, multi, write_concern=None, op_id=None, ordered=True): """Internal delete helper.""" common.validate_is_mapping("filter", criteria) concern = (write_concern or self.write_concern).document acknowledged = concern.get("w") != 0 command = SON([('delete', self.name), ('ordered', ordered), ('deletes', [SON([('q', criteria), ('limit', int(not multi))])])]) if concern: command['writeConcern'] = concern if sock_info.max_wire_version > 1 and acknowledged: # Delete command. result = sock_info.command(self.__database.name, command, codec_options=self.codec_options) _check_write_command_response([(0, result)]) return result else: # Legacy OP_DELETE. return self._legacy_write( sock_info, 'delete', command, acknowledged, op_id, False, message.delete, self.__full_name, criteria, acknowledged, concern, self.codec_options, int(not multi)) def delete_one(self, filter): """Delete a single document matching the filter. >>> db.test.count({'x': 1}) 3 >>> result = db.test.delete_one({'x': 1}) >>> result.deleted_count 1 >>> db.test.count({'x': 1}) 2 :Parameters: - `filter`: A query that matches the document to delete. :Returns: - An instance of :class:`~pymongo.results.DeleteResult`. .. versionadded:: 3.0 """ with self._socket_for_writes() as sock_info: return DeleteResult(self._delete(sock_info, filter, False), self.write_concern.acknowledged) def delete_many(self, filter): """Delete one or more documents matching the filter. >>> db.test.count({'x': 1}) 3 >>> result = db.test.delete_many({'x': 1}) >>> result.deleted_count 3 >>> db.test.count({'x': 1}) 0 :Parameters: - `filter`: A query that matches the documents to delete. :Returns: - An instance of :class:`~pymongo.results.DeleteResult`. .. versionadded:: 3.0 """ with self._socket_for_writes() as sock_info: return DeleteResult(self._delete(sock_info, filter, True), self.write_concern.acknowledged) def find_one(self, filter=None, *args, **kwargs): """Get a single document from the database. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single document, or ``None`` if no matching document is found. The :meth:`find_one` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `filter` (optional): a dictionary specifying the query to be performed OR any other type to be used as the value for a query for ``"_id"``. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. - `max_time_ms` (optional): a value for max_time_ms may be specified as part of `**kwargs`, e.g. >>> find_one(max_time_ms=100) """ if (filter is not None and not isinstance(filter, collections.Mapping)): filter = {"_id": filter} max_time_ms = kwargs.pop("max_time_ms", None) cursor = self.find(filter, *args, **kwargs).max_time_ms(max_time_ms) for result in cursor.limit(-1): return result return None def find(self, *args, **kwargs): """Query the database. The `filter` argument is a prototype document that all results must match. For example: >>> db.test.find({"hello": "world"}) only matches documents that have a key "hello" with value "world". Matches can have other keys *in addition* to "hello". The `projection` argument is used to specify a subset of fields that should be included in the result documents. By limiting results to a certain subset of fields you can cut down on network traffic and decoding time. Raises :class:`TypeError` if any of the arguments are of improper type. Returns an instance of :class:`~pymongo.cursor.Cursor` corresponding to this query. The :meth:`find` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `filter` (optional): a SON object specifying elements which must be present for a document to be included in the result set - `projection` (optional): a list of field names that should be returned in the result set or a dict specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a dict to exclude fields from the result (e.g. projection={'_id': False}). - `skip` (optional): the number of documents to omit (from the start of the result set) when returning the results - `limit` (optional): the maximum number of results to return - `no_cursor_timeout` (optional): if False (the default), any returned cursor is closed by the server after 10 minutes of inactivity. If set to True, the returned cursor will never time out on the server. Care should be taken to ensure that cursors with no_cursor_timeout turned on are properly closed. - `cursor_type` (optional): the type of cursor to return. The valid options are defined by :class:`~pymongo.cursor.CursorType`: - :attr:`~pymongo.cursor.CursorType.NON_TAILABLE` - the result of this find call will return a standard cursor over the result set. - :attr:`~pymongo.cursor.CursorType.TAILABLE` - the result of this find call will be a tailable cursor - tailable cursors are only for use with capped collections. They are not closed when the last data is retrieved but are kept open and the cursor location marks the final document position. If more data is received iteration of the cursor will continue from the last document received. For details, see the `tailable cursor documentation `_. - :attr:`~pymongo.cursor.CursorType.TAILABLE_AWAIT` - the result of this find call will be a tailable cursor with the await flag set. The server will wait for a few seconds after returning the full result set so that it can capture and return additional data added during the query. - :attr:`~pymongo.cursor.CursorType.EXHAUST` - the result of this find call will be an exhaust cursor. MongoDB will stream batched results to the client without waiting for the client to request each batch, reducing latency. See notes on compatibility below. - `sort` (optional): a list of (key, direction) pairs specifying the sort order for this query. See :meth:`~pymongo.cursor.Cursor.sort` for details. - `allow_partial_results` (optional): if True, mongos will return partial results if some shards are down instead of returning an error. - `oplog_replay` (optional): If True, set the oplogReplay query flag. - `modifiers` (optional): A dict specifying the MongoDB `query modifiers`_ that should be used for this query. For example:: >>> db.test.find(modifiers={"$maxTimeMS": 500}) - `batch_size` (optional): Limits the number of documents returned in a single batch. - `manipulate` (optional): **DEPRECATED** - If True (the default), apply any outgoing SON manipulators before returning. .. note:: There are a number of caveats to using :attr:`~pymongo.cursor.CursorType.EXHAUST` as cursor_type: - The `limit` option can not be used with an exhaust cursor. - Exhaust cursors are not supported by mongos and can not be used with a sharded cluster. - A :class:`~pymongo.cursor.Cursor` instance created with the :attr:`~pymongo.cursor.CursorType.EXHAUST` cursor_type requires an exclusive :class:`~socket.socket` connection to MongoDB. If the :class:`~pymongo.cursor.Cursor` is discarded without being completely iterated the underlying :class:`~socket.socket` connection will be closed and discarded without being returned to the connection pool. .. versionchanged:: 3.0 Changed the parameter names `spec`, `fields`, `timeout`, and `partial` to `filter`, `projection`, `no_cursor_timeout`, and `allow_partial_results` respectively. Added the `cursor_type`, `oplog_replay`, and `modifiers` options. Removed the `network_timeout`, `read_preference`, `tag_sets`, `secondary_acceptable_latency_ms`, `max_scan`, `snapshot`, `tailable`, `await_data`, `exhaust`, `as_class`, and slave_okay parameters. Removed `compile_re` option: PyMongo now always represents BSON regular expressions as :class:`~bson.regex.Regex` objects. Use :meth:`~bson.regex.Regex.try_compile` to attempt to convert from a BSON regular expression to a Python regular expression object. Soft deprecated the `manipulate` option. .. versionchanged:: 2.7 Added `compile_re` option. If set to False, PyMongo represented BSON regular expressions as :class:`~bson.regex.Regex` objects instead of attempting to compile BSON regular expressions as Python native regular expressions, thus preventing errors for some incompatible patterns, see `PYTHON-500`_. .. versionadded:: 2.3 The `tag_sets` and `secondary_acceptable_latency_ms` parameters. .. _PYTHON-500: https://jira.mongodb.org/browse/PYTHON-500 .. _query modifiers: http://docs.mongodb.org/manual/reference/operator/query-modifier/ .. mongodoc:: find """ return Cursor(self, *args, **kwargs) def parallel_scan(self, num_cursors): """Scan this entire collection in parallel. Returns a list of up to ``num_cursors`` cursors that can be iterated concurrently. As long as the collection is not modified during scanning, each document appears once in one of the cursors result sets. For example, to process each document in a collection using some thread-safe ``process_document()`` function: >>> def process_cursor(cursor): ... for document in cursor: ... # Some thread-safe processing function: ... process_document(document) >>> >>> # Get up to 4 cursors. ... >>> cursors = collection.parallel_scan(4) >>> threads = [ ... threading.Thread(target=process_cursor, args=(cursor,)) ... for cursor in cursors] >>> >>> for thread in threads: ... thread.start() >>> >>> for thread in threads: ... thread.join() >>> >>> # All documents have now been processed. The :meth:`parallel_scan` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `num_cursors`: the number of cursors to return .. note:: Requires server version **>= 2.5.5**. .. versionchanged:: 3.0 Removed support for arbitrary keyword arguments, since the parallelCollectionScan command has no optional arguments. """ cmd = SON([('parallelCollectionScan', self.__name), ('numCursors', num_cursors)]) with self._socket_for_reads() as (sock_info, slave_ok): result = self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern) return [CommandCursor(self, cursor['cursor'], sock_info.address) for cursor in result['cursors']] def _count(self, cmd): """Internal count helper.""" with self._socket_for_reads() as (sock_info, slave_ok): res = self._command(sock_info, cmd, slave_ok, allowable_errors=["ns missing"], codec_options=self.codec_options._replace( document_class=dict), read_concern=self.read_concern) if res.get("errmsg", "") == "ns missing": return 0 return int(res["n"]) def count(self, filter=None, **kwargs): """Get the number of documents in this collection. All optional count parameters should be passed as keyword arguments to this method. Valid options include: - `hint` (string or list of tuples): The index to use. Specify either the index name as a string or the index specification as a list of tuples (e.g. [('a', pymongo.ASCENDING), ('b', pymongo.ASCENDING)]). - `limit` (int): The maximum number of documents to count. - `skip` (int): The number of matching documents to skip before returning results. - `maxTimeMS` (int): The maximum amount of time to allow the count command to run, in milliseconds. The :meth:`count` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `filter` (optional): A query document that selects which documents to count in the collection. - `**kwargs` (optional): See list of options above. """ cmd = SON([("count", self.__name)]) if filter is not None: if "query" in kwargs: raise ConfigurationError("can't pass both filter and query") kwargs["query"] = filter if "hint" in kwargs and not isinstance(kwargs["hint"], string_type): kwargs["hint"] = helpers._index_document(kwargs["hint"]) cmd.update(kwargs) return self._count(cmd) def create_indexes(self, indexes): """Create one or more indexes on this collection. >>> from pymongo import IndexModel, ASCENDING, DESCENDING >>> index1 = IndexModel([("hello", DESCENDING), ... ("world", ASCENDING)], name="hello_world") >>> index2 = IndexModel([("goodbye", DESCENDING)]) >>> db.test.create_indexes([index1, index2]) ["hello_world"] :Parameters: - `indexes`: A list of :class:`~pymongo.operations.IndexModel` instances. .. note:: `create_indexes` uses the ``createIndexes`` command introduced in MongoDB **2.6** and cannot be used with earlier versions. .. versionadded:: 3.0 """ if not isinstance(indexes, list): raise TypeError("indexes must be a list") names = [] def gen_indexes(): for index in indexes: if not isinstance(index, IndexModel): raise TypeError("%r is not an instance of " "pymongo.operations.IndexModel" % (index,)) document = index.document names.append(document["name"]) yield document cmd = SON([('createIndexes', self.name), ('indexes', list(gen_indexes()))]) with self._socket_for_writes() as sock_info: self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY) return names def __create_index(self, keys, index_options): """Internal create index helper. :Parameters: - `keys`: a list of tuples [(key, type), (key, type), ...] - `index_options`: a dict of index options. """ index_doc = helpers._index_document(keys) index = {"key": index_doc} index.update(index_options) with self._socket_for_writes() as sock_info: cmd = SON([('createIndexes', self.name), ('indexes', [index])]) try: self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY) except OperationFailure as exc: if exc.code in common.COMMAND_NOT_FOUND_CODES: index["ns"] = self.__full_name wcn = (self.write_concern if self.write_concern.acknowledged else WriteConcern()) self.__database.system.indexes._insert( sock_info, index, True, False, False, wcn) else: raise def create_index(self, keys, **kwargs): """Creates an index on this collection. Takes either a single key or a list of (key, direction) pairs. The key(s) must be an instance of :class:`basestring` (:class:`str` in python 3), and the direction(s) must be one of (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, :data:`~pymongo.TEXT`). To create a single key ascending index on the key ``'mike'`` we just use a string argument:: >>> my_collection.create_index("mike") For a compound index on ``'mike'`` descending and ``'eliot'`` ascending we need to use a list of tuples:: >>> my_collection.create_index([("mike", pymongo.DESCENDING), ... ("eliot", pymongo.ASCENDING)]) All optional index creation parameters should be passed as keyword arguments to this method. For example:: >>> my_collection.create_index([("mike", pymongo.DESCENDING)], ... background=True) Valid options include, but are not limited to: - `name`: custom name to use for this index - if none is given, a name will be generated. - `unique`: if ``True`` creates a uniqueness constraint on the index. - `background`: if ``True`` this index should be created in the background. - `sparse`: if ``True``, omit from the index any documents that lack the indexed field. - `bucketSize`: for use with geoHaystack indexes. Number of documents to group together within a certain proximity to a given longitude and latitude. - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` index. - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` index. - `expireAfterSeconds`: Used to create an expiring (TTL) collection. MongoDB will automatically delete documents from this collection after seconds. The indexed field must be a UTC datetime or the data will not expire. - `partialFilterExpression`: A document that specifies a filter for a partial index. See the MongoDB documentation for a full list of supported options by server version. .. warning:: `dropDups` is not supported by MongoDB 2.7.5 or newer. The option is silently ignored by the server and unique index builds using the option will fail if a duplicate value is detected. .. note:: `expireAfterSeconds` requires server version **>= 2.2** .. note:: `partialFilterExpression` requires server version **>= 3.2** :Parameters: - `keys`: a single key or a list of (key, direction) pairs specifying the index to create - `**kwargs` (optional): any additional index creation options (see the above list) should be passed as keyword arguments .. versionchanged:: 3.2 Added partialFilterExpression to support partial indexes. .. versionchanged:: 3.0 Renamed `key_or_list` to `keys`. Removed the `cache_for` option. :meth:`create_index` no longer caches index names. Removed support for the drop_dups and bucket_size aliases. .. mongodoc:: indexes """ keys = helpers._index_list(keys) name = kwargs.setdefault("name", helpers._gen_index_name(keys)) self.__create_index(keys, kwargs) return name def ensure_index(self, key_or_list, cache_for=300, **kwargs): """**DEPRECATED** - Ensures that an index exists on this collection. .. versionchanged:: 3.0 **DEPRECATED** """ warnings.warn("ensure_index is deprecated. Use create_index instead.", DeprecationWarning, stacklevel=2) # The types supported by datetime.timedelta. if not (isinstance(cache_for, integer_types) or isinstance(cache_for, float)): raise TypeError("cache_for must be an integer or float.") if "drop_dups" in kwargs: kwargs["dropDups"] = kwargs.pop("drop_dups") if "bucket_size" in kwargs: kwargs["bucketSize"] = kwargs.pop("bucket_size") keys = helpers._index_list(key_or_list) name = kwargs.setdefault("name", helpers._gen_index_name(keys)) if not self.__database.client._cached(self.__database.name, self.__name, name): self.__create_index(keys, kwargs) self.__database.client._cache_index(self.__database.name, self.__name, name, cache_for) return name return None def drop_indexes(self): """Drops all indexes on this collection. Can be used on non-existant collections or collections with no indexes. Raises OperationFailure on an error. """ self.__database.client._purge_index(self.__database.name, self.__name) self.drop_index("*") def drop_index(self, index_or_name): """Drops the specified index on this collection. Can be used on non-existant collections or collections with no indexes. Raises OperationFailure on an error (e.g. trying to drop an index that does not exist). `index_or_name` can be either an index name (as returned by `create_index`), or an index specifier (as passed to `create_index`). An index specifier should be a list of (key, direction) pairs. Raises TypeError if index is not an instance of (str, unicode, list). .. warning:: if a custom name was used on index creation (by passing the `name` parameter to :meth:`create_index` or :meth:`ensure_index`) the index **must** be dropped by name. :Parameters: - `index_or_name`: index (or name of index) to drop """ name = index_or_name if isinstance(index_or_name, list): name = helpers._gen_index_name(index_or_name) if not isinstance(name, string_type): raise TypeError("index_or_name must be an index name or list") self.__database.client._purge_index( self.__database.name, self.__name, name) cmd = SON([("dropIndexes", self.__name), ("index", name)]) with self._socket_for_writes() as sock_info: self._command(sock_info, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=["ns not found"]) def reindex(self): """Rebuilds all indexes on this collection. .. warning:: reindex blocks all other operations (indexes are built in the foreground) and will be slow for large collections. """ cmd = SON([("reIndex", self.__name)]) with self._socket_for_writes() as sock_info: return self._command( sock_info, cmd, read_preference=ReadPreference.PRIMARY) def list_indexes(self): """Get a cursor over the index documents for this collection. >>> for index in db.test.list_indexes(): ... print(index) ... SON([(u'v', 1), (u'key', SON([(u'_id', 1)])), (u'name', u'_id_'), (u'ns', u'test.test')]) :Returns: An instance of :class:`~pymongo.command_cursor.CommandCursor`. .. versionadded:: 3.0 """ codec_options = CodecOptions(SON) coll = self.with_options(codec_options) with self._socket_for_primary_reads() as (sock_info, slave_ok): cmd = SON([("listIndexes", self.__name), ("cursor", {})]) if sock_info.max_wire_version > 2: cursor = self._command(sock_info, cmd, slave_ok, ReadPreference.PRIMARY, codec_options)["cursor"] return CommandCursor(coll, cursor, sock_info.address) else: namespace = _UJOIN % (self.__database.name, "system.indexes") res = helpers._first_batch( sock_info, self.__database.name, "system.indexes", {"ns": self.__full_name}, 0, slave_ok, codec_options, ReadPreference.PRIMARY, cmd, self.database.client._event_listeners) data = res["data"] cursor = { "id": res["cursor_id"], "firstBatch": data, "ns": namespace, } # Note that a collection can only have 64 indexes, so we don't # technically have to pass len(data) here. There will never be # an OP_GET_MORE call. return CommandCursor( coll, cursor, sock_info.address, len(data)) def index_information(self): """Get information on this collection's indexes. Returns a dictionary where the keys are index names (as returned by create_index()) and the values are dictionaries containing information about each index. The dictionary is guaranteed to contain at least a single key, ``"key"`` which is a list of (key, direction) pairs specifying the index (as passed to create_index()). It will also contain any other metadata about the indexes, except for the ``"ns"`` and ``"name"`` keys, which are cleaned. Example output might look like this: >>> db.test.ensure_index("x", unique=True) u'x_1' >>> db.test.index_information() {u'_id_': {u'key': [(u'_id', 1)]}, u'x_1': {u'unique': True, u'key': [(u'x', 1)]}} """ cursor = self.list_indexes() info = {} for index in cursor: index["key"] = index["key"].items() index = dict(index) info[index.pop("name")] = index return info def options(self): """Get the options set on this collection. Returns a dictionary of options and their values - see :meth:`~pymongo.database.Database.create_collection` for more information on the possible options. Returns an empty dictionary if the collection has not been created yet. """ with self._socket_for_primary_reads() as (sock_info, slave_ok): if sock_info.max_wire_version > 2: criteria = {"name": self.__name} else: criteria = {"name": self.__full_name} cursor = self.__database._list_collections(sock_info, slave_ok, criteria) result = None for doc in cursor: result = doc break if not result: return {} options = result.get("options", {}) if "create" in options: del options["create"] return options def aggregate(self, pipeline, **kwargs): """Perform an aggregation using the aggregation framework on this collection. All optional aggregate parameters should be passed as keyword arguments to this method. Valid options include, but are not limited to: - `allowDiskUse` (bool): Enables writing to temporary files. When set to True, aggregation stages can write data to the _tmp subdirectory of the --dbpath directory. The default is False. - `maxTimeMS` (int): The maximum amount of time to allow the operation to run in milliseconds. - `batchSize` (int): The maximum number of documents to return per batch. Ignored if the connected mongod or mongos does not support returning aggregate results using a cursor, or `useCursor` is ``False``. - `useCursor` (bool): Requests that the `server` provide results using a cursor, if possible. Ignored if the connected mongod or mongos does not support returning aggregate results using a cursor. The default is ``True``. Set this to ``False`` when upgrading a 2.4 or older sharded cluster to 2.6 or newer (see the warning below). The :meth:`aggregate` method obeys the :attr:`read_preference` of this :class:`Collection`. Please note that using the ``$out`` pipeline stage requires a read preference of :attr:`~pymongo.read_preferences.ReadPreference.PRIMARY` (the default). The server will raise an error if the ``$out`` pipeline stage is used with any other read preference. .. warning:: When upgrading a 2.4 or older sharded cluster to 2.6 or newer the `useCursor` option **must** be set to ``False`` until all shards have been upgraded to 2.6 or newer. .. note:: This method does not support the 'explain' option. Please use :meth:`~pymongo.database.Database.command` instead. An example is included in the :ref:`aggregate-examples` documentation. :Parameters: - `pipeline`: a list of aggregation pipeline stages - `**kwargs` (optional): See list of options above. :Returns: A :class:`~pymongo.command_cursor.CommandCursor` over the result set. .. versionchanged:: 3.0 The :meth:`aggregate` method always returns a CommandCursor. The pipeline argument must be a list. .. versionchanged:: 2.7 When the cursor option is used, return :class:`~pymongo.command_cursor.CommandCursor` instead of :class:`~pymongo.cursor.Cursor`. .. versionchanged:: 2.6 Added cursor support. .. versionadded:: 2.3 .. seealso:: :doc:`/examples/aggregation` .. _aggregate command: http://docs.mongodb.org/manual/applications/aggregation """ if not isinstance(pipeline, list): raise TypeError("pipeline must be a list") if "explain" in kwargs: raise ConfigurationError("The explain option is not supported. " "Use Database.command instead.") cmd = SON([("aggregate", self.__name), ("pipeline", pipeline)]) # Remove things that are not command options. batch_size = common.validate_positive_integer_or_none( "batchSize", kwargs.pop("batchSize", None)) use_cursor = common.validate_boolean( "useCursor", kwargs.pop("useCursor", True)) # If the server does not support the "cursor" option we # ignore useCursor and batchSize. with self._socket_for_reads() as (sock_info, slave_ok): if sock_info.max_wire_version > 0: if use_cursor: if "cursor" not in kwargs: kwargs["cursor"] = {} if batch_size is not None: kwargs["cursor"]["batchSize"] = batch_size cmd.update(kwargs) # Apply this Collection's read concern if $out is not in the # pipeline. if sock_info.max_wire_version >= 4 and 'readConcern' not in cmd: if pipeline and '$out' in pipeline[-1]: result = self._command(sock_info, cmd, slave_ok) else: result = self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern) else: result = self._command(sock_info, cmd, slave_ok) if "cursor" in result: cursor = result["cursor"] else: # Pre-MongoDB 2.6. Fake a cursor. cursor = { "id": 0, "firstBatch": result["result"], "ns": self.full_name, } return CommandCursor( self, cursor, sock_info.address).batch_size(batch_size or 0) # key and condition ought to be optional, but deprecation # would be painful as argument order would have to change. def group(self, key, condition, initial, reduce, finalize=None, **kwargs): """Perform a query similar to an SQL *group by* operation. Returns an array of grouped items. The `key` parameter can be: - ``None`` to use the entire document as a key. - A :class:`list` of keys (each a :class:`basestring` (:class:`str` in python 3)) to group by. - A :class:`basestring` (:class:`str` in python 3), or :class:`~bson.code.Code` instance containing a JavaScript function to be applied to each document, returning the key to group by. The :meth:`group` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `key`: fields to group by (see above description) - `condition`: specification of rows to be considered (as a :meth:`find` query specification) - `initial`: initial value of the aggregation counter object - `reduce`: aggregation function as a JavaScript string - `finalize`: function to be called on each object in output list. - `**kwargs` (optional): additional arguments to the group command may be passed as keyword arguments to this helper method .. versionchanged:: 2.2 Removed deprecated argument: command """ group = {} if isinstance(key, string_type): group["$keyf"] = Code(key) elif key is not None: group = {"key": helpers._fields_list_to_dict(key, "key")} group["ns"] = self.__name group["$reduce"] = Code(reduce) group["cond"] = condition group["initial"] = initial if finalize is not None: group["finalize"] = Code(finalize) cmd = SON([("group", group)]) cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): return self._command(sock_info, cmd, slave_ok)["retval"] def rename(self, new_name, **kwargs): """Rename this collection. If operating in auth mode, client must be authorized as an admin to perform this operation. Raises :class:`TypeError` if `new_name` is not an instance of :class:`basestring` (:class:`str` in python 3). Raises :class:`~pymongo.errors.InvalidName` if `new_name` is not a valid collection name. :Parameters: - `new_name`: new name for this collection - `**kwargs` (optional): additional arguments to the rename command may be passed as keyword arguments to this helper method (i.e. ``dropTarget=True``) """ if not isinstance(new_name, string_type): raise TypeError("new_name must be an " "instance of %s" % (string_type.__name__,)) if not new_name or ".." in new_name: raise InvalidName("collection names cannot be empty") if new_name[0] == "." or new_name[-1] == ".": raise InvalidName("collecion names must not start or end with '.'") if "$" in new_name and not new_name.startswith("oplog.$main"): raise InvalidName("collection names must not contain '$'") new_name = "%s.%s" % (self.__database.name, new_name) cmd = SON([("renameCollection", self.__full_name), ("to", new_name)]) cmd.update(kwargs) with self._socket_for_writes() as sock_info: sock_info.command('admin', cmd) def distinct(self, key, filter=None, **kwargs): """Get a list of distinct values for `key` among all documents in this collection. Raises :class:`TypeError` if `key` is not an instance of :class:`basestring` (:class:`str` in python 3). All optional distinct parameters should be passed as keyword arguments to this method. Valid options include: - `maxTimeMS` (int): The maximum amount of time to allow the count command to run, in milliseconds. The :meth:`distinct` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `key`: name of the field for which we want to get the distinct values - `filter` (optional): A query document that specifies the documents from which to retrieve the distinct values. - `**kwargs` (optional): See list of options above. """ if not isinstance(key, string_type): raise TypeError("key must be an " "instance of %s" % (string_type.__name__,)) cmd = SON([("distinct", self.__name), ("key", key)]) if filter is not None: if "query" in kwargs: raise ConfigurationError("can't pass both filter and query") kwargs["query"] = filter cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): return self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern)["values"] def map_reduce(self, map, reduce, out, full_response=False, **kwargs): """Perform a map/reduce operation on this collection. If `full_response` is ``False`` (default) returns a :class:`~pymongo.collection.Collection` instance containing the results of the operation. Otherwise, returns the full response from the server to the `map reduce command`_. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `out`: output collection name or `out object` (dict). See the `map reduce command`_ documentation for available options. Note: `out` options are order sensitive. :class:`~bson.son.SON` can be used to specify multiple options. e.g. SON([('replace', ), ('db', )]) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: >>> db.test.map_reduce(map, reduce, "myresults", limit=2) .. note:: The :meth:`map_reduce` method does **not** obey the :attr:`read_preference` of this :class:`Collection`. To run mapReduce on a secondary use the :meth:`inline_map_reduce` method instead. .. seealso:: :doc:`/examples/aggregation` .. versionchanged:: 2.2 Removed deprecated arguments: merge_output and reduce_output .. _map reduce command: http://docs.mongodb.org/manual/reference/command/mapReduce/ .. mongodoc:: mapreduce """ if not isinstance(out, (string_type, collections.Mapping)): raise TypeError("'out' must be an instance of " "%s or a mapping" % (string_type.__name__,)) cmd = SON([("mapreduce", self.__name), ("map", map), ("reduce", reduce), ("out", out)]) cmd.update(kwargs) with self._socket_for_primary_reads() as (sock_info, slave_ok): if (sock_info.max_wire_version >= 4 and 'readConcern' not in cmd and 'inline' in cmd['out']): response = self._command( sock_info, cmd, slave_ok, ReadPreference.PRIMARY, read_concern=self.read_concern) else: response = self._command( sock_info, cmd, slave_ok, ReadPreference.PRIMARY) if full_response or not response.get('result'): return response elif isinstance(response['result'], dict): dbase = response['result']['db'] coll = response['result']['collection'] return self.__database.client[dbase][coll] else: return self.__database[response["result"]] def inline_map_reduce(self, map, reduce, full_response=False, **kwargs): """Perform an inline map/reduce operation on this collection. Perform the map/reduce operation on the server in RAM. A result collection is not created. The result set is returned as a list of documents. If `full_response` is ``False`` (default) returns the result documents in a list. Otherwise, returns the full response from the server to the `map reduce command`_. The :meth:`inline_map_reduce` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: >>> db.test.inline_map_reduce(map, reduce, limit=2) """ cmd = SON([("mapreduce", self.__name), ("map", map), ("reduce", reduce), ("out", {"inline": 1})]) cmd.update(kwargs) with self._socket_for_reads() as (sock_info, slave_ok): if sock_info.max_wire_version >= 4 and 'readConcern' not in cmd: res = self._command(sock_info, cmd, slave_ok, read_concern=self.read_concern) else: res = self._command(sock_info, cmd, slave_ok) res = self._command(sock_info, cmd, slave_ok) if full_response: return res else: return res.get("results") def __find_and_modify(self, filter, projection, sort, upsert=None, return_document=ReturnDocument.BEFORE, **kwargs): """Internal findAndModify helper.""" common.validate_is_mapping("filter", filter) if not isinstance(return_document, bool): raise ValueError("return_document must be " "ReturnDocument.BEFORE or ReturnDocument.AFTER") cmd = SON([("findAndModify", self.__name), ("query", filter), ("new", return_document)]) cmd.update(kwargs) if projection is not None: cmd["fields"] = helpers._fields_list_to_dict(projection, "projection") if sort is not None: cmd["sort"] = helpers._index_document(sort) if upsert is not None: common.validate_boolean("upsert", upsert) cmd["upsert"] = upsert with self._socket_for_writes() as sock_info: if sock_info.max_wire_version >= 4 and 'writeConcern' not in cmd: wc_doc = self.write_concern.document if wc_doc: cmd['writeConcern'] = wc_doc out = self._command(sock_info, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=[_NO_OBJ_ERROR]) _check_write_command_response([(0, out)]) return out.get("value") def find_one_and_delete(self, filter, projection=None, sort=None, **kwargs): """Finds a single document and deletes it, returning the document. >>> db.test.count({'x': 1}) 2 >>> db.test.find_one_and_delete({'x': 1}) {u'x': 1, u'_id': ObjectId('54f4e12bfba5220aa4d6dee8')} >>> db.test.count({'x': 1}) 1 If multiple documents match *filter*, a *sort* can be applied. >>> for doc in db.test.find({'x': 1}): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> db.test.find_one_and_delete( ... {'x': 1}, sort=[('_id', pymongo.DESCENDING)]) {u'x': 1, u'_id': 2} The *projection* option can be used to limit the fields returned. >>> db.test.find_one_and_delete({'x': 1}, projection={'_id': False}) {u'x': 1} :Parameters: - `filter`: A query that matches the document to delete. - `projection` (optional): a list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a mapping to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is deleted. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). .. versionchanged:: 3.2 Respects write concern. .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionadded:: 3.0 """ kwargs['remove'] = True return self.__find_and_modify(filter, projection, sort, **kwargs) def find_one_and_replace(self, filter, replacement, projection=None, sort=None, upsert=False, return_document=ReturnDocument.BEFORE, **kwargs): """Finds a single document and replaces it, returning either the original or the replaced document. The :meth:`find_one_and_replace` method differs from :meth:`find_one_and_update` by replacing the document matched by *filter*, rather than modifying the existing document. >>> for doc in db.test.find({}): ... print(doc) ... {u'x': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} >>> db.test.find_one_and_replace({'x': 1}, {'y': 1}) {u'x': 1, u'_id': 0} >>> for doc in db.test.find({}): ... print(doc) ... {u'y': 1, u'_id': 0} {u'x': 1, u'_id': 1} {u'x': 1, u'_id': 2} :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The replacement document. - `projection` (optional): A list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a mapping to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is replaced. - `upsert` (optional): When ``True``, inserts a new document if no document matches the query. Defaults to ``False``. - `return_document`: If :attr:`ReturnDocument.BEFORE` (the default), returns the original document before it was replaced, or ``None`` if no document matches. If :attr:`ReturnDocument.AFTER`, returns the replaced or inserted document. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). .. versionchanged:: 3.2 Respects write concern. .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionadded:: 3.0 """ common.validate_ok_for_replace(replacement) kwargs['update'] = replacement return self.__find_and_modify(filter, projection, sort, upsert, return_document, **kwargs) def find_one_and_update(self, filter, update, projection=None, sort=None, upsert=False, return_document=ReturnDocument.BEFORE, **kwargs): """Finds a single document and updates it, returning either the original or the updated document. >>> db.test.find_one_and_update( ... {'_id': 665}, {'$inc': {'count': 1}, '$set': {'done': True}}) {u'_id': 665, u'done': False, u'count': 25}} By default :meth:`find_one_and_update` returns the original version of the document before the update was applied. To return the updated version of the document instead, use the *return_document* option. >>> from pymongo import ReturnDocument >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... return_document=ReturnDocument.AFTER) {u'_id': u'userid', u'seq': 1} You can limit the fields returned with the *projection* option. >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... projection={'seq': True, '_id': False}, ... return_document=ReturnDocument.AFTER) {u'seq': 2} The *upsert* option can be used to create the document if it doesn't already exist. >>> db.example.delete_many({}).deleted_count 1 >>> db.example.find_one_and_update( ... {'_id': 'userid'}, ... {'$inc': {'seq': 1}}, ... projection={'seq': True, '_id': False}, ... upsert=True, ... return_document=ReturnDocument.AFTER) {u'seq': 1} If multiple documents match *filter*, a *sort* can be applied. >>> for doc in db.test.find({'done': True}): ... print(doc) ... {u'_id': 665, u'done': True, u'result': {u'count': 26}} {u'_id': 701, u'done': True, u'result': {u'count': 17}} >>> db.test.find_one_and_update( ... {'done': True}, ... {'$set': {'final': True}}, ... sort=[('_id', pymongo.DESCENDING)]) {u'_id': 701, u'done': True, u'result': {u'count': 17}} :Parameters: - `filter`: A query that matches the document to update. - `update`: The update operations to apply. - `projection` (optional): A list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a dict to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is updated. - `upsert` (optional): When ``True``, inserts a new document if no document matches the query. Defaults to ``False``. - `return_document`: If :attr:`ReturnDocument.BEFORE` (the default), returns the original document before it was updated, or ``None`` if no document matches. If :attr:`ReturnDocument.AFTER`, returns the updated or inserted document. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). .. versionchanged:: 3.2 Respects write concern. .. warning:: Starting in PyMongo 3.2, this command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionadded:: 3.0 """ common.validate_ok_for_update(update) kwargs['update'] = update return self.__find_and_modify(filter, projection, sort, upsert, return_document, **kwargs) def save(self, to_save, manipulate=True, check_keys=True, **kwargs): """Save a document in this collection. **DEPRECATED** - Use :meth:`insert_one` or :meth:`replace_one` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("save is deprecated. Use insert_one or replace_one " "instead", DeprecationWarning, stacklevel=2) common.validate_is_document_type("to_save", to_save) write_concern = None if kwargs: write_concern = WriteConcern(**kwargs) with self._socket_for_writes() as sock_info: if not (isinstance(to_save, RawBSONDocument) or "_id" in to_save): return self._insert(sock_info, to_save, True, check_keys, manipulate, write_concern) else: self._update(sock_info, {"_id": to_save["_id"]}, to_save, True, check_keys, False, manipulate, write_concern) return to_save.get("_id") def insert(self, doc_or_docs, manipulate=True, check_keys=True, continue_on_error=False, **kwargs): """Insert a document(s) into this collection. **DEPRECATED** - Use :meth:`insert_one` or :meth:`insert_many` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("insert is deprecated. Use insert_one or insert_many " "instead.", DeprecationWarning, stacklevel=2) write_concern = None if kwargs: write_concern = WriteConcern(**kwargs) with self._socket_for_writes() as sock_info: return self._insert(sock_info, doc_or_docs, not continue_on_error, check_keys, manipulate, write_concern) def update(self, spec, document, upsert=False, manipulate=False, multi=False, check_keys=True, **kwargs): """Update a document(s) in this collection. **DEPRECATED** - Use :meth:`replace_one`, :meth:`update_one`, or :meth:`update_many` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("update is deprecated. Use replace_one, update_one or " "update_many instead.", DeprecationWarning, stacklevel=2) common.validate_is_mapping("spec", spec) common.validate_is_mapping("document", document) if document: # If a top level key begins with '$' this is a modify operation # and we should skip key validation. It doesn't matter which key # we check here. Passing a document with a mix of top level keys # starting with and without a '$' is invalid and the server will # raise an appropriate exception. first = next(iter(document)) if first.startswith('$'): check_keys = False write_concern = None if kwargs: write_concern = WriteConcern(**kwargs) with self._socket_for_writes() as sock_info: return self._update(sock_info, spec, document, upsert, check_keys, multi, manipulate, write_concern) def remove(self, spec_or_id=None, multi=True, **kwargs): """Remove a document(s) from this collection. **DEPRECATED** - Use :meth:`delete_one` or :meth:`delete_many` instead. .. versionchanged:: 3.0 Removed the `safe` parameter. Pass ``w=0`` for unacknowledged write operations. """ warnings.warn("remove is deprecated. Use delete_one or delete_many " "instead.", DeprecationWarning, stacklevel=2) if spec_or_id is None: spec_or_id = {} if not isinstance(spec_or_id, collections.Mapping): spec_or_id = {"_id": spec_or_id} write_concern = None if kwargs: write_concern = WriteConcern(**kwargs) with self._socket_for_writes() as sock_info: return self._delete(sock_info, spec_or_id, multi, write_concern) def find_and_modify(self, query={}, update=None, upsert=False, sort=None, full_response=False, manipulate=False, **kwargs): """Update and return an object. **DEPRECATED** - Use :meth:`find_one_and_delete`, :meth:`find_one_and_replace`, or :meth:`find_one_and_update` instead. """ warnings.warn("find_and_modify is deprecated, use find_one_and_delete" ", find_one_and_replace, or find_one_and_update instead", DeprecationWarning, stacklevel=2) if not update and not kwargs.get('remove', None): raise ValueError("Must either update or remove") if update and kwargs.get('remove', None): raise ValueError("Can't do both update and remove") # No need to include empty args if query: kwargs['query'] = query if update: kwargs['update'] = update if upsert: kwargs['upsert'] = upsert if sort: # Accept a list of tuples to match Cursor's sort parameter. if isinstance(sort, list): kwargs['sort'] = helpers._index_document(sort) # Accept OrderedDict, SON, and dict with len == 1 so we # don't break existing code already using find_and_modify. elif (isinstance(sort, _ORDERED_TYPES) or isinstance(sort, dict) and len(sort) == 1): warnings.warn("Passing mapping types for `sort` is deprecated," " use a list of (key, direction) pairs instead", DeprecationWarning, stacklevel=2) kwargs['sort'] = sort else: raise TypeError("sort must be a list of (key, direction) " "pairs, a dict of len 1, or an instance of " "SON or OrderedDict") fields = kwargs.pop("fields", None) if fields is not None: kwargs["fields"] = helpers._fields_list_to_dict(fields, "fields") cmd = SON([("findAndModify", self.__name)]) cmd.update(kwargs) with self._socket_for_writes() as sock_info: if sock_info.max_wire_version >= 4 and 'writeConcern' not in cmd: wc_doc = self.write_concern.document if wc_doc: cmd['writeConcern'] = wc_doc out = self._command(sock_info, cmd, read_preference=ReadPreference.PRIMARY, allowable_errors=[_NO_OBJ_ERROR]) _check_write_command_response([(0, out)]) if not out['ok']: if out["errmsg"] == _NO_OBJ_ERROR: return None else: # Should never get here b/c of allowable_errors raise ValueError("Unexpected Error: %s" % (out,)) if full_response: return out else: document = out.get('value') if manipulate: document = self.__database._fix_outgoing(document, self) return document def __iter__(self): return self def __next__(self): raise TypeError("'Collection' object is not iterable") next = __next__ def __call__(self, *args, **kwargs): """This is only here so that some API misusages are easier to debug. """ if "." not in self.__name: raise TypeError("'Collection' object is not callable. If you " "meant to call the '%s' method on a 'Database' " "object it is failing because no such method " "exists." % self.__name) raise TypeError("'Collection' object is not callable. If you meant to " "call the '%s' method on a 'Collection' object it is " "failing because no such method exists." % self.__name.split(".")[-1]) pymongo-3.2/pymongo/_cmessagemodule.c0000644000175000017500000012724412630145074021760 0ustar behackettbehackett00000000000000/* * Copyright 2009-2015 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains C implementations of some of the functions * needed by the message module. If possible, these implementations * should be used to speed up message creation. */ #include "Python.h" #include "_cbsonmodule.h" #include "buffer.h" struct module_state { PyObject* _cbson; }; /* See comments about module initialization in _cbsonmodule.c */ #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif #if PY_MAJOR_VERSION >= 3 #define BYTES_FORMAT_STRING "y#" #else #define BYTES_FORMAT_STRING "s#" #endif #define DOC_TOO_LARGE_FMT "BSON document too large (%d bytes)" \ " - the connected server supports" \ " BSON document sizes up to %ld bytes." /* Get an error class from the pymongo.errors module. * * Returns a new ref */ static PyObject* _error(char* name) { PyObject* error; PyObject* errors = PyImport_ImportModule("pymongo.errors"); if (!errors) { return NULL; } error = PyObject_GetAttrString(errors, name); Py_DECREF(errors); return error; } /* add a lastError message on the end of the buffer. * returns 0 on failure */ static int add_last_error(PyObject* self, buffer_t buffer, int request_id, char* ns, int nslen, codec_options_t* options, PyObject* args) { struct module_state *state = GETSTATE(self); int message_start; int document_start; int message_length; int document_length; PyObject* key; PyObject* value; Py_ssize_t pos = 0; PyObject* one; char *p = strchr(ns, '.'); /* Length of the database portion of ns. */ nslen = p ? (int)(p - ns) : nslen; message_start = buffer_save_space(buffer, 4); if (message_start == -1) { PyErr_NoMemory(); return 0; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" /* responseTo */ "\xd4\x07\x00\x00" /* opcode */ "\x00\x00\x00\x00", /* options */ 12) || !buffer_write_bytes(buffer, ns, nslen) || /* database */ !buffer_write_bytes(buffer, ".$cmd\x00" /* collection name */ "\x00\x00\x00\x00" /* skip */ "\xFF\xFF\xFF\xFF", /* limit (-1) */ 14)) { return 0; } /* save space for length */ document_start = buffer_save_space(buffer, 4); if (document_start == -1) { PyErr_NoMemory(); return 0; } /* getlasterror: 1 */ if (!(one = PyLong_FromLong(1))) return 0; if (!write_pair(state->_cbson, buffer, "getlasterror", 12, one, 0, options, 1)) { Py_DECREF(one); return 0; } Py_DECREF(one); /* getlasterror options */ while (PyDict_Next(args, &pos, &key, &value)) { if (!decode_and_write_pair(state->_cbson, buffer, key, value, 0, options, 0)) { return 0; } } /* EOD */ if (!buffer_write_bytes(buffer, "\x00", 1)) { return 0; } message_length = buffer_get_position(buffer) - message_start; document_length = buffer_get_position(buffer) - document_start; memcpy(buffer_get_buffer(buffer) + message_start, &message_length, 4); memcpy(buffer_get_buffer(buffer) + document_start, &document_length, 4); return 1; } static int init_insert_buffer(buffer_t buffer, int request_id, int options, const char* coll_name, int coll_name_len) { /* Save space for message length */ int length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyErr_NoMemory(); return length_location; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd2\x07\x00\x00", 8) || !buffer_write_bytes(buffer, (const char*)&options, 4) || !buffer_write_bytes(buffer, coll_name, coll_name_len + 1)) { return -1; } return length_location; } static PyObject* _cbson_insert_message(PyObject* self, PyObject* args) { /* Used by the Bulk API to insert into pre-2.6 servers. Collection.insert * uses _cbson_do_batched_insert. */ struct module_state *state = GETSTATE(self); /* NOTE just using a random number as the request_id */ int request_id = rand(); char* collection_name = NULL; int collection_name_length; PyObject* docs; PyObject* doc; PyObject* iterator; int before, cur_size, max_size = 0; int flags = 0; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; codec_options_t options; PyObject* last_error_args; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#ObbObO&", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, convert_codec_options, &options)) { return NULL; } if (continue_on_error) { flags += 1; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } length_location = init_insert_buffer(buffer, request_id, flags, collection_name, collection_name_length); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); buffer_free(buffer); return NULL; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } while ((doc = PyIter_Next(iterator)) != NULL) { before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { Py_DECREF(doc); Py_DECREF(iterator); destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } Py_DECREF(doc); cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; } Py_DECREF(iterator); if (PyErr_Occurred()) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } if (!max_size) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, &options, last_error_args)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* _cbson_update_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); char* collection_name = NULL; int collection_name_length; int before, cur_size, max_size = 0; PyObject* doc; PyObject* spec; unsigned char multi; unsigned char upsert; unsigned char safe; unsigned char check_keys; codec_options_t options; PyObject* last_error_args; int flags; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#bbOObObO&", "utf-8", &collection_name, &collection_name_length, &upsert, &multi, &spec, &doc, &safe, &last_error_args, &check_keys, convert_codec_options, &options)) { return NULL; } flags = 0; if (upsert) { flags += 1; } if (multi) { flags += 2; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd1\x07\x00\x00" "\x00\x00\x00\x00", 12) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_bytes(buffer, (const char*)&flags, 4)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, spec, 0, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } max_size = buffer_get_position(buffer) - before; before = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } cur_size = buffer_get_position(buffer) - before; max_size = (cur_size > max_size) ? cur_size : max_size; message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); if (safe) { if (!add_last_error(self, buffer, request_id, collection_name, collection_name_length, &options, last_error_args)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } } PyMem_Free(collection_name); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* _cbson_query_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ struct module_state *state = GETSTATE(self); int request_id = rand(); unsigned int flags; char* collection_name = NULL; int collection_name_length; int begin, cur_size, max_size = 0; int num_to_skip; int num_to_return; PyObject* query; PyObject* field_selector; codec_options_t options; buffer_t buffer; int length_location, message_length; unsigned char check_keys = 0; PyObject* result; if (!PyArg_ParseTuple(args, "Iet#iiOOO&|b", &flags, "utf-8", &collection_name, &collection_name_length, &num_to_skip, &num_to_return, &query, &field_selector, convert_codec_options, &options, &check_keys)) { return NULL; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00\xd4\x07\x00\x00", 8) || !buffer_write_bytes(buffer, (const char*)&flags, 4) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_bytes(buffer, (const char*)&num_to_skip, 4) || !buffer_write_bytes(buffer, (const char*)&num_to_return, 4)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, query, check_keys, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } max_size = buffer_get_position(buffer) - begin; if (field_selector != Py_None) { begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, field_selector, 0, &options, 1)) { destroy_codec_options(&options); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } cur_size = buffer_get_position(buffer) - begin; max_size = (cur_size > max_size) ? cur_size : max_size; } PyMem_Free(collection_name); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING "i", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), max_size); destroy_codec_options(&options); buffer_free(buffer); return result; } static PyObject* _cbson_get_more_message(PyObject* self, PyObject* args) { /* NOTE just using a random number as the request_id */ int request_id = rand(); char* collection_name = NULL; int collection_name_length; int num_to_return; long long cursor_id; buffer_t buffer; int length_location, message_length; PyObject* result; if (!PyArg_ParseTuple(args, "et#iL", "utf-8", &collection_name, &collection_name_length, &num_to_return, &cursor_id)) { return NULL; } buffer = buffer_new(); if (!buffer) { PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } // save space for message length length_location = buffer_save_space(buffer, 4); if (length_location == -1) { PyMem_Free(collection_name); PyErr_NoMemory(); return NULL; } if (!buffer_write_bytes(buffer, (const char*)&request_id, 4) || !buffer_write_bytes(buffer, "\x00\x00\x00\x00" "\xd5\x07\x00\x00" "\x00\x00\x00\x00", 12) || !buffer_write_bytes(buffer, collection_name, collection_name_length + 1) || !buffer_write_bytes(buffer, (const char*)&num_to_return, 4) || !buffer_write_bytes(buffer, (const char*)&cursor_id, 8)) { buffer_free(buffer); PyMem_Free(collection_name); return NULL; } PyMem_Free(collection_name); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); /* objectify buffer */ result = Py_BuildValue("i" BYTES_FORMAT_STRING, request_id, buffer_get_buffer(buffer), buffer_get_position(buffer)); buffer_free(buffer); return result; } static void _set_document_too_large(int size, long max) { PyObject* DocumentTooLarge = _error("DocumentTooLarge"); if (DocumentTooLarge) { #if PY_MAJOR_VERSION >= 3 PyObject* error = PyUnicode_FromFormat(DOC_TOO_LARGE_FMT, size, max); #else PyObject* error = PyString_FromFormat(DOC_TOO_LARGE_FMT, size, max); #endif if (error) { PyErr_SetObject(DocumentTooLarge, error); Py_DECREF(error); } Py_DECREF(DocumentTooLarge); } } static PyObject* _send_insert(PyObject* self, PyObject* ctx, PyObject* gle_args, buffer_t buffer, char* coll_name, int coll_len, int request_id, int safe, codec_options_t* options, PyObject* to_publish) { if (safe) { if (!add_last_error(self, buffer, request_id, coll_name, coll_len, options, gle_args)) { return NULL; } } /* The max_doc_size parameter for legacy_write is the max size of any * document in buffer. We enforced max size already, pass 0 here. */ return PyObject_CallMethod(ctx, "legacy_write", "i" BYTES_FORMAT_STRING "iNO", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), 0, PyBool_FromLong((long)safe), to_publish); } static PyObject* _cbson_do_batched_insert(PyObject* self, PyObject* args) { struct module_state *state = GETSTATE(self); /* NOTE just using a random number as the request_id */ int request_id = rand(); int send_safe, flags = 0; int length_location, message_length; int collection_name_length; char* collection_name = NULL; PyObject* docs; PyObject* doc; PyObject* iterator; PyObject* ctx; PyObject* last_error_args; PyObject* result; PyObject* max_bson_size_obj; PyObject* max_message_size_obj; PyObject* to_publish = NULL; unsigned char check_keys; unsigned char safe; unsigned char continue_on_error; codec_options_t options; unsigned char empty = 1; long max_bson_size; long max_message_size; buffer_t buffer; PyObject *exc_type = NULL, *exc_value = NULL, *exc_trace = NULL; if (!PyArg_ParseTuple(args, "et#ObbObO&O", "utf-8", &collection_name, &collection_name_length, &docs, &check_keys, &safe, &last_error_args, &continue_on_error, convert_codec_options, &options, &ctx)) { return NULL; } if (continue_on_error) { flags += 1; } /* * If we are doing unacknowledged writes *and* continue_on_error * is True it's pointless (and slower) to send GLE. */ send_safe = (safe || !continue_on_error); max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size"); #if PY_MAJOR_VERSION >= 3 max_bson_size = PyLong_AsLong(max_bson_size_obj); #else max_bson_size = PyInt_AsLong(max_bson_size_obj); #endif Py_XDECREF(max_bson_size_obj); if (max_bson_size == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } max_message_size_obj = PyObject_GetAttrString(ctx, "max_message_size"); #if PY_MAJOR_VERSION >= 3 max_message_size = PyLong_AsLong(max_message_size_obj); #else max_message_size = PyInt_AsLong(max_message_size_obj); #endif Py_XDECREF(max_message_size_obj); if (max_message_size == -1) { destroy_codec_options(&options); PyMem_Free(collection_name); return NULL; } buffer = buffer_new(); if (!buffer) { destroy_codec_options(&options); PyErr_NoMemory(); PyMem_Free(collection_name); return NULL; } length_location = init_insert_buffer(buffer, request_id, flags, collection_name, collection_name_length); if (length_location == -1) { goto insertfail; } if (!(to_publish = PyList_New(0))) { goto insertfail; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } goto insertfail; } while ((doc = PyIter_Next(iterator)) != NULL) { int before = buffer_get_position(buffer); int cur_size; if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { goto iterfail; } cur_size = buffer_get_position(buffer) - before; if (cur_size > max_bson_size) { /* If we've encoded anything send it before raising. */ if (!empty) { buffer_update_position(buffer, before); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, send_safe, &options, to_publish); if (!result) goto iterfail; Py_DECREF(result); } _set_document_too_large(cur_size, max_bson_size); goto iterfail; } empty = 0; /* We have enough data, send this batch. */ if (buffer_get_position(buffer) > max_message_size) { int new_request_id = rand(); int message_start; buffer_t new_buffer = buffer_new(); if (!new_buffer) { PyErr_NoMemory(); goto iterfail; } message_start = init_insert_buffer(new_buffer, new_request_id, flags, collection_name, collection_name_length); if (message_start == -1) { buffer_free(new_buffer); goto iterfail; } /* Copy the overflow encoded document into the new buffer. */ if (!buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer) + before, cur_size)) { buffer_free(new_buffer); goto iterfail; } /* Roll back to the beginning of this document. */ buffer_update_position(buffer, before); message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, send_safe, &options, to_publish); buffer_free(buffer); buffer = new_buffer; request_id = new_request_id; length_location = message_start; Py_DECREF(to_publish); if (!(to_publish = PyList_New(0))) { goto insertfail; } if (!result) { PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; PyObject* OperationFailure; PyErr_Fetch(&etype, &evalue, &etrace); OperationFailure = _error("OperationFailure"); if (OperationFailure) { if (PyErr_GivenExceptionMatches(etype, OperationFailure)) { if (!safe || continue_on_error) { Py_DECREF(OperationFailure); if (!safe) { /* We're doing unacknowledged writes and * continue_on_error is False. Just return. */ Py_DECREF(etype); Py_XDECREF(evalue); Py_XDECREF(etrace); Py_DECREF(to_publish); Py_DECREF(iterator); Py_DECREF(doc); buffer_free(buffer); PyMem_Free(collection_name); Py_RETURN_NONE; } /* continue_on_error is True, store the error * details to re-raise after the final batch */ Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); exc_type = etype; exc_value = evalue; exc_trace = etrace; if (PyList_Append(to_publish, doc) < 0) { goto iterfail; } Py_CLEAR(doc); continue; } } Py_DECREF(OperationFailure); } /* This isn't OperationFailure, we couldn't * import OperationFailure, or we are doing * acknowledged writes. Re-raise immediately. */ PyErr_Restore(etype, evalue, etrace); goto iterfail; } else { Py_DECREF(result); } } if (PyList_Append(to_publish, doc) < 0) { goto iterfail; } Py_CLEAR(doc); } Py_DECREF(iterator); if (PyErr_Occurred()) { goto insertfail; } if (empty) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk insert"); Py_DECREF(InvalidOperation); } goto insertfail; } message_length = buffer_get_position(buffer) - length_location; memcpy(buffer_get_buffer(buffer) + length_location, &message_length, 4); /* Send the last (or only) batch */ result = _send_insert(self, ctx, last_error_args, buffer, collection_name, collection_name_length, request_id, safe, &options, to_publish); Py_DECREF(to_publish); PyMem_Free(collection_name); buffer_free(buffer); if (!result) { Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); return NULL; } else { Py_DECREF(result); } if (exc_type) { /* Re-raise any previously stored exception * due to continue_on_error being True */ PyErr_Restore(exc_type, exc_value, exc_trace); return NULL; } Py_RETURN_NONE; iterfail: Py_XDECREF(doc); Py_DECREF(iterator); insertfail: Py_XDECREF(exc_type); Py_XDECREF(exc_value); Py_XDECREF(exc_trace); Py_XDECREF(to_publish); buffer_free(buffer); PyMem_Free(collection_name); return NULL; } static PyObject* _send_write_command(PyObject* ctx, buffer_t buffer, int lst_len_loc, int cmd_len_loc, unsigned char* errors, PyObject* to_publish) { PyObject* result; int request_id = rand(); int position = buffer_get_position(buffer); int length = position - lst_len_loc - 1; memcpy(buffer_get_buffer(buffer) + lst_len_loc, &length, 4); length = position - cmd_len_loc; memcpy(buffer_get_buffer(buffer) + cmd_len_loc, &length, 4); memcpy(buffer_get_buffer(buffer), &position, 4); memcpy(buffer_get_buffer(buffer) + 4, &request_id, 4); /* Send the current batch */ result = PyObject_CallMethod(ctx, "write_command", "i" BYTES_FORMAT_STRING "O", request_id, buffer_get_buffer(buffer), buffer_get_position(buffer), to_publish); if (result && PyDict_GetItemString(result, "writeErrors")) *errors = 1; return result; } static buffer_t _command_buffer_new(char* ns, int ns_len) { buffer_t buffer; if (!(buffer = buffer_new())) { PyErr_NoMemory(); return NULL; } /* Save space for message length and request id */ if ((buffer_save_space(buffer, 8)) == -1) { PyErr_NoMemory(); buffer_free(buffer); return NULL; } if (!buffer_write_bytes(buffer, "\x00\x00\x00\x00" /* responseTo */ "\xd4\x07\x00\x00" /* opcode */ "\x00\x00\x00\x00", /* flags */ 12) || !buffer_write_bytes(buffer, ns, ns_len + 1) || /* namespace */ !buffer_write_bytes(buffer, "\x00\x00\x00\x00" /* skip */ "\xFF\xFF\xFF\xFF", /* limit (-1) */ 8)) { buffer_free(buffer); return NULL; } return buffer; } #define _INSERT 0 #define _UPDATE 1 #define _DELETE 2 static PyObject* _cbson_do_batched_write_command(PyObject* self, PyObject* args) { struct module_state *state = GETSTATE(self); long max_bson_size; long max_cmd_size; long max_write_batch_size; long idx_offset = 0; int idx = 0; int cmd_len_loc; int lst_len_loc; int ns_len; int ordered; char *ns = NULL; PyObject* max_bson_size_obj; PyObject* max_write_batch_size_obj; PyObject* command; PyObject* doc; PyObject* docs; PyObject* ctx; PyObject* iterator; PyObject* result; PyObject* results; PyObject* to_publish = NULL; unsigned char op; unsigned char check_keys; codec_options_t options; unsigned char empty = 1; unsigned char errors = 0; buffer_t buffer; if (!PyArg_ParseTuple(args, "et#bOObO&O", "utf-8", &ns, &ns_len, &op, &command, &docs, &check_keys, convert_codec_options, &options, &ctx)) { return NULL; } max_bson_size_obj = PyObject_GetAttrString(ctx, "max_bson_size"); #if PY_MAJOR_VERSION >= 3 max_bson_size = PyLong_AsLong(max_bson_size_obj); #else max_bson_size = PyInt_AsLong(max_bson_size_obj); #endif Py_XDECREF(max_bson_size_obj); if (max_bson_size == -1) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } /* * Max BSON object size + 16k - 2 bytes for ending NUL bytes * XXX: This should come from the server - SERVER-10643 */ max_cmd_size = max_bson_size + 16382; max_write_batch_size_obj = PyObject_GetAttrString(ctx, "max_write_batch_size"); #if PY_MAJOR_VERSION >= 3 max_write_batch_size = PyLong_AsLong(max_write_batch_size_obj); #else max_write_batch_size = PyInt_AsLong(max_write_batch_size_obj); #endif Py_XDECREF(max_write_batch_size_obj); if (max_write_batch_size == -1) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } /* Default to True */ ordered = !((PyDict_GetItemString(command, "ordered")) == Py_False); if (!(results = PyList_New(0))) { destroy_codec_options(&options); PyMem_Free(ns); return NULL; } if (!(to_publish = PyList_New(0))) { destroy_codec_options(&options); PyMem_Free(ns); Py_DECREF(results); return NULL; } if (!(buffer = _command_buffer_new(ns, ns_len))) { destroy_codec_options(&options); PyMem_Free(ns); Py_DECREF(results); Py_DECREF(to_publish); return NULL; } PyMem_Free(ns); /* Position of command document length */ cmd_len_loc = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, command, 0, &options, 0)) { goto cmdfail; } /* Write type byte for array */ *(buffer_get_buffer(buffer) + (buffer_get_position(buffer) - 1)) = 0x4; switch (op) { case _INSERT: { if (!buffer_write_bytes(buffer, "documents\x00", 10)) goto cmdfail; break; } case _UPDATE: { /* MongoDB does key validation for update. */ check_keys = 0; if (!buffer_write_bytes(buffer, "updates\x00", 8)) goto cmdfail; break; } case _DELETE: { /* Never check keys in a delete command. */ check_keys = 0; if (!buffer_write_bytes(buffer, "deletes\x00", 8)) goto cmdfail; break; } default: { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "Unknown command"); Py_DECREF(InvalidOperation); } goto cmdfail; } } /* Save space for list document */ lst_len_loc = buffer_save_space(buffer, 4); if (lst_len_loc == -1) { PyErr_NoMemory(); goto cmdfail; } iterator = PyObject_GetIter(docs); if (iterator == NULL) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "input is not iterable"); Py_DECREF(InvalidOperation); } goto cmdfail; } while ((doc = PyIter_Next(iterator)) != NULL) { int sub_doc_begin = buffer_get_position(buffer); int cur_doc_begin; int cur_size; int enough_data = 0; int enough_documents = 0; char key[16]; empty = 0; INT2STRING(key, idx); if (!buffer_write_bytes(buffer, "\x03", 1) || !buffer_write_bytes(buffer, key, (int)strlen(key) + 1)) { goto cmditerfail; } cur_doc_begin = buffer_get_position(buffer); if (!write_dict(state->_cbson, buffer, doc, check_keys, &options, 1)) { goto cmditerfail; } /* We have enough data, maybe send this batch. */ enough_data = (buffer_get_position(buffer) > max_cmd_size); enough_documents = (idx >= max_write_batch_size); if (enough_data || enough_documents) { buffer_t new_buffer; cur_size = buffer_get_position(buffer) - cur_doc_begin; /* This single document is too large for the command. */ if (!idx) { if (op == _INSERT) { _set_document_too_large(cur_size, max_bson_size); } else { PyObject* DocumentTooLarge = _error("DocumentTooLarge"); if (DocumentTooLarge) { /* * There's nothing intelligent we can say * about size for update and remove. */ PyErr_SetString(DocumentTooLarge, "command document too large"); Py_DECREF(DocumentTooLarge); } } goto cmditerfail; } if (!(new_buffer = buffer_new())) { PyErr_NoMemory(); goto cmditerfail; } /* New buffer including the current overflow document */ if (!buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer), lst_len_loc + 5) || !buffer_write_bytes(new_buffer, "0\x00", 2) || !buffer_write_bytes(new_buffer, (const char*)buffer_get_buffer(buffer) + cur_doc_begin, cur_size)) { buffer_free(new_buffer); goto cmditerfail; } /* * Roll the existing buffer back to the beginning * of the last document encoded. */ buffer_update_position(buffer, sub_doc_begin); if (!buffer_write_bytes(buffer, "\x00\x00", 2)) { buffer_free(new_buffer); goto cmditerfail; } result = _send_write_command(ctx, buffer, lst_len_loc, cmd_len_loc, &errors, to_publish); buffer_free(buffer); buffer = new_buffer; if (!result) goto cmditerfail; #if PY_MAJOR_VERSION >= 3 result = Py_BuildValue("NN", PyLong_FromLong(idx_offset), result); #else result = Py_BuildValue("NN", PyInt_FromLong(idx_offset), result); #endif if (!result) goto cmditerfail; if (PyList_Append(results, result) < 0) { Py_DECREF(result); goto cmditerfail; } Py_DECREF(result); if (errors && ordered) { destroy_codec_options(&options); Py_DECREF(iterator); buffer_free(buffer); return results; } idx_offset += idx; idx = 0; Py_DECREF(to_publish); if (!(to_publish = PyList_New(0))) { goto cmditerfail; } } if (PyList_Append(to_publish, doc) < 0) { goto cmditerfail; } Py_CLEAR(doc); idx += 1; } Py_DECREF(iterator); if (PyErr_Occurred()) { goto cmdfail; } if (empty) { PyObject* InvalidOperation = _error("InvalidOperation"); if (InvalidOperation) { PyErr_SetString(InvalidOperation, "cannot do an empty bulk write"); Py_DECREF(InvalidOperation); } goto cmdfail; } if (!buffer_write_bytes(buffer, "\x00\x00", 2)) goto cmdfail; result = _send_write_command(ctx, buffer, lst_len_loc, cmd_len_loc, &errors, to_publish); if (!result) goto cmdfail; #if PY_MAJOR_VERSION >= 3 result = Py_BuildValue("NN", PyLong_FromLong(idx_offset), result); #else result = Py_BuildValue("NN", PyInt_FromLong(idx_offset), result); #endif if (!result) goto cmdfail; if (PyList_Append(results, result) < 0) { Py_DECREF(result); goto cmdfail; } Py_DECREF(result); Py_DECREF(to_publish); buffer_free(buffer); destroy_codec_options(&options); return results; cmditerfail: Py_XDECREF(doc); Py_DECREF(iterator); cmdfail: destroy_codec_options(&options); Py_DECREF(results); Py_XDECREF(to_publish); buffer_free(buffer); return NULL; } static PyMethodDef _CMessageMethods[] = { {"_insert_message", _cbson_insert_message, METH_VARARGS, "Create an insert message to be sent to MongoDB"}, {"_update_message", _cbson_update_message, METH_VARARGS, "create an update message to be sent to MongoDB"}, {"_query_message", _cbson_query_message, METH_VARARGS, "create a query message to be sent to MongoDB"}, {"_get_more_message", _cbson_get_more_message, METH_VARARGS, "create a get more message to be sent to MongoDB"}, {"_do_batched_insert", _cbson_do_batched_insert, METH_VARARGS, "insert a batch of documents, splitting the batch as needed"}, {"_do_batched_write_command", _cbson_do_batched_write_command, METH_VARARGS, "execute a batch of insert, update, or delete commands"}, {NULL, NULL, 0, NULL} }; #if PY_MAJOR_VERSION >= 3 #define INITERROR return NULL static int _cmessage_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->_cbson); return 0; } static int _cmessage_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->_cbson); return 0; } static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_cmessage", NULL, sizeof(struct module_state), _CMessageMethods, NULL, _cmessage_traverse, _cmessage_clear, NULL }; PyMODINIT_FUNC PyInit__cmessage(void) #else #define INITERROR return PyMODINIT_FUNC init_cmessage(void) #endif { PyObject *_cbson; PyObject *c_api_object; PyObject *m; struct module_state *state; /* Store a reference to the _cbson module since it's needed to call some * of its functions */ _cbson = PyImport_ImportModule("bson._cbson"); if (_cbson == NULL) { INITERROR; } /* Import C API of _cbson * The header file accesses _cbson_API to call the functions */ c_api_object = PyObject_GetAttrString(_cbson, "_C_API"); if (c_api_object == NULL) { Py_DECREF(_cbson); INITERROR; } #if PY_VERSION_HEX >= 0x03010000 _cbson_API = (void **)PyCapsule_GetPointer(c_api_object, "_cbson._C_API"); #else _cbson_API = (void **)PyCObject_AsVoidPtr(c_api_object); #endif if (_cbson_API == NULL) { Py_DECREF(c_api_object); Py_DECREF(_cbson); INITERROR; } #if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&moduledef); #else m = Py_InitModule("_cmessage", _CMessageMethods); #endif if (m == NULL) { Py_DECREF(c_api_object); Py_DECREF(_cbson); INITERROR; } state = GETSTATE(m); state->_cbson = _cbson; Py_DECREF(c_api_object); #if PY_MAJOR_VERSION >= 3 return m; #endif } pymongo-3.2/pymongo/response.py0000644000175000017500000000656612630145074020673 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Represent a response from the server.""" class Response(object): __slots__ = ('_data', '_address', '_request_id', '_duration', '_from_command') def __init__(self, data, address, request_id, duration, from_command): """Represent a response from the server. :Parameters: - `data`: Raw BSON bytes. - `address`: (host, port) of the source server. - `request_id`: The request id of this operation. - `duration`: The duration of the operation. - `from_command`: if the response is the result of a db command. """ self._data = data self._address = address self._request_id = request_id self._duration = duration self._from_command = from_command @property def data(self): """Server response's raw BSON bytes.""" return self._data @property def address(self): """(host, port) of the source server.""" return self._address @property def request_id(self): """The request id of this operation.""" return self._request_id @property def duration(self): """The duration of the operation.""" return self._duration @property def from_command(self): """If the response is a result from a db command.""" return self._from_command class ExhaustResponse(Response): __slots__ = ('_socket_info', '_pool') def __init__(self, data, address, socket_info, pool, request_id, duration, from_command): """Represent a response to an exhaust cursor's initial query. :Parameters: - `data`: Raw BSON bytes. - `address`: (host, port) of the source server. - `socket_info`: The SocketInfo used for the initial query. - `pool`: The Pool from which the SocketInfo came. - `request_id`: The request id of this operation. - `duration`: The duration of the operation. - `from_command`: If the response is the result of a db command. """ super(ExhaustResponse, self).__init__(data, address, request_id, duration, from_command) self._socket_info = socket_info self._pool = pool @property def socket_info(self): """The SocketInfo used for the initial query. The server will send batches on this socket, without waiting for getMores from the client, until the result set is exhausted or there is an error. """ return self._socket_info @property def pool(self): """The Pool from which the SocketInfo came.""" return self._pool pymongo-3.2/pymongo/son_manipulator.py0000644000175000017500000001404412630145074022235 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Manipulators that can edit SON objects as they enter and exit a database. New manipulators should be defined as subclasses of SONManipulator and can be installed on a database by calling `pymongo.database.Database.add_son_manipulator`.""" import collections from bson.dbref import DBRef from bson.objectid import ObjectId from bson.son import SON class SONManipulator(object): """A base son manipulator. This manipulator just saves and restores objects without changing them. """ def will_copy(self): """Will this SON manipulator make a copy of the incoming document? Derived classes that do need to make a copy should override this method, returning True instead of False. All non-copying manipulators will be applied first (so that the user's document will be updated appropriately), followed by copying manipulators. """ return False def transform_incoming(self, son, collection): """Manipulate an incoming SON object. :Parameters: - `son`: the SON object to be inserted into the database - `collection`: the collection the object is being inserted into """ if self.will_copy(): return SON(son) return son def transform_outgoing(self, son, collection): """Manipulate an outgoing SON object. :Parameters: - `son`: the SON object being retrieved from the database - `collection`: the collection this object was stored in """ if self.will_copy(): return SON(son) return son class ObjectIdInjector(SONManipulator): """A son manipulator that adds the _id field if it is missing. .. versionchanged:: 2.7 ObjectIdInjector is no longer used by PyMongo, but remains in this module for backwards compatibility. """ def transform_incoming(self, son, collection): """Add an _id field if it is missing. """ if not "_id" in son: son["_id"] = ObjectId() return son # This is now handled during BSON encoding (for performance reasons), # but I'm keeping this here as a reference for those implementing new # SONManipulators. class ObjectIdShuffler(SONManipulator): """A son manipulator that moves _id to the first position. """ def will_copy(self): """We need to copy to be sure that we are dealing with SON, not a dict. """ return True def transform_incoming(self, son, collection): """Move _id to the front if it's there. """ if not "_id" in son: return son transformed = SON({"_id": son["_id"]}) transformed.update(son) return transformed class NamespaceInjector(SONManipulator): """A son manipulator that adds the _ns field. """ def transform_incoming(self, son, collection): """Add the _ns field to the incoming object """ son["_ns"] = collection.name return son class AutoReference(SONManipulator): """Transparently reference and de-reference already saved embedded objects. This manipulator should probably only be used when the NamespaceInjector is also being used, otherwise it doesn't make too much sense - documents can only be auto-referenced if they have an *_ns* field. NOTE: this will behave poorly if you have a circular reference. TODO: this only works for documents that are in the same database. To fix this we'll need to add a DatabaseInjector that adds *_db* and then make use of the optional *database* support for DBRefs. """ def __init__(self, db): self.database = db def will_copy(self): """We need to copy so the user's document doesn't get transformed refs. """ return True def transform_incoming(self, son, collection): """Replace embedded documents with DBRefs. """ def transform_value(value): if isinstance(value, collections.MutableMapping): if "_id" in value and "_ns" in value: return DBRef(value["_ns"], transform_value(value["_id"])) else: return transform_dict(SON(value)) elif isinstance(value, list): return [transform_value(v) for v in value] return value def transform_dict(object): for (key, value) in object.items(): object[key] = transform_value(value) return object return transform_dict(SON(son)) def transform_outgoing(self, son, collection): """Replace DBRefs with embedded documents. """ def transform_value(value): if isinstance(value, DBRef): return self.database.dereference(value) elif isinstance(value, list): return [transform_value(v) for v in value] elif isinstance(value, collections.MutableMapping): return transform_dict(SON(value)) return value def transform_dict(object): for (key, value) in object.items(): object[key] = transform_value(value) return object return transform_dict(SON(son)) # TODO make a generic translator for custom types. Take encode, decode, # should_encode and should_decode functions and just encode and decode where # necessary. See examples/custom_type.py for where this would be useful. # Alternatively it could take a should_encode, to_binary, from_binary and # binary subtype. pymongo-3.2/pymongo/ssl_support.py0000644000175000017500000001233512630145074021421 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Support for SSL in PyMongo.""" import atexit import sys import threading HAVE_SSL = True try: import ssl except ImportError: HAVE_SSL = False HAVE_CERTIFI = False try: import certifi HAVE_CERTIFI = True except ImportError: pass HAVE_WINCERTSTORE = False try: from wincertstore import CertFile HAVE_WINCERTSTORE = True except ImportError: pass from bson.py3compat import string_type from pymongo.errors import ConfigurationError _WINCERTSLOCK = threading.Lock() _WINCERTS = None if HAVE_SSL: try: # Python 3.2 and above. from ssl import SSLContext except ImportError: from pymongo.ssl_context import SSLContext def validate_cert_reqs(option, value): """Validate the cert reqs are valid. It must be None or one of the three values ``ssl.CERT_NONE``, ``ssl.CERT_OPTIONAL`` or ``ssl.CERT_REQUIRED``. """ if value is None: return value elif isinstance(value, string_type) and hasattr(ssl, value): value = getattr(ssl, value) if value in (ssl.CERT_NONE, ssl.CERT_OPTIONAL, ssl.CERT_REQUIRED): return value raise ValueError("The value of %s must be one of: " "`ssl.CERT_NONE`, `ssl.CERT_OPTIONAL` or " "`ssl.CERT_REQUIRED" % (option,)) def _load_wincerts(): """Set _WINCERTS to an instance of wincertstore.Certfile.""" global _WINCERTS certfile = CertFile() certfile.addstore("CA") certfile.addstore("ROOT") atexit.register(certfile.close) _WINCERTS = certfile # XXX: Possible future work. # - Support CRL files? Only supported by CPython >= 2.7.9 and >= 3.4 # http://bugs.python.org/issue8813 # - OCSP? Not supported by python at all. # http://bugs.python.org/issue17123 # - Setting OP_NO_COMPRESSION? The server doesn't yet. # - Adding an ssl_context keyword argument to MongoClient? This might # be useful for sites that have unusual requirements rather than # trying to expose every SSLContext option through a keyword/uri # parameter. def get_ssl_context(*args): """Create and return an SSLContext object.""" certfile, keyfile, ca_certs, cert_reqs = args # Note PROTOCOL_SSLv23 is about the most misleading name imaginable. # This configures the server and client to negotiate the # highest protocol version they both support. A very good thing. ctx = SSLContext(ssl.PROTOCOL_SSLv23) if hasattr(ctx, "options"): # Explicitly disable SSLv2 and SSLv3. Note that up to # date versions of MongoDB 2.4 and above already do this, # python disables SSLv2 by default in >= 2.7.7 and >= 3.3.4 # and SSLv3 in >= 3.4.3. There is no way for us to do this # explicitly for python 2.6 or 2.7 before 2.7.9. ctx.options |= getattr(ssl, "OP_NO_SSLv2", 0) ctx.options |= getattr(ssl, "OP_NO_SSLv3", 0) if certfile is not None: ctx.load_cert_chain(certfile, keyfile) if ca_certs is not None: ctx.load_verify_locations(ca_certs) elif cert_reqs != ssl.CERT_NONE: # CPython >= 2.7.9 or >= 3.4.0, pypy >= 2.5.1 if hasattr(ctx, "load_default_certs"): ctx.load_default_certs() # Python >= 3.2.0, useless on Windows. elif (sys.platform != "win32" and hasattr(ctx, "set_default_verify_paths")): ctx.set_default_verify_paths() elif sys.platform == "win32" and HAVE_WINCERTSTORE: with _WINCERTSLOCK: if _WINCERTS is None: _load_wincerts() ctx.load_verify_locations(_WINCERTS.name) elif HAVE_CERTIFI: ctx.load_verify_locations(certifi.where()) else: raise ConfigurationError( "`ssl_cert_reqs` is not ssl.CERT_NONE and no system " "CA certificates could be loaded. `ssl_ca_certs` is " "required.") ctx.verify_mode = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs return ctx else: def validate_cert_reqs(option, dummy): """No ssl module, raise ConfigurationError.""" raise ConfigurationError("The value of %s is set but can't be " "validated. The ssl module is not available" % (option,)) def get_ssl_context(*dummy): """No ssl module, raise ConfigurationError.""" raise ConfigurationError("The ssl module is not available.") pymongo-3.2/pymongo/__init__.py0000644000175000017500000000570012631420463020560 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Python driver for MongoDB.""" ASCENDING = 1 """Ascending sort order.""" DESCENDING = -1 """Descending sort order.""" GEO2D = "2d" """Index specifier for a 2-dimensional `geospatial index`_. .. _geospatial index: http://docs.mongodb.org/manual/core/2d/ """ GEOHAYSTACK = "geoHaystack" """Index specifier for a 2-dimensional `haystack index`_. .. versionadded:: 2.1 .. _haystack index: http://docs.mongodb.org/manual/core/geohaystack/ """ GEOSPHERE = "2dsphere" """Index specifier for a `spherical geospatial index`_. .. versionadded:: 2.5 .. note:: 2dsphere indexing requires server version **>= 2.4.0**. .. _spherical geospatial index: http://docs.mongodb.org/manual/core/2dsphere/ """ HASHED = "hashed" """Index specifier for a `hashed index`_. .. versionadded:: 2.5 .. note:: hashed indexing requires server version **>= 2.4.0**. .. _hashed index: http://docs.mongodb.org/manual/core/index-hashed/ """ TEXT = "text" """Index specifier for a `text index`_. .. versionadded:: 2.7.1 .. note:: text search requires server version **>= 2.4.0**. .. _text index: http://docs.mongodb.org/manual/core/index-text/ """ OFF = 0 """No database profiling.""" SLOW_ONLY = 1 """Only profile slow operations.""" ALL = 2 """Profile all operations.""" version_tuple = (3, 2) def get_version_string(): if isinstance(version_tuple[-1], str): return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1] return '.'.join(map(str, version_tuple)) __version__ = version = get_version_string() """Current version of PyMongo.""" from pymongo.collection import ReturnDocument from pymongo.common import (MIN_SUPPORTED_WIRE_VERSION, MAX_SUPPORTED_WIRE_VERSION) from pymongo.cursor import CursorType from pymongo.mongo_client import MongoClient from pymongo.mongo_replica_set_client import MongoReplicaSetClient from pymongo.operations import (IndexModel, InsertOne, DeleteOne, DeleteMany, UpdateOne, UpdateMany, ReplaceOne) from pymongo.read_preferences import ReadPreference from pymongo.write_concern import WriteConcern def has_c(): """Is the C extension installed?""" try: from pymongo import _cmessage return True except ImportError: return False pymongo-3.2/pymongo/common.py0000644000175000017500000004452012630145074020315 0ustar behackettbehackett00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Functions and classes common to multiple pymongo modules.""" import collections import warnings from bson.binary import (STANDARD, PYTHON_LEGACY, JAVA_LEGACY, CSHARP_LEGACY) from bson.codec_options import CodecOptions from bson.py3compat import string_type, integer_types, iteritems from bson.raw_bson import RawBSONDocument from pymongo.auth import MECHANISMS from pymongo.errors import ConfigurationError from pymongo.monitoring import _validate_event_listeners from pymongo.read_concern import ReadConcern from pymongo.read_preferences import (read_pref_mode_from_name, _ServerMode) from pymongo.ssl_support import validate_cert_reqs from pymongo.write_concern import WriteConcern # Defaults until we connect to a server and get updated limits. MAX_BSON_SIZE = 16 * (1024 ** 2) MAX_MESSAGE_SIZE = 2 * MAX_BSON_SIZE MIN_WIRE_VERSION = 0 MAX_WIRE_VERSION = 0 MAX_WRITE_BATCH_SIZE = 1000 # What this version of PyMongo supports. MIN_SUPPORTED_WIRE_VERSION = 0 MAX_SUPPORTED_WIRE_VERSION = 3 # Frequency to call ismaster on servers, in seconds. HEARTBEAT_FREQUENCY = 10 # Frequency to process kill-cursors, in seconds. See MongoClient.close_cursor. KILL_CURSOR_FREQUENCY = 1 # How long to wait, in seconds, for a suitable server to be found before # aborting an operation. For example, if the client attempts an insert # during a replica set election, SERVER_SELECTION_TIMEOUT governs the # longest it is willing to wait for a new primary to be found. SERVER_SELECTION_TIMEOUT = 30 # Spec requires at least 500ms between ismaster calls. MIN_HEARTBEAT_INTERVAL = 0.5 # Default connectTimeout in seconds. CONNECT_TIMEOUT = 20.0 # Default value for maxPoolSize. MAX_POOL_SIZE = 100 # Default value for localThresholdMS. LOCAL_THRESHOLD_MS = 15 # mongod/s 2.6 and above return code 59 when a # command doesn't exist. mongod versions previous # to 2.6 and mongos 2.4.x return no error code # when a command does exist. mongos versions previous # to 2.4.0 return code 13390 when a command does not # exist. COMMAND_NOT_FOUND_CODES = (59, 13390, None) # Error codes to ignore if GridFS calls createIndex on a secondary UNAUTHORIZED_CODES = (13, 16547, 16548) def partition_node(node): """Split a host:port string into (host, int(port)) pair.""" host = node port = 27017 idx = node.rfind(':') if idx != -1: host, port = node[:idx], int(node[idx + 1:]) if host.startswith('['): host = host[1:-1] return host, port def clean_node(node): """Split and normalize a node name from an ismaster response.""" host, port = partition_node(node) # Normalize hostname to lowercase, since DNS is case-insensitive: # http://tools.ietf.org/html/rfc4343 # This prevents useless rediscovery if "foo.com" is in the seed list but # "FOO.com" is in the ismaster response. return host.lower(), port def raise_config_error(key, dummy): """Raise ConfigurationError with the given key name.""" raise ConfigurationError("Unknown option %s" % (key,)) # Mapping of URI uuid representation options to valid subtypes. _UUID_REPRESENTATIONS = { 'standard': STANDARD, 'pythonLegacy': PYTHON_LEGACY, 'javaLegacy': JAVA_LEGACY, 'csharpLegacy': CSHARP_LEGACY } def validate_boolean(option, value): """Validates that 'value' is True or False.""" if isinstance(value, bool): return value raise TypeError("%s must be True or False" % (option,)) def validate_boolean_or_string(option, value): """Validates that value is True, False, 'true', or 'false'.""" if isinstance(value, string_type): if value not in ('true', 'false'): raise ValueError("The value of %s must be " "'true' or 'false'" % (option,)) return value == 'true' return validate_boolean(option, value) def validate_integer(option, value): """Validates that 'value' is an integer (or basestring representation). """ if isinstance(value, integer_types): return value elif isinstance(value, string_type): if not value.isdigit(): raise ValueError("The value of %s must be " "an integer" % (option,)) return int(value) raise TypeError("Wrong type for %s, value must be an integer" % (option,)) def validate_positive_integer(option, value): """Validate that 'value' is a positive integer, which does not include 0. """ val = validate_integer(option, value) if val <= 0: raise ValueError("The value of %s must be " "a positive integer" % (option,)) return val def validate_non_negative_integer(option, value): """Validate that 'value' is a positive integer or 0. """ val = validate_integer(option, value) if val < 0: raise ValueError("The value of %s must be " "a non negative integer" % (option,)) return val def validate_readable(option, value): """Validates that 'value' is file-like and readable. """ if value is None: return value # First make sure its a string py3.3 open(True, 'r') succeeds # Used in ssl cert checking due to poor ssl module error reporting value = validate_string(option, value) open(value, 'r').close() return value def validate_positive_integer_or_none(option, value): """Validate that 'value' is a positive integer or None. """ if value is None: return value return validate_positive_integer(option, value) def validate_non_negative_integer_or_none(option, value): """Validate that 'value' is a positive integer or 0 or None. """ if value is None: return value return validate_non_negative_integer(option, value) def validate_string(option, value): """Validates that 'value' is an instance of `basestring` for Python 2 or `str` for Python 3. """ if isinstance(value, string_type): return value raise TypeError("Wrong type for %s, value must be " "an instance of %s" % (option, string_type.__name__)) def validate_string_or_none(option, value): """Validates that 'value' is an instance of `basestring` or `None`. """ if value is None: return value return validate_string(option, value) def validate_int_or_basestring(option, value): """Validates that 'value' is an integer or string. """ if isinstance(value, integer_types): return value elif isinstance(value, string_type): if value.isdigit(): return int(value) return value raise TypeError("Wrong type for %s, value must be an " "integer or a string" % (option,)) def validate_positive_float(option, value): """Validates that 'value' is a float, or can be converted to one, and is positive. """ errmsg = "%s must be an integer or float" % (option,) try: value = float(value) except ValueError: raise ValueError(errmsg) except TypeError: raise TypeError(errmsg) # float('inf') doesn't work in 2.4 or 2.5 on Windows, so just cap floats at # one billion - this is a reasonable approximation for infinity if not 0 < value < 1e9: raise ValueError("%s must be greater than 0 and " "less than one billion" % (option,)) return value def validate_positive_float_or_zero(option, value): """Validates that 'value' is 0 or a positive float, or can be converted to 0 or a positive float. """ if value == 0 or value == "0": return 0 return validate_positive_float(option, value) def validate_timeout_or_none(option, value): """Validates a timeout specified in milliseconds returning a value in floating point seconds. """ if value is None: return value return validate_positive_float(option, value) / 1000.0 def validate_timeout_or_zero(option, value): """Validates a timeout specified in milliseconds returning a value in floating point seconds for the case where None is an error and 0 is valid. Setting the timeout to nothing in the URI string is a config error. """ if value is None: raise ConfigurationError("%s cannot be None" % (option, )) if value == 0 or value == "0": return 0 return validate_positive_float(option, value) / 1000.0 def validate_read_preference(dummy, value): """Validate a read preference. """ if not isinstance(value, _ServerMode): raise TypeError("%r is not a read preference." % (value,)) return value def validate_read_preference_mode(dummy, name): """Validate read preference mode for a MongoReplicaSetClient. """ try: return read_pref_mode_from_name(name) except ValueError: raise ValueError("%s is not a valid read preference" % (name,)) def validate_auth_mechanism(option, value): """Validate the authMechanism URI option. """ # CRAM-MD5 is for server testing only. Undocumented, # unsupported, may be removed at any time. You have # been warned. if value not in MECHANISMS and value != 'CRAM-MD5': raise ValueError("%s must be in %s" % (option, tuple(MECHANISMS))) return value def validate_uuid_representation(dummy, value): """Validate the uuid representation option selected in the URI. """ try: return _UUID_REPRESENTATIONS[value] except KeyError: raise ValueError("%s is an invalid UUID representation. " "Must be one of " "%s" % (value, tuple(_UUID_REPRESENTATIONS))) def validate_read_preference_tags(name, value): """Parse readPreferenceTags if passed as a client kwarg. """ if not isinstance(value, list): value = [value] tag_sets = [] for tag_set in value: if tag_set == '': tag_sets.append({}) continue try: tag_sets.append(dict([tag.split(":") for tag in tag_set.split(",")])) except Exception: raise ValueError("%r not a valid " "value for %s" % (tag_set, name)) return tag_sets _MECHANISM_PROPS = frozenset(['SERVICE_NAME']) def validate_auth_mechanism_properties(option, value): """Validate authMechanismProperties.""" value = validate_string(option, value) props = {} for opt in value.split(','): try: key, val = opt.split(':') except ValueError: raise ValueError("auth mechanism properties must be " "key:value pairs like SERVICE_NAME:" "mongodb, not %s." % (opt,)) if key not in _MECHANISM_PROPS: raise ValueError("%s is not a supported auth " "mechanism property. Must be one of " "%s." % (key, tuple(_MECHANISM_PROPS))) props[key] = val return props def validate_document_class(option, value): """Validate the document_class option.""" if not issubclass(value, (collections.MutableMapping, RawBSONDocument)): raise TypeError("%s must be dict, bson.son.SON, " "bson.raw_bson.RawBSONDocument, or a " "sublass of collections.MutableMapping" % (option,)) return value def validate_is_mapping(option, value): """Validate the type of method arguments that expect a document.""" if not isinstance(value, collections.Mapping): raise TypeError("%s must be an instance of dict, bson.son.SON, or " "other type that inherits from " "collections.Mapping" % (option,)) def validate_is_document_type(option, value): """Validate the type of method arguments that expect a MongoDB document.""" if not isinstance(value, (collections.MutableMapping, RawBSONDocument)): raise TypeError("%s must be an instance of dict, bson.son.SON, " "bson.raw_bson.RawBSONDocument, or " "a type that inherits from " "collections.MutableMapping" % (option,)) def validate_ok_for_replace(replacement): """Validate a replacement document.""" validate_is_mapping("replacement", replacement) # Replacement can be {} if replacement and not isinstance(replacement, RawBSONDocument): first = next(iter(replacement)) if first.startswith('$'): raise ValueError('replacement can not include $ operators') def validate_ok_for_update(update): """Validate an update document.""" validate_is_mapping("update", update) # Update can not be {} if not update: raise ValueError('update only works with $ operators') first = next(iter(update)) if not first.startswith('$'): raise ValueError('update only works with $ operators') # journal is an alias for j, # wtimeoutms is an alias for wtimeout, VALIDATORS = { 'replicaset': validate_string_or_none, 'w': validate_int_or_basestring, 'wtimeout': validate_integer, 'wtimeoutms': validate_integer, 'fsync': validate_boolean_or_string, 'j': validate_boolean_or_string, 'journal': validate_boolean_or_string, 'connecttimeoutms': validate_timeout_or_none, 'maxpoolsize': validate_positive_integer_or_none, 'socketkeepalive': validate_boolean_or_string, 'sockettimeoutms': validate_timeout_or_none, 'waitqueuetimeoutms': validate_timeout_or_none, 'waitqueuemultiple': validate_non_negative_integer_or_none, 'ssl': validate_boolean_or_string, 'ssl_keyfile': validate_readable, 'ssl_certfile': validate_readable, 'ssl_cert_reqs': validate_cert_reqs, 'ssl_ca_certs': validate_readable, 'ssl_match_hostname': validate_boolean_or_string, 'readconcernlevel': validate_string_or_none, 'read_preference': validate_read_preference, 'readpreference': validate_read_preference_mode, 'readpreferencetags': validate_read_preference_tags, 'localthresholdms': validate_positive_float_or_zero, 'serverselectiontimeoutms': validate_timeout_or_zero, 'authmechanism': validate_auth_mechanism, 'authsource': validate_string, 'authmechanismproperties': validate_auth_mechanism_properties, 'document_class': validate_document_class, 'tz_aware': validate_boolean_or_string, 'uuidrepresentation': validate_uuid_representation, 'connect': validate_boolean, 'event_listeners': _validate_event_listeners } _AUTH_OPTIONS = frozenset(['authmechanismproperties']) def validate_auth_option(option, value): """Validate optional authentication parameters. """ lower, value = validate(option, value) if lower not in _AUTH_OPTIONS: raise ConfigurationError('Unknown ' 'authentication option: %s' % (option,)) return lower, value def validate(option, value): """Generic validation function. """ lower = option.lower() validator = VALIDATORS.get(lower, raise_config_error) value = validator(option, value) return lower, value def get_validated_options(options): """Validate each entry in options and raise a warning if it is not valid. Returns a copy of options with invalid entries removed """ validated_options = {} for opt, value in iteritems(options): lower = opt.lower() try: validator = VALIDATORS.get(lower, raise_config_error) value = validator(opt, value) except (ValueError, ConfigurationError) as exc: warnings.warn(str(exc)) else: validated_options[lower] = value return validated_options WRITE_CONCERN_OPTIONS = frozenset([ 'w', 'wtimeout', 'wtimeoutms', 'fsync', 'j', 'journal' ]) class BaseObject(object): """A base class that provides attributes and methods common to multiple pymongo classes. SHOULD NOT BE USED BY DEVELOPERS EXTERNAL TO MONGODB. """ def __init__(self, codec_options, read_preference, write_concern, read_concern): if not isinstance(codec_options, CodecOptions): raise TypeError("codec_options must be an instance of " "bson.codec_options.CodecOptions") self.__codec_options = codec_options if not isinstance(read_preference, _ServerMode): raise TypeError("%r is not valid for read_preference. See " "pymongo.read_preferences for valid " "options." % (read_preference,)) self.__read_preference = read_preference if not isinstance(write_concern, WriteConcern): raise TypeError("write_concern must be an instance of " "pymongo.write_concern.WriteConcern") self.__write_concern = write_concern if not isinstance(read_concern, ReadConcern): raise TypeError("read_concern must be an instance of " "pymongo.read_concern.ReadConcern") self.__read_concern = read_concern @property def codec_options(self): """Read only access to the :class:`~bson.codec_options.CodecOptions` of this instance. """ return self.__codec_options @property def write_concern(self): """Read only access to the :class:`~pymongo.write_concern.WriteConcern` of this instance. .. versionchanged:: 3.0 The :attr:`write_concern` attribute is now read only. """ return self.__write_concern @property def read_preference(self): """Read only access to the read preference of this instance. .. versionchanged:: 3.0 The :attr:`read_preference` attribute is now read only. """ return self.__read_preference @property def read_concern(self): """Read only access to the read concern of this instance. .. versionadded:: 3.2 """ return self.__read_concern pymongo-3.2/pymongo/helpers.py0000644000175000017500000003137512630145074020473 0ustar behackettbehackett00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Bits and pieces used by the driver that don't really fit elsewhere.""" import collections import datetime import struct import bson from bson.codec_options import CodecOptions from bson.py3compat import itervalues, string_type, iteritems, u from bson.son import SON from pymongo import ASCENDING from pymongo.errors import (CursorNotFound, DuplicateKeyError, ExecutionTimeout, NotMasterError, OperationFailure, WriteError, WriteConcernError, WTimeoutError) from pymongo.message import _Query, _convert_exception from pymongo.read_concern import DEFAULT_READ_CONCERN _UUNDER = u("_") def _gen_index_name(keys): """Generate an index name from the set of fields it is over.""" return _UUNDER.join(["%s_%s" % item for item in keys]) def _index_list(key_or_list, direction=None): """Helper to generate a list of (key, direction) pairs. Takes such a list, or a single key, or a single key and direction. """ if direction is not None: return [(key_or_list, direction)] else: if isinstance(key_or_list, string_type): return [(key_or_list, ASCENDING)] elif not isinstance(key_or_list, (list, tuple)): raise TypeError("if no direction is specified, " "key_or_list must be an instance of list") return key_or_list def _index_document(index_list): """Helper to generate an index specifying document. Takes a list of (key, direction) pairs. """ if isinstance(index_list, collections.Mapping): raise TypeError("passing a dict to sort/create_index/hint is not " "allowed - use a list of tuples instead. did you " "mean %r?" % list(iteritems(index_list))) elif not isinstance(index_list, (list, tuple)): raise TypeError("must use a list of (key, direction) pairs, " "not: " + repr(index_list)) if not len(index_list): raise ValueError("key_or_list must not be the empty list") index = SON() for (key, value) in index_list: if not isinstance(key, string_type): raise TypeError("first item in each key pair must be a string") if not isinstance(value, (string_type, int, collections.Mapping)): raise TypeError("second item in each key pair must be 1, -1, " "'2d', 'geoHaystack', or another valid MongoDB " "index specifier.") index[key] = value return index def _unpack_response(response, cursor_id=None, codec_options=CodecOptions()): """Unpack a response from the database. Check the response for errors and unpack, returning a dictionary containing the response data. Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or OperationFailure. :Parameters: - `response`: byte string as returned from the database - `cursor_id` (optional): cursor_id we sent to get this response - used for raising an informative exception when we get cursor id not valid at server response - `codec_options` (optional): an instance of :class:`~bson.codec_options.CodecOptions` """ response_flag = struct.unpack(">> from pymongo import MongoClient >>> c = MongoClient() >>> c.test_database Database(MongoClient('localhost', 27017), u'test_database') >>> c['test-database'] Database(MongoClient('localhost', 27017), u'test-database') """ import contextlib import datetime import threading import warnings import weakref from collections import defaultdict from bson.codec_options import DEFAULT_CODEC_OPTIONS from bson.py3compat import (integer_types, string_type) from bson.son import SON from pymongo import (common, database, helpers, message, periodic_executor, uri_parser) from pymongo.client_options import ClientOptions from pymongo.cursor_manager import CursorManager from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, InvalidOperation, InvalidURI, NetworkTimeout, NotMasterError, OperationFailure) from pymongo.read_preferences import ReadPreference from pymongo.server_selectors import (writable_preferred_server_selector, writable_server_selector) from pymongo.server_type import SERVER_TYPE from pymongo.topology import Topology from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.settings import TopologySettings from pymongo.write_concern import WriteConcern class MongoClient(common.BaseObject): HOST = "localhost" PORT = 27017 # Define order to retrieve options from ClientOptions for __repr__. # No host/port; these are retrieved from TopologySettings. _constructor_args = ('document_class', 'tz_aware', 'connect') def __init__( self, host=None, port=None, document_class=dict, tz_aware=False, connect=True, **kwargs): """Client for a MongoDB instance, a replica set, or a set of mongoses. The client object is thread-safe and has connection-pooling built in. If an operation fails because of a network error, :class:`~pymongo.errors.ConnectionFailure` is raised and the client reconnects in the background. Application code should handle this exception (recognizing that the operation failed) and then continue to execute. The `host` parameter can be a full `mongodb URI `_, in addition to a simple hostname. It can also be a list of hostnames or URIs. Any port specified in the host string(s) will override the `port` parameter. If multiple mongodb URIs containing database or auth information are passed, the last database, username, and password present will be used. For username and passwords reserved characters like ':', '/', '+' and '@' must be escaped following RFC 2396. .. warning:: When using PyMongo in a multiprocessing context, please read :ref:`multiprocessing` first. :Parameters: - `host` (optional): hostname or IP address of the instance to connect to, or a mongodb URI, or a list of hostnames / mongodb URIs. If `host` is an IPv6 literal it must be enclosed in '[' and ']' characters following the RFC2732 URL syntax (e.g. '[::1]' for localhost) - `port` (optional): port number on which to connect - `document_class` (optional): default class to use for documents returned from queries on this client - `tz_aware` (optional): if ``True``, :class:`~datetime.datetime` instances returned as values in a document by this :class:`MongoClient` will be timezone aware (otherwise they will be naive) - `connect` (optional): if ``True`` (the default), immediately begin connecting to MongoDB in the background. Otherwise connect on the first operation. | **Other optional parameters can be passed as keyword arguments:** - `maxPoolSize` (optional): The maximum number of connections that the pool will open simultaneously. If this is set, operations will block if there are `maxPoolSize` outstanding connections from the pool. Defaults to 100. Cannot be 0. - `socketTimeoutMS`: (integer or None) Controls how long (in milliseconds) the driver will wait for a response after sending an ordinary (non-monitoring) database operation before concluding that a network error has occurred. Defaults to ``None`` (no timeout). - `connectTimeoutMS`: (integer or None) Controls how long (in milliseconds) the driver will wait during server monitoring when connecting a new socket to a server before concluding the server is unavailable. Defaults to ``20000`` (20 seconds). - `serverSelectionTimeoutMS`: (integer) Controls how long (in milliseconds) the driver will wait to find an available, appropriate server to carry out a database operation; while it is waiting, multiple server monitoring operations may be carried out, each controlled by `connectTimeoutMS`. Defaults to ``30000`` (30 seconds). - `waitQueueTimeoutMS`: (integer or None) How long (in milliseconds) a thread will wait for a socket from the pool if the pool has no free sockets. Defaults to ``None`` (no timeout). - `waitQueueMultiple`: (integer or None) Multiplied by maxPoolSize to give the number of threads allowed to wait for a socket at one time. Defaults to ``None`` (no limit). - `socketKeepAlive`: (boolean) Whether to send periodic keep-alive packets on connected sockets. Defaults to ``False`` (do not send keep-alive packets). - `event_listeners`: a list or tuple of event listeners. See :mod:`~pymongo.monitoring` for details. | **Write Concern options:** | (Only set if passed. No default values.) - `w`: (integer or string) If this is a replica set, write operations will block until they have been replicated to the specified number or tagged set of servers. `w=` always includes the replica set primary (e.g. w=3 means write to the primary and wait until replicated to **two** secondaries). Passing w=0 **disables write acknowledgement** and all other write concern options. - `wtimeout`: (integer) Used in conjunction with `w`. Specify a value in milliseconds to control how long to wait for write propagation to complete. If replication does not complete in the given timeframe, a timeout exception is raised. - `j`: If ``True`` block until write operations have been committed to the journal. Cannot be used in combination with `fsync`. Prior to MongoDB 2.6 this option was ignored if the server was running without journaling. Starting with MongoDB 2.6 write operations will fail with an exception if this option is used when the server is running without journaling. - `fsync`: If ``True`` and the server is running without journaling, blocks until the server has synced all data files to disk. If the server is running with journaling, this acts the same as the `j` option, blocking until write operations have been committed to the journal. Cannot be used in combination with `j`. | **Replica set keyword arguments for connecting with a replica set - either directly or via a mongos:** - `replicaSet`: (string or None) The name of the replica set to connect to. The driver will verify that all servers it connects to match this name. Implies that the hosts specified are a seed list and the driver should attempt to find all members of the set. Defaults to ``None``. - `read_preference`: The read preference for this client. See :class:`~pymongo.read_preferences.ReadPreference` for all available read preference options. Defaults to ``PRIMARY``. | **SSL configuration:** - `ssl`: If ``True``, create the connection to the server using SSL. Defaults to ``False``. - `ssl_keyfile`: The private keyfile used to identify the local connection against mongod. If included with the ``certfile`` then only the ``ssl_certfile`` is needed. Implies ``ssl=True``. Defaults to ``None``. - `ssl_certfile`: The certificate file used to identify the local connection against mongod. Implies ``ssl=True``. Defaults to ``None``. - `ssl_cert_reqs`: Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided. It must be one of the three values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL`` (not required, but validated if provided), or ``ssl.CERT_REQUIRED`` (required and validated). If the value of this parameter is not ``ssl.CERT_NONE`` and a value is not provided for ``ssl_ca_certs`` PyMongo will attempt to load system provided CA certificates. If the python version in use does not support loading system CA certificates then the ``ssl_ca_certs`` parameter must point to a file of CA certificates. Implies ``ssl=True``. Defaults to ``ssl.CERT_REQUIRED`` if not provided and ``ssl=True``. - `ssl_ca_certs`: The ca_certs file contains a set of concatenated "certification authority" certificates, which are used to validate certificates passed from the other end of the connection. Implies ``ssl=True``. Defaults to ``None``. - `ssl_match_hostname`: If ``True`` (the default), and `ssl_cert_reqs` is not ``ssl.CERT_NONE``, enables hostname verification using the :func:`~ssl.match_hostname` function from python's :mod:`~ssl` module. Think very carefully before setting this to ``False`` as that could make your application vulnerable to man-in-the-middle attacks. | **Read Concern options:** | (If not set explicitly, this will use the server default) - `readConcernLevel`: (string) The read concern level specifies the level of isolation for read operations. For example, a read operation using a read concern level of ``majority`` will only return data that has been written to a majority of nodes. If the level is left unspecified, the server default will be used. .. mongodoc:: connections .. versionchanged:: 3.0 :class:`~pymongo.mongo_client.MongoClient` is now the one and only client class for a standalone server, mongos, or replica set. It includes the functionality that had been split into :class:`~pymongo.mongo_client.MongoReplicaSetClient`: it can connect to a replica set, discover all its members, and monitor the set for stepdowns, elections, and reconfigs. The :class:`~pymongo.mongo_client.MongoClient` constructor no longer blocks while connecting to the server or servers, and it no longer raises :class:`~pymongo.errors.ConnectionFailure` if they are unavailable, nor :class:`~pymongo.errors.ConfigurationError` if the user's credentials are wrong. Instead, the constructor returns immediately and launches the connection process on background threads. Therefore the ``alive`` method is removed since it no longer provides meaningful information; even if the client is disconnected, it may discover a server in time to fulfill the next operation. In PyMongo 2.x, :class:`~pymongo.MongoClient` accepted a list of standalone MongoDB servers and used the first it could connect to:: MongoClient(['host1.com:27017', 'host2.com:27017']) A list of multiple standalones is no longer supported; if multiple servers are listed they must be members of the same replica set, or mongoses in the same sharded cluster. The behavior for a list of mongoses is changed from "high availability" to "load balancing". Before, the client connected to the lowest-latency mongos in the list, and used it until a network error prompted it to re-evaluate all mongoses' latencies and reconnect to one of them. In PyMongo 3, the client monitors its network latency to all the mongoses continuously, and distributes operations evenly among those with the lowest latency. See :ref:`mongos-load-balancing` for more information. The ``connect`` option is added. The ``start_request``, ``in_request``, and ``end_request`` methods are removed, as well as the ``auto_start_request`` option. The ``copy_database`` method is removed, see the :doc:`copy_database examples ` for alternatives. The :meth:`MongoClient.disconnect` method is removed; it was a synonym for :meth:`~pymongo.MongoClient.close`. :class:`~pymongo.mongo_client.MongoClient` no longer returns an instance of :class:`~pymongo.database.Database` for attribute names with leading underscores. You must use dict-style lookups instead:: client['__my_database__'] Not:: client.__my_database__ """ if host is None: host = self.HOST if isinstance(host, string_type): host = [host] if port is None: port = self.PORT if not isinstance(port, int): raise TypeError("port must be an instance of int") seeds = set() username = None password = None dbase = None opts = {} for entity in host: if "://" in entity: if entity.startswith("mongodb://"): res = uri_parser.parse_uri(entity, port, warn=True) seeds.update(res["nodelist"]) username = res["username"] or username password = res["password"] or password dbase = res["database"] or dbase opts = res["options"] else: idx = entity.find("://") raise InvalidURI("Invalid URI scheme: " "%s" % (entity[:idx],)) else: seeds.update(uri_parser.split_hosts(entity, port)) if not seeds: raise ConfigurationError("need to specify at least one host") # _pool_class, _monitor_class, and _condition_class are for deep # customization of PyMongo, e.g. Motor. pool_class = kwargs.pop('_pool_class', None) monitor_class = kwargs.pop('_monitor_class', None) condition_class = kwargs.pop('_condition_class', None) keyword_opts = kwargs keyword_opts['document_class'] = document_class keyword_opts['tz_aware'] = tz_aware keyword_opts['connect'] = connect # Validate all keyword options. keyword_opts = dict(common.validate(k, v) for k, v in keyword_opts.items()) opts.update(keyword_opts) self.__options = options = ClientOptions( username, password, dbase, opts) self.__default_database_name = dbase self.__lock = threading.Lock() self.__cursor_manager = CursorManager(self) self.__kill_cursors_queue = [] self._event_listeners = options.pool_options.event_listeners # Cache of existing indexes used by ensure_index ops. self.__index_cache = {} super(MongoClient, self).__init__(options.codec_options, options.read_preference, options.write_concern, options.read_concern) self.__all_credentials = {} creds = options.credentials if creds: self._cache_credentials(creds.source, creds) self._topology_settings = TopologySettings( seeds=seeds, replica_set_name=options.replica_set_name, pool_class=pool_class, pool_options=options.pool_options, monitor_class=monitor_class, condition_class=condition_class, local_threshold_ms=options.local_threshold_ms, server_selection_timeout=options.server_selection_timeout) self._topology = Topology(self._topology_settings) if connect: self._topology.open() def target(): client = self_ref() if client is None: return False # Stop the executor. MongoClient._process_kill_cursors_queue(client) return True executor = periodic_executor.PeriodicExecutor( interval=common.KILL_CURSOR_FREQUENCY, min_interval=0.5, target=target, name="pymongo_kill_cursors_thread") # We strongly reference the executor and it weakly references us via # this closure. When the client is freed, stop the executor soon. self_ref = weakref.ref(self, executor.close) self._kill_cursors_executor = executor executor.open() def _cache_credentials(self, source, credentials, connect=False): """Save a set of authentication credentials. The credentials are used to login a socket whenever one is created. If `connect` is True, verify the credentials on the server first. """ # Don't let other threads affect this call's data. all_credentials = self.__all_credentials.copy() if source in all_credentials: # Nothing to do if we already have these credentials. if credentials == all_credentials[source]: return raise OperationFailure('Another user is already authenticated ' 'to this database. You must logout first.') if connect: server = self._get_topology().select_server( writable_preferred_server_selector) # get_socket() logs out of the database if logged in with old # credentials, and logs in with new ones. with server.get_socket(all_credentials) as sock_info: sock_info.authenticate(credentials) # If several threads run _cache_credentials at once, last one wins. self.__all_credentials[source] = credentials def _purge_credentials(self, source): """Purge credentials from the authentication cache.""" self.__all_credentials.pop(source, None) def _cached(self, dbname, coll, index): """Test if `index` is cached.""" cache = self.__index_cache now = datetime.datetime.utcnow() return (dbname in cache and coll in cache[dbname] and index in cache[dbname][coll] and now < cache[dbname][coll][index]) def _cache_index(self, dbname, collection, index, cache_for): """Add an index to the index cache for ensure_index operations.""" now = datetime.datetime.utcnow() expire = datetime.timedelta(seconds=cache_for) + now if database not in self.__index_cache: self.__index_cache[dbname] = {} self.__index_cache[dbname][collection] = {} self.__index_cache[dbname][collection][index] = expire elif collection not in self.__index_cache[dbname]: self.__index_cache[dbname][collection] = {} self.__index_cache[dbname][collection][index] = expire else: self.__index_cache[dbname][collection][index] = expire def _purge_index(self, database_name, collection_name=None, index_name=None): """Purge an index from the index cache. If `index_name` is None purge an entire collection. If `collection_name` is None purge an entire database. """ if not database_name in self.__index_cache: return if collection_name is None: del self.__index_cache[database_name] return if not collection_name in self.__index_cache[database_name]: return if index_name is None: del self.__index_cache[database_name][collection_name] return if index_name in self.__index_cache[database_name][collection_name]: del self.__index_cache[database_name][collection_name][index_name] def _server_property(self, attr_name): """An attribute of the current server's description. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. Not threadsafe if used multiple times in a single method, since the server may change. In such cases, store a local reference to a ServerDescription first, then use its properties. """ server = self._topology.select_server( writable_server_selector) return getattr(server.description, attr_name) @property def event_listeners(self): """The event listeners registered for this client. See :mod:`~pymongo.monitoring` for details. """ return self._event_listeners.event_listeners @property def address(self): """(host, port) of the current standalone, primary, or mongos, or None. Accessing :attr:`address` raises :exc:`~.errors.InvalidOperation` if the client is load-balancing among mongoses, since there is no single address. Use :attr:`nodes` instead. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. .. versionadded:: 3.0 """ topology_type = self._topology._description.topology_type if topology_type == TOPOLOGY_TYPE.Sharded: raise InvalidOperation( 'Cannot use "address" property when load balancing among' ' mongoses, use "nodes" instead.') if topology_type not in (TOPOLOGY_TYPE.ReplicaSetWithPrimary, TOPOLOGY_TYPE.Single): return None return self._server_property('address') @property def primary(self): """The (host, port) of the current primary of the replica set. Returns ``None`` if this client is not connected to a replica set, there is no primary, or this client was created without the `replicaSet` option. .. versionadded:: 3.0 MongoClient gained this property in version 3.0 when MongoReplicaSetClient's functionality was merged in. """ return self._topology.get_primary() @property def secondaries(self): """The secondary members known to this client. A sequence of (host, port) pairs. Empty if this client is not connected to a replica set, there are no visible secondaries, or this client was created without the `replicaSet` option. .. versionadded:: 3.0 MongoClient gained this property in version 3.0 when MongoReplicaSetClient's functionality was merged in. """ return self._topology.get_secondaries() @property def arbiters(self): """Arbiters in the replica set. A sequence of (host, port) pairs. Empty if this client is not connected to a replica set, there are no arbiters, or this client was created without the `replicaSet` option. """ return self._topology.get_arbiters() @property def is_primary(self): """If this client is connected to a server that can accept writes. True if the current server is a standalone, mongos, or the primary of a replica set. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. """ return self._server_property('is_writable') @property def is_mongos(self): """If this client is connected to mongos. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available.. """ return self._server_property('server_type') == SERVER_TYPE.Mongos @property def max_pool_size(self): """The maximum number of sockets the pool will open concurrently. When the pool has reached `max_pool_size`, operations block waiting for a socket to be returned to the pool. If ``waitQueueTimeoutMS`` is set, a blocked operation will raise :exc:`~pymongo.errors.ConnectionFailure` after a timeout. By default ``waitQueueTimeoutMS`` is not set. """ return self.__options.pool_options.max_pool_size @property def nodes(self): """Set of all currently connected servers. .. warning:: When connected to a replica set the value of :attr:`nodes` can change over time as :class:`MongoClient`'s view of the replica set changes. :attr:`nodes` can also be an empty set when :class:`MongoClient` is first instantiated and hasn't yet connected to any servers, or a network partition causes it to lose connection to all servers. """ description = self._topology.description return frozenset(s.address for s in description.known_servers) @property def max_bson_size(self): """The largest BSON object the connected server accepts in bytes. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. """ return self._server_property('max_bson_size') @property def max_message_size(self): """The largest message the connected server accepts in bytes. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. """ return self._server_property('max_message_size') @property def max_write_batch_size(self): """The maxWriteBatchSize reported by the server. If the client is not connected, this will block until a connection is established or raise ServerSelectionTimeoutError if no server is available. Returns a default value when connected to server versions prior to MongoDB 2.6. """ return self._server_property('max_write_batch_size') @property def local_threshold_ms(self): """The local threshold for this instance.""" return self.__options.local_threshold_ms @property def server_selection_timeout(self): """The server selection timeout for this instance in seconds.""" return self.__options.server_selection_timeout def _is_writable(self): """Attempt to connect to a writable server, or return False. """ topology = self._get_topology() # Starts monitors if necessary. try: svr = topology.select_server(writable_server_selector) # When directly connected to a secondary, arbiter, etc., # select_server returns it, whatever the selector. Check # again if the server is writable. return svr.description.is_writable except ConnectionFailure: return False def close(self): """Disconnect from MongoDB. Close all sockets in the connection pools and stop the monitor threads. If this instance is used again it will be automatically re-opened and the threads restarted. """ self._topology.close() def set_cursor_manager(self, manager_class): """Set this client's cursor manager. Raises :class:`TypeError` if `manager_class` is not a subclass of :class:`~pymongo.cursor_manager.CursorManager`. A cursor manager handles closing cursors. Different managers can implement different policies in terms of when to actually kill a cursor that has been closed. :Parameters: - `manager_class`: cursor manager to use .. versionchanged:: 3.0 Undeprecated. """ manager = manager_class(self) if not isinstance(manager, CursorManager): raise TypeError("manager_class must be a subclass of " "CursorManager") self.__cursor_manager = manager def _get_topology(self): """Get the internal :class:`~pymongo.topology.Topology` object. If this client was created with "connect=False", calling _get_topology launches the connection process in the background. """ self._topology.open() return self._topology @contextlib.contextmanager def _get_socket(self, selector): server = self._get_topology().select_server(selector) try: with server.get_socket(self.__all_credentials) as sock_info: yield sock_info except NetworkTimeout: # The socket has been closed. Don't reset the server. # Server Discovery And Monitoring Spec: "When an application # operation fails because of any network error besides a socket # timeout...." raise except NotMasterError: # "When the client sees a "not master" error it MUST replace the # server's description with type Unknown. It MUST request an # immediate check of the server." self._reset_server_and_request_check(server.description.address) raise except ConnectionFailure: # "Client MUST replace the server's description with type Unknown # ... MUST NOT request an immediate check of the server." self.__reset_server(server.description.address) raise def _socket_for_writes(self): return self._get_socket(writable_server_selector) @contextlib.contextmanager def _socket_for_reads(self, read_preference): preference = read_preference or ReadPreference.PRIMARY # Get a socket for a server matching the read preference, and yield # sock_info, slave_ok. Server Selection Spec: "slaveOK must be sent to # mongods with topology type Single. If the server type is Mongos, # follow the rules for passing read preference to mongos, even for # topology type Single." # Thread safe: if the type is single it cannot change. topology = self._get_topology() single = topology.description.topology_type == TOPOLOGY_TYPE.Single with self._get_socket(read_preference) as sock_info: slave_ok = (single and not sock_info.is_mongos) or ( preference != ReadPreference.PRIMARY) yield sock_info, slave_ok def _send_message_with_response(self, operation, read_preference=None, exhaust=False, address=None): """Send a message to MongoDB and return a Response. :Parameters: - `operation`: a _Query or _GetMore object. - `read_preference` (optional): A ReadPreference. - `exhaust` (optional): If True, the socket used stays checked out. It is returned along with its Pool in the Response. - `address` (optional): Optional address when sending a message to a specific server, used for getMore. """ with self.__lock: # If needed, restart kill-cursors thread after a fork. self._kill_cursors_executor.open() topology = self._get_topology() if address: server = topology.select_server_by_address(address) if not server: raise AutoReconnect('server %s:%d no longer available' % address) else: selector = read_preference or writable_server_selector server = topology.select_server(selector) # A _Query's slaveOk bit is already set for queries with non-primary # read preference. If this is a direct connection to a mongod, override # and *always* set the slaveOk bit. See bullet point 2 in # server-selection.rst#topology-type-single. set_slave_ok = ( topology.description.topology_type == TOPOLOGY_TYPE.Single and server.description.server_type != SERVER_TYPE.Mongos) return self._reset_on_error( server, server.send_message_with_response, operation, set_slave_ok, self.__all_credentials, self._event_listeners, exhaust) def _reset_on_error(self, server, func, *args, **kwargs): """Execute an operation. Reset the server on network error. Returns fn()'s return value on success. On error, clears the server's pool and marks the server Unknown. Re-raises any exception thrown by fn(). """ try: return func(*args, **kwargs) except NetworkTimeout: # The socket has been closed. Don't reset the server. raise except ConnectionFailure: self.__reset_server(server.description.address) raise def __reset_server(self, address): """Clear our connection pool for a server and mark it Unknown.""" self._topology.reset_server(address) def _reset_server_and_request_check(self, address): """Clear our pool for a server, mark it Unknown, and check it soon.""" self._topology.reset_server_and_request_check(address) def __eq__(self, other): if isinstance(other, self.__class__): return self.address == other.address return NotImplemented def __ne__(self, other): return not self == other def _repr_helper(self): def option_repr(option, value): """Fix options whose __repr__ isn't usable in a constructor.""" if option == 'document_class': if value is dict: return 'document_class=dict' else: return 'document_class=%s.%s' % (value.__module__, value.__name__) if "ms" in option: return "%s='%s'" % (option, int(value * 1000)) return '%s=%r' % (option, value) # Host first... options = ['host=%r' % [ '%s:%d' % (host, port) for host, port in self._topology_settings.seeds]] # ... then everything in self._constructor_args... options.extend( option_repr(key, self.__options._options[key]) for key in self._constructor_args) # ... then everything else. options.extend( option_repr(key, self.__options._options[key]) for key in self.__options._options if key not in set(self._constructor_args)) return ', '.join(options) def __repr__(self): return ("MongoClient(%s)" % (self._repr_helper(),)) def __getattr__(self, name): """Get a database by name. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. :Parameters: - `name`: the name of the database to get """ if name.startswith('_'): raise AttributeError( "MongoClient has no attribute %r. To access the %s" " database, use client[%r]." % (name, name, name)) return self.__getitem__(name) def __getitem__(self, name): """Get a database by name. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. :Parameters: - `name`: the name of the database to get """ return database.Database(self, name) def close_cursor(self, cursor_id, address=None): """Close a single database cursor. Raises :class:`TypeError` if `cursor_id` is not an instance of ``(int, long)``. What closing the cursor actually means depends on this client's cursor manager. :Parameters: - `cursor_id`: id of cursor to close - `address` (optional): (host, port) pair of the cursor's server. If it is not provided, the client attempts to close the cursor on the primary or standalone, or a mongos server. .. versionchanged:: 3.0 Added ``address`` parameter. """ if not isinstance(cursor_id, integer_types): raise TypeError("cursor_id must be an instance of (int, long)") self.__cursor_manager.close(cursor_id, address) def kill_cursors(self, cursor_ids, address=None): """Send a kill cursors message soon with the given ids. Raises :class:`TypeError` if `cursor_ids` is not an instance of ``list``. This method may be called from a :class:`~pymongo.cursor.Cursor` destructor during garbage collection, so it isn't safe to take a lock or do network I/O. Instead, we schedule the cursor to be closed soon on a background thread. :Parameters: - `cursor_ids`: list of cursor ids to kill - `address` (optional): (host, port) pair of the cursor's server. If it is not provided, the client attempts to close the cursor on the primary or standalone, or a mongos server. .. versionchanged:: 3.0 Now accepts an `address` argument. Schedules the cursors to be closed on a background thread instead of sending the message immediately. """ if not isinstance(cursor_ids, list): raise TypeError("cursor_ids must be a list") # "Atomic", needs no lock. self.__kill_cursors_queue.append((address, cursor_ids)) # This method is run periodically by a background thread. def _process_kill_cursors_queue(self): """Process any pending kill cursors requests.""" address_to_cursor_ids = defaultdict(list) # Other threads or the GC may append to the queue concurrently. while True: try: address, cursor_ids = self.__kill_cursors_queue.pop() except IndexError: break address_to_cursor_ids[address].extend(cursor_ids) # Don't re-open topology if it's closed and there's no pending cursors. if address_to_cursor_ids: listeners = self._event_listeners publish = listeners.enabled_for_commands topology = self._get_topology() for address, cursor_ids in address_to_cursor_ids.items(): try: if address: # address could be a tuple or _CursorAddress, but # select_server_by_address needs (host, port). server = topology.select_server_by_address( tuple(address)) else: # Application called close_cursor() with no address. server = topology.select_server( writable_server_selector) try: namespace = address.namespace db, coll = namespace.split('.', 1) except AttributeError: namespace = None db = coll = "OP_KILL_CURSORS" spec = SON([('killCursors', coll), ('cursors', cursor_ids)]) with server.get_socket(self.__all_credentials) as sock_info: if (sock_info.max_wire_version >= 4 and namespace is not None): sock_info.command(db, spec) else: if publish: start = datetime.datetime.now() request_id, msg = message.kill_cursors(cursor_ids) if publish: duration = datetime.datetime.now() - start listeners.publish_command_start( spec, db, request_id, address) start = datetime.datetime.now() try: sock_info.send_message(msg, 0) except Exception as exc: if publish: dur = ((datetime.datetime.now() - start) + duration) listeners.publish_command_failure( dur, message._convert_exception(exc), 'killCursors', request_id, address) raise if publish: duration = ((datetime.datetime.now() - start) + duration) # OP_KILL_CURSORS returns no reply, fake one. reply = {'cursorsUnknown': cursor_ids, 'ok': 1} listeners.publish_command_success( duration, reply, 'killCursors', request_id, address) except ConnectionFailure as exc: warnings.warn("couldn't close cursor on %s: %s" % (address, exc)) def server_info(self): """Get information about the MongoDB server we're connected to.""" return self.admin.command("buildinfo", read_preference=ReadPreference.PRIMARY) def database_names(self): """Get a list of the names of all databases on the connected server.""" return [db["name"] for db in self._database_default_options('admin').command( "listDatabases")["databases"]] def drop_database(self, name_or_database): """Drop a database. Raises :class:`TypeError` if `name_or_database` is not an instance of :class:`basestring` (:class:`str` in python 3) or :class:`~pymongo.database.Database`. :Parameters: - `name_or_database`: the name of a database to drop, or a :class:`~pymongo.database.Database` instance representing the database to drop """ name = name_or_database if isinstance(name, database.Database): name = name.name if not isinstance(name, string_type): raise TypeError("name_or_database must be an instance " "of %s or a Database" % (string_type.__name__,)) self._purge_index(name) self[name].command("dropDatabase", read_preference=ReadPreference.PRIMARY) def get_default_database(self): """Get the database named in the MongoDB connection URI. >>> uri = 'mongodb://host/my_database' >>> client = MongoClient(uri) >>> db = client.get_default_database() >>> assert db.name == 'my_database' Useful in scripts where you want to choose which database to use based only on the URI in a configuration file. """ if self.__default_database_name is None: raise ConfigurationError('No default database defined') return self[self.__default_database_name] def get_database(self, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None): """Get a :class:`~pymongo.database.Database` with the given name and options. Useful for creating a :class:`~pymongo.database.Database` with different codec options, read preference, and/or write concern from this :class:`MongoClient`. >>> client.read_preference Primary() >>> db1 = client.test >>> db1.read_preference Primary() >>> from pymongo import ReadPreference >>> db2 = client.get_database( ... 'test', read_preference=ReadPreference.SECONDARY) >>> db2.read_preference Secondary(tag_sets=None) :Parameters: - `name`: The name of the database - a string. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`MongoClient` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`MongoClient` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`MongoClient` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`MongoClient` is used. """ return database.Database( self, name, codec_options, read_preference, write_concern, read_concern) def _database_default_options(self, name): """Get a Database instance with the default settings.""" return self.get_database( name, codec_options=DEFAULT_CODEC_OPTIONS, read_preference=ReadPreference.PRIMARY, write_concern=WriteConcern()) @property def is_locked(self): """Is this server locked? While locked, all write operations are blocked, although read operations may still be allowed. Use :meth:`unlock` to unlock. """ ops = self._database_default_options('admin').current_op() return bool(ops.get('fsyncLock', 0)) def fsync(self, **kwargs): """Flush all pending writes to datafiles. :Parameters: Optional parameters can be passed as keyword arguments: - `lock`: If True lock the server to disallow writes. - `async`: If True don't block while synchronizing. .. warning:: `async` and `lock` can not be used together. .. warning:: MongoDB does not support the `async` option on Windows and will raise an exception on that platform. """ self.admin.command("fsync", read_preference=ReadPreference.PRIMARY, **kwargs) def unlock(self): """Unlock a previously locked server. """ cmd = {"fsyncUnlock": 1} with self._socket_for_writes() as sock_info: if sock_info.max_wire_version >= 4: try: sock_info.command("admin", cmd) except OperationFailure as exc: # Ignore "DB not locked" to replicate old behavior if exc.code != 125: raise else: helpers._first_batch(sock_info, "admin", "$cmd.sys.unlock", {}, -1, True, self.codec_options, ReadPreference.PRIMARY, cmd, self._event_listeners) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def __iter__(self): return self def __next__(self): raise TypeError("'MongoClient' object is not iterable") next = __next__ pymongo-3.2/pymongo/monitoring.py0000644000175000017500000003255412630145074021216 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools to monitor driver events. Use :func:`register` to register global listeners for specific events. Currently only command events are published. Listeners must be a subclass of :class:`CommandListener` and implement :meth:`~CommandListener.started`, :meth:`~CommandListener.succeeded`, and :meth:`~CommandListener.failed`. For example, a simple command logger might be implemented like this:: import logging from pymongo import monitoring class CommandLogger(monitoring.CommandListener): def started(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} started on server " "{0.connection_id}".format(event)) def succeeded(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "succeeded in {0.duration_micros} " "microseconds".format(event)) def failed(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "failed in {0.duration_micros} " "microseconds".format(event)) monitoring.register(CommandLogger()) Event listeners can also be registered per instance of :class:`~pymongo.mongo_client.MongoClient`:: client = MongoClient(event_listeners=[CommandLogger()]) Note that previously registered global listeners are automatically included when configuring per client event listeners. Registering a new global listener will not add that listener to existing client instances. .. note:: Events are delivered **synchronously**. Application threads block waiting for event handlers (e.g. :meth:`~CommandListener.started`) to return. Care must be taken to ensure that your event handlers are efficient enough to not adversely affect overall application performance. .. warning:: The command documents published through this API are *not* copies. If you intend to modify them in any way you must copy them in your event handler first. """ import sys import traceback from collections import namedtuple, Sequence _Listeners = namedtuple('Listeners', ('command_listeners',)) _LISTENERS = _Listeners([]) class CommandListener(object): """Abstract base class for command listeners.""" def started(self, event): """Abstract method to handle CommandStartedEvent. :Parameters: - `event`: An instance of :class:`CommandStartedEvent` """ raise NotImplementedError def succeeded(self, event): """Abstract method to handle CommandSucceededEvent. :Parameters: - `event`: An instance of :class:`CommandSucceededEvent` """ raise NotImplementedError def failed(self, event): """Abstract method to handle CommandFailedEvent. :Parameters: - `event`: An instance of :class:`CommandFailedEvent` """ raise NotImplementedError def _to_micros(dur): """Convert duration 'dur' to microseconds.""" if hasattr(dur, 'total_seconds'): return int(dur.total_seconds() * 10e5) # Python 2.6 return dur.microseconds + (dur.seconds + dur.days * 24 * 3600) * 1000000 def _validate_event_listeners(option, listeners): """Validate event listeners""" if not isinstance(listeners, Sequence): raise TypeError("%s must be a list or tuple" % (option,)) for listener in listeners: if not isinstance(listener, CommandListener): raise TypeError("Only subclasses of " "pymongo.monitoring.CommandListener are supported") return listeners def register(listener): """Register a global event listener. :Parameters: - `listener`: A subclass of :class:`CommandListener`. """ _validate_event_listeners('listener', [listener]) _LISTENERS.command_listeners.append(listener) def _handle_exception(): """Print exceptions raised by subscribers to stderr.""" # Heavily influenced by logging.Handler.handleError. # See note here: # https://docs.python.org/3.4/library/sys.html#sys.__stderr__ if sys.stderr: einfo = sys.exc_info() try: traceback.print_exception(einfo[0], einfo[1], einfo[2], None, sys.stderr) except IOError: pass finally: del einfo # Note - to avoid bugs from forgetting which if these is all lowercase and # which are camelCase, and at the same time avoid having to add a test for # every command, use all lowercase here and test against command_name.lower(). _SENSITIVE_COMMANDS = set( ["authenticate", "saslstart", "saslcontinue", "getnonce", "createuser", "updateuser", "copydbgetnonce", "copydbsaslstart", "copydb"]) class _CommandEvent(object): """Base class for command events.""" __slots__ = ("__cmd_name", "__rqst_id", "__conn_id", "__op_id") def __init__(self, command_name, request_id, connection_id, operation_id): self.__cmd_name = command_name self.__rqst_id = request_id self.__conn_id = connection_id self.__op_id = operation_id @property def command_name(self): """The command name.""" return self.__cmd_name @property def request_id(self): """The request id for this operation.""" return self.__rqst_id @property def connection_id(self): """The address (host, port) of the server this command was sent to.""" return self.__conn_id @property def operation_id(self): """An id for this series of events or None.""" return self.__op_id class CommandStartedEvent(_CommandEvent): """Event published when a command starts. :Parameters: - `command`: The command document. - `database_name`: The name of the database this command was run against. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `operation_id`: An optional identifier for a series of related events. """ __slots__ = ("__cmd", "__db") def __init__(self, command, database_name, *args): if not command: raise ValueError("%r is not a valid command" % (command,)) # Command name must be first key. command_name = next(iter(command)) super(CommandStartedEvent, self).__init__(command_name, *args) if command_name.lower() in _SENSITIVE_COMMANDS: self.__cmd = {} else: self.__cmd = command self.__db = database_name @property def command(self): """The command document.""" return self.__cmd @property def database_name(self): """The name of the database this command was run against.""" return self.__db class CommandSucceededEvent(_CommandEvent): """Event published when a command succeeds. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `reply`: The server reply document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `operation_id`: An optional identifier for a series of related events. """ __slots__ = ("__duration_micros", "__reply") def __init__(self, duration, reply, command_name, request_id, connection_id, operation_id): super(CommandSucceededEvent, self).__init__( command_name, request_id, connection_id, operation_id) self.__duration_micros = _to_micros(duration) if command_name.lower() in _SENSITIVE_COMMANDS: self.__reply = {} else: self.__reply = reply @property def duration_micros(self): """The duration of this operation in microseconds.""" return self.__duration_micros @property def reply(self): """The server failure document for this operation.""" return self.__reply class CommandFailedEvent(_CommandEvent): """Event published when a command fails. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `failure`: The server reply document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `operation_id`: An optional identifier for a series of related events. """ __slots__ = ("__duration_micros", "__failure") def __init__(self, duration, failure, *args): super(CommandFailedEvent, self).__init__(*args) self.__duration_micros = _to_micros(duration) self.__failure = failure @property def duration_micros(self): """The duration of this operation in microseconds.""" return self.__duration_micros @property def failure(self): """The server failure document for this operation.""" return self.__failure class _EventListeners(object): """Configure event listeners for a client instance. Any event listeners registered globally are included by default. :Parameters: - `listeners`: A list of event listeners. """ def __init__(self, listeners): self.__command_listeners = _LISTENERS.command_listeners[:] if listeners is not None: self.__command_listeners.extend(listeners) self.__enabled_for_commands = bool(self.__command_listeners) @property def enabled_for_commands(self): """Are any CommandListener instances registered?""" return self.__enabled_for_commands @property def event_listeners(self): """List of registered event listeners.""" return self.__command_listeners[:] def publish_command_start(self, command, database_name, request_id, connection_id, op_id=None): """Publish a CommandStartedEvent to all command listeners. :Parameters: - `command`: The command document. - `database_name`: The name of the database this command was run against. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `op_id`: The (optional) operation id for this operation. """ if op_id is None: op_id = request_id event = CommandStartedEvent( command, database_name, request_id, connection_id, op_id) for subscriber in self.__command_listeners: try: subscriber.started(event) except Exception: _handle_exception() def publish_command_success(self, duration, reply, command_name, request_id, connection_id, op_id=None): """Publish a CommandSucceededEvent to all command listeners. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `reply`: The server reply document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `op_id`: The (optional) operation id for this operation. """ if op_id is None: op_id = request_id event = CommandSucceededEvent( duration, reply, command_name, request_id, connection_id, op_id) for subscriber in self.__command_listeners: try: subscriber.succeeded(event) except Exception: _handle_exception() def publish_command_failure(self, duration, failure, command_name, request_id, connection_id, op_id=None): """Publish a CommandFailedEvent to all command listeners. :Parameters: - `duration`: The command duration as a datetime.timedelta. - `failure`: The server reply document or failure description document. - `command_name`: The command name. - `request_id`: The request id for this operation. - `connection_id`: The address (host, port) of the server this command was sent to. - `op_id`: The (optional) operation id for this operation. """ if op_id is None: op_id = request_id event = CommandFailedEvent( duration, failure, command_name, request_id, connection_id, op_id) for subscriber in self.__command_listeners: try: subscriber.failed(event) except Exception: _handle_exception() pymongo-3.2/pymongo/results.py0000644000175000017500000001713512571426507020537 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Result class definitions.""" from pymongo.errors import InvalidOperation class _WriteResult(object): """Base class for write result classes.""" def __init__(self, acknowledged): self.__acknowledged = acknowledged def _raise_if_unacknowledged(self, property_name): """Raise an exception on property access if unacknowledged.""" if not self.__acknowledged: raise InvalidOperation("A value for %s is not available when " "the write is unacknowledged. Check the " "acknowledged attribute to avoid this " "error." % (property_name,)) @property def acknowledged(self): """Is this the result of an acknowledged write operation? The :attr:`acknowledged` attribute will be ``False`` when using ``WriteConcern(w=0)``, otherwise ``True``. .. note:: If the :attr:`acknowledged` attribute is ``False`` all other attibutes of this class will raise :class:`~pymongo.errors.InvalidOperation` when accessed. Values for other attributes cannot be determined if the write operation was unacknowledged. .. seealso:: :class:`~pymongo.write_concern.WriteConcern` """ return self.__acknowledged class InsertOneResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.insert_one`. """ __slots__ = ("__inserted_id", "__acknowledged") def __init__(self, inserted_id, acknowledged): self.__inserted_id = inserted_id super(InsertOneResult, self).__init__(acknowledged) @property def inserted_id(self): """The inserted document's _id.""" return self.__inserted_id class InsertManyResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.insert_many`. """ __slots__ = ("__inserted_ids", "__acknowledged") def __init__(self, inserted_ids, acknowledged): self.__inserted_ids = inserted_ids super(InsertManyResult, self).__init__(acknowledged) @property def inserted_ids(self): """A list of _ids of the inserted documents, in the order provided. .. note:: If ``False`` is passed for the `ordered` parameter to :meth:`~pymongo.collection.Collection.insert_many` the server may have inserted the documents in a different order than what is presented here. """ return self.__inserted_ids class UpdateResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.update_one`, :meth:`~pymongo.collection.Collection.update_many`, and :meth:`~pymongo.collection.Collection.replace_one`. """ __slots__ = ("__raw_result", "__acknowledged") def __init__(self, raw_result, acknowledged): self.__raw_result = raw_result super(UpdateResult, self).__init__(acknowledged) @property def raw_result(self): """The raw result document returned by the server.""" return self.__raw_result @property def matched_count(self): """The number of documents matched for this update.""" self._raise_if_unacknowledged("matched_count") if self.upserted_id is not None: return 0 return self.__raw_result.get("n", 0) @property def modified_count(self): """The number of documents modified. .. note:: modified_count is only reported by MongoDB 2.6 and later. When connected to an earlier server version, or in certain mixed version sharding configurations, this attribute will be set to ``None``. """ self._raise_if_unacknowledged("modified_count") return self.__raw_result.get("nModified") @property def upserted_id(self): """The _id of the inserted document if an upsert took place. Otherwise ``None``. """ self._raise_if_unacknowledged("upserted_id") return self.__raw_result.get("upserted") class DeleteResult(_WriteResult): """The return type for :meth:`~pymongo.collection.Collection.delete_one` and :meth:`~pymongo.collection.Collection.delete_many`""" __slots__ = ("__raw_result", "__acknowledged") def __init__(self, raw_result, acknowledged): self.__raw_result = raw_result super(DeleteResult, self).__init__(acknowledged) @property def raw_result(self): """The raw result document returned by the server.""" return self.__raw_result @property def deleted_count(self): """The number of documents deleted.""" self._raise_if_unacknowledged("deleted_count") return self.__raw_result.get("n", 0) class BulkWriteResult(_WriteResult): """An object wrapper for bulk API write results.""" __slots__ = ("__bulk_api_result", "__acknowledged") def __init__(self, bulk_api_result, acknowledged): """Create a BulkWriteResult instance. :Parameters: - `bulk_api_result`: A result dict from the bulk API - `acknowledged`: Was this write result acknowledged? If ``False`` then all properties of this object will raise :exc:`~pymongo.errors.InvalidOperation`. """ self.__bulk_api_result = bulk_api_result super(BulkWriteResult, self).__init__(acknowledged) @property def bulk_api_result(self): """The raw bulk API result.""" return self.__bulk_api_result @property def inserted_count(self): """The number of documents inserted.""" self._raise_if_unacknowledged("inserted_count") return self.__bulk_api_result.get("nInserted") @property def matched_count(self): """The number of documents matched for an update.""" self._raise_if_unacknowledged("matched_count") return self.__bulk_api_result.get("nMatched") @property def modified_count(self): """The number of documents modified. .. note:: modified_count is only reported by MongoDB 2.6 and later. When connected to an earlier server version, or in certain mixed version sharding configurations, this attribute will be set to ``None``. """ self._raise_if_unacknowledged("modified_count") return self.__bulk_api_result.get("nModified") @property def deleted_count(self): """The number of documents deleted.""" self._raise_if_unacknowledged("deleted_count") return self.__bulk_api_result.get("nRemoved") @property def upserted_count(self): """The number of documents upserted.""" self._raise_if_unacknowledged("upserted_count") return self.__bulk_api_result.get("nUpserted") @property def upserted_ids(self): """A map of operation index to the _id of the upserted document.""" self._raise_if_unacknowledged("upserted_ids") if self.__bulk_api_result: return dict((upsert["index"], upsert["_id"]) for upsert in self.bulk_api_result["upserted"]) pymongo-3.2/pymongo/ismaster.py0000644000175000017500000000741712630145074020660 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Parse a response to the 'ismaster' command.""" import itertools from bson.py3compat import imap from pymongo import common from pymongo.server_type import SERVER_TYPE def _get_server_type(doc): """Determine the server type from an ismaster response.""" if not doc.get('ok'): return SERVER_TYPE.Unknown if doc.get('isreplicaset'): return SERVER_TYPE.RSGhost elif doc.get('setName'): if doc.get('hidden'): return SERVER_TYPE.RSOther elif doc.get('ismaster'): return SERVER_TYPE.RSPrimary elif doc.get('secondary'): return SERVER_TYPE.RSSecondary elif doc.get('arbiterOnly'): return SERVER_TYPE.RSArbiter else: return SERVER_TYPE.RSOther elif doc.get('msg') == 'isdbgrid': return SERVER_TYPE.Mongos else: return SERVER_TYPE.Standalone class IsMaster(object): __slots__ = ('_doc', '_server_type', '_is_writable', '_is_readable') def __init__(self, doc): """Parse an ismaster response from the server.""" self._server_type = _get_server_type(doc) self._doc = doc self._is_writable = self._server_type in ( SERVER_TYPE.RSPrimary, SERVER_TYPE.Standalone, SERVER_TYPE.Mongos) self._is_readable = ( self.server_type == SERVER_TYPE.RSSecondary or self._is_writable) @property def server_type(self): return self._server_type @property def all_hosts(self): """List of hosts, passives, and arbiters known to this server.""" return set(imap(common.clean_node, itertools.chain( self._doc.get('hosts', []), self._doc.get('passives', []), self._doc.get('arbiters', [])))) @property def tags(self): """Replica set member tags or empty dict.""" return self._doc.get('tags', {}) @property def primary(self): """This server's opinion about who the primary is, or None.""" if self._doc.get('primary'): return common.partition_node(self._doc['primary']) else: return None @property def replica_set_name(self): """Replica set name or None.""" return self._doc.get('setName') @property def max_bson_size(self): return self._doc.get('maxBsonObjectSize', common.MAX_BSON_SIZE) @property def max_message_size(self): return self._doc.get('maxMessageSizeBytes', 2 * self.max_bson_size) @property def max_write_batch_size(self): return self._doc.get('maxWriteBatchSize', common.MAX_WRITE_BATCH_SIZE) @property def min_wire_version(self): return self._doc.get('minWireVersion', common.MIN_WIRE_VERSION) @property def max_wire_version(self): return self._doc.get('maxWireVersion', common.MAX_WIRE_VERSION) @property def election_id(self): return self._doc.get('electionId') @property def is_writable(self): return self._is_writable @property def is_readable(self): return self._is_readable @property def me(self): me = self._doc.get('me') if me: return common.clean_node(me) pymongo-3.2/pymongo/client_options.py0000644000175000017500000001525012630145074022054 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Tools to parse mongo client options.""" from bson.codec_options import _parse_codec_options from pymongo.auth import _build_credentials_tuple from pymongo.common import validate_boolean from pymongo import common from pymongo.errors import ConfigurationError from pymongo.monitoring import _EventListeners from pymongo.pool import PoolOptions from pymongo.read_concern import ReadConcern from pymongo.read_preferences import make_read_preference from pymongo.ssl_support import get_ssl_context from pymongo.write_concern import WriteConcern def _parse_credentials(username, password, database, options): """Parse authentication credentials.""" if username is None: return None mechanism = options.get('authmechanism', 'DEFAULT') source = options.get('authsource', database or 'admin') return _build_credentials_tuple( mechanism, source, username, password, options) def _parse_read_preference(options): """Parse read preference options.""" if 'read_preference' in options: return options['read_preference'] mode = options.get('readpreference', 0) tags = options.get('readpreferencetags') return make_read_preference(mode, tags) def _parse_write_concern(options): """Parse write concern options.""" concern = options.get('w') wtimeout = options.get('wtimeout') j = options.get('j', options.get('journal')) fsync = options.get('fsync') return WriteConcern(concern, wtimeout, j, fsync) def _parse_read_concern(options): """Parse read concern options.""" concern = options.get('readconcernlevel') return ReadConcern(concern) def _parse_ssl_options(options): """Parse ssl options.""" use_ssl = options.get('ssl') if use_ssl is not None: validate_boolean('ssl', use_ssl) certfile = options.get('ssl_certfile') keyfile = options.get('ssl_keyfile') ca_certs = options.get('ssl_ca_certs') cert_reqs = options.get('ssl_cert_reqs') match_hostname = options.get('ssl_match_hostname', True) ssl_kwarg_keys = [k for k in options if k.startswith('ssl_') and options[k]] if use_ssl == False and ssl_kwarg_keys: raise ConfigurationError("ssl has not been enabled but the " "following ssl parameters have been set: " "%s. Please set `ssl=True` or remove." % ', '.join(ssl_kwarg_keys)) if ssl_kwarg_keys and use_ssl is None: # ssl options imply ssl = True use_ssl = True if use_ssl is True: ctx = get_ssl_context(certfile, keyfile, ca_certs, cert_reqs) return ctx, match_hostname return None, match_hostname def _parse_pool_options(options): """Parse connection pool options.""" max_pool_size = options.get('maxpoolsize', common.MAX_POOL_SIZE) connect_timeout = options.get('connecttimeoutms', common.CONNECT_TIMEOUT) socket_keepalive = options.get('socketkeepalive', False) socket_timeout = options.get('sockettimeoutms') wait_queue_timeout = options.get('waitqueuetimeoutms') wait_queue_multiple = options.get('waitqueuemultiple') event_listeners = options.get('event_listeners') ssl_context, ssl_match_hostname = _parse_ssl_options(options) return PoolOptions(max_pool_size, connect_timeout, socket_timeout, wait_queue_timeout, wait_queue_multiple, ssl_context, ssl_match_hostname, socket_keepalive, _EventListeners(event_listeners)) class ClientOptions(object): """ClientOptions""" def __init__(self, username, password, database, options): self.__options = options self.__codec_options = _parse_codec_options(options) self.__credentials = _parse_credentials( username, password, database, options) self.__local_threshold_ms = options.get( 'localthresholdms', common.LOCAL_THRESHOLD_MS) # self.__server_selection_timeout is in seconds. Must use full name for # common.SERVER_SELECTION_TIMEOUT because it is set directly by tests. self.__server_selection_timeout = options.get( 'serverselectiontimeoutms', common.SERVER_SELECTION_TIMEOUT) self.__pool_options = _parse_pool_options(options) self.__read_preference = _parse_read_preference(options) self.__replica_set_name = options.get('replicaset') self.__write_concern = _parse_write_concern(options) self.__read_concern = _parse_read_concern(options) self.__connect = options.get('connect') @property def _options(self): """The original options used to create this ClientOptions.""" return self.__options @property def connect(self): """Whether to begin discovering a MongoDB topology automatically.""" return self.__connect @property def codec_options(self): """A :class:`~bson.codec_options.CodecOptions` instance.""" return self.__codec_options @property def credentials(self): """A :class:`~pymongo.auth.MongoCredentials` instance or None.""" return self.__credentials @property def local_threshold_ms(self): """The local threshold for this instance.""" return self.__local_threshold_ms @property def server_selection_timeout(self): """The server selection timeout for this instance in seconds.""" return self.__server_selection_timeout @property def pool_options(self): """A :class:`~pymongo.pool.PoolOptions` instance.""" return self.__pool_options @property def read_preference(self): """A read preference instance.""" return self.__read_preference @property def replica_set_name(self): """Replica set name or None.""" return self.__replica_set_name @property def write_concern(self): """A :class:`~pymongo.write_concern.WriteConcern` instance.""" return self.__write_concern @property def read_concern(self): """A :class:`~pymongo.read_concern.ReadConcern` instance.""" return self.__read_concern pymongo-3.2/pymongo/topology_description.py0000644000175000017500000003125612630145074023306 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Represent the topology of servers.""" from collections import namedtuple from pymongo import common from pymongo.server_type import SERVER_TYPE from pymongo.errors import ConfigurationError from pymongo.server_description import ServerDescription TOPOLOGY_TYPE = namedtuple('TopologyType', ['Single', 'ReplicaSetNoPrimary', 'ReplicaSetWithPrimary', 'Sharded', 'Unknown'])(*range(5)) class TopologyDescription(object): def __init__( self, topology_type, server_descriptions, replica_set_name, max_election_id): """Represent a topology of servers. :Parameters: - `topology_type`: initial type - `server_descriptions`: dict of (address, ServerDescription) for all seeds - `replica_set_name`: replica set name or None - `max_election_id`: greatest electionId seen from a primary, or None """ self._topology_type = topology_type self._replica_set_name = replica_set_name self._server_descriptions = server_descriptions self._max_election_id = max_election_id # Is PyMongo compatible with all servers' wire protocols? self._incompatible_err = None for s in self._server_descriptions.values(): # s.min/max_wire_version is the server's wire protocol. # MIN/MAX_SUPPORTED_WIRE_VERSION is what PyMongo supports. server_too_new = ( # Server too new. s.min_wire_version is not None and s.min_wire_version > common.MAX_SUPPORTED_WIRE_VERSION) server_too_old = ( # Server too old. s.max_wire_version is not None and s.max_wire_version < common.MIN_SUPPORTED_WIRE_VERSION) if server_too_new or server_too_old: self._incompatible_err = ( "Server at %s:%d " "uses wire protocol versions %d through %d, " "but PyMongo only supports %d through %d" % (s.address[0], s.address[1], s.min_wire_version, s.max_wire_version, common.MIN_SUPPORTED_WIRE_VERSION, common.MAX_SUPPORTED_WIRE_VERSION)) break def check_compatible(self): """Raise ConfigurationError if any server is incompatible. A server is incompatible if its wire protocol version range does not overlap with PyMongo's. """ if self._incompatible_err: raise ConfigurationError(self._incompatible_err) def has_server(self, address): return address in self._server_descriptions def reset_server(self, address): """A copy of this description, with one server marked Unknown.""" return updated_topology_description(self, ServerDescription(address)) def reset(self): """A copy of this description, with all servers marked Unknown.""" if self._topology_type == TOPOLOGY_TYPE.ReplicaSetWithPrimary: topology_type = TOPOLOGY_TYPE.ReplicaSetNoPrimary else: topology_type = self._topology_type # The default ServerDescription's type is Unknown. sds = dict((address, ServerDescription(address)) for address in self._server_descriptions) return TopologyDescription( topology_type, sds, self._replica_set_name, self._max_election_id) def server_descriptions(self): """Dict of (address, ServerDescription).""" return self._server_descriptions.copy() @property def topology_type(self): return self._topology_type @property def replica_set_name(self): """The replica set name.""" return self._replica_set_name @property def max_election_id(self): """Greatest electionId seen from a primary, or None.""" return self._max_election_id @property def known_servers(self): """List of Servers of types besides Unknown.""" return [s for s in self._server_descriptions.values() if s.is_server_type_known] # If topology type is Unknown and we receive an ismaster response, what should # the new topology type be? _SERVER_TYPE_TO_TOPOLOGY_TYPE = { SERVER_TYPE.Mongos: TOPOLOGY_TYPE.Sharded, SERVER_TYPE.RSPrimary: TOPOLOGY_TYPE.ReplicaSetWithPrimary, SERVER_TYPE.RSSecondary: TOPOLOGY_TYPE.ReplicaSetNoPrimary, SERVER_TYPE.RSArbiter: TOPOLOGY_TYPE.ReplicaSetNoPrimary, SERVER_TYPE.RSOther: TOPOLOGY_TYPE.ReplicaSetNoPrimary, } def updated_topology_description(topology_description, server_description): """Return an updated copy of a TopologyDescription. :Parameters: - `topology_description`: the current TopologyDescription - `server_description`: a new ServerDescription that resulted from an ismaster call Called after attempting (successfully or not) to call ismaster on the server at server_description.address. Does not modify topology_description. """ address = server_description.address # These values will be updated, if necessary, to form the new # TopologyDescription. topology_type = topology_description.topology_type set_name = topology_description.replica_set_name max_election_id = topology_description.max_election_id server_type = server_description.server_type # Don't mutate the original dict of server descriptions; copy it. sds = topology_description.server_descriptions() # Replace this server's description with the new one. sds[address] = server_description if topology_type == TOPOLOGY_TYPE.Single: # Single type never changes. return TopologyDescription( TOPOLOGY_TYPE.Single, sds, set_name, max_election_id) if topology_type == TOPOLOGY_TYPE.Unknown: if server_type == SERVER_TYPE.Standalone: sds.pop(address) elif server_type not in (SERVER_TYPE.Unknown, SERVER_TYPE.RSGhost): topology_type = _SERVER_TYPE_TO_TOPOLOGY_TYPE[server_type] if topology_type == TOPOLOGY_TYPE.Sharded: if server_type not in (SERVER_TYPE.Mongos, SERVER_TYPE.Unknown): sds.pop(address) elif topology_type == TOPOLOGY_TYPE.ReplicaSetNoPrimary: if server_type in (SERVER_TYPE.Standalone, SERVER_TYPE.Mongos): sds.pop(address) elif server_type == SERVER_TYPE.RSPrimary: topology_type, set_name, max_election_id = _update_rs_from_primary( sds, set_name, server_description, max_election_id) elif server_type in ( SERVER_TYPE.RSSecondary, SERVER_TYPE.RSArbiter, SERVER_TYPE.RSOther): topology_type, set_name = _update_rs_no_primary_from_member( sds, set_name, server_description) elif topology_type == TOPOLOGY_TYPE.ReplicaSetWithPrimary: if server_type in (SERVER_TYPE.Standalone, SERVER_TYPE.Mongos): sds.pop(address) topology_type = _check_has_primary(sds) elif server_type == SERVER_TYPE.RSPrimary: topology_type, set_name, max_election_id = _update_rs_from_primary( sds, set_name, server_description, max_election_id) elif server_type in ( SERVER_TYPE.RSSecondary, SERVER_TYPE.RSArbiter, SERVER_TYPE.RSOther): topology_type = _update_rs_with_primary_from_member( sds, set_name, server_description) else: # Server type is Unknown or RSGhost: did we just lose the primary? topology_type = _check_has_primary(sds) # Return updated copy. return TopologyDescription(topology_type, sds, set_name, max_election_id) def _update_rs_from_primary( sds, replica_set_name, server_description, max_election_id): """Update topology description from a primary's ismaster response. Pass in a dict of ServerDescriptions, current replica set name, the ServerDescription we are processing, and the TopologyDescription's max_election_id if any. Returns (new topology type, new replica_set_name, new max_election_id). """ if replica_set_name is None: replica_set_name = server_description.replica_set_name elif replica_set_name != server_description.replica_set_name: # We found a primary but it doesn't have the replica_set_name # provided by the user. sds.pop(server_description.address) return _check_has_primary(sds), replica_set_name, max_election_id if server_description.election_id is not None: if max_election_id and max_election_id > server_description.election_id: # Stale primary, set to type Unknown. address = server_description.address sds[address] = ServerDescription(address) return _check_has_primary(sds), replica_set_name, max_election_id max_election_id = server_description.election_id # We've heard from the primary. Is it the same primary as before? for server in sds.values(): if (server.server_type is SERVER_TYPE.RSPrimary and server.address != server_description.address): # Reset old primary's type to Unknown. sds[server.address] = ServerDescription(server.address) # There can be only one prior primary. break # Discover new hosts from this primary's response. for new_address in server_description.all_hosts: if new_address not in sds: sds[new_address] = ServerDescription(new_address) # Remove hosts not in the response. for addr in set(sds) - server_description.all_hosts: sds.pop(addr) # If the host list differs from the seed list, we may not have a primary # after all. return _check_has_primary(sds), replica_set_name, max_election_id def _update_rs_with_primary_from_member( sds, replica_set_name, server_description): """RS with known primary. Process a response from a non-primary. Pass in a dict of ServerDescriptions, current replica set name, and the ServerDescription we are processing. Returns new topology type. """ assert replica_set_name is not None if replica_set_name != server_description.replica_set_name: sds.pop(server_description.address) elif (server_description.me and server_description.address != server_description.me): sds.pop(server_description.address) # Had this member been the primary? return _check_has_primary(sds) def _update_rs_no_primary_from_member( sds, replica_set_name, server_description): """RS without known primary. Update from a non-primary's response. Pass in a dict of ServerDescriptions, current replica set name, and the ServerDescription we are processing. Returns (new topology type, new replica_set_name). """ topology_type = TOPOLOGY_TYPE.ReplicaSetNoPrimary if replica_set_name is None: replica_set_name = server_description.replica_set_name elif replica_set_name != server_description.replica_set_name: sds.pop(server_description.address) return topology_type, replica_set_name # This isn't the primary's response, so don't remove any servers # it doesn't report. Only add new servers. for address in server_description.all_hosts: if address not in sds: sds[address] = ServerDescription(address) if (server_description.me and server_description.address != server_description.me): sds.pop(server_description.address) return topology_type, replica_set_name def _check_has_primary(sds): """Current topology type is ReplicaSetWithPrimary. Is primary still known? Pass in a dict of ServerDescriptions. Returns new topology type. """ for s in sds.values(): if s.server_type == SERVER_TYPE.RSPrimary: return TOPOLOGY_TYPE.ReplicaSetWithPrimary else: return TOPOLOGY_TYPE.ReplicaSetNoPrimary pymongo-3.2/pymongo/auth.py0000644000175000017500000003640312630145074017767 0ustar behackettbehackett00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Authentication helpers.""" import hmac HAVE_KERBEROS = True try: import kerberos except ImportError: HAVE_KERBEROS = False from base64 import standard_b64decode, standard_b64encode from collections import namedtuple from hashlib import md5, sha1 from random import SystemRandom from bson.binary import Binary from bson.py3compat import b, string_type, _unicode, PY3 from bson.son import SON from pymongo.errors import ConfigurationError, OperationFailure MECHANISMS = frozenset( ['GSSAPI', 'MONGODB-CR', 'MONGODB-X509', 'PLAIN', 'SCRAM-SHA-1', 'DEFAULT']) """The authentication mechanisms supported by PyMongo.""" MongoCredential = namedtuple( 'MongoCredential', ['mechanism', 'source', 'username', 'password', 'mechanism_properties']) """A hashable namedtuple of values used for authentication.""" GSSAPIProperties = namedtuple('GSSAPIProperties', ['service_name']) """Mechanism properties for GSSAPI authentication.""" def _build_credentials_tuple(mech, source, user, passwd, extra): """Build and return a mechanism specific credentials tuple. """ user = _unicode(user) if mech == 'GSSAPI': properties = extra.get('authmechanismproperties', {}) service_name = properties.get('SERVICE_NAME', 'mongodb') props = GSSAPIProperties(service_name=service_name) # No password, source is always $external. return MongoCredential(mech, '$external', user, None, props) elif mech == 'MONGODB-X509': return MongoCredential(mech, '$external', user, None, None) else: if passwd is None: raise ConfigurationError("A password is required.") return MongoCredential(mech, source, user, _unicode(passwd), None) if PY3: def _xor(fir, sec): """XOR two byte strings together (python 3.x).""" return b"".join([bytes([x ^ y]) for x, y in zip(fir, sec)]) _from_bytes = int.from_bytes _to_bytes = int.to_bytes else: from binascii import (hexlify as _hexlify, unhexlify as _unhexlify) def _xor(fir, sec): """XOR two byte strings together (python 2.x).""" return b"".join([chr(ord(x) ^ ord(y)) for x, y in zip(fir, sec)]) def _from_bytes(value, dummy, int=int, _hexlify=_hexlify): """An implementation of int.from_bytes for python 2.x.""" return int(_hexlify(value), 16) def _to_bytes(value, dummy0, dummy1, _unhexlify=_unhexlify): """An implementation of int.to_bytes for python 2.x.""" return _unhexlify('%040x' % value) try: # The fastest option, if it's been compiled to use OpenSSL's HMAC. from backports.pbkdf2 import pbkdf2_hmac def _hi(data, salt, iterations): return pbkdf2_hmac('sha1', data, salt, iterations) except ImportError: try: # Python 2.7.8+, or Python 3.4+. from hashlib import pbkdf2_hmac def _hi(data, salt, iterations): return pbkdf2_hmac('sha1', data, salt, iterations) except ImportError: def _hi(data, salt, iterations): """A simple implementation of PBKDF2.""" mac = hmac.HMAC(data, None, sha1) def _digest(msg, mac=mac): """Get a digest for msg.""" _mac = mac.copy() _mac.update(msg) return _mac.digest() from_bytes = _from_bytes to_bytes = _to_bytes _u1 = _digest(salt + b'\x00\x00\x00\x01') _ui = from_bytes(_u1, 'big') for _ in range(iterations - 1): _u1 = _digest(_u1) _ui ^= from_bytes(_u1, 'big') return to_bytes(_ui, 20, 'big') try: from hmac import compare_digest except ImportError: if PY3: def _xor_bytes(a, b): return a ^ b else: def _xor_bytes(a, b, _ord=ord): return _ord(a) ^ _ord(b) # Python 2.x < 2.7.7 and Python 3.x < 3.3 # References: # - http://bugs.python.org/issue14532 # - http://bugs.python.org/issue14955 # - http://bugs.python.org/issue15061 def compare_digest(a, b, _xor_bytes=_xor_bytes): left = None right = b if len(a) == len(b): left = a result = 0 if len(a) != len(b): left = b result = 1 for x, y in zip(left, right): result |= _xor_bytes(x, y) return result == 0 def _parse_scram_response(response): """Split a scram response into key, value pairs.""" return dict(item.split(b"=", 1) for item in response.split(b",")) def _authenticate_scram_sha1(credentials, sock_info): """Authenticate using SCRAM-SHA-1.""" username = credentials.username password = credentials.password source = credentials.source # Make local _hmac = hmac.HMAC _sha1 = sha1 user = username.encode("utf-8").replace(b"=", b"=3D").replace(b",", b"=2C") nonce = standard_b64encode( (("%s" % (SystemRandom().random(),))[2:]).encode("utf-8")) first_bare = b"n=" + user + b",r=" + nonce cmd = SON([('saslStart', 1), ('mechanism', 'SCRAM-SHA-1'), ('payload', Binary(b"n,," + first_bare)), ('autoAuthorize', 1)]) res = sock_info.command(source, cmd) server_first = res['payload'] parsed = _parse_scram_response(server_first) iterations = int(parsed[b'i']) salt = parsed[b's'] rnonce = parsed[b'r'] if not rnonce.startswith(nonce): raise OperationFailure("Server returned an invalid nonce.") without_proof = b"c=biws,r=" + rnonce salted_pass = _hi(_password_digest(username, password).encode("utf-8"), standard_b64decode(salt), iterations) client_key = _hmac(salted_pass, b"Client Key", _sha1).digest() stored_key = _sha1(client_key).digest() auth_msg = b",".join((first_bare, server_first, without_proof)) client_sig = _hmac(stored_key, auth_msg, _sha1).digest() client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig)) client_final = b",".join((without_proof, client_proof)) server_key = _hmac(salted_pass, b"Server Key", _sha1).digest() server_sig = standard_b64encode( _hmac(server_key, auth_msg, _sha1).digest()) cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(client_final))]) res = sock_info.command(source, cmd) parsed = _parse_scram_response(res['payload']) if not compare_digest(parsed[b'v'], server_sig): raise OperationFailure("Server returned an invalid signature.") # Depending on how it's configured, Cyrus SASL (which the server uses) # requires a third empty challenge. if not res['done']: cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(b''))]) res = sock_info.command(source, cmd) if not res['done']: raise OperationFailure('SASL conversation failed to complete.') def _password_digest(username, password): """Get a password digest to use for authentication. """ if not isinstance(password, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) if len(password) == 0: raise ValueError("password can't be empty") if not isinstance(username, string_type): raise TypeError("password must be an " "instance of %s" % (string_type.__name__,)) md5hash = md5() data = "%s:mongo:%s" % (username, password) md5hash.update(data.encode('utf-8')) return _unicode(md5hash.hexdigest()) def _auth_key(nonce, username, password): """Get an auth key to use for authentication. """ digest = _password_digest(username, password) md5hash = md5() data = "%s%s%s" % (nonce, username, digest) md5hash.update(data.encode('utf-8')) return _unicode(md5hash.hexdigest()) def _authenticate_gssapi(credentials, sock_info): """Authenticate using GSSAPI. """ if not HAVE_KERBEROS: raise ConfigurationError('The "kerberos" module must be ' 'installed to use GSSAPI authentication.') try: username = credentials.username gsn = credentials.mechanism_properties.service_name # Starting here and continuing through the while loop below - establish # the security context. See RFC 4752, Section 3.1, first paragraph. host = sock_info.address[0] result, ctx = kerberos.authGSSClientInit( gsn + '@' + host, gssflags=kerberos.GSS_C_MUTUAL_FLAG) if result != kerberos.AUTH_GSS_COMPLETE: raise OperationFailure('Kerberos context failed to initialize.') try: # pykerberos uses a weird mix of exceptions and return values # to indicate errors. # 0 == continue, 1 == complete, -1 == error # Only authGSSClientStep can return 0. if kerberos.authGSSClientStep(ctx, '') != 0: raise OperationFailure('Unknown kerberos ' 'failure in step function.') # Start a SASL conversation with mongod/s # Note: pykerberos deals with base64 encoded byte strings. # Since mongo accepts base64 strings as the payload we don't # have to use bson.binary.Binary. payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslStart', 1), ('mechanism', 'GSSAPI'), ('payload', payload), ('autoAuthorize', 1)]) response = sock_info.command('$external', cmd) # Limit how many times we loop to catch protocol / library issues for _ in range(10): result = kerberos.authGSSClientStep(ctx, str(response['payload'])) if result == -1: raise OperationFailure('Unknown kerberos ' 'failure in step function.') payload = kerberos.authGSSClientResponse(ctx) or '' cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) response = sock_info.command('$external', cmd) if result == kerberos.AUTH_GSS_COMPLETE: break else: raise OperationFailure('Kerberos ' 'authentication failed to complete.') # Once the security context is established actually authenticate. # See RFC 4752, Section 3.1, last two paragraphs. if kerberos.authGSSClientUnwrap(ctx, str(response['payload'])) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Unwrap step.') if kerberos.authGSSClientWrap(ctx, kerberos.authGSSClientResponse(ctx), username) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Wrap step.') payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) sock_info.command('$external', cmd) finally: kerberos.authGSSClientClean(ctx) except kerberos.KrbError as exc: raise OperationFailure(str(exc)) def _authenticate_plain(credentials, sock_info): """Authenticate using SASL PLAIN (RFC 4616) """ source = credentials.source username = credentials.username password = credentials.password payload = ('\x00%s\x00%s' % (username, password)).encode('utf-8') cmd = SON([('saslStart', 1), ('mechanism', 'PLAIN'), ('payload', Binary(payload)), ('autoAuthorize', 1)]) sock_info.command(source, cmd) def _authenticate_cram_md5(credentials, sock_info): """Authenticate using CRAM-MD5 (RFC 2195) """ source = credentials.source username = credentials.username password = credentials.password # The password used as the mac key is the # same as what we use for MONGODB-CR passwd = _password_digest(username, password) cmd = SON([('saslStart', 1), ('mechanism', 'CRAM-MD5'), ('payload', Binary(b'')), ('autoAuthorize', 1)]) response = sock_info.command(source, cmd) # MD5 as implicit default digest for digestmod is deprecated # in python 3.4 mac = hmac.HMAC(key=passwd.encode('utf-8'), digestmod=md5) mac.update(response['payload']) challenge = username.encode('utf-8') + b' ' + b(mac.hexdigest()) cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', Binary(challenge))]) sock_info.command(source, cmd) def _authenticate_x509(credentials, sock_info): """Authenticate using MONGODB-X509. """ query = SON([('authenticate', 1), ('mechanism', 'MONGODB-X509'), ('user', credentials.username)]) sock_info.command('$external', query) def _authenticate_mongo_cr(credentials, sock_info): """Authenticate using MONGODB-CR. """ source = credentials.source username = credentials.username password = credentials.password # Get a nonce response = sock_info.command(source, {'getnonce': 1}) nonce = response['nonce'] key = _auth_key(nonce, username, password) # Actually authenticate query = SON([('authenticate', 1), ('user', username), ('nonce', nonce), ('key', key)]) sock_info.command(source, query) def _authenticate_default(credentials, sock_info): if sock_info.max_wire_version >= 3: return _authenticate_scram_sha1(credentials, sock_info) else: return _authenticate_mongo_cr(credentials, sock_info) _AUTH_MAP = { 'CRAM-MD5': _authenticate_cram_md5, 'GSSAPI': _authenticate_gssapi, 'MONGODB-CR': _authenticate_mongo_cr, 'MONGODB-X509': _authenticate_x509, 'PLAIN': _authenticate_plain, 'SCRAM-SHA-1': _authenticate_scram_sha1, 'DEFAULT': _authenticate_default, } def authenticate(credentials, sock_info): """Authenticate sock_info.""" mechanism = credentials.mechanism auth_func = _AUTH_MAP.get(mechanism) auth_func(credentials, sock_info) def logout(source, sock_info): """Log out from a database.""" sock_info.command(source, {'logout': 1}) pymongo-3.2/pymongo/network.py0000644000175000017500000001423412630145074020515 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Internal network layer helper methods.""" import datetime import errno import select import struct _HAS_POLL = True _poller = None _EVENT_MASK = 0 try: from select import poll _poller = poll() _EVENT_MASK = (select.POLLIN | select.POLLPRI | select.POLLERR | select.POLLHUP | select.POLLNVAL) except ImportError: _HAS_POLL = False from pymongo import helpers, message from pymongo.errors import AutoReconnect, NotMasterError, OperationFailure from pymongo.read_concern import DEFAULT_READ_CONCERN _UNPACK_INT = struct.Struct(" max_bson_size + message._COMMAND_OVERHEAD): message._raise_document_too_large( name, size, max_bson_size + message._COMMAND_OVERHEAD) if publish: encoding_duration = datetime.datetime.now() - start listeners.publish_command_start(orig, dbname, request_id, address) start = datetime.datetime.now() try: sock.sendall(msg) response = receive_message(sock, 1, request_id) unpacked = helpers._unpack_response( response, codec_options=codec_options) response_doc = unpacked['data'][0] if check: msg = "command %s on namespace %s failed: %%s" % ( repr(spec).replace("%", "%%"), ns) helpers._check_command_response(response_doc, msg, allowable_errors) except Exception as exc: if publish: duration = (datetime.datetime.now() - start) + encoding_duration if isinstance(exc, (NotMasterError, OperationFailure)): failure = exc.details else: failure = message._convert_exception(exc) listeners.publish_command_failure( duration, failure, name, request_id, address) raise if publish: duration = (datetime.datetime.now() - start) + encoding_duration listeners.publish_command_success( duration, response_doc, name, request_id, address) return response_doc def receive_message(sock, operation, request_id): """Receive a raw BSON message or raise socket.error.""" header = _receive_data_on_socket(sock, 16) length = _UNPACK_INT(header[:4])[0] actual_op = _UNPACK_INT(header[12:])[0] assert operation == actual_op, ("wire protocol error: " "unknown opcode %r" % (actual_op,)) # No request_id for exhaust cursor "getMore". if request_id is not None: response_id = _UNPACK_INT(header[8:12])[0] assert request_id == response_id, ( "wire protocol error: got response id %r but expected %r" % (response_id, request_id)) assert length > 16, ("wire protocol error: message length is shorter" " than standard message header: %r" % (length,)) return _receive_data_on_socket(sock, length - 16) def _receive_data_on_socket(sock, length): msg = b"" while length: try: chunk = sock.recv(length) except (IOError, OSError) as exc: err = None if hasattr(exc, 'errno'): err = exc.errno elif exc.args: err = exc.args[0] if err == errno.EINTR: continue raise if chunk == b"": raise AutoReconnect("connection closed") length -= len(chunk) msg += chunk return msg def socket_closed(sock): """Return True if we know socket has been closed, False otherwise. """ try: if _HAS_POLL: _poller.register(sock, _EVENT_MASK) rd = _poller.poll(0) _poller.unregister(sock) else: rd, _, _ = select.select([sock], [], [], 0) # Any exception here is equally bad (select.error, ValueError, etc.). except: return True return len(rd) > 0 pymongo-3.2/pymongo/periodic_executor.py0000644000175000017500000001065512630145074022543 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Run a target function on a background thread.""" import atexit import threading import time import weakref from pymongo.monotonic import time as _time class PeriodicExecutor(object): def __init__(self, interval, min_interval, target, name=None): """"Run a target function periodically on a background thread. If the target's return value is false, the executor stops. :Parameters: - `interval`: Seconds between calls to `target`. - `min_interval`: Minimum seconds between calls if `wake` is called very often. - `target`: A function. - `name`: A name to give the underlying thread. """ # threading.Event and its internal condition variable are expensive # in Python 2, see PYTHON-983. Use a boolean to know when to wake. # The executor's design is constrained by several Python issues, see # "periodic_executor.rst" in this repository. self._event = False self._interval = interval self._min_interval = min_interval self._target = target self._stopped = False self._thread = None self._name = name def open(self): """Start. Multiple calls have no effect. Not safe to call from multiple threads at once. """ self._stopped = False started = False try: started = self._thread and self._thread.is_alive() except ReferenceError: # Thread terminated. pass if not started: thread = threading.Thread(target=self._run, name=self._name) thread.daemon = True self._thread = weakref.proxy(thread) _register_executor(self) thread.start() def close(self, dummy=None): """Stop. To restart, call open(). The dummy parameter allows an executor's close method to be a weakref callback; see monitor.py. """ self._stopped = True def join(self, timeout=None): if self._thread is not None: try: self._thread.join(timeout) except ReferenceError: # Thread already terminated. pass def wake(self): """Execute the target function soon.""" self._event = True def _run(self): while not self._stopped: try: if not self._target(): self._stopped = True break except: self._stopped = True raise deadline = _time() + self._interval while not self._stopped and _time() < deadline: time.sleep(self._min_interval) if self._event: break # Early wake. self._event = False # _EXECUTORS has a weakref to each running PeriodicExecutor. Once started, # an executor is kept alive by a strong reference from its thread and perhaps # from other objects. When the thread dies and all other referrers are freed, # the executor is freed and removed from _EXECUTORS. If any threads are # running when the interpreter begins to shut down, we try to halt and join # them to avoid spurious errors. _EXECUTORS = set() def _register_executor(executor): ref = weakref.ref(executor, _on_executor_deleted) _EXECUTORS.add(ref) def _on_executor_deleted(ref): _EXECUTORS.remove(ref) def _shutdown_executors(): # Copy the set. Stopping threads has the side effect of removing executors. executors = list(_EXECUTORS) # First signal all executors to close... for ref in executors: executor = ref() if executor: executor.close() # ...then try to join them. for ref in executors: executor = ref() if executor: executor.join(1) executor = None atexit.register(_shutdown_executors) pymongo-3.2/pymongo/settings.py0000644000175000017500000000643112630145074020664 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Represent MongoClient's configuration.""" import threading from pymongo import monitor, pool from pymongo.common import LOCAL_THRESHOLD_MS, SERVER_SELECTION_TIMEOUT from pymongo.topology_description import TOPOLOGY_TYPE from pymongo.pool import PoolOptions from pymongo.server_description import ServerDescription class TopologySettings(object): def __init__(self, seeds=None, replica_set_name=None, pool_class=None, pool_options=None, monitor_class=None, condition_class=None, local_threshold_ms=LOCAL_THRESHOLD_MS, server_selection_timeout=SERVER_SELECTION_TIMEOUT): """Represent MongoClient's configuration. Take a list of (host, port) pairs and optional replica set name. """ self._seeds = seeds or [('localhost', 27017)] self._replica_set_name = replica_set_name self._pool_class = pool_class or pool.Pool self._pool_options = pool_options or PoolOptions() self._monitor_class = monitor_class or monitor.Monitor self._condition_class = condition_class or threading.Condition self._local_threshold_ms = local_threshold_ms self._server_selection_timeout = server_selection_timeout self._direct = (len(self._seeds) == 1 and not replica_set_name) @property def seeds(self): """List of server addresses.""" return self._seeds @property def replica_set_name(self): return self._replica_set_name @property def pool_class(self): return self._pool_class @property def pool_options(self): return self._pool_options @property def monitor_class(self): return self._monitor_class @property def condition_class(self): return self._condition_class @property def local_threshold_ms(self): return self._local_threshold_ms @property def server_selection_timeout(self): return self._server_selection_timeout @property def direct(self): """Connect directly to a single server, or use a set of servers? True if there is one seed and no replica_set_name. """ return self._direct def get_topology_type(self): if self.direct: return TOPOLOGY_TYPE.Single elif self.replica_set_name is not None: return TOPOLOGY_TYPE.ReplicaSetNoPrimary else: return TOPOLOGY_TYPE.Unknown def get_server_descriptions(self): """Initial dict of (address, ServerDescription) for all seeds.""" return dict([ (address, ServerDescription(address)) for address in self.seeds]) pymongo-3.2/pymongo/read_concern.py0000644000175000017500000000444112630145074021445 0ustar behackettbehackett00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License", # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tools for working with read concerns.""" from bson.py3compat import string_type class ReadConcern(object): """ReadConcern :Parameters: - `level`: (string) The read concern level specifies the level of isolation for read operations. For example, a read operation using a read concern level of ``majority`` will only return data that has been written to a majority of nodes. If the level is left unspecified, the server default will be used. .. versionadded:: 3.2 """ def __init__(self, level=None): if level is None or isinstance(level, string_type): self.__level = level else: raise TypeError( 'level must be a string or None.') @property def level(self): """The read concern level.""" return self.__level @property def ok_for_legacy(self): """Return ``True`` if this read concern is compatible with old wire protocol versions.""" return self.level is None or self.level == 'local' @property def document(self): """The document representation of this read concern. .. note:: :class:`ReadConcern` is immutable. Mutating the value of :attr:`document` does not mutate this :class:`ReadConcern`. """ doc = {} if self.__level: doc['level'] = self.level return doc def __eq__(self, other): if isinstance(other, ReadConcern): return self.document == other.document return NotImplemented def __repr__(self): if self.level: return 'ReadConcern(%s)' % self.level return 'ReadConcern()' DEFAULT_READ_CONCERN = ReadConcern() pymongo-3.2/pymongo/topology.py0000644000175000017500000004153112630145074020700 0ustar behackettbehackett00000000000000# Copyright 2014-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You # may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. See the License for the specific language governing # permissions and limitations under the License. """Internal class to monitor a topology of one or more servers.""" import os import random import threading import warnings from bson.py3compat import itervalues from pymongo import common from pymongo.pool import PoolOptions from pymongo.topology_description import (updated_topology_description, TOPOLOGY_TYPE, TopologyDescription) from pymongo.errors import ServerSelectionTimeoutError, InvalidOperation from pymongo.monotonic import time as _time from pymongo.server import Server from pymongo.server_selectors import (any_server_selector, apply_local_threshold, arbiter_server_selector, secondary_server_selector, writable_server_selector) class Topology(object): """Monitor a topology of one or more servers.""" def __init__(self, topology_settings): self._settings = topology_settings topology_description = TopologyDescription( topology_settings.get_topology_type(), topology_settings.get_server_descriptions(), topology_settings.replica_set_name, None) self._description = topology_description # Store the seed list to help diagnose errors in _error_message(). self._seed_addresses = list(topology_description.server_descriptions()) self._opened = False self._lock = threading.Lock() self._condition = self._settings.condition_class(self._lock) self._servers = {} self._pid = None def open(self): """Start monitoring, or restart after a fork. No effect if called multiple times. .. warning:: To avoid a deadlock during Python's getaddrinfo call, will generate a warning if open() is called from a different process than the one that initialized the Topology. To prevent this from happening, MongoClient must be created after any forking OR MongoClient must be started with connect=False. """ with self._lock: if self._pid is None: self._pid = os.getpid() else: if os.getpid() != self._pid: warnings.warn( "MongoClient opened before fork. Create MongoClient " "with connect=False, or create client after forking. " "See PyMongo's documentation for details: http://api." "mongodb.org/python/current/faq.html#using-pymongo-" "with-multiprocessing>") self._ensure_opened() def select_servers(self, selector, server_selection_timeout=None, address=None): """Return a list of Servers matching selector, or time out. :Parameters: - `selector`: function that takes a list of Servers and returns a subset of them. - `server_selection_timeout` (optional): maximum seconds to wait. If not provided, the default value common.SERVER_SELECTION_TIMEOUT is used. - `address`: optional server address to select. Calls self.open() if needed. Raises exc:`ServerSelectionTimeoutError` after `server_selection_timeout` if no matching servers are found. """ if server_selection_timeout is None: server_timeout = self._settings.server_selection_timeout else: server_timeout = server_selection_timeout with self._lock: self._description.check_compatible() now = _time() end_time = now + server_timeout server_descriptions = self._apply_selector(selector, address) while not server_descriptions: # No suitable servers. if server_timeout == 0 or now > end_time: raise ServerSelectionTimeoutError( self._error_message(selector)) self._ensure_opened() self._request_check_all() # Release the lock and wait for the topology description to # change, or for a timeout. We won't miss any changes that # came after our most recent _apply_selector call, since we've # held the lock until now. self._condition.wait(common.MIN_HEARTBEAT_INTERVAL) self._description.check_compatible() now = _time() server_descriptions = self._apply_selector(selector, address) return [self.get_server_by_address(sd.address) for sd in server_descriptions] def select_server(self, selector, server_selection_timeout=None, address=None): """Like select_servers, but choose a random server if several match.""" return random.choice(self.select_servers(selector, server_selection_timeout, address)) def select_server_by_address(self, address, server_selection_timeout=None): """Return a Server for "address", reconnecting if necessary. If the server's type is not known, request an immediate check of all servers. Time out after "server_selection_timeout" if the server cannot be reached. :Parameters: - `address`: A (host, port) pair. - `server_selection_timeout` (optional): maximum seconds to wait. If not provided, the default value common.SERVER_SELECTION_TIMEOUT is used. Calls self.open() if needed. Raises exc:`ServerSelectionTimeoutError` after `server_selection_timeout` if no matching servers are found. """ return self.select_server(any_server_selector, server_selection_timeout, address) def on_change(self, server_description): """Process a new ServerDescription after an ismaster call completes.""" # We do no I/O holding the lock. with self._lock: # Any monitored server was definitely in the topology description # once. Check if it's still in the description or if some state- # change removed it. E.g., we got a host list from the primary # that didn't include this server. if self._description.has_server(server_description.address): self._description = updated_topology_description( self._description, server_description) self._update_servers() # Wake waiters in select_servers(). self._condition.notify_all() def get_server_by_address(self, address): """Get a Server or None. Returns the current version of the server immediately, even if it's Unknown or absent from the topology. Only use this in unittests. In driver code, use select_server_by_address, since then you're assured a recent view of the server's type and wire protocol version. """ return self._servers.get(address) def has_server(self, address): return address in self._servers def get_primary(self): """Return primary's address or None.""" # Implemented here in Topology instead of MongoClient, so it can lock. with self._lock: topology_type = self._description.topology_type if topology_type != TOPOLOGY_TYPE.ReplicaSetWithPrimary: return None description = writable_server_selector( self._description.known_servers)[0] return description.address def _get_replica_set_members(self, selector): """Return set of replica set member addresses.""" # Implemented here in Topology instead of MongoClient, so it can lock. with self._lock: topology_type = self._description.topology_type if topology_type not in (TOPOLOGY_TYPE.ReplicaSetWithPrimary, TOPOLOGY_TYPE.ReplicaSetNoPrimary): return set() descriptions = selector(self._description.known_servers) return set([d.address for d in descriptions]) def get_secondaries(self): """Return set of secondary addresses.""" return self._get_replica_set_members(secondary_server_selector) def get_arbiters(self): """Return set of arbiter addresses.""" return self._get_replica_set_members(arbiter_server_selector) def request_check_all(self, wait_time=5): """Wake all monitors, wait for at least one to check its server.""" with self._lock: self._request_check_all() self._condition.wait(wait_time) def reset_pool(self, address): with self._lock: server = self._servers.get(address) if server: server.pool.reset() def reset_server(self, address): """Clear our pool for a server and mark it Unknown. Do *not* request an immediate check. """ with self._lock: self._reset_server(address) def reset_server_and_request_check(self, address): """Clear our pool for a server, mark it Unknown, and check it soon.""" with self._lock: self._reset_server(address) self._request_check(address) def close(self): """Clear pools and terminate monitors. Topology reopens on demand.""" with self._lock: for server in self._servers.values(): server.close() # Mark all servers Unknown. self._description = self._description.reset() self._update_servers() @property def description(self): return self._description def _ensure_opened(self): """Start monitors, or restart after a fork. Hold the lock when calling this. """ if not self._opened: self._opened = True self._update_servers() else: # Restart monitors if we forked since previous call. for server in itervalues(self._servers): server.open() def _reset_server(self, address): """Clear our pool for a server and mark it Unknown. Hold the lock when calling this. Does *not* request an immediate check. """ server = self._servers.get(address) # "server" is None if another thread removed it from the topology. if server: server.reset() # Mark this server Unknown. self._description = self._description.reset_server(address) self._update_servers() def _request_check(self, address): """Wake one monitor. Hold the lock when calling this.""" server = self._servers.get(address) # "server" is None if another thread removed it from the topology. if server: server.request_check() def _request_check_all(self): """Wake all monitors. Hold the lock when calling this.""" for server in self._servers.values(): server.request_check() def _apply_selector(self, selector, address): if self._description.topology_type == TOPOLOGY_TYPE.Single: # Ignore the selector. return self._description.known_servers elif address: sd = self._description.server_descriptions().get(address) return [sd] if sd else [] elif self._description.topology_type == TOPOLOGY_TYPE.Sharded: return apply_local_threshold(self._settings.local_threshold_ms, self._description.known_servers) else: sds = selector(self._description.known_servers) return apply_local_threshold( self._settings.local_threshold_ms, sds) def _update_servers(self): """Sync our Servers from TopologyDescription.server_descriptions. Hold the lock while calling this. """ for address, sd in self._description.server_descriptions().items(): if address not in self._servers: monitor = self._settings.monitor_class( server_description=sd, topology=self, pool=self._create_pool_for_monitor(address), topology_settings=self._settings) server = Server( server_description=sd, pool=self._create_pool_for_server(address), monitor=monitor) self._servers[address] = server server.open() else: self._servers[address].description = sd for address, server in list(self._servers.items()): if not self._description.has_server(address): server.close() self._servers.pop(address) def _create_pool_for_server(self, address): return self._settings.pool_class(address, self._settings.pool_options) def _create_pool_for_monitor(self, address): options = self._settings.pool_options # According to the Server Discovery And Monitoring Spec, monitors use # connect_timeout for both connect_timeout and socket_timeout. The # pool only has one socket so maxPoolSize and so on aren't needed. monitor_pool_options = PoolOptions( connect_timeout=options.connect_timeout, socket_timeout=options.connect_timeout, ssl_context=options.ssl_context, ssl_match_hostname=options.ssl_match_hostname, socket_keepalive=True) return self._settings.pool_class(address, monitor_pool_options, handshake=False) def _error_message(self, selector): """Format an error message if server selection fails. Hold the lock when calling this. """ is_replica_set = self._description.topology_type in ( TOPOLOGY_TYPE.ReplicaSetWithPrimary, TOPOLOGY_TYPE.ReplicaSetNoPrimary) if is_replica_set: server_plural = 'replica set members' elif self._description.topology_type == TOPOLOGY_TYPE.Sharded: server_plural = 'mongoses' else: server_plural = 'servers' if self._description.known_servers: # We've connected, but no servers match the selector. if selector is writable_server_selector: if is_replica_set: return 'No primary available for writes' else: return 'No %s available for writes' % server_plural else: return 'No %s match selector "%s"' % (server_plural, selector) else: addresses = list(self._description.server_descriptions()) servers = list(self._description.server_descriptions().values()) if not servers: if is_replica_set: # We removed all servers because of the wrong setName? return 'No %s available for replica set name "%s"' % ( server_plural, self._settings.replica_set_name) else: return 'No %s available' % server_plural # 1 or more servers, all Unknown. Are they unknown for one reason? error = servers[0].error same = all(server.error == error for server in servers[1:]) if same: if error is None: # We're still discovering. return 'No %s found yet' % server_plural if (is_replica_set and not set(addresses).intersection(self._seed_addresses)): # We replaced our seeds with new hosts but can't reach any. return ( 'Could not reach any servers in %s. Replica set is' ' configured with internal hostnames or IPs?' % addresses) return str(error) else: return ','.join(str(server.error) for server in servers if server.error)