pax_global_header00006660000000000000000000000064132300766270014520gustar00rootroot0000000000000052 comment=68136d8546b0cb52fb080ff4d1bc7ab5af0f6818 motor-1.2.1/000077500000000000000000000000001323007662700126615ustar00rootroot00000000000000motor-1.2.1/.evergreen/000077500000000000000000000000001323007662700147215ustar00rootroot00000000000000motor-1.2.1/.evergreen/config.yml000066400000000000000000000666461323007662700167330ustar00rootroot00000000000000######################################## # Evergreen Template for MongoDB Drivers ######################################## # When a task that used to pass starts to fail # Go through all versions that may have been skipped to detect # when the task started failing stepback: true # Mark a failure as a system/bootstrap failure (purple box) rather then a task # failure by default. # Actual testing tasks are marked with `type: test` command_type: system # Protect ourself against rogue test case, or curl gone wild, that runs forever # Good rule of thumb: the averageish length a task takes, times 5 # That roughly accounts for variable system performance for various buildvariants exec_timeout_secs: 1800 # 6 minutes is the longest we'll ever run # What to do when evergreen hits the timeout (`post:` tasks are run automatically) timeout: - command: shell.exec params: script: | ls -la functions: "fetch source": # Executes git clone and applies the submitted patch, if any - command: git.get_project params: directory: "src" # Make an evergreen exapanstion file with dynamic values - command: shell.exec params: working_dir: "src" script: | # Get the current unique version of this checkout if [ "${is_patch}" = "true" ]; then CURRENT_VERSION=$(git describe)-patch-${version_id} else CURRENT_VERSION=latest fi export DRIVERS_TOOLS="$(pwd)/../drivers-tools" export PROJECT_DIRECTORY="$(pwd)" # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) export PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) fi export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" export UPLOAD_BUCKET="${project}" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" DRIVERS_TOOLS: "$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" MONGODB_BINARIES: "$MONGODB_BINARIES" UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" PREPARE_SHELL: | set -o errexit set -o xtrace export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" export ASYNC_TEST_TIMEOUT=30 EOT # See what we've done cat expansion.yml # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update params: file: src/expansion.yml "prepare resources": - command: shell.exec params: working_dir: "src" script: | ${PREPARE_SHELL} rm -rf $DRIVERS_TOOLS if [ "${project}" = "drivers-tools" ]; then # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else git clone git://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config "upload release": - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${project}.tar.gz remote_file: ${UPLOAD_BUCKET}/${project}-${CURRENT_VERSION}.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} # Upload build artifacts that other tasks may depend on # Note this URL needs to be totally unique, while predictable for the next task # so it can automatically download the artifacts "upload build": # Compress and upload the entire build directory - command: archive.targz_pack params: # Example: mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz target: "${build_id}.tar.gz" source_dir: ${PROJECT_DIRECTORY}/ include: - "./**" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${build_id}.tar.gz # Example: /mciuploads/${UPLOAD_BUCKET}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${task_name}/${build_id}.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} "fetch build": - command: shell.exec params: continue_on_err: true script: "set -o xtrace && rm -rf ${PROJECT_DIRECTORY}" - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${BUILD_NAME}/${build_id}.tar.gz bucket: mciuploads local_file: build.tar.gz - command: shell.exec params: continue_on_err: true # EVG-1105: Use s3.get extract_to: ./ script: "set -o xtrace && cd .. && rm -rf ${PROJECT_DIRECTORY} && mkdir ${PROJECT_DIRECTORY}/ && tar xf build.tar.gz -C ${PROJECT_DIRECTORY}/" "exec compile script" : - command: shell.exec type: test params: working_dir: "src" script: | ${PREPARE_SHELL} [ -f ${PROJECT_DIRECTORY}/${file} ] && BUILDTOOL="${buildtool}" sh ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping" "exec script" : - command: shell.exec type: test params: working_dir: "src" script: | ${PREPARE_SHELL} [ -f ${PROJECT_DIRECTORY}/${file} ] && sh ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping" "upload docs" : - command: shell.exec params: silent: true script: | export AWS_ACCESS_KEY_ID=${aws_key} export AWS_SECRET_ACCESS_KEY=${aws_secret} aws s3 cp ${PROJECT_DIRECTORY}/doc/html s3://mciuploads/${UPLOAD_BUCKET}/docs/${CURRENT_VERSION} --recursive --acl public-read --region us-east-1 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${PROJECT_DIRECTORY}/doc/html/index.html remote_file: ${UPLOAD_BUCKET}/docs/${CURRENT_VERSION}/index.html bucket: mciuploads permissions: public-read content_type: text/html display_name: "Rendered docs" "upload coverage" : - command: shell.exec params: silent: true script: | export AWS_ACCESS_KEY_ID=${aws_key} export AWS_SECRET_ACCESS_KEY=${aws_secret} aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${PROJECT_DIRECTORY}/coverage/index.html remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html bucket: mciuploads permissions: public-read content_type: text/html display_name: "Coverage Report" "upload scan artifacts" : - command: shell.exec type: test params: script: | cd if find ${PROJECT_DIRECTORY}/scan -name \*.html | grep -q html; then (cd ${PROJECT_DIRECTORY}/scan && find . -name index.html -exec echo "
  • {}
    • " \;) >> scan.html else echo "No issues found" > scan.html fi - command: shell.exec params: silent: true script: | export AWS_ACCESS_KEY_ID=${aws_key} export AWS_SECRET_ACCESS_KEY=${aws_secret} aws s3 cp ${PROJECT_DIRECTORY}/scan s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1 - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${PROJECT_DIRECTORY}/scan.html remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html bucket: mciuploads permissions: public-read content_type: text/html display_name: "Scan Build Report" "upload mo artifacts": - command: shell.exec params: script: | ${PREPARE_SHELL} find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: mongodb-logs.tar.gz remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "mongodb-logs.tar.gz" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log bucket: mciuploads permissions: public-read content_type: ${content_type|text/plain} display_name: "orchestration.log" "upload working dir": - command: archive.targz_pack params: target: "working-dir.tar.gz" source_dir: ${PROJECT_DIRECTORY}/ include: - "./**" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: working-dir.tar.gz remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "working-dir.tar.gz" - command: archive.targz_pack params: target: "drivers-dir.tar.gz" source_dir: ${DRIVERS_TOOLS} include: - "./**" exclude_files: # Windows cannot read the mongod *.lock files because they are locked. - "*.lock" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: drivers-dir.tar.gz remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "drivers-dir.tar.gz" "upload test results": - command: attach.results params: file_location: "${DRIVERS_TOOLS}/results.json" - command: attach.xunit_results params: file: "src/xunit-results/TEST-*.xml" "bootstrap mongo-orchestration": - command: shell.exec params: script: | ${PREPARE_SHELL} MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh # run-orchestration generates expansion file with the MONGODB_URI for the cluster - command: expansions.update params: file: mo-expansion.yml "stop mongo-orchestration": - command: shell.exec params: script: | ${PREPARE_SHELL} sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh "run tox": - command: shell.exec type: test params: working_dir: "src" script: | ${PREPARE_SHELL} PYTHON_BINARY=${PYTHON_BINARY} TOX_BINARY=${TOX_BINARY} TOX_ENV=${TOX_ENV} AUTH=${AUTH} SSL=${SSL} sh ${PROJECT_DIRECTORY}/.evergreen/run-tox.sh "run enterprise auth tests": - command: shell.exec type: test params: silent: true working_dir: "src" script: | # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) PYTHON_BINARY=${PYTHON_BINARY} TOX_ENV=${TOX_ENV} SASL_HOST=${sasl_host} SASL_PORT=${sasl_port} SASL_USER=${sasl_user} SASL_PASS=${sasl_pass} SASL_DB=${sasl_db} PRINCIPAL=${principal} GSSAPI_DB=${gssapi_db} KEYTAB_BASE64=${keytab_base64} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-enterprise-auth-tests.sh "cleanup": - command: shell.exec params: script: | ${PREPARE_SHELL} rm -rf $DRIVERS_TOOLS || true "fix absolute paths": - command: shell.exec params: script: | ${PREPARE_SHELL} for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename done "windows fix": - command: shell.exec params: script: | ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do cat $i | tr -d '\r' > $i.new mv $i.new $i done # Copy client certificate because symlinks do not work on Windows. cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem "make files executable": - command: shell.exec params: script: | ${PREPARE_SHELL} for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do chmod +x $i done "init test-results": - command: shell.exec params: script: | ${PREPARE_SHELL} echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json "install dependencies": - command: shell.exec params: working_dir: "src" script: | ${PREPARE_SHELL} file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. [ -f "$file" ] && sh $file || echo "$file not available, skipping" pre: - func: "fetch source" - func: "prepare resources" - func: "windows fix" - func: "fix absolute paths" - func: "init test-results" - func: "make files executable" - func: "install dependencies" post: # Disabled, causing timeouts # - func: "upload working dir" - func: "upload mo artifacts" - func: "upload test results" - func: "stop mongo-orchestration" - func: "cleanup" tasks: # Wildcard task. Do you need to find out what tools are available and where? # Throw it here, and execute this task on all buildvariants - name: getdata commands: - command: shell.exec type: test params: script: | set -o xtrace . ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true get_distro || true echo $DISTRO echo $MARCH echo $OS uname -a || true ls /etc/*release* || true cc --version || true gcc --version || true clang --version || true gcov --version || true lcov --version || true llvm-cov --version || true echo $PATH ls -la /usr/local/Cellar/llvm/*/bin/ || true ls -la /usr/local/Cellar/ || true scan-build --version || true genhtml --version || true valgrind --version || true # Standard test tasks {{{ - name: "test-2.6-standalone" tags: ["2.6", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "2.6" TOPOLOGY: "server" - func: "run tox" - name: "test-2.6-replica_set" tags: ["2.6", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "2.6" TOPOLOGY: "replica_set" - func: "run tox" - name: "test-2.6-sharded_cluster" tags: ["2.6", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "2.6" TOPOLOGY: "sharded_cluster" - func: "run tox" - name: "test-3.0-standalone" tags: ["3.0", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.0" TOPOLOGY: "server" - func: "run tox" - name: "test-3.0-replica_set" tags: ["3.0", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.0" TOPOLOGY: "replica_set" - func: "run tox" - name: "test-3.0-sharded_cluster" tags: ["3.0", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.0" TOPOLOGY: "sharded_cluster" - func: "run tox" - name: "test-3.2-standalone" tags: ["3.2", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.2" TOPOLOGY: "server" - func: "run tox" - name: "test-3.2-replica_set" tags: ["3.2", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.2" TOPOLOGY: "replica_set" - func: "run tox" - name: "test-3.2-sharded_cluster" tags: ["3.2", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.2" TOPOLOGY: "sharded_cluster" - func: "run tox" - name: "test-3.4-standalone" tags: ["3.4", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.4" TOPOLOGY: "server" - func: "run tox" - name: "test-3.4-replica_set" tags: ["3.4", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.4" TOPOLOGY: "replica_set" - func: "run tox" - name: "test-3.4-sharded_cluster" tags: ["3.4", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.4" TOPOLOGY: "sharded_cluster" - func: "run tox" - name: "test-3.6-standalone" tags: ["3.6", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.6" TOPOLOGY: "server" - func: "run tox" - name: "test-3.6-replica_set" tags: ["3.6", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.6" TOPOLOGY: "replica_set" - func: "run tox" - name: "test-3.6-sharded_cluster" tags: ["3.6", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.6" TOPOLOGY: "sharded_cluster" - func: "run tox" - name: "test-latest-standalone" tags: ["latest", "standalone"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "latest" TOPOLOGY: "server" - func: "run tox" - name: "test-latest-replica_set" tags: ["latest", "replica_set"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "latest" TOPOLOGY: "replica_set" - func: "run tox" - name: "test-latest-sharded_cluster" tags: ["latest", "sharded_cluster"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "latest" TOPOLOGY: "sharded_cluster" - func: "run tox" - name: "test-enterprise-auth" tags: ["enterprise-auth"] commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "latest" TOPOLOGY: "server" - func: "run enterprise auth tests" - name: "docs" commands: - func: "run tox" vars: TOX_ENV: py3-sphinx-docs - name: "doctest" commands: - func: "bootstrap mongo-orchestration" vars: VERSION: "3.6" TOPOLOGY: "server" - func: "run tox" vars: TOX_ENV: py3-sphinx-doctest # }}} axes: - id: versions display_name: MongoDB Version values: - id: "latest" display_name: "latest" variables: VERSION: "latest" - id: "3.6" display_name: "3.6" variables: VERSION: "3.6" - id: "3.4" display_name: "3.4" variables: VERSION: "3.4" - id: "3.2" display_name: "3.2" variables: VERSION: "3.2" - id: "3.0" display_name: "3.0" variables: VERSION: "3.0" - id: "2.6" display_name: "2.6" variables: VERSION: "2.6" - id: topology display_name: Topology values: - id: standalone display_name: Standalone variables: TOPOLOGY: "server" - id: replicaset display_name: Replica Set variables: TOPOLOGY: "replica_set" - id: sharded-cluster display_name: Sharded Cluster variables: TOPOLOGY: "sharded_cluster" - id: ssl display_name: SSL values: - id: ssl display_name: SSL variables: SSL: "ssl" AUTH: "auth" - id: nossl display_name: NoSSL variables: SSL: "nossl" AUTH: "noauth" - id: tox-env display_name: "Tox Env" values: # TODO: avoid repetition? - id: "tornado4-py27" variables: TOX_ENV: "tornado4-py27" PYTHON_BINARY: "/opt/python/2.7/bin/python" - id: "tornado4-pypy" variables: TOX_ENV: "tornado4-pypy" PYTHON_BINARY: "/opt/python/pypy/bin/pypy" - id: "tornado4-pypy3" variables: TOX_ENV: "tornado4-pypy3" PYTHON_BINARY: "/opt/python/pypy3.5/bin/pypy3" - id: "tornado4-py34" variables: TOX_ENV: "tornado4-py34" PYTHON_BINARY: "/opt/python/3.4/bin/python3" - id: "tornado4-py35" variables: TOX_ENV: "tornado4-py35" PYTHON_BINARY: "/opt/python/3.5/bin/python3" - id: "tornado4-py36" variables: TOX_ENV: "tornado4-py36" PYTHON_BINARY: "/opt/python/3.6/bin/python3" - id: "tornado_git-py27" variables: TOX_ENV: "tornado_git-py27" PYTHON_BINARY: "/opt/python/2.7/bin/python" - id: "tornado_git-py34" variables: TOX_ENV: "tornado_git-py34" PYTHON_BINARY: "/opt/python/3.4/bin/python3" - id: "tornado_git-py35" variables: TOX_ENV: "tornado_git-py35" PYTHON_BINARY: "/opt/python/3.5/bin/python3" - id: "tornado_git-py36" variables: TOX_ENV: "tornado_git-py36" PYTHON_BINARY: "/opt/python/3.6/bin/python3" - id: "asyncio-py34" variables: TOX_ENV: "asyncio-py34" PYTHON_BINARY: "/opt/python/3.4/bin/python3" - id: "asyncio-py35" variables: TOX_ENV: "asyncio-py35" PYTHON_BINARY: "/opt/python/3.5/bin/python3" - id: "asyncio-py36" variables: TOX_ENV: "asyncio-py36" PYTHON_BINARY: "/opt/python/3.6/bin/python3" - id: "py3-pymongo-master" variables: TOX_ENV: "py3-pymongo-master" PYTHON_BINARY: "/opt/python/3.6/bin/python3" - id: "synchro" variables: TOX_ENV: "synchro" # The synchro tests are configured for py27 in tox.ini. PYTHON_BINARY: "/opt/python/2.7/bin/python" - id: tox-env-osx display_name: "Tox Env OSX" values: - id: "tornado4-py27" variables: TOX_ENV: "tornado4-py27" PYTHON_BINARY: "python2.7" - id: "tornado_git-py27" variables: TOX_ENV: "tornado_git-py27" PYTHON_BINARY: "python2.7" # TODO: Reenable once we've diagnosed why this hangs, MOTOR-178 # Also reenable the exclude_spec in the "test-mongodb-versions-macos" matrix # - id: "synchro" # variables: # TOX_ENV: "synchro" # # The synchro tests are configured for py27 in tox.ini. # PYTHON_BINARY: "python2.7" - id: os display_name: "Operating System" values: - id: "rhel" display_name: "RHEL 6.2" run_on: "rhel62-small" variables: TOX_BINARY: "/opt/python/3.6/bin/tox" - id: "osx" display_name: "OSX 10.10" run_on: "macos-1012" variables: TOX_BINARY: "tox" buildvariants: - matrix_name: "test-mongodb-versions" matrix_spec: {"os": "rhel", "tox-env": "*", ssl: "*"} exclude_spec: # TODO: synchro needs PyMongo master's updated SSL test certs, # which may require Motor test suite changes. - os: "*" tox-env: ["synchro"] ssl: "ssl" display_name: "${os} ${tox-env} ${ssl}" tasks: - ".latest" - ".3.6" - ".3.4" - ".3.2" - ".3.0" - ".2.6" - matrix_name: "enterprise-auth" matrix_spec: {"tox-env": ["synchro"], ssl: "ssl"} display_name: "Enterprise Auth" run_on: - rhel62-small tasks: - name: "test-enterprise-auth" - name: "docs" display_name: "Docs - Build" run_on: - rhel62-small expansions: TOX_ENV: "py3-sphinx-docs" TOX_BINARY: "/opt/python/3.6/bin/tox" PYTHON_BINARY: "/opt/python/3.6/bin/python3" tasks: - name: "docs" - name: "doctests" display_name: "Docs - Test" run_on: - rhel62-small expansions: TOX_ENV: "py3-sphinx-doctest" TOX_BINARY: "/opt/python/3.6/bin/tox" PYTHON_BINARY: "/opt/python/3.6/bin/python3" tasks: - name: "doctest" - matrix_name: "test-mongodb-versions-macos" matrix_spec: {"os": "osx", "tox-env-osx": "*", ssl: "*"} # exclude_spec: # # TODO: synchro needs PyMongo master's updated SSL test certs, # # which may require Motor test suite changes. # - os: "*" # tox-env-osx: ["synchro"] # ssl: "ssl" display_name: "${os} ${tox-env-osx} ${ssl}" tasks: - ".latest" - ".3.6" - ".3.4" - ".3.2" rules: # There are no builds of MongoDB <= 3.0 with SSL for macOS - if: os: "*" tox-env-osx: "*" ssl: "nossl" then: add_tasks: - ".3.0" - ".2.6" motor-1.2.1/.evergreen/install-dependencies.sh000066400000000000000000000012241323007662700213460ustar00rootroot00000000000000#!/bin/sh set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail # Copy Motor's test certificates over driver-evergreen-tools' cp ${PROJECT_DIRECTORY}/test/certificates/* ${DRIVERS_TOOLS}/.evergreen/x509gen/ # Replace MongoOrchestration's client certificate. cp ${PROJECT_DIRECTORY}/test/certificates/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem if [ -w /etc/hosts ]; then SUDO="" else SUDO="sudo" fi # Add 'server' and 'hostname_not_in_cert' as a hostnames echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts motor-1.2.1/.evergreen/run-enterprise-auth-tests.sh000066400000000000000000000015621323007662700223420ustar00rootroot00000000000000#!/bin/bash # Don't trace to avoid secrets showing up in the logs set -o errexit echo "Running enterprise authentication tests" export DB_USER="bob" export DB_PASSWORD="pwd123" # BUILD-3830 touch ${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty export KRB5_CONFIG=${PROJECT_DIRECTORY}/.evergreen/krb5.conf.empty echo "Writing keytab" echo ${KEYTAB_BASE64} | base64 -d > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab echo "Running kinit" kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p ${PRINCIPAL} echo "Setting GSSAPI variables" export GSSAPI_HOST=${SASL_HOST} export GSSAPI_PORT=${SASL_PORT} export GSSAPI_PRINCIPAL=${PRINCIPAL} # Pass needed env variables to the test environment. export TOX_TESTENV_PASSENV=* # --sitepackages allows use of pykerberos without a test dep. /opt/python/3.6/bin/python3 -m tox -e "$TOX_ENV" --sitepackages -- -x test.test_auth motor-1.2.1/.evergreen/run-tox.sh000077500000000000000000000016121323007662700166740ustar00rootroot00000000000000#!/bin/sh set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail # Supported/used environment variables: # AUTH Set to enable authentication. Defaults to "noauth" # SSL Set to enable SSL. Defaults to "nossl" # TOX_ENV Tox environment name, e.g. "tornado4-py36" AUTH=${AUTH:-noauth} SSL=${SSL:-nossl} if [ "$AUTH" != "noauth" ]; then export DB_USER="bob" export DB_PASSWORD="pwd123" fi if [ "$SSL" != "nossl" ]; then export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" fi if [ "$TOX_ENV" = "synchro" ]; then SETUP_ARGS="-- --check-exclude-patterns" fi # Run the tests, and store the results in Evergreen compatible XUnit XML ${TOX_BINARY} -e ${TOX_ENV} ${SETUP_ARGS} motor-1.2.1/.gitignore000066400000000000000000000001531323007662700146500ustar00rootroot00000000000000*~ *#* .DS* *.pyc *.pyd build/ dist/ motor.egg-info/ setup.cfg *.egg .tox doc/_build/ .idea/ xunit-results motor-1.2.1/CONTRIBUTING.rst000066400000000000000000000031421323007662700153220ustar00rootroot00000000000000Contributing to Motor ===================== Contributions are encouraged. Please read these guidelines before sending a pull request. Bugfixes and New Features ------------------------- Before starting to write code, look for existing tickets or create one in `Jira `_ for your specific issue or feature request. Running Tests ------------- Install a recent version of MongoDB and run it on the default port from a clean data directory. Pass "--setParameter enableTestCommands=1" to mongod to enable testing MotorCursor's ``max_time_ms`` method. Control how the tests connect to MongoDB with these environment variables: - ``DB_IP``: Defaults to "localhost", can be a domain name or IP - ``DB_PORT``: Defaults to 27017 - ``DB_USER``, ``DB_PASSWORD``: To test with authentication, create an admin user and set these environment variables to the username and password - ``CERT_DIR``: Path with alternate client.pem and ca.pem for testing. Otherwise the suite uses those in test/certificates/. Install `tox`_ and run it from the command line in the repository directory. You will need a variety of Python interpreters installed. For a minimal test, ensure you have Python 2.7 and 3.5, and run:: > tox -e tornado4-py27,tornado4-py36 The doctests pass with Python 3.6 and a MongoDB 3.6 instance running on port 27017: > tox -e py3-sphinx-doctest .. _tox: https://testrun.org/tox/ General Guidelines ------------------ - Avoid backward breaking changes if at all possible. - Write inline documentation for new classes and methods. - Add yourself to doc/contributors.rst :) motor-1.2.1/LICENSE000066400000000000000000000261361323007662700136760ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. motor-1.2.1/MANIFEST.in000066400000000000000000000003121323007662700144130ustar00rootroot00000000000000include README.rst include LICENSE recursive-include doc *.rst recursive-include doc *.py recursive-include test *.py recursive-include doc *.conf recursive-include doc *.css recursive-include doc *.js motor-1.2.1/README.rst000066400000000000000000000067311323007662700143570ustar00rootroot00000000000000===== Motor ===== .. image:: https://raw.github.com/mongodb/motor/master/doc/_static/motor.png :Info: Motor is a full-featured, non-blocking MongoDB_ driver for Python Tornado_ and asyncio_ applications. :Author: A\. Jesse Jiryu Davis About ===== Motor presents a callback- or Future-based API for non-blocking access to MongoDB. The source is `on GitHub `_ and the docs are on ReadTheDocs_. "We use Motor in high throughput environments, processing tens of thousands of requests per second. It allows us to take full advantage of modern hardware, ensuring we utilise the entire capacity of our purchased CPUs. This helps us be more efficient with computing power, compute spend and minimises the environmental impact of our infrastructure as a result." --*David Mytton, Server Density* "We develop easy-to-use sensors and sensor systems with open source software to ensure every innovator, from school child to laboratory researcher, has the same opportunity to create. We integrate Motor into our software to guarantee massively scalable sensor systems for everyone." --*Ryan Smith, inXus Interactive* Installation ============ $ pip install motor Dependencies ============ Motor works in all the environments officially supported by Tornado or by asyncio. It requires: * Unix, including Mac OS X. Windows is not supported. * PyMongo_ 3.4 or later. * Python 2.7 or later. * `futures`_ on Python 2.7. * `backports.pbkdf2`_ for faster authentication with MongoDB 3.0+, especially on Python older than 2.7.8, or on Python 3 before Python 3.4. See `requirements `_ for details about compatibility. How To Ask For Help =================== Issues with, questions about, or feedback for Motor should be sent to the `mongodb-user list on Google Groups`_. For confirmed issues or feature requests, open a case in `Jira `_ in the "MOTOR" project. Please include all of the following information: - Detailed steps to reproduce the problem, including your code and a full traceback, if possible. - What you expected to happen, and what actually happened. - The exact python version used, with patch level:: $ python -c "import sys; print(sys.version)" - The exact version of PyMongo used:: $ python -c "import pymongo; print(pymongo.version); print(pymongo.has_c())" - The exact Tornado version, if you are using Tornado:: $ python -c "import tornado; print(tornado.version)" - The operating system and version (e.g. RedHat Enterprise Linux 6.4, OSX 10.9.5, ...) Documentation ============= Motor's documentation is on ReadTheDocs_. Build the documentation with Python 3.5. Install sphinx, Tornado, and aiohttp, and do ``cd doc; make html``. Examples ======== See the `examples on ReadTheDocs `_. Testing ======= Run ``python setup.py test``. Tests are located in the ``test/`` directory. .. _PyMongo: http://pypi.python.org/pypi/pymongo/ .. _MongoDB: http://mongodb.org/ .. _Tornado: http://tornadoweb.org/ .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _futures: https://pypi.python.org/pypi/futures .. _backports.pbkdf2: https://pypi.python.org/pypi/backports.pbkdf2/ .. _ReadTheDocs: https://motor.readthedocs.io/ .. _mongodb-user list on Google Groups: https://groups.google.com/forum/?fromgroups#!forum/mongodb-user .. _sphinx: http://sphinx.pocoo.org/ motor-1.2.1/doc/000077500000000000000000000000001323007662700134265ustar00rootroot00000000000000motor-1.2.1/doc/Makefile000066400000000000000000000060641323007662700150740ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Motor.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Motor.qhc" latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." motor-1.2.1/doc/__init__.py000066400000000000000000000000011323007662700155260ustar00rootroot00000000000000 motor-1.2.1/doc/_static/000077500000000000000000000000001323007662700150545ustar00rootroot00000000000000motor-1.2.1/doc/_static/motor.png000066400000000000000000001576221323007662700167370ustar00rootroot00000000000000PNG  IHDRatEXtSoftwareAdobe ImageReadyqe<iTXtXML:com.adobe.xmp a PLTE0.މ,)KZj'j,,FbhzP(1Zk_5jY#n+Y#"Z-*N6T4157EJ2)%ǷK{k+ABIx)ZI]it;!(i=~C K:cv4! j@gzS\d)77= gPSZWblFIP$ ³9*wcr|^ja8J|_'=HJOW%KZ[n!0:Rt'/2傜ǚ]"(ptlcd*O_ig|fL5RbEr2.-xx~򆠱z/*-'BOme'M]P3JW +#'l[ĉ=]llO>Dms(FUx13]og.60"l#*RJJ(*wEklaYZ==B/DPtll,,?,'mP%2/10w3WiQev&mh8`r ehhh턮*WiOV_&eeD-Hwgh/\=޿>ߙށ;ˮ/+%.TݍSoz.,m=m{?;PhQyPuV/N7H!FZ.T[/CQ~ߕ~2zZfϞM D]#ra\'1Nͧ_?@!> tw'ҢoFI,"Ts)S'/ٟ^q&ȭ_un Ι|rszng#̀8Htmoz#b~u sE>j[ju7ѳtPW9B {~ٳolB2w6Ϥ$>D<c)>19ޣJ]hq 8=GyGלi#cSVj vRV]\|:ȭW~s(k96c-b' Qֽ`U5 +&-Uu\YgM'41 nSrٗ`\Fgh4ԅ "8zPѨ4 ](*`ϥ)SO38x><ǠĊFF'z!&ӕQx;֒lWe)!΅X"5xRqBy.&שoݶ9+Ĥ+U(­H W͆Ʌ/ق {q# C`g G E\E֫Pb40 (H,rw@CEd@ٷVKQ!&:Q%\b Fhz3<  rl(NߴP3LA*m(1gD#(2)^F_bɍஈ }X@)PsT_p]xfU NM(Fv Lr.gWD00) )UmY$r"Uv}UA5zQ9EPR-,Xl3ϵA;l4~ƐFKMq/haC\[эCk쥿벘 $>e3)nM-+ . $jԲ 8&H_P͔‡oJ.> 5ΖLb' I`aJ<2p@ I1c/Sl6-X%*QNM|BmA1jRˍrzڞPx38!(>eE(j܍ID+y?GL.LR ll%\~Jnmd@A2 pKxpceMZ&T jN;$/$g($Z11 .}.1%I{9X!}s)xtQBfUpy!A;ARn+*ũ[3̜v?8(uR*|QB0{#!hd$񮄹D/8],ȍhz EE%kˢ,ԒL6ʾ~Q8'B $]K'λtEͽ`& 0)2`vm01X6P%Q`5L e` {*4B']i%cRPB/rDB@PƏH7gb9+5Py9睳p0V'cyel]!Lay) ^/@ZJЦ$CzH\o_d@<`笼>=jvͼk-@8E-48U4Y0U1`')E9T%.q1^@(M"qy_T\*/ 8B)t0#p>$+Pe] yj)\-aa p¥IGXxoQq9JXh˦_Y?:ηu.R#ڮ]?zqӖb) g/zB1J!>ďڸD :{dT:u*2A4ZMP&HSd)!VihxT$OԹWW{O{.+zy"Bhk 5F_H)2ׂ: AQkȺ ğP ch\L%.Hgzo׹BIѼ޹DRs(O=N 6BT p,.Mc%ꞸXt\‰'[Poʡ")Pp%WCO{uv])/q)@(om*rq [ CG" ,x.L aY 2~+QjJcAAFzaم\ fwc}]_{ʯgSB"N径  x /wG^`)Vmo{,tuI$1A Q o$4+\ȅJ O] _}ݸzasZY:IVژ5A#/"@*[JA !_U2 vnAGFTɕ(Yq+@@^L$zBi 7Zqe3fҁM8H8?JwLz)F&LS+P߰kO0]qGlu$n=7zɅsOc_X ӕJ"wtDp3-;{L dr"g Bqz)t`檃WLGq1cNEm!*#d-°k|^ZS~AD^*3W\6\W)02kԻ{.@"B]ÎV,5'o4Mܰ=^ґy]07 bx r@^|˺aނN7Ig.òVsaaqzV\< vQB0׋xՙ:$ PHT@},z0WL%a졐;//;8KvΉ xZ멷 a6K‰ \q2`^0k#e{F3C5 ظxA<ʭky1,U taskLE#Ai.q xdAioV dA)&'u(bs??ʼnp9:ű &~sV\WLBhe?ӫLgh ,+p )>>Ik?Y=XHF0!ƣC_]ܐHOa=t/F ^-VL_R ٤ ,ޒA7{ ]旭Ji$ظU1eg\AJ"I&X B]pۑ2՛u{6JKAUǤ޳W $H@2}/+D:P5 sF"H\(w+Y/fq+f.;4Nh M$S2!bhP4If+q /J}Xu 3bYIYE 䟾@Tn8gX&H홹jTqsz,za@fh,Ϧ &~0" ͵V G35:, )Yc ̙ wGg}nܧtDp!_ܥ8P\tD Tţ;Wn앥VŒ#brMQl@KCZO7B՛>,%.ۓ`nLFd=ٗ/Rac**;"b;#,0VAJqol8"ӿJv_CK0 _UR`=8g-lIܣd!gV6Cr'zO:UOǽʥx~E+B9z<M -ђҸBJm!*[.+C!~V*Ld>g0A񳽽&ӫGk`UȕR WO͠ `bt3D(N {.SX69 *"F2Hd~fز{˰lvcJ"ݐPpP&+'ߘ-[xx"R9G,g\%i͈DJf-G3M Դp! c.7q}t[n8E ޕPxI;-/o'7!{3+٣{?ɧ^}n$Ąty )TZ"^0 )"!!%x#uwx!KFSZV 谖 Ļ ,r!a$RM-~g Sȫr˴dQHp)b] 4 Ц^ O༁djQc W\/[S,x#`1cf$y?}Oq:)dPffeǴ8_}z`?JkRM4PkN,mE& Բ#Ĕ)]a 3Op|w?ؕWwu狏owOzA ؈k{eM5q wt`m$ ĵB1/H ԅ. i|*< WV#İ7SsK,1'{whӖС|xWD@ b1 m?π\pN fxvĠEd'a{p !(〴.7P0^55CmL%JHm(n޸AC gKtgfHGO OM[2:B՛c;pA&+ z`u=Ri hC…jRRScNj E 6zI QRi$ѫ 8I¢?Sz3 Mw |ᡀ,uB8b(A)1y~﯑jC`!K8b\(ETߔ`p3 S2@Uys%a6ݛfUZ,-z 2HRTcoD|؃ƊMѲؤ#&hS0JA|?SjxQiS,,fEFd|3[V7ֶ6?RR\~s *-X0Cf4Y ATsZTYƴTheQڃ(-7 qԵSu#q${"g;>ڻĶ*>'ߡm'wo[޲:OEJ Y`0A\&ψ ^<׵8[E.$*E#OD UOZgS DP<D"{a X[dW֕w9ɓ'&N*a8Lhz9 ij ``"g9ºiDMLd!7X u q$y艣H18Za{&8*:nt0/WfXݠl)#'aKJ{ښ* Xm@GKl {{xfTnLRI&mmRSsι= J< BJHH )YFK |E6~XT ~/@KR@Lk݆}v`Car}6d =9m/@(O$P :% !R u-7x1G`F1 !4vI0 3Dr ='aq?=Er̵ I&4XTNmsuI۰Ğ4f N4<3EXpcј6Ͽ /춣.bՅv%EgXKCUC>Q(yſvyghJ jhw K-ZbJe6EO/Y>;۲tBsEG R@R_a{X(H@\d#W] )!mLLɏt:eWw;ڔd?wir1ER;dDt%`].+r-jZ bӠV,JJ՗-`,żCw)wQOݻH-Nۤ:dtVŠ1!Ԅ9' v)T!(j"֐1YY#~%mΰ*A.e{C]yϾƺLpq.R,>eVuܨy ˆPH0KQo̟رc/N=x cbHyr)X5fjƅe[-9G\ µHtM+ "N!5H\>'d$xE-.pxdƤH\ޚ)ÞZ>6ԗ^aF1LqׯZ9PU~L8nS2y9ut$`-S`}to}JBC>CEC:<93^JgT"qj鋵{MTcPyԌ93O6 Mr<]\.lU3ν}BQ*ψ<[L[13u3DLΎYKD:3믍˰E9bޫ/V=@tt$c,si l]^+~|Wv0)*3Uzy;\1TD2y^Yvr%=2vbQJ43\'69oyx X#hӵOG+~"2#[n|'S8ʏtr⻧NYf׵J%O]}7sZxb_]e ڛP̵Epy* =%$Rp}}Vk4r!`Mj ,$ ^É({ƳU':81^_?|Lgaa>RQ0/MO Tn}dzqf0ɼ Lر&1gRVQw7[f;BRfqT]1fx!pmN{[ǁk!(uQR= ۪r+9*3rƲ0;ᧇ]tu?-.矿nal}n1ԩF(~e=HNBJIt-dKmj ;0xZ˲?j,ښCoTʞݱX}cxBGsu!qE#翉xʙ|OdŜGpmx}=W_ٿ䒙6췏P<8Q~2`NV}&RvJy zZey5۫6"n]b֑E\9vmNmVSyHc#r_[>OtW\䒷WɌw;U孋{R,Y 0ϵ0^O> c?om76v[0dޢT傍e*%&Xԃieߘ~vONO$\avyuًRNjo..8Nm>`<^L6{u"q0(K4 TRx,@h)X7ZRtsVwNʱ2O֬{6^v_nC];5eg-rz'2'[U=ʷ#B߫Zkv-~L.  {K8s4Ls%\}TU|l+~Eg5O{O޸D4w%\,?yi.[2N1%K)S(}QHCAq(^PcfBy60 @y_XY$QƗX戳~}*wsomdysVT]w]<.2=Ӧb/$k=_6K+W`F ҵ R@ܔ>R^_iآEԡ Hi@ l+|CrTG jWdc"5 ߞs'ΧXeТ%?qy` x+[lr׽}ɴPr s!Uy ڹ#)8KM`]N^&ZQS!5"&H z*ȿ\V| P\(ëBPfu8K*8\zx8s~_"q/U%&#,m^\(ּ8\GYVR4-K\{PWxdQp6r[KJOu)^kR'5TJ_βW,bo+\ kSylL(_rϚW_Ʋ?'|*mV Z\Oy"¬Vݠx0GU;)U* {Tj9ׂo,:T/:TF&w7vHp#D140q]׿\>3vpmb0 "ºxN'qQ$U*3q<YWwޙ;wC+1o *H/DR/Ok$#DT@^ڊZxLV,NS &]ޝZwrw>\'G~z^zJv_waޠcFWnuѦ{(14p;!C`F`ъVR"΅K0&G_@ccMP,:0<<\yxom _{ #Ej?.02PDzҧ{"hɠБܠ2جb AE=$NI0J`np}mrǙc,9K*rOWUgeJoꫢKX~_p)Hm%GEt3"xee6 BzI66ZkeD&:-Ug?<e?20[Ӟ˅L"ߺY&ݷz}?U:Nj⃏LX7@"CG<^ !P uc 6M ^#eazb»)M.^U~eg("fMʒbgOGuE듧۵^~>) k/@i0~l:FXU Bo|b;UJ0X@V?q [)FpНj9ül쌯d#Mj, 6F;]B11ٽb{}D!=B1,Ӄsݴ5kJ5;iӦ񞡸V>ܹWKo$.4jPkUK9_*#t;]m6^+;ʡ%#۞9jɢX0U]2缕Ӧ=}yI*Rܗ>ӻئ(U2A\225-!D Yj&{a/',Mx?Uwd4}gIy5xaqo/ʰXd#ϭ_x7įŋ{9DʼnT.bQ'8$[s /p߬uGBh`5OhIm\Wz\2RktxCE@ʾ.w" ݶ/Bu]CdLDR>a9#VdFٷӯ7%G/~D5p9κӊmd=uٜ&ܒ~d?gtĖ_J(1dnvPkZCW{ q'xr=JQe"hWN] ›"Gs)./ ۲^s٦[vO*t4)%TЖ 78B˺r^[{J1fb A`i "׿ʹ% 'C,ty*VQJÞ()  s||xٞ_S7$cRa0ŊeiC/|.~Qsk`-n̉=bRƦ \7%pNG#&LJWVl@"wxp|NJ!SP^=TXT|yA|ϳ-^~zy--x#m V(onbqaИ/B.b8pv]H?Q&`ѭ9&FPDN`x.key.z_)>-<Tf癶 X<>6gmȥš OgZ:?5;ޡ"e!g^XfQQ~M )` 3,YەGgȃ賉˥⹕PmNT|+zVLUH@IeWu6K a[*;g;ׯ`68+^ҟF;,<- nUqaUJ"rm"ވ(E}[ȋ=ob&rW39mw%>Y^=&Jj-C3K +哎cuuu]2u/ﻯ~oyKGu&'X~P=o/ܓ C*FIY[~<)W*2%v>G. }=+1n/z^P(#e55y@N5dX'Ry|s S:7yĄLcChypsZd4STXՋqp'oiu;f1dꄸ &`؊~B:ik[ٴkOɄ|=gOomG*E:vs3y)#yKWޣȦ]CجΠ7>rׯVR n[yq.u]=xUs.Fh bI?~_x(Wdv#w"x2yee<] Y1XOrL.\yv )B#1Fϛ Kś<!Pp}FES *v>a0uZ$羵2R rgcMaY8 ʌ|ftC_#tS|AY0 rkF'/"0`h(4!>WI<ޛ#I8b ;ApX7پ].ĪEW\kal7¨,[E.Hޓv ۈM|:>{,]#4@ISrS']l+V^)xBٵ/Rdz,YC؞7h/,m>PZ%!!(N5֊;j4OW) ]VMT:~顽-yI%5PvrZ\X- ,Qp{>%|cޡhnFEU.=~g `#@3Xյa/o{{%H jK'w fd'kΊ9GmtZ kX։[RjEfnC6kFW"E5t퍷2v"3֤Z+% YK+媻by/0Ѿ u%Wx4$ cU;l.oxMQ0jp : jĖ>^ նh):TQFѺg V041(:Q:PRue>$cMWL$:/yoi͆2P?G4DxN | )@l5 JMxW/Ū-VwtwKhvtm+P|bqg/_|1Ѽ;[[W矬X[Q|WϠIv_LtӛShmOl'mE$@̲̚o֋ B<]8i^#+*gagp.ɷ5WTR-CF=sSN5Ta|knY[c,ivx+Zu#xworHnjQw\Q|ڛ= wd6DLc@22yA%(D{޵_q{Fn]ty*Ri hcY.x*z6pSVG* }!_FkIq'kV)Ǽv^c~tOfQؔoUCӐ)oXI4U3]Rp?G+bXù!alg>N?)Ss)\مF 6#H ,Vr3 3/|n]לA ӒH8*z8/'qyWOf՗Yϊ=ӟ};髮?-% 3y8{ow>RVl-e.HP7$1.7orx%D)GjcAy_-v0SijHLߞӻ+הTN [7H6ukWMC[L{jO <ͨ{cwr&6cN3nrSYft/Cl=‹?\\_ߩU+{?:IeypEx*jQ ۳d1@+>N~4Îr#fe!j7B!>n@F R|l=0EJZ( t6&r_v~'a³R&`í In3^ k ԩyr^379XcvĬV bp =IL) `2K/"zSYB5ɞCM@T8^gzb}rs-Qem9yT!WR jjzYDmg^b[*UaMqrg=ǔZ{qC(Yyd:P.h) 1 ȑhtfzHMC}]T/{ucy.Ƿ;tuRivg"9[EɱUf@Q3!ĚA'~Z<*E?62S5=\Htq=3>Er_LtUQ_k&?oj3)O?]%ao{ķ<礎unX5ɺYkg{օݿ`PP\EB]EwO @PpoD 윍۶JyԲHEOZ$ca`sbKjȔ<O9Z篓:27$ߋxgW!٨g5$NPkuu^d!y]HZ3X:L-}쎰*_4nZ7t%4 TsIy7],pn}y=ցo}L:D= z)a{1ǢoeY:ռw+:ߝT :gJiLoR `ɮynrU$:L$T38V|`_CFSrM+8qAʸmcmU1'tiH)t\[;\8P~v5ټ57g }ri:'=3H}^FPXW=ujQ^siEoS.$Z02'OE| T3AEI SG@<8P6 ;][~J%|\+4|HzNdYm\ӮR-rQ QUqFJ .fFӖ[ZqD Z[qWG( PucGmuH rS/t4 s8oq*{}ħ0ςvD$&PC=VЙf"$-§GT!+<+O1?Y8.딦y;R}=iמrRHȅGpj*oLR{nק7#1`rЧO?fSЦ'.%cHoERnĂ#!@w=rO uG%,,p;ixGi,wDVQmd[ӓt:~Em Q(ɔ77)֚ ?K@ IKT{u«U KMeЌ|L}nh?=J-lNe$}`_sd8x#P؁22?]3E5d{CQ_ 'e>2kŞh7ɜ԰]aib} 2pj/ wӝ2!'7;aog# EρM]?2/:Ln[Ag `kBSR68ejtVfuMGts"yuHj \P~vOv,J*-|*}$egV(:-hyE>^js?YL46O 7K/0^/ҐY1eD6tA3RRj%%pתمv{"!5)5$Uvz_άnlğk:N4 eF"/p:Ocr!n+>s%U(/mrM%m'RށaIk^}fQg,&aYp⍅t6uؙշ-@X|L~RyGp\),U(߃H3Cp,Y}5+4kl4%MB̗tNLD4C<\Y]-c,UNi1בn;+]a4snzC~믓rV*k \..s:u HJd3dTYMy+SJ1NGI\S~ȏ&p|IQh(cp==NbϼD\5E%wN|GK*HJP'>19 O)Ts,ZZZO jC%AA"UL`AV%pZ8'N\ZYhg{FTTl߷nݺ}eNQQej@SPr'^fH  j X!n T#E.Rb&}7>ۺ[oƢVJA9W탚>S3Xu˾ɴurՋ|P1p~ G^p2O;A4$ٻkuᜂ|ޙ7YucJ|w?S5k5Xv-S{^OoqkE6o>oņ"H{>ЗT^T08Uwa@Ý88\z @.4 {ysyWY#HÉ,}X^;xk\]M}uͅ[Y]O+&\˯T\?W晡o T*Rܰ͋{jv(j&-꧘a`׵z5Z,Kw17dh*{MbLɛk*jPetz$xz0P;fʩ暊SNp暚ʫL&VlCU}͸ߘuєVtN%b0]RIn޻34Ke|}-nT1̆&_-ձXuCW+j7`U[qbjJ~+汸;djr(A5#gm/OBu ո@j@FBaRq^ULPh "TҐ1L+Hcn[O423dkxsMMM+G;'~+RU3/tB.fTþ̓VUIޏ|ýPW, ӵQ]|J0R W{ {f9d-+vi8XCLg!kPQ # <{luK3 Tuu BO^xUw?ކl,m'\Td蛹W|7Ґ:!"p8zmX8ߦZQE4c@#bdӐfOUBBDZw9S&D UdY5MVe&RU ^0wٿ3Gl?1?o7b{ȬkZױ#pA脻SzE"٠ŅtTe7 q/]7aSJ{5=3ՙLIrejlv\_㤤ۮy٦\>ĤQ[oŜWZLO7j/w3Q- k@ RI͛=ԠN,n_A ދz"hLpJje5W8>!ygzwך3e퍡[+V;Ƿp]^t]HY9kUHH.7n~)x#v_q޿DiGG`F4[z V@T(g>||n [͐Gܖ׭v إQƩ-v]XpBPUcrZk+s0r"E}cR^^QVW/)a)^vj5Pkd '.SDq˽f{{mu< JAőñjme{D ܊:+ñ2jN+Dz'Dt"xc.=&CPwwA'M,  ]ZTA5@EHzw1R 17O,!atZ[yF FF'x< \ŠxE<>-=ȘpbFDgCh`cmV@ ,܆!p<DBe"k4>#5J-ÑHZ3s,b_qb0@|RSZxOú#2 ex"J$j6&"ëHs WBA9lչfФ߹7M&Ss\E gT:ub@zWs$vl8 jZt4; {Ӛ mdBuc\{!eؑO_Ulpc O[`f˃hr/,ZR- H愆[Z֣)߽̈́ Y^2Eጒuއf=R]=3&8M~f68p֦l~vo< +Lfu9J#֢(ːP㻯{O }q~~e ;ۨ'|\;a(Dҋl-56}9*B1(rD=/֍fta4~X5#^|-T{ėr;'Cŕg ؛KO:[YN~ⵑ`XO<<ڮ1tI2aȉh!Q;w@K[b[z03\yS 2 l_\ KkDL]qqkS.]ZCx{CgJ^ڕW1B q!"RoQC;V.hEWĝt!)tjqN;[3=;6aBM,!"Ż@:Vr1&_*ݢ 4t%"~ vn2Y>^~qDq Rr#G`d$J-(~uLA AjH}FKb#TpRdV!i˰ȦLK$J΄9*E;NvQunv[!,ZM}|I=bH`‹.Z^m DD}Y!-p}Z=oiX u_(ˏ+qEao)`%%dH Z3}o|T.d96!c?"%S='p6[=jGw-*PVŞvHD>r!- dJR@rhz`_%n@ {ƼT Nu8;ÎSxtA׀QC5ƺJ <Vֺ  /}yNyߵ5F @G+1NiuEOGFCvYOLoȌI!IP -50ѡLwv5AjP\bqH%`2U\>[=رeݺPEABQc܅ <~Z\,kw׼S5KPSڵk~kV@Cey>$3ʝvT<i~ޫ.Qn$9Y6[r_FI#yolcXfM33NJ/:6{ ٗ$'Ƣ[*&xRVOqg#.Eh`TT4Y*X hoF~|NTj/';n@ 8&w c`_J"q7(ƓY[t<9'[X{D^X^1Z{[,U)]m[ )M,a'zAo@=x!ҟmhi e3( hM.47 ;QC.v͎HmlPLve#8ۂ\Gّbu#j`?'uV@9F3X3ⴖǕũ)Sy޼y~J>Zs4؀fHu8KOr#S3탡X,3N[Xa'^aLsEOx{uWʅ̄Rϗ/J<64X} d LrU0׿`k}.bHeyG%dqI___ڿk>)8(reJ nw+˟X܍7w] ̟Y{x^0a0>cf$SWU"WyiOk6ˇ[śxq%VuĸLMLT7mWilf6$Ho]󚶉> >7e_ђ^"j"Dwfr<[ IG'U+Ae7/m , b<#(uMɗZĊ*'RR.Jpvf&L8N98U=- 76ZĶ?gV?YS*!葒{2tz܎rI-ZifOJa$u>oþ'NNC8Ht}C,_={-z_\)xS. v ξ =S|>#ƒySMMt HWQ׾\Px&]\J'¡$H kf7dυ+od<ָ>2u:;BL뾛k̰aMR(pk0&Tuy[iL$ZԼ6+`=+n|z+%,˼L Ԝ-o>Nߩ@9(`/+b\ѾqVt/"GHҒC Ig4H\N o7lr&*.iźZ*TS1YJoV]Uc[骇EfmUWk~ Au BH&|#.dT/]|isL\&Qδ*e8gdMǬGf4# 5Cp0X,8H'TZJZY980KWN({ӑH{ pb+:z7n߾ꆜ(DLVV.'tO@i/#(JVU/N,?@Qhxe)D6X'h|f!y#><WHҦ#"T_V jSh緳]憬} @u_s=0 u!2~כ /~=(-fC96BوBH]>)`kVN$[ݖo D=1L2om^N'ꆬ 0* /eMn=x?5^TwtcWV,qqPUHLYiCg@C:%tlK{D::$I.zyHkb!-6K_X͒Nq@pa-'7z^4#ffBt|ޱGob! dDRΈ Y\cHb_{hNavcKCܺb1 ӧyY`ȿjolZ# ¢eˏlEQ$1jۘjUrrj^`V5L2 )?e*e_JQy/(<(u!gzby8ڳi#3]zHx WU5`P͏=!1! kj[/T#Z+>艢`=Ȫ"+?H&=uu2砋) 5x>mwl EilayQBv(wB&&&&#+x `&=qc,n"!48M 9^zW%m]Q ߾?=WtVPb^dZ3\EQq|J&y| oz<ٷ-zE7=#&o2Gʢ43C#L0J \j/W4L -^twrqg`e#GԽr#~ =7n:xFNe P5V4%SIKIY* 01zw^ݣܝ~^mǤTqy+)8I"@_(KI {a(+_3ujhM՛@2~n_+BF0]*9Y5߾.{onb;=BZ`m3EO Y*5θQ! иFϩ |gB:ox{.`eo9>Bfy5~O"`;`) 0`:,e%BP(@yu9<ŇAcNL{}_`(/g']짇LRY/5}?/6Og6?p޹g~ޢm7==We $0qvԸHH G2 JЂ. hOD*@E{g*5҉[hG^zgϞ}yΊ.`xY:AA*(Nk.9>P-AgkˮJIn6%[+Y{~T ?Yuo/Ʋ4K+ъ0E/H *B.=T7Qb2eȀNDqqD C)Z:S-K-6a(S*Hz)/+{G҇cʂ% .O]m%,۵>U #o tLӇܗ:HIOuEL_y dMiN+F0\ qr_H QCeµ[] <@X4ħ*MOJHb? +vdjm{%ߘ/zG⤝s; CR<&9嚎'" AB%2&8%jcr4¥3z _u8%9oe,iO#[P-^2NurJդ~U`6 ُJe[B1~ ّ4Vk tsԐߢ N/Dsu {A\#nxi\NYZrDbQyO<@!"H9Lq.sJ^)^=?1i ~3+|t",q=uWf~Tx\^Ӵ΢#4Ig1ÁH2KmڛZꉠ_+ZwbqT)J:'^lY=G0bS jIv yIJp,i>^dԫQěĶֈ@[[ϜK_vNJg"p}Cpڣi)'g- -LOFc s}rzf䊛9GӱX,x:-VrTJ%_EZi@[L$35`zJvf|oه;ƋIL6C_{n:q:*wxwsȼH>V>@жv;*)^׽o﹖Fqoܳzd UPC0zr96 qa^{0B0LˣC;Mr3p6ٟ7}ZxsUgC<`~H25T&"l/]>W7󛽞);hhY SœJE8"݆[zGNlAEƲՌD$JP (:qBY`13}Q hsYPolAgBGh5ʧ5tH׭&a |; 1ZiZ4riS6kK" 汀…7**h9gJk5<h*~c%[WEˆ19gchT $DmsJ'2mAH6xn*/O=ʏrc܋<0}4zªXV&Ө S[c?"6K\!޸PP1pI5@q1&rfޠ:o-Ԩ<iN\1^qY̳eT}V4ghjv;M/I 5GMFˋα B"A{s5ݺz'v?$Q*7j/fH41Q%s7L]|"ĒKۤKg!qcw.-SΘfKiuoрU޴&k~D4Ph߬1>:%Y䆵{55י{K7(>vʯʕkE[lh[RKS $@+%Wr֊9pxoLOݤ*<'兢z2X Atۼ*B[q h-i2YІ-l_}5c '$B+pZ<7o׾g Qus<=ݔi'[Gwc^+QxNuZsee$+ej{$# O-C^*b9˽^+v[+v^#G(" !<h9|ŲzbCsǀ>|Iffs >Є=2ѹgqVWcoQ1o;y;fY0D"OeZC(ա6HKMV8RsE!nF!sGhYM/0m27D,ion$>od+9mV[H@𥝔ZxhfaV+@}Ϝtl JJU=Q_9WnxkױLE4rx7Y{7s# Jm2o( >I ^sF>XxqTN29r zw\/~% )w5q&l5 }~jyӞw.?_3 cB:!_ͮY`RbіIhz({1[Y?.RKܭs̅^_r˼e1Ue^ݞ7_nzq}HGr$e_gU7e:p@k+U-%̇r*ߘbg_;zWS_G(sT0oX-wl̊"ې4blokqUKլs_tJ٬yIl=vd/ydtle}SSӏ汻dM:4?ql]$jKSJ TYfj.+|U5H"M6sz yz{+y? Ϗ!folj_@$OB QQ7s^ ,X̙G/0KҠEnEߌW?'_݋'SHLZoG~}xEK0{·z=/30[8žAgEu[o)"){*إL]oUH T6e 3Y9wA7fo|C`9Lyveh]ӄ-LPck5? >jMl:`铒piE@e<J/mr䉥:GzרHTDģw ~Uļ[uחM&K՗֨x՞[;Z/)Pu|Aq% |x9|lj6&7>^Gmҭ۠Py-͌|lL"`Q3<;[oɮƌ Fcoon.W7T2"i)W3]߼7,F__˲//Nv 2Ph7j}J"6_C@ֲxJ6|{T_ <{-ML1V\kQbBk'o8 .UN_IJmuN ?:I:w`Ê|ϊC7i-M$EP$D6^ڞpi 1޼_ϕ3H{o$iv#8b RQ"efUϹOM" (:PǼER,󸐗zeN(ˊa4>AʙL8뻕Yp,J pO@C<cYG*~m)zU~Hm|N]5fUGOfu*% #7&O%J[[έW7WVғӒ)o%e\?d;A^g. qE𯘳M_@x\OZlx3H0³{RE0 r9\+܍,e^WtEKH|xjKTz*hojo/ؼ#[k') 2?)q'*7xRcҺ>p `yS;LDu*|׫"Nu r>u%yel˪k,iTlO 4荽򫨗c7ZLhL{YJ.v]~׍\g>c#R\i~,c@sr+2p3ѓ&eS0xٶ1=g}LX3,O1@0z1\պZ@0 $F~[+(c3/ݠ"b W&W7)VaͫW&{E}DW1";$qa, 6ZPeY\lK7zqDj ʋ9t|[Nr"M8x"Ό#.!},| w;W$[2W&,H<6弰8OĎ4|k qok1JH.mo[UmъkQ ]VA1ջrWO^r{/rBcPfXb]*qPCeM͕!L;(Zbc9Tⴷx-~_Es-Űhc=SnIi8 {7]1f݉Ozw%2^xC#p &W)H|q3i(.L})ŸpĿzC}W,.aVYhAbK`p~S.ë*^ՔLGd5/ƞY4Eđa/g.Q`o,z❉Uc3Ɓ8%9EhIwQ]VS=H$f1 fq-4ķ$IjU `7Fe#,bR9FX{3ջڊ^"Sbm ~=DX/iaP:Q&K΍]K,)?ֳHS[HbHjYe-aVAzN+K׳ Wk͹/4 (crn0:j2_V,ps',-S^ӟy%}G=o굲?5LW20,sĘBT8rlD6Rie˲snж!ĸīIo U[yu%0MMY;R0!` վڷn`k(B9v!khKD$fA3[99o|!38QmuU*1#]Zq:ϕd#&1j N%6y2-IɣS OFYb=buN,ݒJ(.ͪW;zqL<,uJI ($LH^:Ns_H],GSA4I^f^U`5XQ@6#OM;/ӼRi=fI. JI2ڷfin3.\"LV_x%Ҧh6D 1yp*5hUU20Ut=3%oG#U W7*ih;Pɑ~#{,Yioȷx<7wDBV1IE. "[vى)Ef߅"\ <,6!sBp$][q޶S gx5(1`6%!9zɠ7&W`KG˂\4{c˖u,9gY2/S?7Ubxb eLWwFJjH>Ӧ_n=bot_nM҉0^b.B0H.ǽr%U/m^u Dp/p$e&57o}Ջ~znLmG$pĩ#% yEi]|}d2ܩ픀|g>G{b^TcN7_ܚwgS>)V|;!C0 e eOE)7o uXMԈ+grKǑ5:z9m؜%unt2-m(9|\H#&.uk](Akcn$-ˌ3h'@qxK7pJ3&̗Ԉ@U,`U('eo%e Y[k=!D$5ZR"JDxI1S20-Hy5+zۮPm=NrCCɻ"^2 '!}̜z* 72,IJ)E)܁ĵ}PFCJTO/dGGc|M}Qt/2U^AIlҵL*|]v$T/+jyI.(ٹt)..y-mSEwuC?H%KЯI> eۼMXȓ qc 8qߺO8YH$ϩƲ[2@͠&gBLlm^T:iZPYItqW)fWA0SABi}i4l+be=œuuhTJ8ⵗN WmO[Br -=4ZY&%CAe5ȡt@WyHfU`!4Hon 7h<,ڀ(ߎ?e'G{}GUjM}Vp=Y\w3{R[,8/J?E+BvPL˦ o0/=^H`\3UFYAv{GGt %Uk˹ wzݺuc.fKC"$4صe {́8#J@~Sx|| 깸=*)bUzbzaz,y֭Ǐ]*U<1/S"򍔗_CC[G:>"pAP"#b֜HFܥ:TLgbK:LL<1P:\ٹF^++KRqG5 kGV/ 0 -@?NLP!O$EhFdQ&ӭs5DFx,!J.a.|HiB0%P@"?ǁQα q@.9L6DdžWyˊ\pыҾy#5*z BLJAӏ.S!(_ G C^ c(_EF$¢1QAF|Yͼz5FnE\1#\/#‘tŭ^ Nu7f%];bd&T/=L2)MFsa&wwDTnqL4qP(:%`%d״UoDYF4]2Z-iYH~Zm|Kqd 7?~wQ>|('H4^%2r'ݰ+E/)NxId]Dbu9T%R̬ K=M=1T÷:S.US  ^?. |<_ySI$RWJ(!z:_O+[4>-ۯqN {-pwzT|3,'{d[þ?j0k*q:>P W^|A{'i$׷) G|FIҿa@NAkB,eOʤԴL3Bvr!-${v ^ܤ2)_^lp! E|U'~| t>ֺobX[^c%u3Ä"B̚;V`]iDO/=-P~$#*C9ZP`̊X7 c*Ppe#>E:'T(XtT=ΨYz{4Z ѧb.‹ ; L-;]ؑpScs+&N%0ْ./jMa:<׿7Lv%i|`eZN;\U˼GȸJn%;E`¹ʏ[9زy7Y1O tEYMp.m()PˡÛn e+¡0 8 @72ZQX``QN\&6g|_O#]eB}Q |@UB) (RdT3ŗ&Mvu\''9֊2) iMo!!u'SYc^ivKDB}ި86?3C_h~֔DR2;` *uq:S_bV0b%w~bW;8~=]~GzI0CJP+/+gErfdJ"搢+l2q8˨>$bi"[MhW˰0D$)a[$gkX+Z?qLW@%D2$"'āpl \8o(UdԫUgCE\, ٧8yzc3uH-T ݻ C6r"4sP)I)SW@9U r\k'C{B@aWP <]H(8BnQ@'439Xy `03<`ɢ06{`츼{m(;T3=K tcGf=O~ /@dܽIuplbN jZ{e;~ nMGz/"xY4:X2fQ#(0zǫ@9}TG ĶaGAeKqsQ^{hXx(g'Z[d2*qQYK3R$ђ%)^L]sǙ3-&?Ȫ%U~o[GÒ>ئSX^&.C< S!8O$3Y22˝ȪuE\MTHNv,s*W M{1R0VRQf)s-+-1!b$j|U?r^A)Ⱥa~_upح3ypDsfdr'Kdž\tiFEWݒTP7\ixR|'Z `,4E\^o"{P&D l$TK#EN\4Ue ˫JkD܇{.K1i9p(;w1=YyM 1Ku0bڑypu=r%H䮅-^< 2 ŖwE@[Rne2>}ߨذY ǔbGg7n|:a;"mG޾r9KI0O+bLC=TX.3,VhݘT@Y7NqK㔏tBn F9i,JpŠlEI_gv3bגZIHƳLQnν^]4%wUk„1]5(5`.0Z^2A _'ΓEB{lT PB:d0jRn]Z˘t ot㳍1DڿY0r*^5hDHFKOM ny||H^+7c\:$!sk,ЎPVJGZkӎ2>ej *i6ѢXPZjD`H*DwBt$y#V%ikRs.G@Tԙ)\L{#-&1@4J=ж'Nq]܇4n FA/^o4 sq %0}ם<0qQ&vLA&ڑwg 8~v2q#a檁@(TpL })$]5.EUkY~rBۋyԴ>>q6nܸ# xgƯ0ƙR!N.i^D_C G|Ԕa TqMa 9,$DڪюE! #R~&~7>( dtʻ„3ɮ/+ idd'- 3!,tbuc&DD) $ R`IvQ2xUc&wTcˮ);ث:g #z)CEZ<]YG>1P8nP0ktS'`T$uz#Vjd7H&^&1f⼑ 3 ֗>ձjT֍/ #Ku;)$93B]B[:Ayu.sX+eĘ'wQ$wVXW`(󾹡`07SHTO]hu-C8RQDUtN>1ByPq{6NR8*dDdr"a鉐J&6P}^qy1b`) OPFΜ:PТ6D+NFxVIqpTg!Ҏƨ)ꄁj& GVF.USy&SATL2R:|}󆽧zك))"O wZas{՚ҁX<t'W_4ђщWSD&3|*PK`5DoTOvkr?"NuH$}~ٍJۥ@PܤZ#ݪ\ڐ1u;NQk8nBXU/E)WJsPRD1_`4LvYנˇぶ'ރ9!P *^!^PP jV(6 V"?g~+LG?E: LK<9b %Сp`Y޶Ng#l_5D]& VǕ,RY(\wҥΝ|ʙ/l⭗k^Fh߄9ĚnA.5a8n =Qmr:xAAQ<+u!ٳDZW(ts\-^m< A(ada<8".&`4@E"*ve腫`t(ēW㮥p\}%;#|%Ip 4%ﰗ9@\3DQѱa]M7DTњ;0 y"gP҂{9#J}wЀ:f[:cƜe7*nͺ~KiE3G¹xnA/ea\`["ԩخRT\52XbȄ X.9{+3Z3 ߣJf^R6u1 { Ro~ӇpbiEv2Qep!-ʫ7Us qη^ sY1)>NݥO&H$lihkfC=;k$$6xҀI^б]!^![݃h;ޡ"65ՐE%mHdhf tFk#/:o \ϮZY燜fڦPs 92Q PJ洌x3ϭfjt3~zqVE:˝ bOb?OYqתϊzkIGB\5Aú9E  }0rE$cS +9z0VYqݚee+ 0w ޿Ix!Ү#bIg2g2EKP7ʂeM#fѾ)B_!B2+Mr,?𫫑}vtTvt6~t4bpJSDqN3h:G;EuP觿4 ,d) op<hH' el7I] V7 f DTH=&>u|B cpfpC;E+ka Av\ 5^xCx\ ʭv [.stf_: ,3K .bM|#J JVOPjGh9H⥃LI[tS( A%_[7<`<2`+W_C)BsgC~瞛\GP tc uiۨgdFQ3HaǗl } qoL"~;[PtΡb$?Lc4mc ae1 '/\|d…q8l/lO3fݻz {=d7zŐ5&\e3P]Ui'` 8rbZ΄bg ԸX|ec݁~ 1_<:bfE`@uOU`>W()C豪k5C`u>ܕ;0,ao&(!zp"MP@w2&fb4pa!Wt=!sswH8zB}&ɉq3Tǐ$*:6p2,=9EdPgZ> n"]Qe}\upm{eqJP!xK@}X{IT}t2ET{Ao"44yDJqH}o}3;Nl6 QYK~b[;bR"!XAټtԍTIp (µ T(%x⹸*CH#b<$Z/fa.0>ȰK̴!z8xthx4׏sg\Jfj郭K[O#PJC_yp%MhM#NiSاpשe(ձOMLlpkt3vs, fS}OJg\3!(LpQU">ٙXb8Rk'Sb]ZHdP|?yN!غSNPo-x>'%`{r'6kw,>f&/h`ۯ1u"ޜW~-B cQ? 웩2V* A6 [ mS_X; 1\q ߜ 9">ur4bQ`#\Cx3kx@ҋCz 캷2N9f:A\ *X `&ӈ_Cio9ZB(uh: m~Tg|sTD*uQD1E!d2Y'Tk n![y =vV4'Ⱥk{i23"<:8jJ*/~PVL O,Ъ3u<=M1)ufd2#}ƛUEtɱp.2҇IFɡ^8|3IO͐C~ k U 3$:@\G)ޤS}ŷ)RUw%qK$vߘ(AJgR'~Eć %D?+f7$M al1+ ע +;[R is98ނK tu- \;C_nSJ'xB A^@)GKz(D%wgO*:!pE| @+]]8o6ghp?6-f<%Ɨĉ GgF ϷWEqVkDL$&w>_ *+[׬j/صa7E$ ċ!Vo| ]ayH22Ɗ=wٝ66F"ك`,;W$Ropk%<~haOJMH#MOˊmUV^'~Wo7>yq\.yt]RRR MD),; PQj10p!6v[՜TAXƱ~?Z\=s䙓's%~S%U?jn)U{cu(W5g*d-įV.6%mN+>/Ek%ba/![+#@@ piImBys;LDC?s.}W}ohί7A-4SZ'f?v{!qjP )bIMG}2pH-%%];YiͧZoՊV\@p[`&jo0ݒwb*i?MH >2\_c@ĮBT" jN~W~jcN@"lO6#Pvڰ{zz˾}[/.:}UKZW1~gClnoL) AFi\t11I">a&*TQ@g)2PXYٹZ%Vs5jt`AL]-+,؆R``ǼÕj]/aU2K3i̖ 9y\g~ l ZERE LU:vvq Vqpvyq9ߪ. οG!T c>AJ/\ @ E@!*a!2 ;hA.R7ćv:2`0ѩARF(^rC&4~w\"@Yς,UQwcKcײG{W'Ա-gľy鱥s/_C,3zR֤lI}6%E@x7T?M .^\Qҥ=zM+&,+ H\\rp?9ʊ=}bB 2Y*U<7QGM Mn:|oBL?@q-q .[UHu$ ,ե-Q27GxgmڳiŁk=f_?z1W e%T0b$AHk# poTZ[>PݑZ'QAV,w LNU?ZII;w \K7s+;@a }|'Pn J]._AԐyD;̳N`tYʤaĚ@9Yqp1є@΀F!"AnR+ p\ &q(9:̷n$ٔ .oFYwUz81R~Thsw¬AvJ>!.]0ppA I 8UI`i@y.-3dTDe%DI2!]8@-Lw`;kIԣ4L'3̢xZ b_8 sAK?0 n1S҇^pS![ FCbqDh vIB0Iς`+XQc!CȺM= n}+5 {|ȘOVLNBt@7.SR@"D tvXaZIی8Fad>z(V"]6L 51f'[[n셰3vͮ 59<>~tK`%i81 i, /s,N4>Cy_E0)I.ylSuSصgwCiCixi6N^a5?ڔN.V zgqhrp|ΗG;ym8a Tgh(k HA0W:Azoa[Kf0C(s/Dh7^ʥ84+Ø)wP[Bt@GlN5*4 -p[(pV?ulOKacWd@M[>\> \0߂M1g. Q nX0E+~86s LK.^0%A&60>2ujqkP{knM6b88P|g"BI7gA*Ƣ̅|'_ KV΃MQY&APq>H\.=v-+A:~ܖR)gm&DS᝭@qoʑW@Ͳ#CfNZ+5!-za/ )fi~;~ )unYhT)-kDP*jc1JvzR0N0S4(A,H,އ#.}3Pߑ4{aY@ 87aV qX[-#nÙ19}U DukT nԓ1,$>~.iܻ5)^4lBG"!9%:'B4j%t)Eק2xCᎏZwz()C $1 :bPy2q-,8B,62CQYiGȷx#{ aMr6 JĶ*qJ ݈s6Ab|c{Np$F\ƆY6691btA@>*n9Zv+ ZJFvE?@]Rqj}-B4ju:_z065(ߖ@ <$Fw ( }u4dQ>]Ppp(sd26_~T(pEA>+ #_lC_a2وUE#;AN}gPvc)(D-Q}SP@ (V p}; P?#*aitNd۩!Hg3(Զvꤝ7C D-9DͥY=YFM5=ņz*IH;EP͘"x (ILXH)[-ǞINGsq6J3Tu]<@uMqRV"ze aƆ<) saM|g9`YP54 uԐ@uu;p&;$AU45y0?B@@?2 Tԁ3E)^c LA 4& ζ% 1ĉ'< Om Bݲ%Obxo}C l}4 6Xr;wu,(^%4NqpJ_&Ta1WL܆>N=h&(ug(N K!w)Qp`_. .. seealso:: :ref:`Differences between PyMongo's and Motor's GridFS APIs `. .. class:: AsyncIOMotorGridFSBucket Create a new instance of :class:`AsyncIOMotorGridFSBucket`. Raises :exc:`TypeError` if `database` is not an instance of :class:`AsyncIOMotorDatabase`. Raises :exc:`~pymongo.errors.ConfigurationError` if `write_concern` is not acknowledged. :Parameters: - `database`: database to use. - `bucket_name` (optional): The name of the bucket. Defaults to 'fs'. - `chunk_size_bytes` (optional): The chunk size in bytes. Defaults to 255KB. - `write_concern` (optional): The :class:`~pymongo.write_concern.WriteConcern` to use. If ``None`` (the default) db.write_concern is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) db.read_preference is used. .. mongodoc:: gridfs .. coroutinemethod:: delete(self, file_id) Delete a file's metadata and data chunks from a GridFS bucket:: async def delete(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) # Get _id of file to delete file_id = await fs.upload_from_stream("test_file", b"data I want to store!") await fs.delete(file_id) Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be deleted. .. coroutinemethod:: download_to_stream(self, file_id, destination) Downloads the contents of the stored file specified by file_id and writes the contents to `destination`:: async def download(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) # Get _id of file to read file_id = await fs.upload_from_stream("test_file", b"data I want to store!") # Get file to write to file = open('myfile','wb+') await fs.download_to_stream(file_id, file) file.seek(0) contents = file.read() Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be downloaded. - `destination`: a file-like object implementing :meth:`write`. .. coroutinemethod:: download_to_stream_by_name(self, filename, destination, revision=-1) Write the contents of `filename` (with optional `revision`) to `destination`. For example:: async def download_by_name(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) # Get file to write to file = open('myfile','wb') await fs.download_to_stream_by_name("test_file", file) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to read from. - `destination`: A file-like object that implements :meth:`write`. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). :Note: Revision numbers are defined as follows: - 0 = the original stored file - 1 = the first revision - 2 = the second revision - etc... - -2 = the second most recent revision - -1 = the most recent revision .. method:: find(self, *args, **kwargs) Find and return the files collection documents that match ``filter``. Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: async def find(): cursor = fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True) async for grid_data in cursor: data = grid_data.read() iterates through all versions of "lisa.txt" stored in GridFS. Setting no_cursor_timeout may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) returns a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~AsyncIOMotorCollection.find` in :class:`AsyncIOMotorCollection`. :Parameters: - `filter`: Search query. - `batch_size` (optional): The number of documents to return per batch. - `limit` (optional): The maximum number of documents to return. - `no_cursor_timeout` (optional): The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to True prevent that. - `skip` (optional): The number of documents to skip before returning. - `sort` (optional): The order by which to sort results. Defaults to None. .. coroutinemethod:: open_download_stream(self, file_id) Opens a stream to read the contents of the stored file specified by file_id:: async def download_stream(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) # get _id of file to read. file_id = await fs.upload_from_stream("test_file", b"data I want to store!") grid_out = await fs.open_download_stream(file_id) contents = await grid_out.read() Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be downloaded. Returns a :class:`AsyncIOMotorGridOut`. .. coroutinemethod:: open_download_stream_by_name(self, filename, revision=-1) Opens a stream to read the contents of `filename` and optional `revision`:: async def download_by_name(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) # get _id of file to read. file_id = await fs.upload_from_stream("test_file", b"data I want to store!") grid_out = await fs.open_download_stream_by_name(file_id) contents = await grid_out.read() Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` filename is not a string. :Parameters: - `filename`: The name of the file to read from. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). Returns a :class:`AsyncIOMotorGridOut`. :Note: Revision numbers are defined as follows: - 0 = the original stored file - 1 = the first revision - 2 = the second revision - etc... - -2 = the second most recent revision - -1 = the most recent revision .. method:: open_upload_stream(self, filename, chunk_size_bytes=None, metadata=None) Opens a stream for writing. Specify the filename, and add any additional information in the metadata field of the file document or modify the chunk size:: async def upload(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream( "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) await grid_in.write(b"data I want to store!") await grid_in.close() # uploaded on close Returns an instance of :class:`AsyncIOMotorGridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. In a Python 3.5 native coroutine, the "async with" statement calls :meth:`~AsyncIOMotorGridIn.close` automatically:: async def upload(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) async with await fs.new_file() as gridin: await gridin.write(b'First part\n') await gridin.write(b'Second part') # gridin is now closed automatically. :Parameters: - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`AsyncIOMotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. .. method:: open_upload_stream_with_id(self, file_id, filename, chunk_size_bytes=None, metadata=None) Opens a stream for writing. Specify the filed_id and filename, and add any additional information in the metadata field of the file document, or modify the chunk size:: async def upload(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream_with_id( ObjectId(), "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) await grid_in.write(b"data I want to store!") await grid_in.close() # uploaded on close Returns an instance of :class:`AsyncIOMotorGridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `file_id`: The id to use for this file. The id must not have already been used for another file. - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`AsyncIOMotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. .. coroutinemethod:: rename(self, file_id, new_filename) Renames the stored file with the specified file_id. For example:: async def rename(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) # get _id of file to read. file_id = await fs.upload_from_stream("test_file", b"data I want to store!") await fs.rename(file_id, "new_test_name") Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be renamed. - `new_filename`: The new name of the file. .. coroutinemethod:: upload_from_stream(self, filename, source, chunk_size_bytes=None, metadata=None) Uploads a user file to a GridFS bucket. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: async def upload_from_stream(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) file_id = await fs.upload_from_stream( "test_file", b"data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`AsyncIOMotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. Returns the _id of the uploaded file. .. coroutinemethod:: upload_from_stream_with_id(self, file_id, filename, source, chunk_size_bytes=None, metadata=None) Uploads a user file to a GridFS bucket with a custom file id. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: async def upload_from_stream_with_id(): my_db = AsyncIOMotorClient().test fs = AsyncIOMotorGridFSBucket(my_db) file_id = await fs.upload_from_stream_with_id( ObjectId(), "test_file", b"data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `file_id`: The id to use for this file. The id must not have already been used for another file. - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`AsyncIOMotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. .. autoclass:: AsyncIOMotorGridFS :members: :exclude-members: find_one, put .. coroutinemethod:: find_one(self, filter=None, *args, **kwargs) Get a single file from gridfs. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single :class:`AsyncIOMotorGridOut`, or ``None`` if no matching file is found. For example:: file = await fs.find_one({"filename": "lisa.txt"}) :Parameters: - `filter` (optional): a dictionary specifying the query to be performing OR any other type to be used as the value for a query for ``"_id"`` in the file collection. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. .. coroutinemethod:: put(self, data, **kwargs) Put data in GridFS as a new file. Equivalent to doing:: try: f = await fs.new_file(**kwargs) await f.write(data) finally: await f.close() `data` can be a :class:`bytes` instance or a file-like object providing a :meth:`read` method. If an `encoding` keyword argument is passed, `data` can also be a :class:`str`, which will be encoded as `encoding` before being written. Any keyword arguments will be passed through to the created file - see :class:`AsyncIOMotorGridIn` for possible arguments. Returns the ``"_id"`` of the created file. If the ``"_id"`` of the file is manually specified, it must not already exist in GridFS. Otherwise :class:`~gridfs.errors.FileExists` is raised. :Parameters: - `data`: data to be written as a file. - `**kwargs` (optional): keyword arguments for file creation .. autoclass:: AsyncIOMotorGridIn :members: .. autoclass:: AsyncIOMotorGridOut :members: .. autoclass:: AsyncIOMotorGridOutCursor :members: motor-1.2.1/doc/api-asyncio/asyncio_motor_change_stream.rst000066400000000000000000000003131323007662700241360ustar00rootroot00000000000000:class:`~motor.motor_asyncio.AsyncIOMotorChangeStream` ====================================================== .. currentmodule:: motor.motor_asyncio .. autoclass:: AsyncIOMotorChangeStream :members: motor-1.2.1/doc/api-asyncio/asyncio_motor_client.rst000066400000000000000000000007001323007662700226140ustar00rootroot00000000000000:class:`~motor.motor_asyncio.AsyncIOMotorClient` -- Connection to MongoDB ========================================================================= .. autoclass:: motor.motor_asyncio.AsyncIOMotorClient :members: .. describe:: client[db_name] || client.db_name Get the `db_name` :class:`AsyncIOMotorDatabase` on :class:`AsyncIOMotorClient` `client`. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. motor-1.2.1/doc/api-asyncio/asyncio_motor_collection.rst000066400000000000000000000121531323007662700234760ustar00rootroot00000000000000:class:`~motor.motor_asyncio.AsyncIOMotorCollection` ==================================================== .. currentmodule:: motor.motor_asyncio .. autoclass:: AsyncIOMotorCollection :members: :exclude-members: create_index, inline_map_reduce .. describe:: c[name] || c.name Get the `name` sub-collection of :class:`AsyncIOMotorCollection` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. .. attribute:: database The :class:`AsyncIOMotorDatabase` that this :class:`AsyncIOMotorCollection` is a part of. .. coroutinemethod:: create_index(self, keys, **kwargs) Creates an index on this collection. Takes either a single key or a list of (key, direction) pairs. The key(s) must be an instance of :class:`basestring` (:class:`str` in python 3), and the direction(s) must be one of (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, :data:`~pymongo.TEXT`). To create a single key ascending index on the key ``'mike'`` we just use a string argument:: await my_collection.create_index("mike") For a compound index on ``'mike'`` descending and ``'eliot'`` ascending we need to use a list of tuples:: await my_collection.create_index([("mike", pymongo.DESCENDING), ("eliot", pymongo.ASCENDING)]) All optional index creation parameters should be passed as keyword arguments to this method. For example:: await my_collection.create_index([("mike", pymongo.DESCENDING)], background=True) Valid options include, but are not limited to: - `name`: custom name to use for this index - if none is given, a name will be generated. - `unique`: if ``True`` creates a uniqueness constraint on the index. - `background`: if ``True`` this index should be created in the background. - `sparse`: if ``True``, omit from the index any documents that lack the indexed field. - `bucketSize`: for use with geoHaystack indexes. Number of documents to group together within a certain proximity to a given longitude and latitude. - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` index. - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` index. - `expireAfterSeconds`: Used to create an expiring (TTL) collection. MongoDB will automatically delete documents from this collection after seconds. The indexed field must be a UTC datetime or the data will not expire. - `partialFilterExpression`: A document that specifies a filter for a partial index. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. See the MongoDB documentation for a full list of supported options by server version. .. warning:: `dropDups` is not supported by MongoDB 3.0 or newer. The option is silently ignored by the server and unique index builds using the option will fail if a duplicate value is detected. .. note:: `partialFilterExpression` requires server version **>= 3.2** .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. :Parameters: - `keys`: a single key or a list of (key, direction) pairs specifying the index to create - `**kwargs` (optional): any additional index creation options (see the above list) should be passed as keyword arguments .. mongodoc:: indexes .. coroutinemethod:: inline_map_reduce(self, map, reduce, full_response=False, **kwargs) Perform an inline map/reduce operation on this collection. Perform the map/reduce operation on the server in RAM. A result collection is not created. The result set is returned as a list of documents. If `full_response` is ``False`` (default) returns the result documents in a list. Otherwise, returns the full response from the server to the `map reduce command`_. The :meth:`inline_map_reduce` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: await db.test.inline_map_reduce(map, reduce, limit=2) .. _map reduce command: http://docs.mongodb.org/manual/reference/command/mapReduce/ .. mongodoc:: mapreduce motor-1.2.1/doc/api-asyncio/asyncio_motor_cursor.rst000066400000000000000000000002711323007662700226560ustar00rootroot00000000000000:class:`~motor.motor_asyncio.AsyncIOMotorCursor` ================================================ .. currentmodule:: motor.motor_asyncio .. autoclass:: AsyncIOMotorCursor :members: motor-1.2.1/doc/api-asyncio/asyncio_motor_database.rst000066400000000000000000000006771323007662700231170ustar00rootroot00000000000000:class:`~motor.motor_asyncio.AsyncIOMotorDatabase` ================================================== .. currentmodule:: motor.motor_asyncio .. autoclass:: AsyncIOMotorDatabase :members: .. describe:: db[collection_name] || db.collection_name Get the `collection_name` :class:`AsyncIOMotorCollection` of :class:`AsyncIOMotorDatabase` `db`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. motor-1.2.1/doc/api-asyncio/index.rst000066400000000000000000000005571323007662700175120ustar00rootroot00000000000000Motor asyncio API ================= .. toctree:: asyncio_motor_client asyncio_motor_database asyncio_motor_collection asyncio_motor_cursor asyncio_motor_change_stream asyncio_gridfs aiohttp .. seealso:: :doc:`../tutorial-asyncio` This page describes using Motor with asyncio. For Tornado integration, see :doc:`../api-tornado/index`. motor-1.2.1/doc/api-tornado/000077500000000000000000000000001323007662700156435ustar00rootroot00000000000000motor-1.2.1/doc/api-tornado/gridfs.rst000066400000000000000000000450111323007662700176540ustar00rootroot00000000000000Motor GridFS Classes ==================== .. currentmodule:: motor.motor_tornado Store blobs of data in `GridFS `_. .. seealso:: :ref:`Differences between PyMongo's and Motor's GridFS APIs `. .. seealso:: :doc:`web` .. class:: MotorGridFSBucket Create a new instance of :class:`MotorGridFSBucket`. Raises :exc:`TypeError` if `database` is not an instance of :class:`MotorDatabase`. Raises :exc:`~pymongo.errors.ConfigurationError` if `write_concern` is not acknowledged. :Parameters: - `database`: database to use. - `bucket_name` (optional): The name of the bucket. Defaults to 'fs'. - `chunk_size_bytes` (optional): The chunk size in bytes. Defaults to 255KB. - `write_concern` (optional): The :class:`~pymongo.write_concern.WriteConcern` to use. If ``None`` (the default) db.write_concern is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) db.read_preference is used. .. mongodoc:: gridfs .. coroutinemethod:: delete(self, file_id, callback=None) Delete a file's metadata and data chunks from a GridFS bucket:: @gen.coroutine def delete(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) # Get _id of file to delete file_id = yield fs.upload_from_stream("test_file", b"data I want to store!") yield fs.delete(file_id) Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be deleted. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future. .. coroutinemethod:: download_to_stream(self, file_id, destination, callback=None) Downloads the contents of the stored file specified by file_id and writes the contents to `destination`:: @gen.coroutine def download(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) # Get _id of file to read file_id = yield fs.upload_from_stream("test_file", b"data I want to store!") # Get file to write to file = open('myfile','wb+') yield fs.download_to_stream(file_id, file) file.seek(0) contents = file.read() Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be downloaded. - `destination`: a file-like object implementing :meth:`write`. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future. .. coroutinemethod:: download_to_stream_by_name(self, filename, destination, revision=-1, callback=None) Write the contents of `filename` (with optional `revision`) to `destination`. For example:: @gen.coroutine def download_by_name(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) # Get file to write to file = open('myfile','wb') yield fs.download_to_stream_by_name("test_file", file) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to read from. - `destination`: A file-like object that implements :meth:`write`. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). :Note: Revision numbers are defined as follows: - 0 = the original stored file - 1 = the first revision - 2 = the second revision - etc... - -2 = the second most recent revision - -1 = the most recent revision .. method:: find(self, *args, **kwargs) Find and return the files collection documents that match ``filter``. Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: @gen.coroutine def find(): cursor = fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True) while (yield cursor.fetch_next): grid_data = cursor.next_object() data = grid_data.read() iterates through all versions of "lisa.txt" stored in GridFS. Setting no_cursor_timeout may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) returns a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~MotorCollection.find` in :class:`MotorCollection`. :Parameters: - `filter`: Search query. - `batch_size` (optional): The number of documents to return per batch. - `limit` (optional): The maximum number of documents to return. - `no_cursor_timeout` (optional): The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to True prevent that. - `skip` (optional): The number of documents to skip before returning. - `sort` (optional): The order by which to sort results. Defaults to None. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future. .. coroutinemethod:: open_download_stream(self, file_id, callback=None) Opens a stream to read the contents of the stored file specified by file_id:: @gen.coroutine def download_stream(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) # get _id of file to read. file_id = yield fs.upload_from_stream("test_file", b"data I want to store!") grid_out = yield fs.open_download_stream(file_id) contents = yield grid_out.read() Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be downloaded. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future that resolves to a :class:`MotorGridOut`. .. coroutinemethod:: open_download_stream_by_name(self, filename, revision=-1, callback=None) Opens a stream to read the contents of `filename` and optional `revision`:: @gen.coroutine def download_by_name(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) # get _id of file to read. file_id = yield fs.upload_from_stream("test_file", b"data I want to store!") grid_out = yield fs.open_download_stream_by_name(file_id) contents = yield grid_out.read() Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` filename is not a string. :Parameters: - `filename`: The name of the file to read from. - `revision` (optional): Which revision (documents with the same filename and different uploadDate) of the file to retrieve. Defaults to -1 (the most recent revision). - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future that resolves to a :class:`MotorGridOut`. :Note: Revision numbers are defined as follows: - 0 = the original stored file - 1 = the first revision - 2 = the second revision - etc... - -2 = the second most recent revision - -1 = the most recent revision .. method:: open_upload_stream(self, filename, chunk_size_bytes=None, metadata=None) Opens a stream for writing. Specify the filename, and add any additional information in the metadata field of the file document or modify the chunk size:: @gen.coroutine def upload(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream( "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) yield grid_in.write(b"data I want to store!") yield grid_in.close() # uploaded on close Returns an instance of :class:`MotorGridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. In a Python 3.5 native coroutine, the "async with" statement calls :meth:`~MotorGridIn.close` automatically:: async def upload(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) async with await fs.new_file() as gridin: await gridin.write(b'First part\n') await gridin.write(b'Second part') # gridin is now closed automatically. :Parameters: - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`MotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. .. method:: open_upload_stream_with_id(self, file_id, filename, chunk_size_bytes=None, metadata=None) Opens a stream for writing. Specify the filed_id and filename, and add any additional information in the metadata field of the file document, or modify the chunk size:: @gen.coroutine def upload(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) grid_in, file_id = fs.open_upload_stream_with_id( ObjectId(), "test_file", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) yield grid_in.write(b"data I want to store!") yield grid_in.close() # uploaded on close Returns an instance of :class:`MotorGridIn`. Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `file_id`: The id to use for this file. The id must not have already been used for another file. - `filename`: The name of the file to upload. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes in :class:`MotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. .. coroutinemethod:: rename(self, file_id, new_filename, callback=None) Renames the stored file with the specified file_id. For example:: @gen.coroutine def rename(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) # get _id of file to read. file_id = yield fs.upload_from_stream("test_file", b"data I want to store!") yield fs.rename(file_id, "new_test_name") Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. :Parameters: - `file_id`: The _id of the file to be renamed. - `new_filename`: The new name of the file. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future. .. coroutinemethod:: upload_from_stream(self, filename, source, chunk_size_bytes=None, metadata=None, callback=None) Uploads a user file to a GridFS bucket. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: @gen.coroutine def upload_from_stream(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) file_id = yield fs.upload_from_stream( "test_file", b"data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`MotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future that resolves to the _id of the uploaded file. .. coroutinemethod:: upload_from_stream_with_id(self, file_id, filename, source, chunk_size_bytes=None, metadata=None, callback=None) Uploads a user file to a GridFS bucket with a custom file id. Reads the contents of the user file from `source` and uploads it to the file `filename`. Source can be a string or file-like object. For example:: @gen.coroutine def upload_from_stream_with_id(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) file_id = yield fs.upload_from_stream_with_id( ObjectId(), "test_file", b"data I want to store!", chunk_size_bytes=4, metadata={"contentType": "text/plain"}) Raises :exc:`~gridfs.errors.NoFile` if no such version of that file exists. Raises :exc:`~ValueError` if `filename` is not a string. :Parameters: - `file_id`: The id to use for this file. The id must not have already been used for another file. - `filename`: The name of the file to upload. - `source`: The source stream of the content to be uploaded. Must be a file-like object that implements :meth:`read` or a string. - `chunk_size_bytes` (options): The number of bytes per chunk of this file. Defaults to the chunk_size_bytes of :class:`MotorGridFSBucket`. - `metadata` (optional): User data for the 'metadata' field of the files collection document. If not provided the metadata field will be omitted from the files collection document. - `callback`: (optional): function taking (result, error), executed when operation completes If a callback is passed, returns None, else returns a Future. .. autoclass:: MotorGridFS :members: :exclude-members: find_one, put .. coroutinemethod:: find_one(self, filter=None, *args, callback=None, **kwargs) Get a single file from gridfs. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single :class:`MotorGridOut`, or ``None`` if no matching file is found. For example:: file = yield fs.find_one({"filename": "lisa.txt"}) :Parameters: - `filter` (optional): a dictionary specifying the query to be performing OR any other type to be used as the value for a query for ``"_id"`` in the file collection. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `callback`: (optional): function taking (result, error), executed when operation completes - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. If a callback is passed, returns None, else returns a Future. .. coroutinemethod:: put(self, data, callback=None, **kwargs) Put data in GridFS as a new file. Equivalent to doing:: try: f = yield fs.new_file(**kwargs) yield f.write(data) finally: yield f.close() `data` can be a :class:`bytes` instance or a file-like object providing a :meth:`read` method. If an `encoding` keyword argument is passed, `data` can also be a :class:`unicode` (:class:`str` in python 3) instance, which will be encoded as `encoding` before being written. Any keyword arguments will be passed through to the created file - see :class:`MotorGridIn` for possible arguments. Returns the ``"_id"`` of the created file. If the ``"_id"`` of the file is manually specified, it must not already exist in GridFS. Otherwise :class:`~gridfs.errors.FileExists` is raised. :Parameters: - `data`: data to be written as a file. - `callback`: (optional): function taking (result, error), executed when operation completes - `**kwargs` (optional): keyword arguments for file creation If a callback is passed, returns None, else returns a Future. .. autoclass:: MotorGridIn :members: .. autoclass:: MotorGridOut :members: .. autoclass:: MotorGridOutCursor :members: motor-1.2.1/doc/api-tornado/index.rst000066400000000000000000000004721323007662700175070ustar00rootroot00000000000000Motor Tornado API ================= .. toctree:: motor_client motor_database motor_collection motor_cursor motor_change_stream gridfs web .. seealso:: :doc:`../tutorial-tornado` This page describes using Motor with Tornado. For asyncio integration, see :doc:`../api-asyncio/index`. motor-1.2.1/doc/api-tornado/motor_change_stream.rst000066400000000000000000000002661323007662700224210ustar00rootroot00000000000000:class:`~motor.motor_tornado.MotorChangeStream` =============================================== .. currentmodule:: motor.motor_tornado .. autoclass:: MotorChangeStream :members: motor-1.2.1/doc/api-tornado/motor_client.rst000066400000000000000000000006611323007662700210760ustar00rootroot00000000000000:class:`~motor.motor_tornado.MotorClient` -- Connection to MongoDB ================================================================== .. currentmodule:: motor.motor_tornado .. autoclass:: MotorClient :members: .. describe:: client[db_name] || client.db_name Get the `db_name` :class:`MotorDatabase` on :class:`MotorClient` `client`. Raises :class:`~pymongo.errors.InvalidName` if an invalid database name is used. motor-1.2.1/doc/api-tornado/motor_collection.rst000066400000000000000000000127071323007662700217570ustar00rootroot00000000000000:class:`~motor.motor_tornado.MotorCollection` ============================================= .. currentmodule:: motor.motor_tornado .. autoclass:: MotorCollection :members: :exclude-members: create_index, inline_map_reduce .. describe:: c[name] || c.name Get the `name` sub-collection of :class:`MotorCollection` `c`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. .. attribute:: database The :class:`MotorDatabase` that this :class:`MotorCollection` is a part of. .. coroutinemethod:: create_index(self, keys, callback=None, **kwargs) Creates an index on this collection. Takes either a single key or a list of (key, direction) pairs. The key(s) must be an instance of :class:`basestring` (:class:`str` in python 3), and the direction(s) must be one of (:data:`~pymongo.ASCENDING`, :data:`~pymongo.DESCENDING`, :data:`~pymongo.GEO2D`, :data:`~pymongo.GEOHAYSTACK`, :data:`~pymongo.GEOSPHERE`, :data:`~pymongo.HASHED`, :data:`~pymongo.TEXT`). To create a single key ascending index on the key ``'mike'`` we just use a string argument:: yield my_collection.create_index("mike") For a compound index on ``'mike'`` descending and ``'eliot'`` ascending we need to use a list of tuples:: yield my_collection.create_index([("mike", pymongo.DESCENDING), ("eliot", pymongo.ASCENDING)]) All optional index creation parameters should be passed as keyword arguments to this method. For example:: yield my_collection.create_index([("mike", pymongo.DESCENDING)], background=True) Valid options include, but are not limited to: - `name`: custom name to use for this index - if none is given, a name will be generated. - `unique`: if ``True`` creates a uniqueness constraint on the index. - `background`: if ``True`` this index should be created in the background. - `sparse`: if ``True``, omit from the index any documents that lack the indexed field. - `bucketSize`: for use with geoHaystack indexes. Number of documents to group together within a certain proximity to a given longitude and latitude. - `min`: minimum value for keys in a :data:`~pymongo.GEO2D` index. - `max`: maximum value for keys in a :data:`~pymongo.GEO2D` index. - `expireAfterSeconds`: Used to create an expiring (TTL) collection. MongoDB will automatically delete documents from this collection after seconds. The indexed field must be a UTC datetime or the data will not expire. - `partialFilterExpression`: A document that specifies a filter for a partial index. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. See the MongoDB documentation for a full list of supported options by server version. .. warning:: `dropDups` is not supported by MongoDB 3.0 or newer. The option is silently ignored by the server and unique index builds using the option will fail if a duplicate value is detected. .. note:: `partialFilterExpression` requires server version **>= 3.2** .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. :Parameters: - `keys`: a single key or a list of (key, direction) pairs specifying the index to create - `callback`: (optional): function taking (result, error), executed when operation completes - `**kwargs` (optional): any additional index creation options (see the above list) should be passed as keyword arguments If a callback is passed, returns None, else returns a Future. .. mongodoc:: indexes .. coroutinemethod:: inline_map_reduce(self, map, reduce, full_response=False, callback=None, **kwargs) Perform an inline map/reduce operation on this collection. Perform the map/reduce operation on the server in RAM. A result collection is not created. The result set is returned as a list of documents. If `full_response` is ``False`` (default) returns the result documents in a list. Otherwise, returns the full response from the server to the `map reduce command`_. The :meth:`inline_map_reduce` method obeys the :attr:`read_preference` of this :class:`Collection`. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `callback`: (optional): function taking (result, error), executed when operation completes - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: yield db.test.inline_map_reduce(map, reduce, limit=2) If a callback is passed, returns None, else returns a Future. .. _map reduce command: http://docs.mongodb.org/manual/reference/command/mapReduce/ .. mongodoc:: mapreduce motor-1.2.1/doc/api-tornado/motor_cursor.rst000066400000000000000000000002441323007662700211320ustar00rootroot00000000000000:class:`~motor.motor_tornado.MotorCursor` ========================================= .. currentmodule:: motor.motor_tornado .. autoclass:: MotorCursor :members: motor-1.2.1/doc/api-tornado/motor_database.rst000066400000000000000000000006341323007662700213640ustar00rootroot00000000000000:class:`~motor.motor_tornado.MotorDatabase` =========================================== .. currentmodule:: motor.motor_tornado .. autoclass:: MotorDatabase :members: .. describe:: db[collection_name] || db.collection_name Get the `collection_name` :class:`MotorCollection` of :class:`MotorDatabase` `db`. Raises :class:`~pymongo.errors.InvalidName` if an invalid collection name is used. motor-1.2.1/doc/api-tornado/web.rst000066400000000000000000000003441323007662700171530ustar00rootroot00000000000000:mod:`motor.web` - Integrate Motor with the Tornado web framework ================================================================= .. currentmodule:: motor.web .. automodule:: motor.web :members: :no-inherited-members: motor-1.2.1/doc/changelog.rst000066400000000000000000001027721323007662700161200ustar00rootroot00000000000000Changelog ========= .. currentmodule:: motor.motor_tornado Motor 1.2.1 ----------- An asyncio application that created a Change Stream with :meth:`MotorCollection.watch` and shut down while the Change Stream was open would print several errors. I have rewritten :meth:`MotorChangeStream.next` and some Motor internals to allow clean shutdown with asyncio. Motor 1.2 --------- Motor 1.2 drops support for MongoDB 2.4 and adds support for MongoDB 3.6 features. It depends on PyMongo 3.6 or later. Motor continues to support MongoDB 2.6 and later. Dropped support for Python 2.6 and 3.3. Motor continues to support Python 2.7, and 3.4+. Dropped support for Tornado 3. A recent version of Tornado 4 is required. Dropped support for the `Python 3.5.0 and Python 3.5.1 "async for" protocol `_. Motor allows "async for" with cursors in Python 3.5.2 and later. See the :ref:`Compatibility Matrix ` for the relationships among Motor, Python, Tornado, and MongoDB versions. Added support for `aiohttp`_ 2.0 and later, and dropped older aiohttp versions. Highlights include: - New method :meth:`MotorCollection.watch` to acquire a Change Stream on a collection. - New Session API to support causal consistency, see :meth:`MotorClient.start_session`. - Support for array_filters in :meth:`~MotorCollection.update_one`, :meth:`~MotorCollection.update_many`, :meth:`~MotorCollection.find_one_and_update`, :meth:`~MotorCollection.bulk_write`. - :meth:`MotorClient.list_databases` and :meth:`MotorClient.list_database_names`. - Support for mongodb+srv:// URIs. See :class:`~pymongo.mongo_client.MongoClient` for details. - Support for retryable writes and the ``retryWrites`` URI option. See :class:`~pymongo.mongo_client.MongoClient` for details. The maximum number of workers in the thread pool can be overridden with an environment variable, see :doc:`configuration`. :class:`MotorCollection` accepts codec_options, read_preference, write_concern, and read_concern arguments. This is rarely needed; you typically create a :class:`MotorCollection` from a :class:`MotorDatabase`, not by calling its constructor directly. Deleted obsolete class ``motor.Op``. Motor 1.1 --------- Motor depends on PyMongo 3.4 or later. It wraps the latest PyMongo code which support the new server features introduced in MongoDB 3.4. (It is a coincidence that the latest MongoDB and PyMongo versions are the same number.) Highlights include: - Complete support for MongoDB 3.4: - Unicode aware string comparison using collations. See :ref:`PyMongo's examples for collation `. - :class:`MotorCursor` and :class:`MotorGridOutCursor` have a new attribute :meth:`~MotorCursor.collation`. - Support for the new :class:`~bson.decimal128.Decimal128` BSON type. - A new maxStalenessSeconds read preference option. - A username is no longer required for the MONGODB-X509 authentication mechanism when connected to MongoDB >= 3.4. - :meth:`~MotorCollection.parallel_scan` supports maxTimeMS. - :class:`~pymongo.write_concern.WriteConcern` is automatically applied by all helpers for commands that write to the database when connected to MongoDB 3.4+. This change affects the following helpers: - :meth:`MotorClient.drop_database` - :meth:`MotorDatabase.create_collection` - :meth:`MotorDatabase.drop_collection` - :meth:`MotorCollection.aggregate` (when using $out) - :meth:`MotorCollection.create_indexes` - :meth:`MotorCollection.create_index` - :meth:`MotorCollection.drop_indexes` - :meth:`MotorCollection.drop_indexes` - :meth:`MotorCollection.drop_index` - :meth:`MotorCollection.map_reduce` (when output is not "inline") - :meth:`MotorCollection.reindex` - :meth:`MotorCollection.rename` - Improved support for logging server discovery and monitoring events. See :mod:`PyMongo's monitoring documentation ` for examples. - Support for matching iPAddress subjectAltName values for TLS certificate verification. - TLS compression is now explicitly disabled when possible. - The Server Name Indication (SNI) TLS extension is used when possible. - PyMongo's `bson` module provides finer control over JSON encoding/decoding with :class:`~bson.json_util.JSONOptions`. - Allow :class:`~bson.code.Code` objects to have a scope of ``None``, signifying no scope. Also allow encoding Code objects with an empty scope (i.e. ``{}``). .. warning:: Starting in PyMongo 3.4, :attr:`bson.code.Code.scope` may return ``None``, as the default scope is ``None`` instead of ``{}``. .. note:: PyMongo 3.4+ attempts to create sockets non-inheritable when possible (i.e. it sets the close-on-exec flag on socket file descriptors). Support is limited to a subset of POSIX operating systems (not including Windows) and the flag usually cannot be set in a single atomic operation. CPython 3.4+ implements `PEP 446`_, creating all file descriptors non-inheritable by default. Users that require this behavior are encouraged to upgrade to CPython 3.4+. .. _PEP 446: https://www.python.org/dev/peps/pep-0446/ Motor 1.0 --------- Motor now depends on PyMongo 3.3 and later. The move from PyMongo 2 to 3 brings a large number of API changes, read :doc:`migrate-to-motor-1` and `the PyMongo 3 changelog`_ carefully. .. _the PyMongo 3 changelog: http://api.mongodb.com/python/current/changelog.html#changes-in-version-3-0 :class:`MotorReplicaSetClient` is removed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In Motor 1.0, :class:`MotorClient` is the only class. Connect to a replica set with a "replicaSet" URI option or parameter:: MotorClient("mongodb://hostname/?replicaSet=my-rs") MotorClient(host, port, replicaSet="my-rs") New features ~~~~~~~~~~~~ New classes :class:`~motor.motor_tornado.MotorGridFSBucket` and :class:`~motor.motor_asyncio.AsyncIOMotorGridFSBucket` conform to the `GridFS API Spec `_ for MongoDB drivers. These classes supersede the old :class:`~motor.motor_tornado.MotorGridFS` and :class:`~motor.motor_asyncio.AsyncIOMotorGridFS`. See `GridFS`_ changes below, especially note the **breaking change** in :class:`~motor.motor_web.GridFSHandler`. Serve GridFS files over HTTP using `aiohttp`_ and :class:`~motor.aiohttp.AIOHTTPGridFS`. .. _aiohttp: https://aiohttp.readthedocs.io/ :class:`MotorClient` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Removed: - :meth:`MotorClient.open`; clients have opened themselves automatically on demand since version 0.2. - :attr:`MotorClient.seeds`, use :func:`pymongo.uri_parser.parse_uri` on your MongoDB URI. - :attr:`MotorClient.alive` Added: - :attr:`MotorClient.event_listeners` - :attr:`MotorClient.max_idle_time_ms` - :attr:`MotorClient.min_pool_size` Unix domain socket paths must be quoted with :func:`urllib.parse.quote_plus` (or ``urllib.quote_plus`` in Python 2) before they are included in a URI: .. code-block:: python path = '/tmp/mongodb-27017.sock' MotorClient('mongodb://%s' % urllib.parse.quote_plus(path)) :class:`~motor.motor_tornado.MotorCollection` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Added: - :meth:`MotorCollection.create_indexes` - :meth:`MotorCollection.list_indexes` New ``bypass_document_validation`` parameter for :meth:`~.MotorCollection.initialize_ordered_bulk_op` and :meth:`~.MotorCollection.initialize_unordered_bulk_op`. Changes to :meth:`~motor.motor_tornado.MotorCollection.find` and :meth:`~motor.motor_tornado.MotorCollection.find_one` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The following find/find_one options have been renamed: These renames only affect your code if you passed these as keyword arguments, like ``find(fields=['fieldname'])``. If you passed only positional parameters these changes are not significant for your application. - spec -> filter - fields -> projection - partial -> allow_partial_results The following find/find_one options have been added: - cursor_type (see :class:`~pymongo.cursor.CursorType` for values) - oplog_replay - modifiers The following find/find_one options have been removed: - network_timeout (use :meth:`~motor.motor_tornado.MotorCursor.max_time_ms` instead) - read_preference (use :meth:`~motor.motor_tornado.MotorCollection.with_options` instead) - tag_sets (use one of the read preference classes from :mod:`~pymongo.read_preferences` and :meth:`~motor.motor_tornado.MotorCollection.with_options` instead) - secondary_acceptable_latency_ms (use the ``localThresholdMS`` URI option instead) - max_scan (use the new ``modifiers`` option instead) - snapshot (use the new ``modifiers`` option instead) - tailable (use the new ``cursor_type`` option instead) - await_data (use the new ``cursor_type`` option instead) - exhaust (use the new ``cursor_type`` option instead) - as_class (use :meth:`~motor.motor_tornado.MotorCollection.with_options` with :class:`~bson.codec_options.CodecOptions` instead) - compile_re (BSON regular expressions are always decoded to :class:`~bson.regex.Regex`) The following find/find_one options are deprecated: - manipulate The following renames need special handling. - timeout -> no_cursor_timeout - By default, MongoDB closes a cursor after 10 minutes of inactivity. In previous Motor versions, you disabled the timeout by passing ``timeout=False`` to :meth:`.MotorCollection.find` or :meth:`.MotorGridFS.find`. The ``timeout`` parameter has been renamed to ``no_cursor_timeout``, it defaults to ``False``, and you must now pass ``no_cursor_timeout=True`` to disable timeouts. :class:`~motor.motor_tornado.MotorCursor` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Added: - :attr:`.MotorCursor.address` - :meth:`.MotorCursor.max_await_time_ms` Removed: - :attr:`.MotorCursor.conn_id`, use :attr:`~.MotorCursor.address` GridFS ~~~~~~ The old GridFS classes :class:`~motor.motor_tornado.MotorGridFS` and :class:`~motor.motor_asyncio.AsyncIOMotorGridFS` are deprecated in favor of :class:`~motor.motor_tornado.MotorGridFSBucket` and :class:`~motor.motor_asyncio.AsyncIOMotorGridFSBucket`, which comply with MongoDB's cross-language driver spec for GridFS. The old classes are still supported, but will be removed in Motor 2.0. **BREAKING CHANGE**: The overridable method :class:`~motor.web.GridFSHandler.get_gridfs_file` of :class:`~motor.web.GridFSHandler` now takes a :class:`~motor.motor_tornado.MotorGridFSBucket`, not a :class:`~motor.motor_tornado.MotorGridFS`. It also takes an additional ``request`` parameter. :class:`~motor.motor_tornado.MotorGridOutCursor` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Added: - :attr:`.MotorGridOutCursor.address` - :meth:`.MotorGridOutCursor.max_await_time_ms` Removed: - :attr:`.MotorGridOutCursor.conn_id`, use :attr:`~.MotorGridOutCursor.address` :class:`~motor.motor_tornado.MotorGridIn` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New method :meth:`.MotorGridIn.abort`. In a Python 3.5 native coroutine, the "async with" statement calls :meth:`~MotorGridIn.close` automatically:: async def upload(): my_db = MotorClient().test fs = MotorGridFSBucket(my_db) async with await fs.new_file() as gridin: await gridin.write(b'First part\n') await gridin.write(b'Second part') # gridin is now closed automatically. :class:`~motor.motor_tornado.MotorGridOut` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :class:`~motor.motor_tornado.MotorGridOut` is now an async iterable, so reading a chunk at a time is much simpler with a Python 3 native coroutine:: async def read_file(file_id): fs = motor.motor_tornado.MotorGridFS(db) gridout = await fs.get(file_id) async for chunk in gridout: sys.stdout.write(chunk) sys.stdout.flush() Documentation ~~~~~~~~~~~~~ The :doc:`/api-asyncio/index` is now fully documented, side by side with the :doc:`/api-tornado/index`. New :doc:`developer-guide` added. Motor 0.7 --------- For asynchronous I/O Motor now uses a thread pool, which is faster and simpler than the prior implementation with greenlets. It no longer requires the ``greenlet`` package, and now requires the ``futures`` backport package on Python 2. This version updates the PyMongo dependency from 2.8.0 to 2.9.x, and wraps PyMongo 2.9's new APIs. Most of Motor 1.0's API is now implemented, and APIs that will be removed in Motor 1.0 are now deprecated and raise warnings. See the :doc:`/migrate-to-motor-1` to prepare your code for Motor 1.0. :class:`MotorClient` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The :class:`~MotorClient.get_database` method is added for getting a :class:`MotorDatabase` instance with its options configured differently than the MotorClient's. New read-only attributes: - :attr:`~MotorClient.codec_options` - :attr:`~MotorClient.local_threshold_ms` - :attr:`~MotorClient.max_write_batch_size` :class:`MotorReplicaSetClient` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The :meth:`~MotorReplicaSetClient.get_database` method is added for getting a :class:`MotorDatabase` instance with its options configured differently than the MotorReplicaSetClient's. New read-only attributes: - :attr:`~MotorReplicaSetClient.codec_options` - :attr:`~MotorReplicaSetClient.local_threshold_ms` :class:`MotorDatabase` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The :meth:`~MotorDatabase.get_collection` method is added for getting a :class:`MotorCollection` instance with its options configured differently than the MotorDatabase's. The ``connection`` property is deprecated in favor of a new read-only attribute :attr:`~MotorDatabase.client`. New read-only attribute: - :attr:`~MotorDatabase.codec_options` :class:`MotorCollection` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The :meth:`~MotorCollection.with_options` method is added for getting a :class:`MotorCollection` instance with its options configured differently than this MotorCollection's. New read-only attribute: - :attr:`~MotorCollection.codec_options` The following methods wrap PyMongo's implementation of the standard `CRUD API Spec`_ for MongoDB Drivers: - :meth:`~MotorCollection.bulk_write` - :meth:`~MotorCollection.insert_one` - :meth:`~MotorCollection.insert_many` - :meth:`~MotorCollection.update_one` - :meth:`~MotorCollection.update_many` - :meth:`~MotorCollection.replace_one` - :meth:`~MotorCollection.delete_one` - :meth:`~MotorCollection.delete_many` - :meth:`~MotorCollection.find_one_and_delete` - :meth:`~MotorCollection.find_one_and_replace` - :meth:`~MotorCollection.find_one_and_update` These new methods do not apply SON Manipulators. .. _CRUD API Spec: https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst :doc:`GridFS ` changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New :class:`MotorGridOutCursor` methods: - :meth:`~MotorGridOutCursor.add_option` - :meth:`~MotorGridOutCursor.remove_option` - :meth:`~MotorGridOutCursor.clone` Added :class:`MotorGridOut` documentation: - :attr:`~MotorGridOut.aliases` - :attr:`~MotorGridOut.chunk_size` - :meth:`~MotorGridOut.close` - :attr:`~MotorGridOut.content_type` - :attr:`~MotorGridOut.filename` - :attr:`~MotorGridOut.length` - :attr:`~MotorGridOut.md5` - :attr:`~MotorGridOut.metadata` - :attr:`~MotorGridOut.name` - :attr:`~MotorGridOut.upload_date` Bugfix ~~~~~~ `MOTOR-124 `_: an import deadlock in Python 2 and Tornado 3 led to an :exc:`~pymongo.errors.AutoReconnect` exception with some replica sets. Motor 0.6.2 ----------- Fix "from motor import \*" for Python 3. Motor 0.6.1 ----------- Fix source distribution, which hadn't included the "frameworks" submodules. Motor 0.6 --------- This is a bugfix release. Fixing these bugs has introduced tiny API changes that may affect some programs. `motor_asyncio` and `motor_tornado` submodules ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ These modules have been moved from: - `motor_asyncio.py` - `motor_tornado.py` To: - `motor_asyncio/__init__.py` - `motor_tornado/__init__.py` Motor had to make this change in order to omit the `motor_asyncio` submodule entirely and avoid a spurious :exc:`SyntaxError` being printed when installing in Python 2. The change should be invisible to application code. Database and collection names with leading underscores ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A database or collection whose name starts with an underscore can no longer be accessed as a property:: # Now raises AttributeError. db = MotorClient()._mydatabase collection = db._mycollection subcollection = collection._subcollection Such databases and collections can still be accessed dict-style:: # Continues to work the same as previous Motor versions. db = MotorClient()['_mydatabase'] collection = db['_mycollection'] To ensure a "sub-collection" with a name that includes an underscore is accessible, Motor collections now allow dict-style access, the same as Motor clients and databases always have:: # New in Motor 0.6 subcollection = collection['_subcollection'] These changes solve problems with iPython code completion and the Python 3 :class:`ABC` abstract base class. Motor 0.5 --------- asyncio ~~~~~~~ Motor can now integrate with asyncio, as an alternative to Tornado. My gratitude to Rémi Jolin, Andrew Svetlov, and Nikolay Novik for their huge contributions to Motor's asyncio integration. Python 3.5 ~~~~~~~~~~ Motor is now compatible with Python 3.5, which required some effort. Motor not only supports users' coroutines, it uses coroutines to implement some of its own features, like :meth:`~MotorClient.open` and :meth:`~MotorGridFS.put`. There is no single way to return a value from a Python 3.5 native coroutine or a Python 2 generator-based coroutine, so Motor internal coroutines that return values were rewritten. (See `commit message dc19418c`_ for an explanation.) .. _commit message dc19418c: https://github.com/mongodb/motor/commit/dc19418c `async` and `await` ~~~~~~~~~~~~~~~~~~~ Motor now supports Python 3.5 native coroutines, written with the `async` and `await` syntax:: async def f(): await collection.insert({'_id': 1}) Cursors from :meth:`~MotorCollection.find`, :meth:`~MotorCollection.aggregate`, or :meth:`~MotorGridFS.find` can be iterated elegantly and very efficiently in native coroutines with `async for`:: async def f(): async for doc in collection.find(): do_something_with(doc) .. _aggregate_changes_0_5: :meth:`~MotorCollection.aggregate` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :meth:`MotorCollection.aggregate` now returns a cursor by default, and the cursor is returned immediately without a `yield`. The old syntax is no longer supported:: # Motor 0.4 and older, no longer supported. cursor = yield collection.aggregate(pipeline, cursor={}) while (yield cursor.fetch_next): doc = cursor.next_object() print(doc) In Motor 0.5, simply do:: # Motor 0.5: no "cursor={}", no "yield". cursor = collection.aggregate(pipeline) while (yield cursor.fetch_next): doc = cursor.next_object() print(doc) Or with Python 3.5 and later:: # Motor 0.5, Python 3.5. async for doc in collection.aggregate(pipeline): print(doc) MongoDB versions 2.4 and older do not support aggregation cursors. For compatibility with older MongoDBs, :meth:`~MotorCollection.aggregate` now takes an argument ``cursor=False``, and returns a Future that you can yield to get all the results in one document:: # Motor 0.5 with MongoDB 2.4 and older. reply = yield collection.aggregate(cursor=False) for doc in reply['results']: print(doc) Deprecations ~~~~~~~~~~~~ Motor 0.5 deprecates a large number of APIs that will be removed in version 1.0: `MotorClient`: - `~MotorClient.host` - `~MotorClient.port` - `~MotorClient.document_class` - `~MotorClient.tz_aware` - `~MotorClient.secondary_acceptable_latency_ms` - `~MotorClient.tag_sets` - `~MotorClient.uuid_subtype` - `~MotorClient.disconnect` - `~MotorClient.alive` `MotorReplicaSetClient`: - `~MotorReplicaSetClient.document_class` - `~MotorReplicaSetClient.tz_aware` - `~MotorReplicaSetClient.secondary_acceptable_latency_ms` - `~MotorReplicaSetClient.tag_sets` - `~MotorReplicaSetClient.uuid_subtype` - `~MotorReplicaSetClient.alive` `MotorDatabase`: - `~MotorDatabase.secondary_acceptable_latency_ms` - `~MotorDatabase.tag_sets` - `~MotorDatabase.uuid_subtype` `MotorCollection`: - `~MotorCollection.secondary_acceptable_latency_ms` - `~MotorCollection.tag_sets` - `~MotorCollection.uuid_subtype` Cursor slicing ~~~~~~~~~~~~~~ Cursors can no longer be indexed like ``cursor[n]`` or sliced like ``cursor[start:end]``, see `MOTOR-84 `_. If you wrote code like this:: cursor = collection.find()[i] yield cursor.fetch_next doc = cursor.next_object() Then instead, write:: cursor = collection.find().skip(i).limit(-1) yield cursor.fetch_next doc = cursor.next_object() The negative limit ensures the server closes the cursor after one result, saving Motor the work of closing it. See `cursor.limit `_. SSL hostname validation error ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When you use Motor with Tornado and SSL hostname validation fails, Motor used to raise a :exc:`~pymongo.errors.ConnectionFailure` with a useful messsage like "hostname 'X' doesn't match 'Y'". The message is now empty and Tornado logs a warning instead. Configuring uuid_subtype ~~~~~~~~~~~~~~~~~~~~~~~~ You can now get and set :attr:`~MotorClient.uuid_subtype` on :class:`MotorClient`, :class:`MotorReplicaSetClient`, and :class:`MotorDatabase` instances, not just on :class:`MotorCollection`. Motor 0.4.1 ----------- Fix `MOTOR-66 `_, deadlock when initiating :class:`MotorReplicaSetClient` connection from multiple operations at once. Motor 0.4 --------- Supports MongoDB 3.0. In particular, supports MongoDB 3.0's new SCRAM-SHA-1 authentication mechanism and updates the implementations of :meth:`MotorClient.database_names` and :meth:`MotorDatabase.collection_names`. Updates PyMongo dependency from 2.7.1 to 2.8, therefore inheriting `PyMongo 2.7.2's bug fixes `_ and `PyMongo 2.8's bug fixes `_ and `features `_. Fixes `a connection-pool timeout when waitQueueMultipleMS is set `_ and `two bugs in replica set monitoring `_. The ``copy_database`` method has been removed. It was overly complex and no one used it, see `MOTOR-56 `_. You can still use the :meth:`MotorDatabase.command` method directly. The only scenario not supported is copying a database from one host to another, if the remote host requires authentication. For this, use PyMongo's `copy_database`_ method, or, since PyMongo's ``copy_database`` will be removed in a future release too, use the mongo shell. .. _copy_database: http://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient.copy_database .. seealso:: `The "copydb" command `_. Motor 0.3.3 ----------- Fix `MOTOR-45 `_, a stack-context leak in domain name resolution that could lead to an infinite loop and rapid memory leak. Document Motor's :doc:`requirements` in detail. Motor 0.3.2 ----------- Fix `MOTOR-44 `_, a socket leak in :class:`MotorClient.copy_database` and :class:`MotorReplicaSetClient.copy_database`. Motor 0.3.1 ----------- Fix `MOTOR-43 `_, a TypeError when using :class:`~motor.web.GridFSHandler` with a timezone-aware :class:`~motor.motor_tornado.MotorClient`. Fix GridFS examples that hadn't been updated for Motor 0.2's new syntax. Fix a unittest that hadn't been running. Motor 0.3 --------- No new features. * Updates PyMongo dependency from 2.7 to 2.7.1, therefore inheriting `PyMongo 2.7.1's bug fixes `_. * Motor continues to support Python 2.6, 2.7, 3.3, and 3.4, but now with single-source. 2to3 no longer runs during installation with Python 3. * ``nosetests`` is no longer required for regular Motor tests. * Fixes `a mistake in the docstring `_ for aggregate(). Motor 0.2.1 ----------- Fixes two bugs: * `MOTOR-32 `_: The documentation for :meth:`MotorCursor.close` claimed it immediately halted execution of :meth:`MotorCursor.each`, but it didn't. * `MOTOR-33 `_: An incompletely iterated cursor's ``__del__`` method sometimes got stuck and cost 100% CPU forever, even though the application was still responsive. Motor 0.2 --------- This version includes API changes that break backward compatibility with applications written for Motor 0.1. For most applications, the migration chores will be minor. In exchange, Motor 0.2 offers a cleaner style, and it wraps the new and improved PyMongo 2.7 instead of 2.5. Changes in Dependencies ~~~~~~~~~~~~~~~~~~~~~~~ Motor now requires PyMongo 2.7.0 exactly and Tornado 3 or later. It drops support for Python 2.5 since Tornado 3 has dropped it. Motor continues to work with Python 2.6 through 3.4. It still requires `Greenlet`_. API Changes ~~~~~~~~~~~ open_sync ''''''''' The ``open_sync`` method has been removed from :class:`MotorClient` and :class:`MotorReplicaSetClient`. Clients now connect to MongoDB automatically on first use. Simply delete the call to ``open_sync`` from your application. If it's important to test that MongoDB is available before continuing your application's startup, use ``IOLoop.run_sync``:: loop = tornado.ioloop.IOLoop.current() client = motor.MotorClient(host, port) try: loop.run_sync(client.open) except pymongo.errors.ConnectionFailure: print "Can't connect" Similarly, calling :meth:`MotorGridOut.open` is now optional. :class:`MotorGridIn` and :class:`MotorGridFS` no longer have an ``open`` method at all. .. _changelog-futures: Futures ''''''' Motor 0.2 takes advantage of Tornado's tidy new coroutine syntax:: # Old style: document = yield motor.Op(collection.find_one, {'_id': my_id}) # New style: document = yield collection.find_one({'_id': my_id}) To make this possible, Motor asynchronous methods (except :meth:`MotorCursor.each`) now return a :class:`~tornado.concurrent.Future`. Using Motor with callbacks is still possible: If a callback is passed, it will be executed with the ``(result, error)`` of the operation, same as in Motor 0.1:: def callback(document, error): if error: logging.error("Oh no!") else: print document collection.find_one({'_id': my_id}, callback=callback) If no callback is passed, a Future is returned that resolves to the method's result or error:: document = yield collection.find_one({'_id': my_id}) ``motor.Op`` works the same as before, but it's deprecated. ``WaitOp`` and ``WaitAllOps`` have been removed. Code that used them can now yield a ``Future`` or a list of them. Consider this function written for Tornado 2 and Motor 0.1:: @gen.engine def get_some_documents(): cursor = collection.find().sort('_id').limit(2) cursor.to_list(callback=(yield gen.Callback('key'))) do_something_while_we_wait() try: documents = yield motor.WaitOp('key') print documents except Exception, e: print e The function now becomes:: @gen.coroutine def f(): cursor = collection.find().sort('_id').limit(2) future = cursor.to_list(2) do_something_while_we_wait() try: documents = yield future print documents except Exception, e: print e Similarly, a function written like so in the old style:: @gen.engine def get_two_documents_in_parallel(collection): collection.find_one( {'_id': 1}, callback=(yield gen.Callback('one'))) collection.find_one( {'_id': 2}, callback=(yield gen.Callback('two'))) try: doc_one, doc_two = yield motor.WaitAllOps(['one', 'two']) print doc_one, doc_two except Exception, e: print e Now becomes:: @gen.coroutine def get_two_documents_in_parallel(collection): future_0 = collection.find_one({'_id': 1}) future_1 = collection.find_one({'_id': 2}) try: doc_one, doc_two = yield [future_0, future_1] print doc_one, doc_two except Exception, e: print e to_list ''''''' Any calls to :meth:`MotorCursor.to_list` that omitted the ``length`` argument must now include it:: result = yield collection.find().to_list(100) ``None`` is acceptable, meaning "unlimited." Use with caution. Connection Pooling '''''''''''''''''' :class:`MotorPool` has been rewritten. It supports the new options introduced in PyMongo 2.6, and drops all Motor-specific options. :class:`MotorClient` and :class:`MotorReplicaSetClient` have an option ``max_pool_size``. It used to mean "minimum idle sockets to keep open", but its meaning has changed to "maximum sockets open per host." Once this limit is reached, operations will pause waiting for a socket to become available. Therefore the default has been raised from 10 to 100. If you pass a value for ``max_pool_size`` make sure it's large enough for the expected load. (Sockets are only opened when needed, so there's no cost to having a ``max_pool_size`` larger than necessary. Err towards a larger value.) If you've been accepting the default, continue to do so. ``max_pool_size`` is now synonymous with Motor's special ``max_concurrent`` option, so ``max_concurrent`` has been removed. ``max_wait_time`` has been renamed ``waitQueueTimeoutMS`` for consistency with PyMongo. If you pass ``max_wait_time``, rename it and multiply by 1000. The :exc:`MotorPoolTimeout` exception is gone; catch PyMongo's :exc:`~pymongo.errors.ConnectionFailure` instead. DNS ''' Motor can take advantage of Tornado 3's `asynchronous resolver interface`_. By default, Motor still uses blocking DNS, but you can enable non-blocking lookup with a threaded resolver:: Resolver.configure('tornado.netutil.ThreadedResolver') Or install `pycares`_ and use the c-ares resolver:: Resolver.configure('tornado.platform.caresresolver.CaresResolver') MotorCursor.tail '''''''''''''''' The ``MotorCursor.tail`` method has been removed. It was complex, diverged from PyMongo's feature set, and encouraged overuse of MongoDB capped collections as message queues when a purpose-built message queue is more appropriate. An example of tailing a capped collection is provided instead: :doc:`examples/tailable-cursors`. MotorClient.is_locked ''''''''''''''''''''' ``is_locked`` has been removed since calling it from Motor would be bizarre. If you called ``MotorClient.is_locked`` like:: locked = yield motor.Op(client.is_locked) you should now do:: result = yield client.admin.current_op() locked = bool(result.get('fsyncLock', None)) The result is ``True`` only if an administrator has called `fsyncLock`_ on the mongod. It is unlikely that you have any use for this. GridFSHandler ''''''''''''' :meth:`~web.GridFSHandler.get_gridfs_file` now returns a Future instead of accepting a callback. .. _Greenlet: http://pypi.python.org/pypi/greenlet/ .. _asynchronous resolver interface: http://www.tornadoweb.org/en/stable/netutil.html#tornado.netutil.Resolver .. _pycares: https://pypi.python.org/pypi/pycares .. _fsyncLock: http://docs.mongodb.org/manual/reference/method/db.fsyncLock/ New Features ~~~~~~~~~~~~ The introduction of a :ref:`Futures-based API ` is the most pervasive new feature. In addition Motor 0.2 includes new features from PyMongo 2.6 and 2.7: * :meth:`MotorCollection.aggregate` can return a cursor. * Support for all current MongoDB authentication mechanisms (see PyMongo's `authentication examples`_). * A new :meth:`MotorCollection.parallel_scan` method. * An :doc:`API for bulk writes `. * Support for wire protocol changes in MongoDB 2.6. * The ability to specify a server-side timeout for operations with :meth:`~MotorCursor.max_time_ms`. * A new :meth:`MotorGridFS.find` method for querying GridFS. .. _authentication examples: http://api.mongodb.org/python/current/examples/authentication.html Bugfixes ~~~~~~~~ ``MotorReplicaSetClient.open`` threw an error if called without a callback. ``MotorCursor.to_list`` `ignored SON manipulators `_. (Thanks to Eren Güven for the report and the fix.) `The full list is in Jira `_. Motor 0.1.2 ----------- Fixes innocuous unittest failures when running against Tornado 3.1.1. Motor 0.1.1 ----------- Fixes issue `MOTOR-12`_ by pinning its PyMongo dependency to PyMongo version 2.5.0 exactly. Motor relies on some of PyMongo's internal details, so changes to PyMongo can break Motor, and a change in PyMongo 2.5.1 did. Eventually PyMongo will expose stable hooks for Motor to use, but for now I changed Motor's dependency from ``PyMongo>=2.4.2`` to ``PyMongo==2.5.0``. .. _MOTOR-12: https://jira.mongodb.org/browse/MOTOR-12 motor-1.2.1/doc/conf.py000066400000000000000000000147131323007662700147330ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Motor documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. import sys, os sys.path[0:0] = [os.path.abspath('..')] from pymongo import version as pymongo_version import motor # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.todo', 'doc.mongo_extensions', 'doc.motor_extensions', 'sphinx.ext.intersphinx', 'doc.coroutine_annotation'] primary_domain = 'py' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Motor' copyright = u'2016-present MongoDB, Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = motor.version # The full version, including alpha/beta/rc tags. release = motor.version # List of documents that shouldn't be included in the build. unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for extensions ---------------------------------------------------- autoclass_content = 'init' doctest_path = [os.path.abspath('..')] # Don't test examples pulled from PyMongo's docstrings just because they start # with '>>>' doctest_test_doctest_blocks = '' doctest_global_setup = """ import pprint import sys from datetime import timedelta from tornado import gen from tornado.ioloop import IOLoop import pymongo from pymongo.mongo_client import MongoClient sync_client = MongoClient() ismaster = sync_client.admin.command('isMaster') server_info = sync_client.server_info() if 'setName' in ismaster: raise Exception( "Run doctests with standalone MongoDB 3.6 server, not a replica set") if ismaster.get('msg') == 'isdbgrid': raise Exception( "Run doctests with standalone MongoDB 3.6 server, not mongos") if server_info['versionArray'][:2] != [3, 6]: raise Exception( "Run doctests with standalone MongoDB 3.6 server, not %s" % ( server_info['version'], )) sync_client.drop_database("doctest_test") db = sync_client.doctest_test import motor from motor import MotorClient """ # -- Options for HTML output --------------------------------------------------- html_copy_source = False # Theme gratefully vendored from CPython source. html_theme = "pydoctheme" html_theme_path = ["."] html_theme_options = {'collapsiblesidebar': True} html_static_path = ['static'] html_sidebars = { 'index': ['globaltoc.html', 'searchbox.html'], } # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'Motor' + release.replace('.', '_') # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'Motor.tex', u'Motor Documentation', u'A. Jesse Jiryu Davis', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_use_modindex = True autodoc_default_flags = ['inherited-members'] autodoc_member_order = 'groupwise' pymongo_inventory = ('http://api.mongodb.com/python/%s/' % pymongo_version, None) intersphinx_mapping = { 'bson': pymongo_inventory, 'gridfs': pymongo_inventory, 'pymongo': pymongo_inventory, 'aiohttp': ('http://aiohttp.readthedocs.io/en/stable/', None), 'tornado': ('http://www.tornadoweb.org/en/stable/', None), 'python': ('https://docs.python.org/3/', None), } motor-1.2.1/doc/configuration.rst000066400000000000000000000007101323007662700170250ustar00rootroot00000000000000Configuration ============= Motor uses the Python standard library's :class:`~concurrent.futures.ThreadPoolExecutor` to defer network operations to threads. By default, the executor uses at most five threads per CPU core on your system; to override the default set the environment variable ``MOTOR_MAX_WORKERS``. Some additional threads are used for monitoring servers and background tasks, so the total count of threads in your process will be greater. motor-1.2.1/doc/contributors.rst000066400000000000000000000004761323007662700167240ustar00rootroot00000000000000Contributors ============ The following is a list of people who have contributed to **Motor**. If you belong here and are missing please let us know (or send a pull request after adding yourself to the list): - A\. Jesse Jiryu Davis - Eren Güven - Jorge Puente Sarrín - Rémi Jolin - Andrew Svetlov - Nikolay Novik motor-1.2.1/doc/coroutine_annotation.py000066400000000000000000000017001323007662700202370ustar00rootroot00000000000000"""Gratefully adapted from aiohttp, provides coroutine support to autodoc.""" from sphinx.domains.python import PyModulelevel, PyClassmember from sphinx import addnodes class PyCoroutineMixin(object): def handle_signature(self, sig, signode): ret = super(PyCoroutineMixin, self).handle_signature(sig, signode) signode.insert(0, addnodes.desc_annotation('coroutine ', 'coroutine ')) return ret class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): def run(self): self.name = 'py:function' return PyModulelevel.run(self) class PyCoroutineMethod(PyCoroutineMixin, PyClassmember): def run(self): self.name = 'py:method' return PyClassmember.run(self) def setup(app): app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod) return {'version': '1.0', 'parallel_read_safe': True} motor-1.2.1/doc/developer-guide.rst000066400000000000000000000103241323007662700172400ustar00rootroot00000000000000=============== Developer Guide =============== Some explanations for those who would like to contribute to Motor development. Compatibility ------------- Motor supports the asyncio module in the standard library of Python 3.4 and later. Motor also works with Tornado 4.5 and later along with all the Python versions it supports. Motor is single-source compatible with all supported Python versions, although there are some tricks for Python 3. There is some code for the ``async`` and ``await`` features of Python 3.5+ that is conditionally compiled with ``eval`` in ``core.py``. In ``setup.py`` there are tricks to conditionally import tests depending on Python version. ``setup.py`` also avoids installing the ``frameworks/asyncio`` directory in a Python 2 environment. Frameworks ---------- Motor abstracts the differences between Tornado and asyncio by wrapping each in a "framework" interface. A Motor framework is a module implementing these properties and functions: - ``CLASS_PREFIX`` - ``add_future`` - ``call_soon`` - ``check_event_loop`` - ``coroutine`` - ``future_or_callback`` - ``get_event_loop`` - ``get_future`` - ``is_event_loop`` - ``is_future`` - ``pymongo_class_wrapper`` - ``run_on_executor`` - ``yieldable`` See the ``frameworks/tornado`` and ``frameworks/asyncio`` modules. A framework-specific class, like ``MotorClient`` for Tornado or ``AsyncIOMotorClient`` for asyncio, is created by the ``create_class_with_framework`` function, which combines a framework with a framework-agnostic class, in this case ``AgnosticClient``. Wrapping PyMongo ---------------- For each PyMongo class, Motor declares an equivalent framework-agnostic class. For example, the ``AgnosticClient`` class is a framework-agnostic equivalent to PyMongo's ``MongoClient``. This agnostic class declares each method and property of the PyMongo class that it intends to wrap. These methods and properties begin life as type ``MotorAttributeFactory``. When ``create_class_with_framework`` creates a framework-specific class from an agnostic class, it creates methods and properties for that class which wrap the equivalent PyMongo methods and properties. For example, the ``AgnosticClient`` class declares that ``drop_database`` is an ``AsyncCommand``, which is a subclass of ``MotorAttributeFactory``. At import time, ``create_class_with_framework`` calls the ``create_attribute`` method of each ``MotorAttributeFactory`` on the ``AgnosticClient``, which results in framework-specific implementations of each method and property. So at import time, ``create_class_with_framework`` generates framework-specific wrappers of ``drop_database`` for ``MotorClient`` and ``AsyncIOMotorClient``. These wrappers use framework-specific features to run the ``drop_database`` method asynchronously. Asynchronization ---------------- This is the heart of Motor's implementation. The ``create_attribute`` method for asynchronous methods like ``drop_database`` wraps the equivalent PyMongo method in a Motor method. This wrapper method uses either the Tornado or asyncio framework to: - get a reference to the framework's event loop - start the PyMongo method on a thread in the global ``ThreadPoolExecutor`` - create a ``Future`` that will be resolved by the event loop when the thread finishes - returns the ``Future`` to the caller This is what allows Tornado or asyncio coroutines to call Motor methods with ``yield``, ``yield from``, or ``await`` to await I/O without blocking the event loop. Synchro ------- A common kind of bug in Motor arises when PyMongo adds a feature, like a new method or new optional behavior, which we forget to wrap with Motor. Since PyMongo adds a test to its suite for each new feature, we could catch these omissions by applying PyMongo's latest tests to Motor. Then a missing method or feature would cause an obvious test failure. But PyMongo is synchronous and Motor is async; how can Motor pass PyMongo's tests? Synchro is a hacky little module that re-synchronizes all Motor methods using the Tornado IOLoop's ``run_sync`` method. ``synchrotest.py`` overrides the Python interpreter's import machinery to allow Synchro to masquerade as PyMongo, and runs PyMongo's test suite against it. Use ``tox -e synchro`` to check out PyMongo test suite and run it with Synchro. motor-1.2.1/doc/differences.rst000066400000000000000000000140011323007662700164310ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado ===================================== Differences between Motor and PyMongo ===================================== .. important:: This page describes using Motor with Tornado. Beginning in version 0.5 Motor can also integrate with asyncio instead of Tornado. Major differences ================= Connecting to MongoDB --------------------- Motor provides a single client class, :class:`MotorClient`. Unlike PyMongo's :class:`~pymongo.mongo_client.MongoClient`, Motor's client class does not begin connecting in the background when it is instantiated. Instead it connects on demand, when you first attempt an operation. Callbacks and Futures --------------------- Motor supports nearly every method PyMongo does, but Motor methods that do network I/O take an optional callback function. The callback must accept two parameters: .. code-block:: python def callback(result, error): pass Motor's asynchronous methods return immediately, and execute the callback, with either a result or an error, when the operation has completed. For example, :meth:`~pymongo.collection.Collection.find_one` is used in PyMongo like: .. code-block:: python db = MongoClient().test user = db.users.find_one({'name': 'Jesse'}) print user But Motor's :meth:`~MotorCollection.find_one` method is asynchronous: .. code-block:: python db = MotorClient().test def got_user(user, error): if error: print 'error getting user!', error else: print user db.users.find_one({'name': 'Jesse'}, callback=got_user) The callback must be passed as a keyword argument, not a positional argument. To find multiple documents, Motor provides :meth:`~MotorCursor.to_list`: .. code-block:: python def got_users(users, error): if error: print 'error getting users!', error else: for user in users: print user db.users.find().to_list(length=10, callback=got_users) .. seealso:: :meth:`~MotorCursor.fetch_next` If you pass no callback to an asynchronous method, it returns a Future for use in a :func:`coroutine `: .. code-block:: python from tornado import gen @gen.coroutine def f(): result = yield motor_db.collection.insert_one({'name': 'Randall'}) doc = yield motor_db.collection.find_one() See :ref:`the coroutine example `. Threading and forking --------------------- Multithreading and forking are not supported; Motor is intended to be used in a single-threaded Tornado application. See Tornado's documentation on `running Tornado in production`_ to take advantage of multiple cores. .. _`running Tornado in production`: http://www.tornadoweb.org/en/stable/guide/running.html Minor differences ================= .. _gridfs-differences: GridFS ------ - File-like PyMongo's :class:`~gridfs.grid_file.GridIn` and :class:`~gridfs.grid_file.GridOut` strive to act like Python's built-in file objects, so they can be passed to many functions that expect files. But the I/O methods of :class:`MotorGridIn` and :class:`MotorGridOut` are asynchronous, so they cannot obey the file API and aren't suitable in the same circumstances as files. - Setting properties In PyMongo, you can set arbitrary attributes on a :class:`~gridfs.grid_file.GridIn` and they're stored as metadata on the server, even after the ``GridIn`` is closed:: fs = gridfs.GridFSBucket(db) grid_in, file_id = fs.open_upload_stream('test_file') grid_in.close() grid_in.my_field = 'my_value' # Sends update to server. Updating metadata on a :class:`MotorGridIn` is asynchronous, so the API is different:: @gen.coroutine def f(): fs = motor.motor_tornado.MotorGridFSBucket(db) grid_in, file_id = fs.open_upload_stream('test_file') yield grid_in.close() # Sends update to server. yield grid_in.set('my_field', 'my_value') .. seealso:: :doc:`../api-tornado/gridfs`. is_locked --------- In PyMongo ``is_locked`` is a property of :class:`~pymongo.mongo_client.MongoClient`. Since determining whether the server has been fsyncLocked requires I/O, Motor has no such convenience method. The equivalent in Motor is:: result = yield client.admin.current_op() locked = bool(result.get('fsyncLock', None)) system_js --------- PyMongo supports Javascript procedures stored in MongoDB with syntax like: .. code-block:: python >>> db.system_js.my_func = 'function(x) { return x * x; }' >>> db.system_js.my_func(2) 4.0 Motor does not. Cursor slicing -------------- In Pymongo, the following raises an ``IndexError`` if the collection has fewer than 101 documents: .. code-block:: python # Can raise IndexError. doc = db.collection.find()[100] In Motor, however, no exception is raised. The query simply has no results: .. code-block:: python @gen.coroutine def f(): cursor = db.collection.find()[100] # Iterates zero or one times. while (yield cursor.fetch_next): doc = cursor.next_object() The difference arises because the PyMongo :class:`~pymongo.cursor.Cursor`'s slicing operator blocks until it has queried the MongoDB server, and determines if a document exists at the desired offset; Motor simply returns a new :class:`MotorCursor` with a skip and limit applied. Creating a collection --------------------- There are two ways to create a capped collection using PyMongo: .. code-block:: python # Typical: db.create_collection( 'collection1', capped=True, size=1000) # Unusual: collection = Collection( db, 'collection2', capped=True, size=1000) Motor can't do I/O in a constructor, so the unusual style is prohibited and only the typical style is allowed: .. code-block:: python @gen.coroutine def f(): yield db.create_collection( 'collection1', capped=True, size=1000) motor-1.2.1/doc/docs-requirements.txt000066400000000000000000000000201323007662700176300ustar00rootroot00000000000000tornado aiohttp motor-1.2.1/doc/examples/000077500000000000000000000000001323007662700152445ustar00rootroot00000000000000motor-1.2.1/doc/examples/aiohttp_example.py000066400000000000000000000027021323007662700210020ustar00rootroot00000000000000# These comments let tutorial-asyncio.rst include this code in sections. # -- setup-start -- import asyncio from aiohttp import web from motor.motor_asyncio import AsyncIOMotorClient @asyncio.coroutine def setup_db(): db = AsyncIOMotorClient().test yield from db.pages.drop() html = '{}' yield from db.pages.insert_one({'_id': 'page-one', 'body': html.format('Hello!')}) yield from db.pages.insert_one({'_id': 'page-two', 'body': html.format('Goodbye.')}) return db # -- setup-end -- # -- handler-start -- @asyncio.coroutine def page_handler(request): # If the visitor gets "/pages/page-one", then page_name is "page-one". page_name = request.match_info.get('page_name') # Retrieve the long-lived database handle. db = request.app['db'] # Find the page by its unique id. document = yield from db.pages.find_one(page_name) if not document: return web.HTTPNotFound(text='No page named {!r}'.format(page_name)) return web.Response(body=document['body'].encode(), content_type='text/html') # -- handler-end -- # -- main-start -- loop = asyncio.get_event_loop() db = loop.run_until_complete(setup_db()) app = web.Application() app['db'] = db # Route requests to the page_handler() coroutine. app.router.add_get('/pages/{page_name}', page_handler) web.run_app(app) # -- main-end -- motor-1.2.1/doc/examples/aiohttp_gridfs_example.py000066400000000000000000000031161323007662700223400ustar00rootroot00000000000000"""Serve pre-compressed static content from GridFS with aiohttp. Requires Python 3.4 or later and aiohttp 2.0 or later. Start a MongoDB server on its default port, run this script, and visit: http://localhost:8080/fs/my_file """ # -- include-start -- import asyncio import gzip import tempfile import aiohttp.web from motor.aiohttp import AIOHTTPGridFS from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFS client = AsyncIOMotorClient() # Use Motor to put compressed data in GridFS, with filename "my_file". async def put_gridfile(): with tempfile.TemporaryFile() as tmp: with gzip.GzipFile(mode='wb', fileobj=tmp) as gzfile: for _ in range(10): gzfile.write(b'Nonesuch nonsense\n') gfs = AsyncIOMotorGridFS(client.my_database) tmp.seek(0) await gfs.put(tmp, filename='my_file', content_type='text', metadata={'compressed': True}) asyncio.get_event_loop().run_until_complete(put_gridfile()) # Add "Content-Encoding: gzip" header for compressed data. def gzip_header(response, gridout): if gridout.metadata.get('compressed'): response.headers['Content-Encoding'] = 'gzip' gridfs_handler = AIOHTTPGridFS(client.my_database, set_extra_headers=gzip_header) app = aiohttp.web.Application() # The GridFS URL pattern must have a "{filename}" variable. resource = app.router.add_resource('/fs/{filename}') resource.add_route('GET', gridfs_handler) resource.add_route('HEAD', gridfs_handler) aiohttp.web.run_app(app) motor-1.2.1/doc/examples/aiohttp_gridfs_example.rst000066400000000000000000000010501323007662700225130ustar00rootroot00000000000000AIOHTTPGridFS Example ===================== Serve pre-compressed static content from GridFS over HTTP. Uses the `aiohttp`_ web framework and :class:`~motor.aiohttp.AIOHTTPGridFS`. .. _aiohttp: https://aiohttp.readthedocs.io/ Instructions ------------ Start a MongoDB server on its default port and run this script. Then visit: http://localhost:8080/fs/my_file Serve compressed static content from GridFS ------------------------------------------- .. literalinclude:: aiohttp_gridfs_example.py :language: python3 :start-after: include-start motor-1.2.1/doc/examples/authentication.rst000066400000000000000000000027041323007662700210200ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado Authentication With Motor ========================= This page describes using Motor with Tornado. Beginning in version 0.5 Motor can also integrate with asyncio instead of Tornado. To use authentication, you must start ``mongod`` with ``--auth`` or, for replica sets or sharded clusters, ``--keyFile``. Create an admin user and optionally normal users or read-only users. .. seealso:: `MongoDB Authentication `_ Authentication at Startup ------------------------- To create an authenticated connection use a `MongoDB connection URI`_:: uri = "mongodb://user:pass@localhost:27017/database_name" client = motor.motor_tornado.MotorClient(uri) Motor logs in to the server on demand, when you first attempt an operation. Asynchronous Authentication --------------------------- Use the non-blocking :meth:`~MotorDatabase.authenticate` method to log in after starting the IOLoop:: client = motor.motor_tornado.MotorClient('localhost', 27017) @gen.coroutine def login(c): yield c.my_database.authenticate("user", "pass") After you've logged in to a database with a given :class:`MotorClient`, all further operations on that database using that client will already be authenticated until you call :meth:`~MotorDatabase.logout`. .. _MongoDB connection URI: http://docs.mongodb.org/manual/reference/connection-string/ motor-1.2.1/doc/examples/bulk.rst000066400000000000000000000142411323007662700167350ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado Bulk Write Operations ===================== .. testsetup:: client = MotorClient() db = client.test_database IOLoop.current().run_sync(db.test.drop) This tutorial explains how to take advantage of Motor's bulk write operation features. Executing write operations in batches reduces the number of network round trips, increasing write throughput. This example describes using Motor with Tornado. Beginning in version 0.5 Motor can also integrate with asyncio instead of Tornado. Bulk Insert ----------- A batch of documents can be inserted by passing a list or generator to the :meth:`~MotorCollection.insert_many` method. Motor will automatically split the batch into smaller sub-batches based on the maximum message size accepted by MongoDB, supporting very large bulk insert operations. .. doctest:: >>> @gen.coroutine ... def f(): ... yield db.test.insert_many(({'i': i} for i in range(10000))) ... count = yield db.test.count() ... print("Final count: %d" % count) >>> >>> IOLoop.current().run_sync(f) Final count: 10000 Mixed Bulk Write Operations --------------------------- .. versionadded:: 0.2 Motor also supports executing mixed bulk write operations. A batch of insert, update, and delete operations can be executed together using the Bulk API. .. _ordered_bulk: Ordered Bulk Write Operations ............................. Ordered bulk write operations are batched and sent to the server in the order provided for serial execution. The return value is a document describing the type and count of operations performed. .. doctest:: >>> from pprint import pprint >>> >>> @gen.coroutine ... def f(): ... bulk = db.test.initialize_ordered_bulk_op() ... # Remove all documents from the previous example. ... bulk.find({}).remove() ... bulk.insert({'_id': 1}) ... bulk.insert({'_id': 2}) ... bulk.insert({'_id': 3}) ... bulk.find({'_id': 1}).update({'$set': {'foo': 'bar'}}) ... bulk.find({'_id': 4}).upsert().update({'$inc': {'j': 1}}) ... bulk.find({'j': 1}).replace_one({'j': 2}) ... result = yield bulk.execute() ... pprint(result) ... >>> IOLoop.current().run_sync(f) {'nInserted': 3, 'nMatched': 2, 'nModified': 2, 'nRemoved': 10000, 'nUpserted': 1, 'upserted': [{'_id': 4, 'index': 5}], 'writeConcernErrors': [], 'writeErrors': []} The first write failure that occurs (e.g. duplicate key error) aborts the remaining operations, and Motor raises :class:`~pymongo.errors.BulkWriteError`. The :attr:`details` attibute of the exception instance provides the execution results up until the failure occurred and details about the failure - including the operation that caused the failure. .. doctest:: >>> from pymongo.errors import BulkWriteError >>> >>> @gen.coroutine ... def f(): ... bulk = db.test.initialize_ordered_bulk_op() ... bulk.find({'j': 2}).replace_one({'i': 5}) ... # Violates the unique key constraint on _id. ... ... bulk.insert({'_id': 4}) ... bulk.find({'i': 5}).remove_one() ... try: ... yield bulk.execute() ... except BulkWriteError as err: ... pprint(err.details) ... >>> IOLoop.current().run_sync(f) {'nInserted': 0, 'nMatched': 1, 'nModified': 1, 'nRemoved': 0, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [{'code': 11000, 'errmsg': '... duplicate key error ...', 'index': 1, 'op': {'_id': 4}}]} .. _unordered_bulk: Unordered Bulk Write Operations ............................... Unordered bulk write operations are batched and sent to the server in **arbitrary order** where they may be executed in parallel. Any errors that occur are reported after all operations are attempted. In the next example the first and third operations fail due to the unique constraint on _id. Since we are doing unordered execution the second and fourth operations succeed. .. doctest:: >>> @gen.coroutine ... def f(): ... bulk = db.test.initialize_unordered_bulk_op() ... bulk.insert({'_id': 1}) ... bulk.find({'_id': 2}).remove_one() ... bulk.insert({'_id': 3}) ... bulk.find({'_id': 4}).replace_one({'i': 1}) ... try: ... yield bulk.execute() ... except BulkWriteError as err: ... pprint(err.details) ... >>> IOLoop.current().run_sync(f) {'nInserted': 0, 'nMatched': 1, 'nModified': 1, 'nRemoved': 1, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [], 'writeErrors': [{'code': 11000, 'errmsg': '... duplicate key error ...', 'index': 0, 'op': {'_id': 1}}, {'code': 11000, 'errmsg': '... duplicate key error ...', 'index': 2, 'op': {'_id': 3}}]} Write Concern ............. By default bulk operations are executed with the :meth:`~MotorCollection.write_concern` of the collection they are executed against, typically the default write concern ``{w: 1}``. A custom write concern can be passed to the :meth:`~MotorBulkOperationBuilder.execute` method. Write concern errors (e.g. wtimeout) will be reported after all operations are attempted, regardless of execution order. .. doctest:: :options: +SKIP .. Standalone MongoDB raises "can't use w>1" with this example, so skip it. >>> @gen.coroutine ... def f(): ... bulk = db.test.initialize_ordered_bulk_op() ... bulk.insert({'a': 0}) ... bulk.insert({'a': 1}) ... bulk.insert({'a': 2}) ... bulk.insert({'a': 3}) ... try: ... # Times out if the replica set has fewer than four members. ... yield bulk.execute({'w': 4, 'wtimeout': 1}) ... except BulkWriteError as err: ... pprint(err.details) ... >>> IOLoop.current().run_sync(f) {'nInserted': 4, 'nMatched': 0, 'nModified': 0, 'nRemoved': 0, 'nUpserted': 0, 'upserted': [], 'writeConcernErrors': [{'code': 64, 'errInfo': {'wtimeout': True}, 'errmsg': 'waiting for replication timed out'}], 'writeErrors': []} motor-1.2.1/doc/examples/callbacks-and-coroutines.rst000066400000000000000000000144201323007662700226460ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado Examples With Callbacks And Coroutines ====================================== Programming with Motor is far easier with Tornado coroutines than with raw callbacks. Here's an example that shows the difference. This page describes using Motor with Tornado. Beginning in version 0.5 Motor can also integrate with asyncio instead of Tornado. .. contents:: With callbacks -------------- An application that can create and display short messages: .. code-block:: python import tornado.web, tornado.ioloop import motor class NewMessageHandler(tornado.web.RequestHandler): def get(self): """Show a 'compose message' form.""" self.write('''
    ''') # Method exits before the HTTP request completes, thus "asynchronous" @tornado.web.asynchronous def post(self): """Insert a message.""" msg = self.get_argument('msg') # Async insert; callback is executed when insert completes self.settings['db'].messages.insert_one( {'msg': msg}, callback=self._on_response) def _on_response(self, result, error): if error: raise tornado.web.HTTPError(500, error) else: self.redirect('/') class MessagesHandler(tornado.web.RequestHandler): @tornado.web.asynchronous def get(self): """Display all messages.""" self.write('    Compose a message
    ') self.write('
      ') db = self.settings['db'] db.messages.find().sort([('_id', -1)]).each(self._got_message) def _got_message(self, message, error): if error: raise tornado.web.HTTPError(500, error) elif message: self.write('
    • %s
      • ' % message['msg']) else: # Iteration complete self.write('
    ') self.finish() db = motor.motor_tornado.MotorClient().test application = tornado.web.Application( [ (r'/compose', NewMessageHandler), (r'/', MessagesHandler) ], db=db ) print('Listening on http://localhost:8888') application.listen(8888) tornado.ioloop.IOLoop.instance().start() The call to :meth:`~MotorCursor.each` could be replaced with :meth:`~MotorCursor.to_list`, which is easier to use with templates because the callback receives the entire result at once: .. code-block:: python from tornado import template messages_template = template.Template('''
      {% for message in messages %}
    • {{ message['msg'] }}
      • {% end %}
    ''') class MessagesHandler(tornado.web.RequestHandler): @tornado.web.asynchronous def get(self): """Display all messages """ self.write('Compose a message
    ') self.write('
      ') db = self.settings['db'] (db.messages.find() .sort([('_id', -1)]) .limit(10) .to_list(length=10, callback=self._got_messages)) def _got_messages(self, messages, error): if error: raise tornado.web.HTTPError(500, error) elif messages: self.write(messages_template.generate(messages=messages)) self.finish() To protect you from buffering huge numbers of documents in memory, ``to_list`` requires a maximum ``length`` argument. .. _coroutine-example: With coroutines --------------- Motor's asynchronous methods return `Futures `. Yield a Future to resolve it into a result or an exception: .. code-block:: python from tornado import gen class NewMessageHandler(tornado.web.RequestHandler): @gen.coroutine def post(self): """Insert a message.""" msg = self.get_argument('msg') db = self.settings['db'] # insert_one() returns a Future. Yield the Future to get the result. result = yield db.messages.insert_one({'msg': msg}) # Success self.redirect('/') class MessagesHandler(tornado.web.RequestHandler): @gen.coroutine def get(self): """Display all messages.""" self.write('Compose a message
      ') self.write('
        ') db = self.settings['db'] cursor = db.messages.find().sort([('_id', -1)]) while (yield cursor.fetch_next): message = cursor.next_object() self.write('
      • %s
        • ' % message['msg']) # Iteration complete self.write('
      ') self.finish() One can parallelize operations and wait for all to complete. To query for two messages at once and wait for both: .. code-block:: python msg = yield db.messages.find_one({'_id': msg_id}) # Start getting the previous. find_one returns a Future. prev_future = db.messages.find_one({'_id': {'$lt': msg_id}}) # Start getting the next. next_future = db.messages.find_one({'_id': {'$gt': msg_id}}) # Wait for both to complete by yielding the Futures. previous_msg, next_msg = yield [prev_future, next_future] `async` and `await` ------------------- Python 3.5 introduces `native coroutines`_ that use the `async` and `await` keywords. Starting in Python 3.5, you can use `async def` instead of the `gen.coroutine` decorator, and replace the while-loop with `async for`: .. code-block:: python class MessagesHandler(tornado.web.RequestHandler): async def get(self): """Display all messages.""" self.write('Compose a message
      ') self.write('
        ') db = self.settings['db'] # New in Python 3.5: async for message in db.messages.find().sort([('_id', -1)]): self.write('
      • %s
        • ' % message['msg']) self.write('
      ') self.finish() This version of the code is dramatically faster. .. _native coroutines: https://www.python.org/dev/peps/pep-0492/ motor-1.2.1/doc/examples/index.rst000066400000000000000000000004331323007662700171050ustar00rootroot00000000000000Motor Examples ============== .. seealso:: :doc:`../tutorial-tornado` .. toctree:: callbacks-and-coroutines bulk monitoring tailable-cursors tornado_change_stream_example authentication aiohttp_gridfs_example See also :ref:`example-web-application-aiohttp`. motor-1.2.1/doc/examples/monitoring.rst000066400000000000000000000110741323007662700201660ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado Application Performance Monitoring (APM) ======================================== Motor implements the same `Command Monitoring`_ and `Topology Monitoring`_ specifications as other MongoDB drivers. Therefore, you can register callbacks to be notified of every MongoDB query or command your program sends, and the server's reply to each, as well as getting a notification whenever the driver checks a server's status or detects a change in your replica set. Motor wraps PyMongo, and it shares PyMongo's API for monitoring. To receive notifications about events, you subclass one of PyMongo's four listener classes, :class:`~pymongo.monitoring.CommandListener`, :class:`~pymongo.monitoring.ServerListener`, :class:`~pymongo.monitoring.TopologyListener`, or :class:`~pymongo.monitoring.ServerHeartbeatListener`. Command Monitoring ------------------ Subclass :class:`~pymongo.monitoring.CommandListener` to be notified whenever a command starts, succeeds, or fails. .. literalinclude:: monitoring_example.py :language: py3 :start-after: command logger start :end-before: command logger end Register an instance of ``MyCommandLogger``: .. literalinclude:: monitoring_example.py :language: py3 :start-after: command logger register start :end-before: command logger register end You can register any number of listeners, of any of the four listener types. Although you use only APIs from PyMongo's :mod:`~pymongo.monitoring` module to configure monitoring, if you create a :class:`MotorClient` its commands are monitored, the same as a PyMongo :class:`~pymongo.mongo_client.MongoClient`. .. literalinclude:: monitoring_example.py :language: py3 :start-after: motorclient start :end-before: motorclient end This logs something like: .. code-block:: text Command insert with request id 50073 started on server ('localhost', 27017) Command insert with request id 50073 on server ('localhost', 27017) succeeded in 362 microseconds See PyMongo's :mod:`~pymongo.monitoring` module for details about the event data your callbacks receive. Server and Topology Monitoring ------------------------------ Subclass :class:`~pymongo.monitoring.ServerListener` to be notified whenever Motor detects a change in the state of a MongoDB server it is connected to. .. literalinclude:: monitoring_example.py :language: py3 :start-after: server logger start :end-before: server logger end Subclass :class:`~pymongo.monitoring.TopologyListener` to be notified whenever Motor detects a change in the state of your server topology. Examples of such topology changes are a replica set failover, or if you are connected to several mongos servers and one becomes unavailable. .. literalinclude:: monitoring_example.py :language: py3 :start-after: topology logger start :end-before: topology logger end Motor monitors MongoDB servers with periodic checks called "heartbeats". Subclass :class:`~pymongo.monitoring.ServerHeartbeatListener` to be notified whenever Motor begins a server check, and whenever a check succeeds or fails. .. literalinclude:: monitoring_example.py :language: py3 :start-after: heartbeat logger start :end-before: heartbeat logger end Thread Safety ------------- Watch out: Your listeners' callbacks are executed on various background threads, *not* the main thread. To interact with Tornado or Motor from a listener callback, you must defer to the main thread using :meth:`IOLoop.add_callback `, which is the only thread-safe :class:`~tornado.ioloop.IOLoop` method. Similarly, if you use asyncio instead of Tornado, defer your action to the main thread with :meth:`~asyncio.AbstractEventLoop.call_soon_threadsafe`. There is probably no need to be concerned about this detail, however: logging is the only reasonable thing to do from a listener, and `the Python logging module is thread-safe `_. Further Information ------------------- See also: * PyMongo's :mod:`~pymongo.monitoring` module * `The Command Monitoring Spec`_ * `The Topology Monitoring Spec`_ * The `monitoring.py`_ example file in the Motor repository .. _The Command Monitoring Spec: .. _Command Monitoring: https://github.com/mongodb/specifications/blob/master/source/command-monitoring/command-monitoring.rst .. _The Topology Monitoring Spec: .. _Topology Monitoring: https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring-monitoring.rst .. _monitoring.py: https://github.com/mongodb/motor/blob/master/doc/examples/monitoring.py motor-1.2.1/doc/examples/monitoring_example.py000066400000000000000000000107321323007662700215210ustar00rootroot00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Application Performance Monitoring (APM) example""" # command logger start import logging import sys from pymongo import monitoring logging.basicConfig(stream=sys.stdout, level=logging.INFO) class CommandLogger(monitoring.CommandListener): def started(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} started on server " "{0.connection_id}".format(event)) def succeeded(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "succeeded in {0.duration_micros} " "microseconds".format(event)) def failed(self, event): logging.info("Command {0.command_name} with request id " "{0.request_id} on server {0.connection_id} " "failed in {0.duration_micros} " "microseconds".format(event)) # command logger end # command logger register start monitoring.register(CommandLogger()) # command logger register end # motorclient start from tornado import gen, ioloop from motor import MotorClient client = MotorClient() async def do_insert(): await client.test.collection.insert({'message': 'hi!'}) # For this example, wait 10 seconds for more monitoring events to fire. await gen.sleep(10) ioloop.IOLoop.current().run_sync(do_insert) # motorclient end # server logger start class ServerLogger(monitoring.ServerListener): def opened(self, event): logging.info("Server {0.server_address} added to topology " "{0.topology_id}".format(event)) def description_changed(self, event): previous_server_type = event.previous_description.server_type new_server_type = event.new_description.server_type if new_server_type != previous_server_type: logging.info( "Server {0.server_address} changed type from " "{0.previous_description.server_type_name} to " "{0.new_description.server_type_name}".format(event)) def closed(self, event): logging.warning("Server {0.server_address} removed from topology " "{0.topology_id}".format(event)) monitoring.register(ServerLogger()) # server logger end # topology logger start class TopologyLogger(monitoring.TopologyListener): def opened(self, event): logging.info("Topology with id {0.topology_id} " "opened".format(event)) def description_changed(self, event): logging.info("Topology description updated for " "topology id {0.topology_id}".format(event)) previous_topology_type = event.previous_description.topology_type new_topology_type = event.new_description.topology_type if new_topology_type != previous_topology_type: logging.info( "Topology {0.topology_id} changed type from " "{0.previous_description.topology_type_name} to " "{0.new_description.topology_type_name}".format(event)) def closed(self, event): logging.info("Topology with id {0.topology_id} " "closed".format(event)) monitoring.register(TopologyLogger()) # topology logger end # heartbeat logger start class HeartbeatLogger(monitoring.ServerHeartbeatListener): def started(self, event): logging.info("Heartbeat sent to server " "{0.connection_id}".format(event)) def succeeded(self, event): logging.info("Heartbeat to server {0.connection_id} " "succeeded with reply " "{0.reply.document}".format(event)) def failed(self, event): logging.warning("Heartbeat to server {0.connection_id} " "failed with error {0.reply}".format(event)) monitoring.register(HeartbeatLogger()) # heartbeat logger end motor-1.2.1/doc/examples/tailable-cursors.rst000066400000000000000000000020541323007662700212520ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado Motor Tailable Cursor Example ============================= This example describes using Motor with Tornado. Beginning in version 0.5 Motor can also integrate with asyncio instead of Tornado. A cursor on a capped collection can be tailed using :meth:`~MotorCursor.fetch_next`: .. code-block:: python @gen.coroutine def tail_example(): results = [] collection = db.my_capped_collection cursor = collection.find(cursor_type=CursorType.TAILABLE, await_data=True) while True: if not cursor.alive: now = datetime.datetime.utcnow() # While collection is empty, tailable cursor dies immediately yield gen.sleep(1) cursor = collection.find(cursor_type=CursorType.TAILABLE, await_data=True) if (yield cursor.fetch_next): results.append(cursor.next_object()) print results .. seealso:: `Tailable cursors `_ motor-1.2.1/doc/examples/tornado_change_stream_example.py000066400000000000000000000070051323007662700236610ustar00rootroot00000000000000import logging import os import sys from base64 import urlsafe_b64encode from pprint import pformat from motor.motor_tornado import MotorClient from bson import json_util # Installed with PyMongo. import tornado.escape import tornado.ioloop import tornado.options import tornado.web import tornado.websocket from tornado.options import define, options define("port", default=8888, help="run on the given port", type=int) define("debug", default=False, help="reload on source changes") define("mongo", default="mongodb://localhost", help="MongoDB URI") define("ns", default="test.test", help="database and collection name") class Application(tornado.web.Application): def __init__(self): handlers = [ (r"/", MainHandler), (r"/socket", ChangesHandler)] templates = os.path.join(os.path.dirname(__file__), "tornado_change_stream_templates") super(Application, self).__init__(handlers, template_path=templates, template_whitespace="all", debug=options.debug) class MainHandler(tornado.web.RequestHandler): def get(self): self.render("index.html", changes=ChangesHandler.cache) class ChangesHandler(tornado.websocket.WebSocketHandler): waiters = set() cache = [] cache_size = 5 def open(self): ChangesHandler.waiters.add(self) def on_close(self): ChangesHandler.waiters.remove(self) @classmethod def update_cache(cls, change): cls.cache.append(change) if len(cls.cache) > cls.cache_size: cls.cache = cls.cache[-cls.cache_size:] @classmethod def send_change(cls, change): change_json = json_util.dumps(change) for waiter in cls.waiters: try: waiter.write_message(change_json) except Exception: logging.error("Error sending message", exc_info=True) @classmethod def on_change(cls, change): logging.info("got change of type '%s'", change.get('operationType')) # Each change notification has a binary _id. Use it to make an HTML # element id, then remove it. html_id = urlsafe_b64encode(change['_id']['_data']).decode().rstrip('=') change.pop('_id') change['html'] = '
      %s
      ' % ( html_id, tornado.escape.xhtml_escape(pformat(change))) change['html_id'] = html_id ChangesHandler.send_change(change) ChangesHandler.update_cache(change) change_stream = None async def watch(collection): global change_stream async with collection.watch() as change_stream: async for change in change_stream: ChangesHandler.on_change(change) def main(): tornado.options.parse_command_line() if '.' not in options.ns: sys.stderr.write('Invalid ns "%s", must contain a "."' % (options.ns,)) sys.exit(1) db_name, collection_name = options.ns.split('.', 1) client = MotorClient(options.mongo) collection = client[db_name][collection_name] app = Application() app.listen(options.port) loop = tornado.ioloop.IOLoop.current() # Start watching collection for changes. loop.add_callback(watch, collection) try: loop.start() except KeyboardInterrupt: pass finally: if change_stream is not None: change_stream.close() if __name__ == "__main__": main() motor-1.2.1/doc/examples/tornado_change_stream_example.rst000066400000000000000000000027561323007662700240510ustar00rootroot00000000000000.. _tornado_change_stream_example: Tornado Change Stream Example ============================= .. currentmodule:: motor.motor_tornado Watch a collection for changes with :meth:`MotorCollection.watch` and display each change notification on a web page using web sockets. Instructions ------------ Start a MongoDB server on its default port and run this script. Then visit: http://localhost:8888 Open a ``mongo`` shell in the terminal and perform some operations on the "test" collection in the "test" database: .. code-block:: none > use test switched to db test > db.test.insertOne({}) > db.test.updateOne({}, {$set: {x: 1}}) > db.test.deleteOne({}) The application receives each change notification and displays it as JSON on the web page: .. code-block:: none Changes {'documentKey': {'_id': ObjectId('5a2a6967ea2dcf7b1c721cfb')}, 'fullDocument': {'_id': ObjectId('5a2a6967ea2dcf7b1c721cfb')}, 'ns': {'coll': 'test', 'db': 'test'}, 'operationType': 'insert'} {'documentKey': {'_id': ObjectId('5a2a6967ea2dcf7b1c721cfb')}, 'ns': {'coll': 'test', 'db': 'test'}, 'operationType': 'update', 'updateDescription': {'removedFields': [], 'updatedFields': {'x': 1.0}}} {'documentKey': {'_id': ObjectId('5a2a6967ea2dcf7b1c721cfb')}, 'ns': {'coll': 'test', 'db': 'test'}, 'operationType': 'delete'} Display change notifications over a web socket ---------------------------------------------- .. literalinclude:: tornado_change_stream_example.py :language: python3 motor-1.2.1/doc/examples/tornado_change_stream_templates/000077500000000000000000000000001323007662700236505ustar00rootroot00000000000000motor-1.2.1/doc/examples/tornado_change_stream_templates/index.html000066400000000000000000000023241323007662700256460ustar00rootroot00000000000000

      Changes

      {% for change in changes %} {% raw change['html'] %} {% end %}
      motor-1.2.1/doc/features.rst000066400000000000000000000026551323007662700160060ustar00rootroot00000000000000============== Motor Features ============== .. currentmodule:: motor.motor_tornado Non-Blocking ============ Motor is an asynchronous driver for MongoDB. It can be used from Tornado_ or asyncio_ applications. Motor never blocks the event loop while connecting to MongoDB or performing I/O. .. _Tornado: http://tornadoweb.org/ .. _asyncio: https://docs.python.org/3/library/asyncio.html Featureful ========== Motor wraps almost all of PyMongo's API and makes it non-blocking. For the few PyMongo features not implemented in Motor, see :doc:`differences`. Convenient With `tornado.gen` ============================= The :mod:`tornado.gen` module lets you use coroutines to simplify asynchronous code. Motor methods return Futures that are convenient to use with coroutines. See :ref:`the coroutine example `. Configurable IOLoops ==================== Motor supports Tornado applications with multiple :class:`IOLoops `. Pass the ``io_loop`` argument to :class:`MotorClient` to configure the loop for a client instance. Streams Static Files from GridFS ================================ Motor can stream data from `GridFS `_ to a Tornado :class:`~tornado.web.RequestHandler` using :meth:`~MotorGridOut.stream_to_handler` or the :class:`~motor.web.GridFSHandler` class. It can also serve GridFS data with aiohttp using the :class:`~motor.aiohttp.AIOHTTPGridFS` class. motor-1.2.1/doc/index.rst000066400000000000000000000041711323007662700152720ustar00rootroot00000000000000Motor: Asynchronous Python driver for MongoDB ============================================= .. image:: _static/motor.png :align: center About ----- Motor presents a callback- or Future-based API for non-blocking access to MongoDB from Tornado_ or asyncio_. The `source is on GitHub `_ and the docs are on `ReadTheDocs `_. "We use Motor in high throughput environments, processing tens of thousands of requests per second. It allows us to take full advantage of modern hardware, ensuring we utilise the entire capacity of our purchased CPUs. This helps us be more efficient with computing power, compute spend and minimises the environmental impact of our infrastructure as a result." --*David Mytton, Server Density* "We develop easy-to-use sensors and sensor systems with open source software to ensure every innovator, from school child to laboratory researcher, has the same opportunity to create. We integrate Motor into our software to guarantee massively scalable sensor systems for everyone." --*Ryan Smith, inXus Interactive* Install with:: $ python -m pip install motor .. _Tornado: http://tornadoweb.org/ .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _PyMongo: http://pypi.python.org/pypi/pymongo/ How To Ask For Help ------------------- Post questions about Motor to the `mongodb-user list on Google Groups `_. For confirmed issues or feature requests, open a case in `Jira `_ in the "MOTOR" project. Contents -------- .. toctree:: :maxdepth: 1 differences features installation requirements configuration tutorial-tornado tutorial-asyncio examples/index changelog migrate-to-motor-1 developer-guide contributors Classes ------- .. toctree:: api-tornado/index api-asyncio/index .. getting the caption italicized with a hyperlink in it requires some RST hackage *Logo by* |musho|_ .. _musho: http://whimsyload.com .. |musho| replace:: *Musho Rodney Alan Greenblat* motor-1.2.1/doc/installation.rst000066400000000000000000000004051323007662700166600ustar00rootroot00000000000000Installation ============ Install Motor from PyPI_ with pip_:: $ python -m pip install motor Pip automatically installs Motor's prerequisite packages. See :doc:`requirements`. .. _PyPI: http://pypi.python.org/pypi/motor .. _pip: http://pip-installer.org motor-1.2.1/doc/make.bat000066400000000000000000000057751323007662700150510ustar00rootroot00000000000000@ECHO OFF REM Command file for Sphinx documentation set SPHINXBUILD=sphinx-build set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. changes to make an overview over all changed/added/deprecated items echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Motor.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Motor.ghc goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) :end motor-1.2.1/doc/migrate-to-motor-1.rst000066400000000000000000000263111323007662700175270ustar00rootroot00000000000000Motor 1.0 Migration Guide ========================= .. currentmodule:: motor.motor_tornado Motor 1.0 brings a number of backward breaking changes to the pre-1.0 API. Follow this guide to migrate an existing application that had used an older version of Motor. Removed features with no migration path --------------------------------------- :class:`MotorReplicaSetClient` is removed .......................................... In Motor 1.0, :class:`MotorClient` is the only class. Connect to a replica set with a "replicaSet" URI option or parameter:: MotorClient("mongodb://localhost/?replicaSet=my-rs") MotorClient(host, port, replicaSet="my-rs") The "compile_re" option is removed .................................. In Motor 1.0 regular expressions are never compiled to Python `re.match` objects. Motor 0.7 --------- The first step in migrating to Motor 1.0 is to upgrade to at least Motor 0.7. If your project has a requirements.txt file, add the line:: motor >= 0.7, < 1.0 Most of the key new methods and options from Motor 1.0 are backported in Motor 0.7 making migration much easier. Enable Deprecation Warnings --------------------------- Starting with Motor 0.7, :exc:`DeprecationWarning` is raised by most methods removed in Motor 1.0. Make sure you enable runtime warnings to see where deprecated functions and methods are being used in your application:: python -Wd Warnings can also be changed to errors:: python -Wd -Werror Not all deprecated features raise `DeprecationWarning` when used. For example, `~motor.motor_tornado.MotorReplicaSetClient` will be removed in Motor 1.0 but it does not raise `DeprecationWarning` in Motor 0.7. See also `Removed features with no migration path`_. CRUD API -------- Changes to find() and find_one() ................................ "spec" renamed "filter" ~~~~~~~~~~~~~~~~~~~~~~~ The ``spec`` option has been renamed to ``filter``. Code like this:: cursor = collection.find(spec={"a": 1}) can be changed to this with Motor 0.7 or later:: cursor = collection.find(filter={"a": 1}) or this with any version of Motor:: cursor = collection.find({"a": 1}) "fields" renamed "projection" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``fields`` option has been renamed to ``projection``. Code like this:: cursor = collection.find({"a": 1}, fields={"_id": False}) can be changed to this with Motor 0.7 or later:: cursor = collection.find({"a": 1}, projection={"_id": False}) or this with any version of Motor:: cursor = collection.find({"a": 1}, {"_id": False}) "partial" renamed "allow_partial_results" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``partial`` option has been renamed to ``allow_partial_results``. Code like this:: cursor = collection.find({"a": 1}, partial=True) can be changed to this with Motor 0.7 or later:: cursor = collection.find({"a": 1}, allow_partial_results=True) "timeout" replaced by "no_cursor_timeout" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``timeout`` option has been replaced by ``no_cursor_timeout``. Code like this:: cursor = collection.find({"a": 1}, timeout=False) can be changed to this with Motor 0.7 or later:: cursor = collection.find({"a": 1}, no_cursor_timeout=True) "snapshot" and "max_scan" replaced by "modifiers" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``snapshot`` and ``max_scan`` options have been removed. They can now be set, along with other $ query modifiers, through the ``modifiers`` option. Code like this:: cursor = collection.find({"a": 1}, snapshot=True) can be changed to this with Motor 0.7 or later:: cursor = collection.find({"a": 1}, modifiers={"$snapshot": True}) or with any version of Motor:: cursor = collection.find({"$query": {"a": 1}, "$snapshot": True}) "network_timeout" is removed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``network_timeout`` option has been removed. This option was always the wrong solution for timing out long running queries and should never be used in production. Starting with **MongoDB 2.6** you can use the $maxTimeMS query modifier. Code like this:: # Set a 5 second select() timeout. cursor = collection.find({"a": 1}, network_timeout=5) can be changed to this with Motor 0.7 or later:: # Set a 5 second (5000 millisecond) server side query timeout. cursor = collection.find({"a": 1}, modifiers={"$maxTimeMS": 5000}) or with any version of Motor:: cursor = collection.find({"$query": {"a": 1}, "$maxTimeMS": 5000}) .. seealso:: `$maxTimeMS `_ Tailable cursors ~~~~~~~~~~~~~~~~ The ``tailable`` and ``await_data`` options have been replaced by ``cursor_type``. Code like this:: cursor = collection.find({"a": 1}, tailable=True) cursor = collection.find({"a": 1}, tailable=True, await_data=True) can be changed to this with Motor 0.7 or later:: from pymongo import CursorType cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE) cursor = collection.find({"a": 1}, cursor_type=CursorType.TAILABLE_AWAIT) Other removed options ~~~~~~~~~~~~~~~~~~~~~ The ``read_preference``, ``tag_sets``, and ``secondary_acceptable_latency_ms`` options have been removed. See the `Read Preferences`_ section for solutions. Read Preferences ---------------- The "read_preference" attribute is immutable ............................................ Code like this:: from pymongo import ReadPreference db = client.my_database db.read_preference = ReadPreference.SECONDARY can be changed to this with Motor 0.7 or later:: db = client.get_database("my_database", read_preference=ReadPreference.SECONDARY) Code like this:: cursor = collection.find({"a": 1}, read_preference=ReadPreference.SECONDARY) can be changed to this with Motor 0.7 or later:: coll2 = collection.with_options(read_preference=ReadPreference.SECONDARY) cursor = coll2.find({"a": 1}) .. seealso:: :meth:`~MotorDatabase.get_collection` The "tag_sets" option and attribute are removed ............................................... The ``tag_sets`` MotorClient option is removed. The ``read_preference`` option can be used instead. Code like this:: client = MotorClient( read_preference=ReadPreference.SECONDARY, tag_sets=[{"dc": "ny"}, {"dc": "sf"}]) can be changed to this with Motor 0.7 or later:: from pymongo.read_preferences import Secondary client = MotorClient(read_preference=Secondary([{"dc": "ny"}])) To change the tags sets for a MotorDatabase or MotorCollection, code like this:: db = client.my_database db.read_preference = ReadPreference.SECONDARY db.tag_sets = [{"dc": "ny"}] can be changed to this with Motor 0.7 or later:: db = client.get_database("my_database", read_preference=Secondary([{"dc": "ny"}])) Code like this:: cursor = collection.find( {"a": 1}, read_preference=ReadPreference.SECONDARY, tag_sets=[{"dc": "ny"}]) can be changed to this with Motor 0.7 or later:: from pymongo.read_preferences import Secondary coll2 = collection.with_options( read_preference=Secondary([{"dc": "ny"}])) cursor = coll2.find({"a": 1}) .. seealso:: :meth:`~MotorDatabase.get_collection` The "secondary_acceptable_latency_ms" option and attribute are removed ...................................................................... Motor 0.x supports ``secondary_acceptable_latency_ms`` as an option to methods throughout the driver, but mongos only supports a global latency option. Motor 1.0 has changed to match the behavior of mongos, allowing migration from a single server, to a replica set, to a sharded cluster without a surprising change in server selection behavior. A new option, ``localThresholdMS``, is available through MotorClient and should be used in place of ``secondaryAcceptableLatencyMS``. Code like this:: client = MotorClient(readPreference="nearest", secondaryAcceptableLatencyMS=100) can be changed to this with Motor 0.7 or later:: client = MotorClient(readPreference="nearest", localThresholdMS=100) Write Concern ------------- The "write_concern" attribute is immutable .......................................... The ``write_concern`` attribute is immutable in Motor 1.0. Code like this:: client = MotorClient() client.write_concern = {"w": "majority"} can be changed to this with any version of Motor:: client = MotorClient(w="majority") Code like this:: db = client.my_database db.write_concern = {"w": "majority"} can be changed to this with Motor 0.7 or later:: from pymongo import WriteConcern db = client.get_database("my_database", write_concern=WriteConcern(w="majority")) The new CRUD API write methods do not accept write concern options. Code like this:: oid = collection.insert({"a": 2}, w="majority") can be changed to this with Motor 0.7 or later:: from pymongo import WriteConcern coll2 = collection.with_options( write_concern=WriteConcern(w="majority")) oid = coll2.insert({"a": 2}) .. seealso:: :meth:`~MotorDatabase.get_collection` Codec Options ------------- The "document_class" attribute is removed ......................................... Code like this:: from bson.son import SON client = MotorClient() client.document_class = SON can be replaced by this in any version of Motor:: from bson.son import SON client = MotorClient(document_class=SON) or to change the ``document_class`` for a :class:`MotorDatabase` with Motor 0.7 or later:: from bson.codec_options import CodecOptions from bson.son import SON db = client.get_database("my_database", CodecOptions(SON)) .. seealso:: :meth:`~MotorDatabase.get_collection` and :meth:`~MotorCollection.with_options` The "uuid_subtype" option and attribute are removed ................................................... Code like this:: from bson.binary import JAVA_LEGACY db = client.my_database db.uuid_subtype = JAVA_LEGACY can be replaced by this with Motor 0.7 or later:: from bson.binary import JAVA_LEGACY from bson.codec_options import CodecOptions db = client.get_database("my_database", CodecOptions(uuid_representation=JAVA_LEGACY)) .. seealso:: :meth:`~MotorDatabase.get_collection` and :meth:`~MotorCollection.with_options` MotorClient ----------- The ``open`` method ................... The :meth:`~MotorClient.open` method is removed in Motor 1.0. Motor clients have opened themselves on demand since Motor 0.2. The max_pool_size parameter is removed ...................................... Motor 1.0 replaced the max_pool_size parameter with support for the MongoDB URI ``maxPoolSize`` option. Code like this:: client = MotorClient(max_pool_size=10) can be replaced by this with Motor 0.7 or later:: client = MotorClient(maxPoolSize=10) client = MotorClient("mongodb://localhost:27017/?maxPoolSize=10") The "disconnect" method is removed .................................. Code like this:: client.disconnect() can be replaced by this:: client.close() The host and port attributes are removed ........................................ Code like this:: host = client.host port = client.port can be replaced by this with Motor 0.7 or later:: address = client.address host, port = address or (None, None) motor-1.2.1/doc/mongo_extensions.py000066400000000000000000000057671323007662700174150ustar00rootroot00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """MongoDB specific extensions to Sphinx.""" from docutils import nodes from sphinx import addnodes from sphinx.util.compat import Directive class mongodoc(nodes.Admonition, nodes.Element): pass class mongoref(nodes.reference): pass def visit_mongodoc_node(self, node): self.visit_admonition(node) def depart_mongodoc_node(self, node): self.depart_admonition(node) def visit_mongoref_node(self, node): atts = {"class": "reference external", "href": node["refuri"], "name": node["name"]} self.body.append(self.starttag(node, 'a', '', **atts)) def depart_mongoref_node(self, node): self.body.append('') if not isinstance(node.parent, nodes.TextElement): self.body.append('\n') class MongodocDirective(Directive): has_content = True required_arguments = 0 optional_arguments = 0 final_argument_whitespace = False option_spec = {} def run(self): node = mongodoc() title = 'See general MongoDB documentation' node += nodes.title(title, title) self.state.nested_parse(self.content, self.content_offset, node) return [node] def process_mongodoc_nodes(app, doctree, fromdocname): for node in doctree.traverse(mongodoc): anchor = None for name in node.parent.parent.traverse(addnodes.desc_signature): anchor = name["ids"][0] break if not anchor: for name in node.parent.traverse(nodes.section): anchor = name["ids"][0] break for para in node.traverse(nodes.paragraph): tag = str(para.traverse()[1]) link = mongoref("", "") link["refuri"] = "http://dochub.mongodb.org/core/%s" % tag link["name"] = anchor link.append(nodes.emphasis(tag, tag)) new_para = nodes.paragraph() new_para += link node.replace(para, new_para) def setup(app): app.add_node(mongodoc, html=(visit_mongodoc_node, depart_mongodoc_node), latex=(visit_mongodoc_node, depart_mongodoc_node), text=(visit_mongodoc_node, depart_mongodoc_node)) app.add_node(mongoref, html=(visit_mongoref_node, depart_mongoref_node)) app.add_directive("mongodoc", MongodocDirective) app.connect("doctree-resolved", process_mongodoc_nodes) return {'parallel_write_safe': True, 'parallel_read_safe': True} motor-1.2.1/doc/motor_extensions.py000066400000000000000000000302761323007662700174270ustar00rootroot00000000000000# Copyright 2012-2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Motor specific extensions to Sphinx.""" import inspect import re from itertools import chain from docutils.nodes import field, list_item, paragraph, title_reference, literal from docutils.nodes import field_list, field_body, bullet_list, Text, field_name from docutils.nodes import literal_block, doctest_block from sphinx import addnodes from sphinx.addnodes import (desc, desc_content, versionmodified, desc_signature, seealso, pending_xref) from sphinx.util.inspect import safe_getattr import motor import motor.core # This is a place to store info while parsing, to be used before generating. motor_info = {} def is_asyncio_api(name): return 'motor_asyncio.' in name def has_node_of_type(root, klass): if isinstance(root, klass): return True for child in root.children: if has_node_of_type(child, klass): return True return False def find_by_path(root, classes): if not classes: return [root] _class = classes[0] rv = [] for child in root.children: if isinstance(child, _class): rv.extend(find_by_path(child, classes[1:])) return rv def get_parameter_names(parameters_node): parameter_names = [] # Most PyMongo methods have bullet lists. for list_item_node in find_by_path(parameters_node, [list_item]): title_ref_nodes = find_by_path( list_item_node, [paragraph, (title_reference, pending_xref)]) parameter_names.append(title_ref_nodes[0].astext()) # Some are just paragraphs. for title_ref_node in find_by_path(parameters_node, [title_reference]): parameter_names.append(title_ref_node.astext()) return parameter_names def insert_callback(parameters_node): # We need to know what params are here already parameter_names = get_parameter_names(parameters_node) if 'callback' not in parameter_names: if '*args' in parameter_names: args_pos = parameter_names.index('*args') else: args_pos = len(parameter_names) if '**kwargs' in parameter_names: kwargs_pos = parameter_names.index('**kwargs') else: kwargs_pos = len(parameter_names) doc = ( " (optional): function taking (result, error), executed when" " operation completes") new_item = paragraph( '', '', literal('', 'callback'), Text(doc)) if parameters_node.children and isinstance(parameters_node.children[0], list_item): # Insert "callback" before *args and **kwargs parameters_node.insert(min(args_pos, kwargs_pos), list_item('', new_item)) else: parameters_node.append(new_item) docstring_warnings = [] def maybe_warn_about_code_block(name, content_node): if has_node_of_type(content_node, (literal_block, doctest_block)): docstring_warnings.append(name) def has_coro_annotation(signature_node): try: return 'coroutine' in signature_node[0][0] except IndexError: return False def process_motor_nodes(app, doctree): # Search doctree for Motor's methods and attributes whose docstrings were # copied from PyMongo, and fix them up for Motor: # 1. Add a 'callback' param (sometimes optional, sometimes required) to # all Motor Tornado methods. If the PyMongo method took no params, we # create a parameter-list from scratch, otherwise we edit PyMongo's # list. # 2. Remove all version annotations like "New in version 2.0" since # PyMongo's version numbers are meaningless in Motor's docs. # 3. Remove "seealso" directives that reference PyMongo's docs. # # We do this here, rather than by registering a callback to Sphinx's # 'autodoc-process-signature' event, because it's way easier to handle the # parsed doctree before it's turned into HTML than it is to update the RST. for objnode in doctree.traverse(desc): if objnode['objtype'] in ('method', 'attribute'): signature_node = find_by_path(objnode, [desc_signature])[0] name = '.'.join([ signature_node['module'], signature_node['fullname']]) assert name.startswith('motor.') obj_motor_info = motor_info.get(name) if obj_motor_info: desc_content_node = find_by_path(objnode, [desc_content])[0] if (desc_content_node.line is None and obj_motor_info['is_pymongo_docstring']): maybe_warn_about_code_block(name, desc_content_node) if obj_motor_info['is_async_method']: # Might be a handwritten RST with "coroutine" already. if not has_coro_annotation(signature_node): coro_annotation = addnodes.desc_annotation( 'coroutine ', 'coroutine ', classes=['coro-annotation']) signature_node.insert(0, coro_annotation) if (not is_asyncio_api(name) and obj_motor_info['coroutine_has_callback']): retval = ("If a callback is passed, returns None, else" " returns a Future.") callback_p = paragraph('', Text(retval)) # Find the parameter list. parameters_nodes = find_by_path( desc_content_node, [ field_list, field, field_body, (bullet_list, paragraph)]) if parameters_nodes: parameters_node = parameters_nodes[0] else: # PyMongo method has no parameters, create an empty # params list parameters_node = bullet_list() parameters_field_list_node = field_list( '', field( '', field_name('', 'Parameters '), field_body('', parameters_node))) desc_content_node.append(parameters_field_list_node) insert_callback(parameters_node) if retval not in str(desc_content_node): desc_content_node.append(callback_p) if obj_motor_info['is_pymongo_docstring']: # Remove all "versionadded", "versionchanged" and # "deprecated" directives from the docs we imported from # PyMongo version_nodes = find_by_path( desc_content_node, [versionmodified]) for version_node in version_nodes: version_node.parent.remove(version_node) # Remove all "seealso" directives that contain :doc: # references from PyMongo's docs seealso_nodes = find_by_path(desc_content_node, [seealso]) for seealso_node in seealso_nodes: if 'reftype="doc"' in str(seealso_node): seealso_node.parent.remove(seealso_node) def get_motor_attr(motor_class, name, *defargs): """If any Motor attributes can't be accessed, grab the equivalent PyMongo attribute. While we're at it, store some info about each attribute in the global motor_info dict. """ attr = safe_getattr(motor_class, name) # Store some info for process_motor_nodes() full_name = '%s.%s.%s' % ( motor_class.__module__, motor_class.__name__, name) full_name_legacy = 'motor.%s.%s.%s' % ( motor_class.__module__, motor_class.__name__, name) # These sub-attributes are set in motor.asynchronize() has_coroutine_annotation = getattr(attr, 'coroutine_annotation', False) is_async_method = getattr(attr, 'is_async_method', False) coroutine_has_callback = has_coroutine_annotation or is_async_method if has_coroutine_annotation: coroutine_has_callback = getattr(attr, 'coroutine_has_callback', True) is_cursor_method = getattr(attr, 'is_motorcursor_chaining_method', False) if is_async_method or is_cursor_method: pymongo_method = getattr( motor_class.__delegate_class__, attr.pymongo_method_name) else: pymongo_method = None # attr.doc is set by statement like 'error = AsyncRead(doc="OBSOLETE")'. is_pymongo_doc = pymongo_method and attr.__doc__ == pymongo_method.__doc__ motor_info[full_name] = motor_info[full_name_legacy] = { 'is_async_method': is_async_method or has_coroutine_annotation, 'coroutine_has_callback': coroutine_has_callback, 'is_pymongo_docstring': is_pymongo_doc, 'pymongo_method': pymongo_method, } return attr def get_motor_argspec(name, method): args, varargs, kwargs, defaults = inspect.getargspec(method) # This part is copied from Sphinx's autodoc.py if args and args[0] in ('cls', 'self'): del args[0] defaults = list(defaults) if defaults else [] add_callback = True if 'callback' in chain(args or [], kwargs or []): add_callback = False elif is_asyncio_api(name): add_callback = False elif (getattr(method, 'coroutine_annotation', False) and not getattr(method, 'coroutine_has_callback', True)): add_callback = False if add_callback: # Add 'callback=None' argument args.append('callback') defaults.append(None) return args, varargs, kwargs, defaults # Adapted from MethodDocumenter.format_args def format_motor_args(name, motor_method, pymongo_method): if pymongo_method: argspec = get_motor_argspec(name, pymongo_method) else: argspec = get_motor_argspec(name, motor_method) formatted_argspec = inspect.formatargspec(*argspec) # escape backslashes for reST return formatted_argspec.replace('\\', '\\\\') pymongo_ref_pat = re.compile(r':doc:`(.*?)`', re.MULTILINE) def _sub_pymongo_ref(match): ref = match.group(1) return ':doc:`%s`' % ref.lstrip('/') def process_motor_docstring(app, what, name, obj, options, lines): if name in motor_info and motor_info[name].get('is_pymongo_docstring'): joined = '\n'.join(lines) subbed = pymongo_ref_pat.sub(_sub_pymongo_ref, joined) lines[:] = subbed.split('\n') def process_motor_signature( app, what, name, obj, options, signature, return_annotation): info = motor_info.get(name) if info: # Real sig obscured by decorator, reconstruct it pymongo_method = info['pymongo_method'] if info['is_async_method']: args = format_motor_args(name, obj, pymongo_method) return args, return_annotation def build_finished(app, exception): if not exception and docstring_warnings: print("PyMongo docstrings with code blocks that need update:") for name in sorted(docstring_warnings): print(name) def setup(app): app.add_autodoc_attrgetter(type(motor.core.AgnosticBase), get_motor_attr) app.connect('autodoc-process-docstring', process_motor_docstring) app.connect('autodoc-process-signature', process_motor_signature) app.connect('doctree-read', process_motor_nodes) app.connect('build-finished', build_finished) return {'parallel_write_safe': True, 'parallel_read_safe': False} motor-1.2.1/doc/pydoctheme/000077500000000000000000000000001323007662700155675ustar00rootroot00000000000000motor-1.2.1/doc/pydoctheme/static/000077500000000000000000000000001323007662700170565ustar00rootroot00000000000000motor-1.2.1/doc/pydoctheme/static/pydoctheme.css000066400000000000000000000056441323007662700217420ustar00rootroot00000000000000@import url("classic.css"); body { background-color: white; margin-left: 1em; margin-right: 1em; } div.related { margin-bottom: 1.2em; padding: 0.5em 0; border-top: 1px solid #ccc; margin-top: 0.5em; } div.related a:hover { color: #0095C4; } div.related:first-child { border-top: 0; border-bottom: 1px solid #ccc; } div.sphinxsidebar { background-color: #eeeeee; border-radius: 5px; line-height: 130%; font-size: smaller; } div.sphinxsidebar h3, div.sphinxsidebar h4 { margin-top: 1.5em; } div.sphinxsidebarwrapper > h3:first-child { margin-top: 0.2em; } div.sphinxsidebarwrapper > ul > li > ul > li { margin-bottom: 0.4em; } div.sphinxsidebar a:hover { color: #0095C4; } div.sphinxsidebar input { font-family: 'Lucida Grande',Arial,sans-serif; border: 1px solid #999999; font-size: smaller; border-radius: 3px; } div.sphinxsidebar input[type=text] { max-width: 150px; } /* override an apparently misguided "hyphens: auto" style in Sphinx basic.css */ div.body p, div.body dd, div.body li, div.body blockquote { -moz-hyphens: none; -ms-hyphens: none; -webkit-hyphens: none; hyphens: none; } div.body { padding: 0 0 0 1.2em; } div.body p { line-height: 140%; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { margin: 0; border: 0; padding: 0.3em 0; } div.body hr { border: 0; background-color: #ccc; height: 1px; } div.body pre { border-radius: 3px; border: 1px solid #ac9; } div.body div.admonition, div.body div.impl-detail { border-radius: 3px; } div.body div.impl-detail > p { margin: 0; } div.body div.seealso { border: 1px solid #dddd66; } div.body a { color: #0072aa; } div.body a:visited { color: #6363bb; } div.body a:hover { color: #00B0E4; } tt, code, pre { font-family: monospace, sans-serif; font-size: 96.5%; } div.body tt, div.body code { border-radius: 3px; } div.body tt.descname, div.body code.descname { font-size: 120%; } div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { font-weight: normal; } .deprecated { border-radius: 3px; } table.docutils { border: 1px solid #ddd; min-width: 20%; border-radius: 3px; margin-top: 10px; margin-bottom: 10px; } table.docutils td, table.docutils th { border: 1px solid #ddd !important; border-radius: 3px; } table p, table li { text-align: left !important; } table.docutils th { background-color: #eee; padding: 0.3em 0.5em; } table.docutils td { background-color: white; padding: 0.3em 0.5em; } table.footnote, table.footnote td { border: 0 !important; } div.footer { line-height: 150%; margin-top: -2em; text-align: right; width: auto; margin-right: 10px; } div.footer a:hover { color: #0095C4; } .refcount { color: #060; } .stableabi { color: #229; } motor-1.2.1/doc/pydoctheme/theme.conf000066400000000000000000000010231323007662700175340ustar00rootroot00000000000000[theme] inherit = classic stylesheet = pydoctheme.css pygments_style = sphinx [options] bodyfont = 'Lucida Grande', Arial, sans-serif headfont = 'Lucida Grande', Arial, sans-serif footerbgcolor = white footertextcolor = #555555 relbarbgcolor = white relbartextcolor = #666666 relbarlinkcolor = #444444 sidebarbgcolor = white sidebartextcolor = #444444 sidebarlinkcolor = #444444 bgcolor = white textcolor = #222222 linkcolor = #0090c0 visitedlinkcolor = #00608f headtextcolor = #1a1a1a headbgcolor = white headlinkcolor = #aaaaaa motor-1.2.1/doc/requirements.rst000066400000000000000000000201451323007662700167050ustar00rootroot00000000000000Requirements ============ The current version of Motor requires: * CPython 2.7, or 3.4 and later. * PyMongo_ 3.6 and later. Motor can integrate with either Tornado or asyncio. Requires the `futures`_ package from PyPI on Python 2. The default authentication mechanism for MongoDB 3.0+ is SCRAM-SHA-1. Install `backports.pbkdf2`_ for faster authentication with MongoDB 3.0+, especially on Python older than 2.7.8. (Python 2.7.9 and later, or Python 3.4 and later, have builtin hash functions nearly as fast as backports.pbkdf2.) Building the docs requires `sphinx`_. .. _PyMongo: https://pypi.python.org/pypi/pymongo/ .. _futures: https://pypi.python.org/pypi/futures .. _backports.pbkdf2: https://pypi.python.org/pypi/backports.pbkdf2/ .. _sphinx: http://sphinx.pocoo.org/ .. _compatibility-matrix: Compatibility Matrix -------------------- Motor and PyMongo ````````````````` Older versions of Motor depended on exact PyMongo versions. Version 0.7 requires the latest PyMongo 2.9.x release beginning with 2.9.4, Version 1.0 works with any PyMongo version beginning with 3.3.0, and Version 1.1 works with any PyMongo version beginning with 3.4.0. +-------------------+-----------------+ | Motor Version | PyMongo Version | +===================+=================+ | 0.1 | 2.5.0 | +-------------------+-----------------+ | 0.2 | 2.7.0 | +-------------------+-----------------+ | 0.3 | 2.7.1 | +-------------------+-----------------+ | 0.4 | 2.8.0 | +-------------------+-----------------+ | 0.5 | 2.8.0 | +-------------------+-----------------+ | 0.6 | 2.8.0 | +-------------------+-----------------+ | 0.7 | 2.9.4+ | +-------------------+-----------------+ | 1.0 | 3.3+ | +-------------------+-----------------+ | 1.1 | 3.4+ | +-------------------+-----------------+ | 1.2 | 3.6+ | +-------------------+-----------------+ Motor and MongoDB ````````````````` All Motor versions are usable with all MongoDB versions as old as 2.2. Where "N" appears there are some incompatibilities and unsupported server features. +---------------------------------------------------------+ | MongoDB Version | +=====================+=====+=====+=====+=====+=====+=====+ | | 2.2 | 2.4 | 2.6 | 3.0 | 3.2 | 3.4 | +---------------+-----+-----+-----+-----+-----+-----+-----+ | Motor Version | 0.1 | Y | Y |**N**|**N**|**N**|**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 0.2 | Y | Y | Y |**N**|**N**|**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 0.3 | Y | Y | Y |**N**|**N**|**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 0.4 | Y | Y | Y | Y |**N**|**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 0.5 | Y | Y | Y | Y |**N**|**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 0.6 | Y | Y | Y | Y |**N**|**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 0.7 | Y | Y | Y | Y | Y |**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 1.0 | Y | Y | Y | Y | Y |**N**| +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 1.1 | Y | Y | Y | Y | Y | Y | +---------------+-----+-----+-----+-----+-----+-----+-----+ | | 1.2 |**N**|**N**| Y | Y | Y | Y | +---------------+-----+-----+-----+-----+-----+-----+-----+ There is no relationship between PyMongo and MongoDB version numbers, although the numbers happen to be close or equal in recent releases of PyMongo and MongoDB. Use `the PyMongo compatibility matrix`_ to determine what MongoDB version is supported by PyMongo. Use the compatibility matrix above to determine what MongoDB version Motor supports. .. _the PyMongo compatibility matrix: https://docs.mongodb.org/ecosystem/drivers/python/#mongodb-compatibility Motor and Tornado ````````````````` Where "N" appears in this matrix, the versions of Motor and Tornado are known to be incompatible, or have not been tested together. +---------------------------------+ | Tornado Version | +=====================+=====+=====+ | | 3.x | 4.x | +---------------+-----+-----+-----+ | Motor Version | 0.1 | Y |**N**| +---------------+-----+-----+-----+ | | 0.2 | Y | Y | +---------------+-----+-----+-----+ | | 0.3 | Y | Y | +---------------+-----+-----+-----+ | | 0.4 | Y | Y | +---------------+-----+-----+-----+ | | 0.5 | Y | Y | +---------------+-----+-----+-----+ | | 0.6 | Y | Y | +---------------+-----+-----+-----+ | | 0.7 | Y | Y | +---------------+-----+-----+-----+ | | 1.0 | Y | Y | +---------------+-----+-----+-----+ | | 1.1 | Y | Y | +---------------+-----+-----+-----+ | | 1.2 |**N**| Y | +---------------+-----+-----+-----+ Motor and Python ```````````````` Until version 0.5, Motor required Tornado, and it supported the same version of Python as its supported Tornado versions did. Beginning in version 0.5, Motor integrates with asyncio or Tornado. Beginning in version 0.5, supports the "async for" syntax with cursors in Python 3.5 and later. Motor 1.2 dropped support for the short-lived version of the "async for" protocol implemented in Python 3.5.0 and 3.5.1. Motor continues to work with "async for" loops in Python 3.5.2 and later. +-------------------------------------------------------------------------+ | Python Version | +=====================+=====+=====+=====+=====+=====+=======+=======+=====+ | | 2.5 | 2.6 | 2.7 | 3.3 | 3.4 | 3.5.0 | 3.5.2 | 3.6 | +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | Motor Version | 0.1 | Y | Y | Y | Y |**N**|**N** |**N** |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 0.2 |**N**| Y | Y | Y |**N**|**N** |**N** |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 0.3 |**N**| Y | Y | Y | Y |**N** |**N** |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 0.4 |**N**| Y | Y | Y | Y |**N** |**N** |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 0.5 |**N**| Y | Y | Y | Y | Y | Y |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 0.6 |**N**| Y | Y | Y | Y | Y | Y |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 0.7 |**N**| Y | Y | Y | Y | Y | Y |**N**| +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 1.0 |**N**| Y | Y | Y | Y | Y | Y | Y | +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 1.1 |**N**| Y | Y | Y | Y | Y | Y | Y | +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ | | 1.2 |**N**|**N**| Y |**N**| Y |**N** | Y | Y | +---------------+-----+-----+-----+-----+-----+-----+-------+-------+-----+ .. _asyncio package from PyPI: https://pypi.python.org/pypi/asyncio Not Supported ------------- Motor does not support Windows: * The author does not test Motor on Windows to ensure it is correct or fast. * Tornado `is not officially supported on Windows `_, so Motor's Tornado integration on Windows is doubly-unsupported. * Since asyncio *does* officially support Windows, Motor's asyncio integration is more likely to work there, but it is untested. Motor also does not support Jython. motor-1.2.1/doc/static/000077500000000000000000000000001323007662700147155ustar00rootroot00000000000000motor-1.2.1/doc/static/sidebar.js000066400000000000000000000142141323007662700166660ustar00rootroot00000000000000/* * sidebar.js * ~~~~~~~~~~ * * This script makes the Sphinx sidebar collapsible and implements intelligent * scrolling. * * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to * collapse and expand the sidebar. * * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the * width of the sidebar and the margin-left of the document are decreased. * When the sidebar is expanded the opposite happens. This script saves a * per-browser/per-session cookie used to remember the position of the sidebar * among the pages. Once the browser is closed the cookie is deleted and the * position reset to the default (expanded). * * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ $(function() { // global elements used by the functions. // the 'sidebarbutton' element is defined as global after its // creation, in the add_sidebar_button function var jwindow = $(window); var jdocument = $(document); var bodywrapper = $('.bodywrapper'); var sidebar = $('.sphinxsidebar'); var sidebarwrapper = $('.sphinxsidebarwrapper'); // original margin-left of the bodywrapper and width of the sidebar // with the sidebar expanded var bw_margin_expanded = bodywrapper.css('margin-left'); var ssb_width_expanded = sidebar.width(); // margin-left of the bodywrapper and width of the sidebar // with the sidebar collapsed var bw_margin_collapsed = '.8em'; var ssb_width_collapsed = '.8em'; // colors used by the current theme var dark_color = '#AAAAAA'; var light_color = '#CCCCCC'; function get_viewport_height() { if (window.innerHeight) return window.innerHeight; else return jwindow.height(); } function sidebar_is_collapsed() { return sidebarwrapper.is(':not(:visible)'); } function toggle_sidebar() { if (sidebar_is_collapsed()) expand_sidebar(); else collapse_sidebar(); // adjust the scrolling of the sidebar scroll_sidebar(); } function collapse_sidebar() { sidebarwrapper.hide(); sidebar.css('width', ssb_width_collapsed); bodywrapper.css('margin-left', bw_margin_collapsed); sidebarbutton.css({ 'margin-left': '0', 'height': bodywrapper.height(), 'border-radius': '5px' }); sidebarbutton.find('span').text('»'); sidebarbutton.attr('title', _('Expand sidebar')); document.cookie = 'sidebar=collapsed'; } function expand_sidebar() { bodywrapper.css('margin-left', bw_margin_expanded); sidebar.css('width', ssb_width_expanded); sidebarwrapper.show(); sidebarbutton.css({ 'margin-left': ssb_width_expanded-12, 'height': bodywrapper.height(), 'border-radius': '0 5px 5px 0' }); sidebarbutton.find('span').text('«'); sidebarbutton.attr('title', _('Collapse sidebar')); //sidebarwrapper.css({'padding-top': // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); document.cookie = 'sidebar=expanded'; } function add_sidebar_button() { sidebarwrapper.css({ 'float': 'left', 'margin-right': '0', 'width': ssb_width_expanded - 28 }); // create the button sidebar.append( '
      «
      ' ); var sidebarbutton = $('#sidebarbutton'); // find the height of the viewport to center the '<<' in the page var viewport_height = get_viewport_height(); var sidebar_offset = sidebar.offset().top; var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); sidebarbutton.find('span').css({ 'display': 'block', 'position': 'fixed', 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 }); sidebarbutton.click(toggle_sidebar); sidebarbutton.attr('title', _('Collapse sidebar')); sidebarbutton.css({ 'border-radius': '0 5px 5px 0', 'color': '#444444', 'background-color': '#CCCCCC', 'font-size': '1.2em', 'cursor': 'pointer', 'height': sidebar_height, 'padding-top': '1px', 'padding-left': '1px', 'margin-left': ssb_width_expanded - 12 }); sidebarbutton.hover( function () { $(this).css('background-color', dark_color); }, function () { $(this).css('background-color', light_color); } ); } function set_position_from_cookie() { if (!document.cookie) return; var items = document.cookie.split(';'); for(var k=0; k wintop && curbot > winbot) { sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); } else if (curtop < wintop && curbot < winbot) { sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, jdocument.height() - sidebar_height - 200])); } } } jwindow.scroll(scroll_sidebar); }); motor-1.2.1/doc/tutorial-asyncio.rst000066400000000000000000000430311323007662700174670ustar00rootroot00000000000000.. currentmodule:: motor.motor_asyncio Tutorial: Using Motor With :mod:`asyncio` ========================================= .. These setups are redundant because I can't figure out how to make doctest run a common setup *before* the setup for the two groups. A "testsetup:: *" is the obvious answer, but it's run *after* group-specific setup. .. testsetup:: before-inserting-2000-docs import pymongo import motor.motor_asyncio import asyncio from asyncio import coroutine db = motor.motor_asyncio.AsyncIOMotorClient().test_database .. testsetup:: after-inserting-2000-docs import pymongo import motor.motor_asyncio import asyncio from asyncio import coroutine db = motor.motor_asyncio.AsyncIOMotorClient().test_database pymongo.MongoClient().test_database.test_collection.insert_many( [{'i': i} for i in range(2000)]) .. testcleanup:: * import pymongo pymongo.MongoClient().test_database.test_collection.delete_many({}) A guide to using MongoDB and asyncio with Motor. .. contents:: Tutorial Prerequisites ---------------------- You can learn about MongoDB with the `MongoDB Tutorial`_ before you learn Motor. Using Python 3.4 or later, do:: $ python3 -m pip install motor This tutorial assumes that a MongoDB instance is running on the default host and port. Assuming you have `downloaded and installed `_ MongoDB, you can start it like so: .. code-block:: bash $ mongod .. _pip: http://www.pip-installer.org/en/latest/installing.html .. _MongoDB Tutorial: http://docs.mongodb.org/manual/tutorial/getting-started/ Object Hierarchy ---------------- Motor, like PyMongo, represents data with a 4-level object hierarchy: * :class:`~motor.motor_asyncio.AsyncIOMotorClient` represents a mongod process, or a cluster of them. You explicitly create one of these client objects, connect it to a running mongod or mongods, and use it for the lifetime of your application. * :class:`~motor.motor_asyncio.AsyncIOMotorDatabase`: Each mongod has a set of databases (distinct sets of data files on disk). You can get a reference to a database from a client. * :class:`~motor.motor_asyncio.AsyncIOMotorCollection`: A database has a set of collections, which contain documents; you get a reference to a collection from a database. * :class:`~motor.motor_asyncio.AsyncIOMotorCursor`: Executing :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.find` on an :class:`~motor.motor_asyncio.AsyncIOMotorCollection` gets an :class:`~motor.motor_asyncio.AsyncIOMotorCursor`, which represents the set of documents matching a query. Creating a Client ----------------- You typically create a single instance of :class:`~motor.motor_asyncio.AsyncIOMotorClient` at the time your application starts up. .. doctest:: before-inserting-2000-docs >>> import motor.motor_asyncio >>> client = motor.motor_asyncio.AsyncIOMotorClient() This connects to a ``mongod`` listening on the default host and port. You can specify the host and port like: .. doctest:: before-inserting-2000-docs >>> client = motor.motor_asyncio.AsyncIOMotorClient('localhost', 27017) Motor also supports `connection URIs`_: .. doctest:: before-inserting-2000-docs >>> client = motor.motor_asyncio.AsyncIOMotorClient('mongodb://localhost:27017') Connect to a replica set like: >>> client = motor.motor_asyncio.AsyncIOMotorClient('mongodb://host1,host2/?replicaSet=my-replicaset-name') .. _connection URIs: http://docs.mongodb.org/manual/reference/connection-string/ Getting a Database ------------------ A single instance of MongoDB can support multiple independent `databases `_. From an open client, you can get a reference to a particular database with dot-notation or bracket-notation: .. doctest:: before-inserting-2000-docs >>> db = client.test_database >>> db = client['test_database'] Creating a reference to a database does no I/O and does not require an ``await`` expression. Getting a Collection -------------------- A `collection `_ is a group of documents stored in MongoDB, and can be thought of as roughly the equivalent of a table in a relational database. Getting a collection in Motor works the same as getting a database: .. doctest:: before-inserting-2000-docs >>> collection = db.test_collection >>> collection = db['test_collection'] Just like getting a reference to a database, getting a reference to a collection does no I/O and doesn't require an ``await`` expression. Inserting a Document -------------------- As in PyMongo, Motor represents MongoDB documents with Python dictionaries. To store a document in MongoDB, call :meth:`~AsyncIOMotorCollection.insert_one` in an ``await`` expression: .. doctest:: before-inserting-2000-docs >>> async def do_insert(): ... document = {'key': 'value'} ... result = await db.test_collection.insert_one(document) ... print('result %s' % repr(result.inserted_id)) ... >>> >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_insert()) result ObjectId('...') .. mongodoc:: insert .. doctest:: before-inserting-2000-docs :hide: >>> # Clean up from previous insert >>> pymongo.MongoClient().test_database.test_collection.delete_many({}) Using native coroutines ----------------------- Starting in Python 3.5, you can define a `native coroutine`_ with `async def` instead of the ``coroutine`` decorator. Within a native coroutine, wait for an async operation with `await` instead of `yield`: .. doctest:: before-inserting-2000-docs >>> async def do_insert(): ... for i in range(2000): ... result = await db.test_collection.insert_one({'i': i}) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_insert()) Within a native coroutine, the syntax to use Motor with Tornado or asyncio is often identical. .. _native coroutine: https://www.python.org/dev/peps/pep-0492/ Getting a Single Document With `find_one` ----------------------------------------- Use :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.find_one` to get the first document that matches a query. For example, to get a document where the value for key "i" is less than 1: .. doctest:: after-inserting-2000-docs >>> async def do_find_one(): ... document = await db.test_collection.find_one({'i': {'$lt': 1}}) ... pprint.pprint(document) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_find_one()) {'_id': ObjectId('...'), 'i': 0} The result is a dictionary matching the one that we inserted previously. .. note:: The returned document contains an ``"_id"``, which was automatically added on insert. .. mongodoc:: find Querying for More Than One Document ----------------------------------- Use :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.find` to query for a set of documents. :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.find` does no I/O and does not require an ``await`` expression. It merely creates an :class:`~motor.motor_asyncio.AsyncIOMotorCursor` instance. The query is actually executed on the server when you call :meth:`~motor.motor_asyncio.AsyncIOMotorCursor.to_list` or execute an ``async for`` loop. To find all documents with "i" less than 5: .. doctest:: after-inserting-2000-docs >>> async def do_find(): ... cursor = db.test_collection.find({'i': {'$lt': 5}}).sort('i') ... for document in await cursor.to_list(length=100): ... pprint.pprint(document) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_find()) {'_id': ObjectId('...'), 'i': 0} {'_id': ObjectId('...'), 'i': 1} {'_id': ObjectId('...'), 'i': 2} {'_id': ObjectId('...'), 'i': 3} {'_id': ObjectId('...'), 'i': 4} A ``length`` argument is required when you call ``to_list`` to prevent Motor from buffering an unlimited number of documents. ``async for`` ~~~~~~~~~~~~~ You can handle one document at a time in an ``async for`` loop: .. doctest:: after-inserting-2000-docs >>> async def do_find(): ... c = db.test_collection ... async for document in c.find({'i': {'$lt': 2}}): ... pprint.pprint(document) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_find()) {'_id': ObjectId('...'), 'i': 0} {'_id': ObjectId('...'), 'i': 1} You can apply a sort, limit, or skip to a query before you begin iterating: .. doctest:: after-inserting-2000-docs >>> async def do_find(): ... cursor = db.test_collection.find({'i': {'$lt': 5}}) ... # Modify the query before iterating ... cursor.sort('i', -1).limit(2).skip(2) ... async for document in cursor: ... pprint.pprint(document) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_find()) {'_id': ObjectId('...'), 'i': 2} {'_id': ObjectId('...'), 'i': 1} The cursor does not actually retrieve each document from the server individually; it gets documents efficiently in `large batches`_. .. _`large batches`: https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches Iteration in Python 3.4 ~~~~~~~~~~~~~~~~~~~~~~~ In Python versions without ``async for``, handle one document at a time with :attr:`~motor.motor_asyncio.AsyncIOMotorCursor.fetch_next` and :meth:`~motor.motor_asyncio.AsyncIOMotorCursor.next_object`: .. doctest:: after-inserting-2000-docs >>> @coroutine ... def do_find(): ... cursor = db.test_collection.find({'i': {'$lt': 5}}) ... while (yield from cursor.fetch_next): ... document = cursor.next_object() ... pprint.pprint(document) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_find()) {'_id': ObjectId('...'), 'i': 0} {'_id': ObjectId('...'), 'i': 1} {'_id': ObjectId('...'), 'i': 2} {'_id': ObjectId('...'), 'i': 3} {'_id': ObjectId('...'), 'i': 4} Counting Documents ------------------ Use :meth:`~motor.motor_asyncio.AsyncIOMotorCursor.count` to determine the number of documents in a collection, or the number of documents that match a query: .. doctest:: after-inserting-2000-docs >>> async def do_count(): ... n = await db.test_collection.find().count() ... print('%s documents in collection' % n) ... n = await db.test_collection.find({'i': {'$gt': 1000}}).count() ... print('%s documents where i > 1000' % n) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_count()) 2000 documents in collection 999 documents where i > 1000 :meth:`~motor.motor_asyncio.AsyncIOMotorCursor.count` uses the *count command* internally; we'll cover commands_ below. .. seealso:: `Count command `_ Updating Documents ------------------ :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.replace_one` changes a document. It requires two parameters: a *query* that specifies which document to replace, and a replacement document. The query follows the same syntax as for :meth:`find` or :meth:`find_one`. To replace a document: .. doctest:: after-inserting-2000-docs >>> async def do_replace(): ... coll = db.test_collection ... old_document = await coll.find_one({'i': 50}) ... print('found document: %s' % pprint.pformat(old_document)) ... _id = old_document['_id'] ... result = await coll.replace_one({'_id': _id}, {'key': 'value'}) ... print('replaced %s document' % result.modified_count) ... new_document = await coll.find_one({'_id': _id}) ... print('document is now %s' % pprint.pformat(new_document)) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_replace()) found document: {'_id': ObjectId('...'), 'i': 50} replaced 1 document document is now {'_id': ObjectId('...'), 'key': 'value'} You can see that :meth:`replace_one` replaced everything in the old document except its ``_id`` with the new document. Use :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.update_one` with MongoDB's modifier operators to update part of a document and leave the rest intact. We'll find the document whose "i" is 51 and use the ``$set`` operator to set "key" to "value": .. doctest:: after-inserting-2000-docs >>> async def do_update(): ... coll = db.test_collection ... result = await coll.update_one({'i': 51}, {'$set': {'key': 'value'}}) ... print('updated %s document' % result.modified_count) ... new_document = await coll.find_one({'i': 51}) ... print('document is now %s' % pprint.pformat(new_document)) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_update()) updated 1 document document is now {'_id': ObjectId('...'), 'i': 51, 'key': 'value'} "key" is set to "value" and "i" is still 51. :meth:`update_one` only affects the first document it finds, you can update all of them with :meth:`update_many`:: await coll.update_many({'i': {'$gt': 100}}, {'$set': {'key': 'value'}}) .. mongodoc:: update Deleting Documents ------------------ :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.delete_many` takes a query with the same syntax as :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.find`. :meth:`delete_many` immediately removes all matching documents. .. doctest:: after-inserting-2000-docs >>> async def do_delete_many(): ... coll = db.test_collection ... n = await coll.count() ... print('%s documents before calling delete_many()' % n) ... result = await db.test_collection.delete_many({'i': {'$gte': 1000}}) ... print('%s documents after' % (await coll.count())) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(do_delete_many()) 2000 documents before calling delete_many() 1000 documents after .. mongodoc:: remove Commands -------- Besides the "CRUD" operations--insert, update, delete, and find--all other operations on MongoDB are commands. Run them using the :meth:`~motor.motor_asyncio.AsyncIOMotorDatabase.command` method on :class:`~motor.motor_asyncio.AsyncIOMotorDatabase`: .. doctest:: after-inserting-2000-docs >>> from bson import SON >>> async def use_count_command(): ... response = await db.command(SON([("count", "test_collection")])) ... print('response: %s' % pprint.pformat(response)) ... >>> loop = asyncio.get_event_loop() >>> loop.run_until_complete(use_count_command()) response: {'n': 1000, 'ok': 1.0...} Since the order of command parameters matters, don't use a Python dict to pass the command's parameters. Instead, make a habit of using :class:`bson.SON`, from the ``bson`` module included with PyMongo:: await db.command(SON([("distinct", "test_collection"), ("key", "my_key"])) Many commands have special helper methods, such as :meth:`~motor.motor_asyncio.AsyncIOMotorDatabase.create_collection` or :meth:`~motor.motor_asyncio.AsyncIOMotorCollection.aggregate`, but these are just conveniences atop the basic :meth:`command` method. .. mongodoc:: commands .. _example-web-application-aiohttp: A Web Application With `aiohttp`_ --------------------------------- Let us create a web application using `aiohttp`_, a popular HTTP package for asyncio. Install it with:: python3 -m pip install aiohttp We are going to make a trivial web site with two pages served from MongoDB. To begin: .. literalinclude:: examples/aiohttp_example.py :language: python3 :start-after: setup-start :end-before: setup-end The ``AsyncIOMotorClient`` constructor does not actually connect to MongoDB. The client connects on demand, when you attempt the first operation. We create it and assign the "test" database's handle to ``db``. The ``setup_db`` coroutine drops the "pages" collection (plainly, this code is for demonstration purposes), then inserts two documents. Each document's page name is its unique id, and the "body" field is a simple HTML page. Finally, ``setup_db`` returns the database handle. We'll use the ``setup_db`` coroutine soon. First, we need a request handler that serves pages from the data we stored in MongoDB. .. literalinclude:: examples/aiohttp_example.py :language: python3 :start-after: handler-start :end-before: handler-end We start the server by running ``setup_db`` and passing the database handle to an :class:`aiohttp.web.Application`: .. literalinclude:: examples/aiohttp_example.py :language: python3 :start-after: main-start :end-before: main-end Note that it is a common mistake to create a new client object for every request; this comes at a dire performance cost. Create the client when your application starts and reuse that one client for the lifetime of the process. You can maintain the client by storing a database handle from the client on your application object, as shown in this example. Visit ``localhost:8080/pages/page-one`` and the server responds "Hello!". At ``localhost:8080/pages/page-two`` it responds "Goodbye." At other URLs it returns a 404. The complete code is in the Motor repository in ``examples/aiohttp_example.py``. .. _aiohttp: https://aiohttp.readthedocs.io/ See also the :doc:`examples/aiohttp_gridfs_example`. Further Reading --------------- The handful of classes and methods introduced here are sufficient for daily tasks. The API documentation for :class:`~motor.motor_asyncio.AsyncIOMotorClient`, :class:`~motor.motor_asyncio.AsyncIOMotorDatabase`, :class:`~motor.motor_asyncio.AsyncIOMotorCollection`, and :class:`~motor.motor_asyncio.AsyncIOMotorCursor` provides a reference to Motor's complete feature set. Learning to use the MongoDB driver is just the beginning, of course. For in-depth instruction in MongoDB itself, see `The MongoDB Manual`_. .. _The MongoDB Manual: http://docs.mongodb.org/manual/ motor-1.2.1/doc/tutorial-tornado.rst000066400000000000000000000517031323007662700174750ustar00rootroot00000000000000.. currentmodule:: motor.motor_tornado Tutorial: Using Motor With Tornado ================================== .. These setups are redundant because I can't figure out how to make doctest run a common setup *before* the setup for the two groups. A "testsetup:: *" is the obvious answer, but it's run *after* group-specific setup. .. testsetup:: before-inserting-2000-docs import pymongo import motor import tornado.web from tornado.ioloop import IOLoop from tornado import gen db = motor.motor_tornado.MotorClient().test_database .. testsetup:: after-inserting-2000-docs import pymongo import motor import tornado.web from tornado.ioloop import IOLoop from tornado import gen db = motor.motor_tornado.MotorClient().test_database sync_db = pymongo.MongoClient().test_database sync_db.test_collection.drop() sync_db.test_collection.insert_many( [{'i': i} for i in range(2000)]) .. testcleanup:: * import pymongo pymongo.MongoClient().test_database.test_collection.delete_many({}) A guide to using MongoDB and Tornado with Motor. .. contents:: Tutorial Prerequisites ---------------------- You can learn about MongoDB with the `MongoDB Tutorial`_ before you learn Motor. Install pip_ and then do:: $ pip install tornado motor Once done, the following should run in the Python shell without raising an exception: .. doctest:: >>> import motor.motor_tornado This tutorial also assumes that a MongoDB instance is running on the default host and port. Assuming you have `downloaded and installed `_ MongoDB, you can start it like so: .. code-block:: bash $ mongod .. _pip: http://www.pip-installer.org/en/latest/installing.html .. _MongoDB Tutorial: http://docs.mongodb.org/manual/tutorial/getting-started/ Object Hierarchy ---------------- Motor, like PyMongo, represents data with a 4-level object hierarchy: * :class:`MotorClient` represents a mongod process, or a cluster of them. You explicitly create one of these client objects, connect it to a running mongod or mongods, and use it for the lifetime of your application. * :class:`MotorDatabase`: Each mongod has a set of databases (distinct sets of data files on disk). You can get a reference to a database from a client. * :class:`MotorCollection`: A database has a set of collections, which contain documents; you get a reference to a collection from a database. * :class:`MotorCursor`: Executing :meth:`~MotorCollection.find` on a :class:`MotorCollection` gets a :class:`MotorCursor`, which represents the set of documents matching a query. Creating a Client ----------------- You typically create a single instance of :class:`MotorClient` at the time your application starts up. .. doctest:: before-inserting-2000-docs >>> client = motor.motor_tornado.MotorClient() This connects to a ``mongod`` listening on the default host and port. You can specify the host and port like: .. doctest:: before-inserting-2000-docs >>> client = motor.motor_tornado.MotorClient('localhost', 27017) Motor also supports `connection URIs`_: .. doctest:: before-inserting-2000-docs >>> client = motor.motor_tornado.MotorClient('mongodb://localhost:27017') Connect to a replica set like: >>> client = motor.motor_tornado.MotorClient('mongodb://host1,host2/?replicaSet=my-replicaset-name') .. _connection URIs: http://docs.mongodb.org/manual/reference/connection-string/ Getting a Database ------------------ A single instance of MongoDB can support multiple independent `databases `_. From an open client, you can get a reference to a particular database with dot-notation or bracket-notation: .. doctest:: before-inserting-2000-docs >>> db = client.test_database >>> db = client['test_database'] Creating a reference to a database does no I/O and does not accept a callback or return a Future. Tornado Application Startup Sequence ------------------------------------ Now that we can create a client and get a database, we're ready to start a Tornado application that uses Motor:: db = motor.motor_tornado.MotorClient().test_database application = tornado.web.Application([ (r'/', MainHandler) ], db=db) application.listen(8888) tornado.ioloop.IOLoop.current().start() There are two things to note in this code. First, the ``MotorClient`` constructor doesn't actually connect to the server; the client will initiate a connection when you attempt the first operation. Second, passing the database as the ``db`` keyword argument to ``Application`` makes it available to request handlers:: class MainHandler(tornado.web.RequestHandler): def get(self): db = self.settings['db'] It is a common mistake to create a new client object for every request; **this comes at a dire performance cost**. Create the client when your application starts and reuse that one client for the lifetime of the process, as shown in these examples. The Tornado :class:`~tornado.httpserver.HTTPServer` class's :meth:`start` method is a simple way to fork multiple web servers and use all of your machine's CPUs. However, you must create your ``MotorClient`` after forking:: # Create the application before creating a MotorClient. application = tornado.web.Application([ (r'/', MainHandler) ]) server = tornado.httpserver.HTTPServer(application) server.bind(8888) # Forks one process per CPU. server.start(0) # Now, in each child process, create a MotorClient. application.settings['db'] = MotorClient().test_database IOLoop.current().start() For production-ready, multiple-CPU deployments of Tornado there are better methods than ``HTTPServer.start()``. See Tornado's guide to :doc:`tornado:guide/running`. Getting a Collection -------------------- A `collection `_ is a group of documents stored in MongoDB, and can be thought of as roughly the equivalent of a table in a relational database. Getting a collection in Motor works the same as getting a database: .. doctest:: before-inserting-2000-docs >>> collection = db.test_collection >>> collection = db['test_collection'] Just like getting a reference to a database, getting a reference to a collection does no I/O and doesn't accept a callback or return a Future. Inserting a Document -------------------- As in PyMongo, Motor represents MongoDB documents with Python dictionaries. To store a document in MongoDB, call :meth:`~MotorCollection.insert_one` with a document and a callback: .. doctest:: before-inserting-2000-docs >>> from tornado.ioloop import IOLoop >>> def my_callback(result, error): ... print('result %s' % repr(result.inserted_id)) ... IOLoop.current().stop() ... >>> document = {'key': 'value'} >>> db.test_collection.insert_one(document, callback=my_callback) >>> IOLoop.current().start() result ObjectId('...') There are several differences to note between Motor and PyMongo. One is that, unlike PyMongo's :meth:`~pymongo.collection.Collection.insert_one`, Motor's has no return value. Another is that ``insert_one`` accepts an optional callback function. The function must take two arguments and it must be passed to ``insert_one`` as a keyword argument, like:: db.test_collection.insert_one(document, callback=some_function) .. warning:: Passing the callback function using the ``callback=`` syntax is required. (This requirement is a side-effect of the technique Motor uses to wrap PyMongo.) If you pass the callback as a positional argument instead, you may see an exception like ``TypeError: method takes exactly 1 argument (2 given)``, or ``TypeError: callable is required``, or some silent misbehavior. :meth:`insert_one` is *asynchronous*. This means it returns immediately, and the actual work of inserting the document into the collection is performed in the background. When it completes, the callback is executed. If the insert succeeded, the ``result`` parameter is a :class:`~pymongo.results.InsertOneResult` with the new document's unique id and the ``error`` parameter is ``None``. If there was an error, ``result`` is ``None`` and ``error`` is an ``Exception`` object. For example, we can trigger a duplicate-key error by trying to insert two documents with the same unique id: .. doctest:: before-inserting-2000-docs >>> loop = IOLoop.current() >>> def my_callback(result, error): ... print('result %s error %s' % (repr(result), repr(error))) ... IOLoop.current().stop() ... >>> def insert_two_documents(): ... db.test_collection.insert_one({'_id': 1}, callback=my_callback) ... >>> IOLoop.current().add_callback(insert_two_documents) >>> IOLoop.current().start() result error None >>> IOLoop.current().add_callback(insert_two_documents) >>> IOLoop.current().start() result None error DuplicateKeyError(...) The first insert results in ``my_callback`` being called with result 1 and error ``None``. The second insert triggers ``my_callback`` with result None and a :class:`~pymongo.errors.DuplicateKeyError`. A typical beginner's mistake with Motor is to insert documents in a loop, not waiting for each insert to complete before beginning the next:: >>> for i in range(2000): ... db.test_collection.insert_one({'i': i}) .. Note that the above is NOT a doctest!! In PyMongo this would insert each document in turn using a single socket, but Motor attempts to run all the :meth:`insert_one` operations at once. This requires up to ``max_pool_size`` open sockets connected to MongoDB, which taxes the client and server. To ensure instead that all inserts use a single connection, wait for acknowledgment of each. This is a bit complex using callbacks: .. doctest:: before-inserting-2000-docs >>> i = 0 >>> def do_insert(result, error): ... global i ... if error: ... raise error ... i += 1 ... if i < 2000: ... db.test_collection.insert_one({'i': i}, callback=do_insert) ... else: ... IOLoop.current().stop() ... >>> # Start >>> db.test_collection.insert_one({'i': i}, callback=do_insert) >>> IOLoop.current().start() You can simplify this code with ``gen.coroutine``. Using Motor with `gen.coroutine` -------------------------------- The :mod:`tornado.gen` module lets you use generators to simplify asynchronous code. There are two parts to coding with generators: :func:`coroutine ` and :class:`~tornado.concurrent.Future`. First, decorate your generator function with ``@gen.coroutine``: >>> @gen.coroutine ... def do_insert(): ... pass If you pass no callback to one of Motor's asynchronous methods, it returns a ``Future``. Yield the ``Future`` instance to wait for an operation to complete and obtain its result: .. doctest:: before-inserting-2000-docs >>> @gen.coroutine ... def do_insert(): ... for i in range(2000): ... future = db.test_collection.insert_one({'i': i}) ... result = yield future ... >>> IOLoop.current().run_sync(do_insert) In the code above, ``result`` is the ``_id`` of each inserted document. .. seealso:: :doc:`examples/bulk`. .. seealso:: :ref:`Detailed example of Motor and gen.coroutine ` .. mongodoc:: insert .. doctest:: before-inserting-2000-docs :hide: >>> # Clean up from previous insert >>> pymongo.MongoClient().test_database.test_collection.delete_many({}) Using native coroutines ----------------------- Starting in Python 3.5, you can define a `native coroutine`_ with `async def` instead of the `gen.coroutine` decorator. Within a native coroutine, wait for an async operation with `await` instead of `yield`: .. doctest:: before-inserting-2000-docs >>> async def do_insert(): ... for i in range(2000): ... result = await db.test_collection.insert_one({'i': i}) ... >>> IOLoop.current().run_sync(do_insert) Within a native coroutine, the syntax to use Motor with Tornado or asyncio is often identical. .. _native coroutine: https://www.python.org/dev/peps/pep-0492/ Getting a Single Document With :meth:`~MotorCollection.find_one` ---------------------------------------------------------------- Use :meth:`~MotorCollection.find_one` to get the first document that matches a query. For example, to get a document where the value for key "i" is less than 1: .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_find_one(): ... document = yield db.test_collection.find_one({'i': {'$lt': 1}}) ... pprint.pprint(document) ... >>> IOLoop.current().run_sync(do_find_one) {'_id': ObjectId('...'), 'i': 0} The result is a dictionary matching the one that we inserted previously. The returned document contains an ``"_id"``, which was automatically added on insert. (We use ``pprint`` here instead of ``print`` to ensure the document's key names are sorted the same in your output as ours.) .. mongodoc:: find Querying for More Than One Document ----------------------------------- Use :meth:`~MotorCollection.find` to query for a set of documents. :meth:`~MotorCollection.find` does no I/O and does not take a callback, it merely creates a :class:`MotorCursor` instance. The query is actually executed on the server when you call :meth:`~MotorCursor.to_list` or :meth:`~MotorCursor.each`, or yield :attr:`~motor.motor_tornado.MotorCursor.fetch_next`. To find all documents with "i" less than 5: .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_find(): ... cursor = db.test_collection.find({'i': {'$lt': 5}}).sort('i') ... for document in (yield cursor.to_list(length=100)): ... pprint.pprint(document) ... >>> IOLoop.current().run_sync(do_find) {'_id': ObjectId('...'), 'i': 0} {'_id': ObjectId('...'), 'i': 1} {'_id': ObjectId('...'), 'i': 2} {'_id': ObjectId('...'), 'i': 3} {'_id': ObjectId('...'), 'i': 4} A ``length`` argument is required when you call to_list to prevent Motor from buffering an unlimited number of documents. To get one document at a time with :attr:`~motor.motor_tornado.MotorCursor.fetch_next` and :meth:`~MotorCursor.next_object`: .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_find(): ... cursor = db.test_collection.find({'i': {'$lt': 5}}) ... while (yield cursor.fetch_next): ... document = cursor.next_object() ... pprint.pprint(document) ... >>> IOLoop.current().run_sync(do_find) {'_id': ObjectId('...'), 'i': 0} {'_id': ObjectId('...'), 'i': 1} {'_id': ObjectId('...'), 'i': 2} {'_id': ObjectId('...'), 'i': 3} {'_id': ObjectId('...'), 'i': 4} You can apply a sort, limit, or skip to a query before you begin iterating: .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_find(): ... c = db.test_collection ... cursor = c.find({'i': {'$lt': 5}}) ... # Modify the query before iterating ... cursor.sort('i', -1).limit(2).skip(2) ... while (yield cursor.fetch_next): ... document = cursor.next_object() ... pprint.pprint(document) ... >>> IOLoop.current().run_sync(do_find) {'_id': ObjectId('...'), 'i': 2} {'_id': ObjectId('...'), 'i': 1} ``fetch_next`` does not actually retrieve each document from the server individually; it gets documents efficiently in `large batches`_. .. _`large batches`: https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches `async for` ----------- In a native coroutine defined with `async def`, replace the while-loop with `async for`: .. doctest:: after-inserting-2000-docs >>> async def do_find(): ... c = db.test_collection ... async for document in c.find({'i': {'$lt': 2}}): ... pprint.pprint(document) ... >>> IOLoop.current().run_sync(do_find) {'_id': ObjectId('...'), 'i': 0} {'_id': ObjectId('...'), 'i': 1} This version of the code is dramatically faster. Counting Documents ------------------ Use :meth:`~MotorCursor.count` to determine the number of documents in a collection, or the number of documents that match a query: .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_count(): ... n = yield db.test_collection.find().count() ... print('%s documents in collection' % n) ... n = yield db.test_collection.find({'i': {'$gt': 1000}}).count() ... print('%s documents where i > 1000' % n) ... >>> IOLoop.current().run_sync(do_count) 2000 documents in collection 999 documents where i > 1000 :meth:`~MotorCursor.count` uses the *count command* internally; we'll cover commands_ below. .. seealso:: `Count command `_ Updating Documents ------------------ :meth:`~MotorCollection.replace_one` changes a document. It requires two parameters: a *query* that specifies which document to replace, and a replacement document. The query follows the same syntax as for :meth:`find` or :meth:`find_one`. To replace a document: .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_replace(): ... coll = db.test_collection ... old_document = yield coll.find_one({'i': 50}) ... print('found document: %s' % pprint.pformat(old_document)) ... _id = old_document['_id'] ... result = yield coll.replace_one({'_id': _id}, {'key': 'value'}) ... print('replaced %s document' % result.modified_count) ... new_document = yield coll.find_one({'_id': _id}) ... print('document is now %s' % pprint.pformat(new_document)) ... >>> IOLoop.current().run_sync(do_replace) found document: {'_id': ObjectId('...'), 'i': 50} replaced 1 document document is now {'_id': ObjectId('...'), 'key': 'value'} You can see that :meth:`replace_one` replaced everything in the old document except its ``_id`` with the new document. Use :meth:`~MotorCollection.update_one` with MongoDB's modifier operators to update part of a document and leave the rest intact. We'll find the document whose "i" is 51 and use the ``$set`` operator to set "key" to "value": .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_update(): ... coll = db.test_collection ... result = yield coll.update_one({'i': 51}, {'$set': {'key': 'value'}}) ... print('updated %s document' % result.modified_count) ... new_document = yield coll.find_one({'i': 51}) ... print('document is now %s' % pprint.pformat(new_document)) ... >>> IOLoop.current().run_sync(do_update) updated 1 document document is now {'_id': ObjectId('...'), 'i': 51, 'key': 'value'} "key" is set to "value" and "i" is still 51. :meth:`update_one` only affects the first document it finds, you can update all of them with :meth:`update_many`:: yield coll.update_many({'i': {'$gt': 100}}, {'$set': {'key': 'value'}}) .. mongodoc:: update Removing Documents ------------------ :meth:`~MotorCollection.delete_many` takes a query with the same syntax as :meth:`~MotorCollection.find`. :meth:`delete_many` immediately removes all matching documents. .. doctest:: after-inserting-2000-docs >>> @gen.coroutine ... def do_delete_many(): ... coll = db.test_collection ... n = yield coll.count() ... print('%s documents before calling delete_many()' % n) ... result = yield db.test_collection.delete_many({'i': {'$gte': 1000}}) ... print('%s documents after' % (yield coll.count())) ... >>> IOLoop.current().run_sync(do_delete_many) 2000 documents before calling delete_many() 1000 documents after .. mongodoc:: remove Commands -------- Besides the "CRUD" operations--insert, update, delete, and find--all other operations on MongoDB are commands. Run them using the :meth:`~MotorDatabase.command` method on :class:`MotorDatabase`: .. doctest:: after-inserting-2000-docs >>> from bson import SON >>> @gen.coroutine ... def use_count_command(): ... response = yield db.command(SON([("count", "test_collection")])) ... print('response: %s' % pprint.pformat(response)) ... >>> IOLoop.current().run_sync(use_count_command) response: {'n': 1000, 'ok': 1.0...} Since the order of command parameters matters, don't use a Python dict to pass the command's parameters. Instead, make a habit of using :class:`bson.SON`, from the ``bson`` module included with PyMongo:: yield db.command(SON([("distinct", "test_collection"), ("key", "my_key")])) Many commands have special helper methods, such as :meth:`~MotorDatabase.create_collection` or :meth:`~MotorCollection.aggregate`, but these are just conveniences atop the basic :meth:`command` method. .. mongodoc:: commands Further Reading --------------- The handful of classes and methods introduced here are sufficient for daily tasks. The API documentation for :class:`MotorClient`, :class:`MotorDatabase`, :class:`MotorCollection`, and :class:`MotorCursor` provides a reference to Motor's complete feature set. Learning to use the MongoDB driver is just the beginning, of course. For in-depth instruction in MongoDB itself, see `The MongoDB Manual`_. .. _The MongoDB Manual: http://docs.mongodb.org/manual/ motor-1.2.1/ez_setup.py000066400000000000000000000301621323007662700150730ustar00rootroot00000000000000#!/usr/bin/env python """ Setuptools bootstrapping installer. Maintained at https://github.com/pypa/setuptools/tree/bootstrap. Run this script to install or upgrade setuptools. """ import os import shutil import sys import tempfile import zipfile import optparse import subprocess import platform import textwrap import contextlib import json import codecs from distutils import log try: from urllib.request import urlopen from urllib.parse import urljoin except ImportError: from urllib2 import urlopen from urlparse import urljoin try: from site import USER_SITE except ImportError: USER_SITE = None LATEST = object() DEFAULT_VERSION = LATEST DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/" DEFAULT_SAVE_DIR = os.curdir def _python_cmd(*args): """ Execute a command. Return True if the command succeeded. """ args = (sys.executable,) + args return subprocess.call(args) == 0 def _install(archive_filename, install_args=()): """Install Setuptools.""" with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 def _build_egg(egg, archive_filename, to_dir): """Build Setuptools egg.""" with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') class ContextualZipFile(zipfile.ZipFile): """Supplement ZipFile class to support context manager for Python 2.6.""" def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __new__(cls, *args, **kwargs): """Construct a ZipFile or ContextualZipFile as appropriate.""" if hasattr(zipfile.ZipFile, '__exit__'): return zipfile.ZipFile(*args, **kwargs) return super(ContextualZipFile, cls).__new__(cls) @contextlib.contextmanager def archive_context(filename): """ Unzip filename to a temporary directory, set to the cwd. The unzipped target is cleaned up after. """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) with ContextualZipFile(filename) as archive: archive.extractall() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _do_download(version, download_base, to_dir, download_delay): """Download Setuptools.""" py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys) tp = 'setuptools-{version}-{py_desig}.egg' egg = os.path.join(to_dir, tp.format(**locals())) if not os.path.exists(egg): archive = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: _unload_pkg_resources() import setuptools setuptools.bootstrap_install_from = egg def use_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=DEFAULT_SAVE_DIR, download_delay=15): """ Ensure that a setuptools version is installed. Return None. Raise SystemExit if the requested version or later cannot be installed. """ version = _resolve_version(version) to_dir = os.path.abspath(to_dir) # prior to importing, capture the module state for # representative modules. rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) try: import pkg_resources pkg_resources.require("setuptools>=" + version) # a suitable version is already installed return except ImportError: # pkg_resources not available; setuptools is not installed; download pass except pkg_resources.DistributionNotFound: # no version of setuptools was found; allow download pass except pkg_resources.VersionConflict as VC_err: if imported: _conflict_bail(VC_err, version) # otherwise, unload pkg_resources to allow the downloaded version to # take precedence. del pkg_resources _unload_pkg_resources() return _do_download(version, download_base, to_dir, download_delay) def _conflict_bail(VC_err, version): """ Setuptools was imported prior to invocation, so it is unsafe to unload it. Bail out. """ conflict_tmpl = textwrap.dedent(""" The required version of setuptools (>={version}) is not available, and can't be installed while this script is running. Please install a more recent version first, using 'easy_install -U setuptools'. (Currently using {VC_err.args[0]!r}) """) msg = conflict_tmpl.format(**locals()) sys.stderr.write(msg) sys.exit(2) def _unload_pkg_resources(): sys.meta_path = [ importer for importer in sys.meta_path if importer.__class__.__module__ != 'pkg_resources.extern' ] del_modules = [ name for name in sys.modules if name.startswith('pkg_resources') ] for mod_name in del_modules: del sys.modules[mod_name] def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell. Powershell will validate trust. Raise an exception if the command cannot complete. """ target = os.path.abspath(target) ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' % locals() ) cmd = [ 'powershell', '-Command', ps_cmd, ] _clean_check(cmd, target) def has_powershell(): """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--location', '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """Use Python to download the file, without connection authentication.""" src = urlopen(url) try: # Read all the data in one block. data = src.read() finally: src.close() # Write all the data in one block to avoid creating a partial file. with open(target, "wb") as dst: dst.write(data) download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ) viable_downloaders = (dl for dl in downloaders if dl.viable()) return next(viable_downloaders, None) def download_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=DEFAULT_SAVE_DIR, delay=15, downloader_factory=get_best_downloader): """ Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ version = _resolve_version(version) # making sure we use the absolute path to_dir = os.path.abspath(to_dir) zip_name = "setuptools-%s.zip" % version url = download_base + zip_name saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _resolve_version(version): """ Resolve LATEST version """ if version is not LATEST: return version meta_url = urljoin(DEFAULT_URL, '/pypi/setuptools/json') resp = urlopen(meta_url) with contextlib.closing(resp): try: charset = resp.info().get_content_charset() except Exception: # Python 2 compat; assume UTF-8 charset = 'UTF-8' reader = codecs.getreader(charset) doc = json.load(reader(resp)) return str(doc['info']['version']) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package. Returns list of command line arguments. """ return ['--user'] if options.user_install else [] def _parse_args(): """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) parser.add_option( '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) parser.add_option( '--to-dir', help="Directory to save (and re-use) package", default=DEFAULT_SAVE_DIR, ) options, args = parser.parse_args() # positional arguments are ignored return options def _download_args(options): """Return args for download_setuptools function from cmdline args.""" return dict( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, to_dir=options.to_dir, ) def main(): """Install or upgrade setuptools and EasyInstall.""" options = _parse_args() archive = download_setuptools(**_download_args(options)) return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) motor-1.2.1/motor/000077500000000000000000000000001323007662700140215ustar00rootroot00000000000000motor-1.2.1/motor/__init__.py000066400000000000000000000040461323007662700161360ustar00rootroot00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Motor, an asynchronous driver for MongoDB.""" from __future__ import unicode_literals, absolute_import import pymongo from motor.motor_py3_compat import text_type version_tuple = (1, 2, 1) def get_version_string(): return '.'.join(map(str, version_tuple)) version = get_version_string() """Current version of Motor.""" pymongo_required = 3, 4 if pymongo.version_tuple[:2] < pymongo_required: major, minor = pymongo_required msg = ( "Motor %s requires PyMongo %s.%s or later. " "You have PyMongo %s. " "Do python -m pip install \"pymongo>=%s.%s,<4\"" ) % (version, major, minor, pymongo.version, major, minor) raise ImportError(msg) try: import tornado except ImportError: tornado = None else: # For backwards compatibility with Motor 0.4, export Motor's Tornado classes # at module root. This may change in Motor 1.0. First get __all__. from .motor_tornado import * # Now some classes that aren't in __all__ but might be expected. from .motor_tornado import (MotorCollection, MotorDatabase, MotorGridFS, MotorGridFSBucket, MotorGridIn, MotorGridOut, MotorBulkOperationBuilder) # Make "from motor import *" the same as "from motor.motor_tornado import *" from .motor_tornado import __all__ motor-1.2.1/motor/aiohttp/000077500000000000000000000000001323007662700154715ustar00rootroot00000000000000motor-1.2.1/motor/aiohttp/__init__.py000066400000000000000000000230231323007662700176020ustar00rootroot00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Serve GridFS files with Motor and aiohttp. Requires Python 3.4 or later and aiohttp 2.0 or later. See the :doc:`/examples/aiohttp_gridfs_example`. """ import asyncio import datetime import mimetypes import aiohttp.web import gridfs from motor.motor_asyncio import (AsyncIOMotorDatabase, AsyncIOMotorGridFSBucket) def get_gridfs_file(bucket, filename, request): """Override to choose a GridFS file to serve at a URL. By default, if a URL pattern like ``/fs/{filename}`` is mapped to this :class:`AIOHTTPGridFS`, then the filename portion of the URL is used as the filename, so a request for "/fs/image.png" results in a call to :meth:`.AsyncIOMotorGridFSBucket.open_download_stream_by_name` with "image.png" as the ``filename`` argument. To customize the mapping of path to GridFS file, override ``get_gridfs_file`` and return a :class:`asyncio.Future` that resolves to a :class:`~motor.motor_asyncio.AsyncIOMotorGridOut`. For example, to retrieve the file by ``_id`` instead of filename:: def get_gridfile_by_id(bucket, filename, request): # "filename" is interpreted as _id instead of name. # Return a Future AsyncIOMotorGridOut. return bucket.open_download_stream(file_id=filename) client = AsyncIOMotorClient() gridfs_handler = AIOHTTPGridFS(client.my_database, get_gridfs_file=get_gridfile_by_id) :Parameters: - `bucket`: An :class:`~motor.motor_asyncio.AsyncIOMotorGridFSBucket` - `filename`: A string, the URL portion matching {filename} in the URL pattern - `request`: An :class:`aiohttp.web.Request` """ # A Future AsyncIOMotorGridOut. return bucket.open_download_stream_by_name(filename) def get_cache_time(filename, modified, mime_type): """Override to customize cache control behavior. Return a positive number of seconds to trigger aggressive caching or 0 to mark resource as cacheable, only. 0 is the default. For example, to allow image caching:: def image_cache_time(filename, modified, mime_type): if mime_type.startswith('image/'): return 3600 return 0 client = AsyncIOMotorClient() gridfs_handler = AIOHTTPGridFS(client.my_database, get_cache_time=image_cache_time) :Parameters: - `filename`: A string, the URL portion matching {filename} in the URL pattern - `modified`: A datetime, when the matching GridFS file was created - `mime_type`: The file's type, a string like "application/octet-stream" """ return 0 def set_extra_headers(response, gridout): """Override to modify the response before sending to client. For example, to allow image caching:: def gzip_header(response, gridout): response.headers['Content-Encoding'] = 'gzip' client = AsyncIOMotorClient() gridfs_handler = AIOHTTPGridFS(client.my_database, set_extra_headers=gzip_header) :Parameters: - `response`: An :class:`aiohttp.web.Response` - `gridout`: The :class:`~motor.motor_asyncio.AsyncIOMotorGridOut` we will serve to the client """ pass def _config_error(request): try: formatter = request.match_info.route.resource.get_info()['formatter'] msg = ('Bad AIOHTTPGridFS route "%s", requires a {filename} variable' % formatter) except (KeyError, AttributeError): # aiohttp API changed? Fall back to simpler error message. msg = ('Bad AIOHTTPGridFS route for request: %s' % request) raise aiohttp.web.HTTPInternalServerError(text=msg) from None class AIOHTTPGridFS: """Serve files from `GridFS`_. This class is a :ref:`request handler ` that serves GridFS files, similar to aiohttp's built-in static file server. .. code-block:: python client = AsyncIOMotorClient() gridfs_handler = AIOHTTPGridFS(client.my_database) app = aiohttp.web.Application() # The GridFS URL pattern must have a "{filename}" variable. resource = app.router.add_resource('/fs/{filename}') resource.add_route('GET', gridfs_handler) resource.add_route('HEAD', gridfs_handler) app_handler = app.make_handler() server = loop.create_server(app_handler, port=80) By default, requests' If-Modified-Since headers are honored, but no specific cache-control timeout is sent to clients. Thus each request for a GridFS file requires a quick check of the file's ``uploadDate`` in MongoDB. Pass a custom :func:`get_cache_time` to customize this. :Parameters: - `database`: An :class:`AsyncIOMotorDatabase` - `get_gridfs_file`: Optional override for :func:`get_gridfs_file` - `get_cache_time`: Optional override for :func:`get_cache_time` - `set_extra_headers`: Optional override for :func:`set_extra_headers` .. _GridFS: https://docs.mongodb.com/manual/core/gridfs/ """ def __init__(self, database, root_collection='fs', get_gridfs_file=get_gridfs_file, get_cache_time=get_cache_time, set_extra_headers=set_extra_headers): if not isinstance(database, AsyncIOMotorDatabase): raise TypeError("First argument to AIOHTTPGridFS must be " "AsyncIOMotorDatabase, not %r" % database) self._database = database self._bucket = AsyncIOMotorGridFSBucket(self._database, root_collection) self._get_gridfs_file = get_gridfs_file self._get_cache_time = get_cache_time self._set_extra_headers = set_extra_headers @asyncio.coroutine def __call__(self, request): """Send filepath to client using request.""" try: filename = request.match_info['filename'] except KeyError: _config_error(request) if request.method not in ('GET', 'HEAD'): raise aiohttp.web.HTTPMethodNotAllowed( method=request.method, allowed_methods={'GET', 'HEAD'}) try: gridout = yield from self._get_gridfs_file(self._bucket, filename, request) except gridfs.NoFile: raise aiohttp.web.HTTPNotFound(text=request.path) resp = aiohttp.web.StreamResponse() self._set_standard_headers(request.path, resp, gridout) # Overridable method set_extra_headers. self._set_extra_headers(resp, gridout) # Check the If-Modified-Since, and don't send the result if the # content has not been modified ims_value = request.if_modified_since if ims_value is not None: # If our MotorClient is tz-aware, assume the naive ims_value is in # its time zone. if_since = ims_value.replace(tzinfo=gridout.upload_date.tzinfo) modified = gridout.upload_date.replace(microsecond=0) if if_since >= modified: resp.set_status(304) return resp # Same for Etag etag = request.headers.get("If-None-Match") if etag is not None and etag.strip('"') == gridout.md5: resp.set_status(304) return resp resp.content_length = gridout.length yield from resp.prepare(request) if request.method == 'GET': resp.set_tcp_cork(True) try: written = 0 while written < gridout.length: # Reading chunk_size at a time minimizes buffering. chunk = yield from gridout.read(gridout.chunk_size) resp.write(chunk) yield from resp.drain() written += len(chunk) finally: resp.set_tcp_nodelay(True) return resp def _set_standard_headers(self, path, resp, gridout): resp.last_modified = gridout.upload_date content_type = gridout.content_type if content_type is None: content_type, encoding = mimetypes.guess_type(path) if content_type: resp.content_type = content_type # MD5 is calculated on the MongoDB server when GridFS file is created. resp.headers["Etag"] = '"%s"' % gridout.md5 # Overridable method get_cache_time. cache_time = self._get_cache_time(path, gridout.upload_date, gridout.content_type) if cache_time > 0: resp.headers["Expires"] = ( datetime.datetime.utcnow() + datetime.timedelta(seconds=cache_time) ).strftime("%a, %d %b %Y %H:%M:%S GMT") resp.headers["Cache-Control"] = "max-age=" + str(cache_time) else: resp.headers["Cache-Control"] = "public" motor-1.2.1/motor/core.py000066400000000000000000001414101323007662700153240ustar00rootroot00000000000000# Copyright 2011-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import """Framework-agnostic core of Motor, an asynchronous driver for MongoDB.""" import functools import sys import textwrap import pymongo import pymongo.auth import pymongo.common import pymongo.database import pymongo.errors import pymongo.mongo_client import pymongo.mongo_replica_set_client import pymongo.son_manipulator from pymongo.bulk import BulkOperationBuilder from pymongo.database import Database from pymongo.change_stream import ChangeStream from pymongo.collection import Collection from pymongo.cursor import Cursor, _QUERY_OPTIONS from pymongo.command_cursor import CommandCursor from .metaprogramming import (AsyncCommand, AsyncRead, AsyncWrite, coroutine_annotation, create_class_with_framework, DelegateMethod, motor_coroutine, MotorCursorChainingMethod, ReadOnlyProperty) from .motor_common import callback_type_error from motor.docstrings import * HAS_SSL = True try: import ssl except ImportError: ssl = None HAS_SSL = False PY35 = sys.version_info >= (3, 5) class AgnosticBase(object): def __eq__(self, other): if (isinstance(other, self.__class__) and hasattr(self, 'delegate') and hasattr(other, 'delegate')): return self.delegate == other.delegate return NotImplemented def __init__(self, delegate): self.delegate = delegate def __repr__(self): return '%s(%r)' % (self.__class__.__name__, self.delegate) class AgnosticBaseProperties(AgnosticBase): codec_options = ReadOnlyProperty() read_preference = ReadOnlyProperty() read_concern = ReadOnlyProperty() write_concern = ReadOnlyProperty() class AgnosticClient(AgnosticBaseProperties): __motor_class_name__ = 'MotorClient' __delegate_class__ = pymongo.mongo_client.MongoClient address = ReadOnlyProperty() arbiters = ReadOnlyProperty() close = DelegateMethod() close_cursor = AsyncCommand() database_names = AsyncRead() drop_database = AsyncCommand().unwrap('MotorDatabase') event_listeners = ReadOnlyProperty() fsync = AsyncCommand() get_database = DelegateMethod(doc=get_database_doc).wrap(Database) get_default_database = DelegateMethod().wrap(Database) HOST = ReadOnlyProperty() is_mongos = ReadOnlyProperty() is_primary = ReadOnlyProperty() kill_cursors = AsyncCommand() list_databases = AsyncRead().wrap(CommandCursor) list_database_names = AsyncRead() local_threshold_ms = ReadOnlyProperty() max_bson_size = ReadOnlyProperty() max_idle_time_ms = ReadOnlyProperty() max_message_size = ReadOnlyProperty() max_pool_size = ReadOnlyProperty() max_write_batch_size = ReadOnlyProperty() min_pool_size = ReadOnlyProperty() nodes = ReadOnlyProperty() PORT = ReadOnlyProperty() primary = ReadOnlyProperty() read_concern = ReadOnlyProperty() retry_writes = ReadOnlyProperty() secondaries = ReadOnlyProperty() server_info = AsyncRead() server_selection_timeout = ReadOnlyProperty() start_session = AsyncRead(doc=start_session_doc) unlock = AsyncCommand() def __init__(self, *args, **kwargs): """Create a new connection to a single MongoDB instance at *host:port*. Takes the same constructor arguments as :class:`~pymongo.mongo_client.MongoClient`, as well as: :Parameters: - `io_loop` (optional): Special :class:`tornado.ioloop.IOLoop` instance to use instead of default """ if 'io_loop' in kwargs: io_loop = kwargs.pop('io_loop') else: io_loop = self._framework.get_event_loop() kwargs.setdefault('connect', False) delegate = self.__delegate_class__(*args, **kwargs) super(AgnosticBaseProperties, self).__init__(delegate) if io_loop: self._framework.check_event_loop(io_loop) self.io_loop = io_loop else: self.io_loop = self._framework.get_event_loop() def get_io_loop(self): return self.io_loop def __getattr__(self, name): if name.startswith('_'): raise AttributeError( "%s has no attribute %r. To access the %s" " database, use client['%s']." % ( self.__class__.__name__, name, name, name)) return self[name] def __getitem__(self, name): db_class = create_class_with_framework( AgnosticDatabase, self._framework, self.__module__) return db_class(self, name) def wrap(self, obj): if obj.__class__ == Database: db_class = create_class_with_framework( AgnosticDatabase, self._framework, self.__module__) return db_class(self, obj.name, _delegate=obj) elif obj.__class__ == CommandCursor: command_cursor_class = create_class_with_framework( AgnosticCommandCursor, self._framework, self.__module__) return command_cursor_class(obj, self) class AgnosticDatabase(AgnosticBaseProperties): __motor_class_name__ = 'MotorDatabase' __delegate_class__ = Database add_user = AsyncCommand() authenticate = AsyncCommand() collection_names = AsyncRead() command = AsyncCommand(doc=cmd_doc) create_collection = AsyncCommand().wrap(Collection) current_op = AsyncRead() dereference = AsyncRead() drop_collection = AsyncCommand().unwrap('MotorCollection') error = AsyncRead(doc="OBSOLETE") eval = AsyncCommand() get_collection = DelegateMethod().wrap(Collection) last_status = AsyncRead(doc="OBSOLETE") list_collection_names = AsyncRead() list_collections = AsyncRead() logout = AsyncCommand() name = ReadOnlyProperty() previous_error = AsyncRead(doc="OBSOLETE") profiling_info = AsyncRead() profiling_level = AsyncRead() remove_user = AsyncCommand() reset_error_history = AsyncCommand(doc="OBSOLETE") set_profiling_level = AsyncCommand() validate_collection = AsyncRead().unwrap('MotorCollection') incoming_manipulators = ReadOnlyProperty() incoming_copying_manipulators = ReadOnlyProperty() outgoing_manipulators = ReadOnlyProperty() outgoing_copying_manipulators = ReadOnlyProperty() def __init__(self, client, name, **kwargs): self._client = client delegate = kwargs.get('_delegate') or Database( client.delegate, name, **kwargs) super(self.__class__, self).__init__(delegate) @property def client(self): """This MotorDatabase's :class:`MotorClient`.""" return self._client def __getattr__(self, name): if name.startswith('_'): raise AttributeError( "%s has no attribute %r. To access the %s" " collection, use database['%s']." % ( self.__class__.__name__, name, name, name)) return self[name] def __getitem__(self, name): collection_class = create_class_with_framework( AgnosticCollection, self._framework, self.__module__) return collection_class(self, name) def __call__(self, *args, **kwargs): database_name = self.delegate.name client_class_name = self._client.__class__.__name__ if database_name == 'open_sync': raise TypeError( "%s.open_sync() is unnecessary Motor 0.2, " "see changelog for details." % client_class_name) raise TypeError( "MotorDatabase object is not callable. If you meant to " "call the '%s' method on a %s object it is " "failing because no such method exists." % ( database_name, client_class_name)) def wrap(self, collection): # Replace pymongo.collection.Collection with MotorCollection. klass = create_class_with_framework( AgnosticCollection, self._framework, self.__module__) return klass(self, collection.name, _delegate=collection) def add_son_manipulator(self, manipulator): """Add a new son manipulator to this database. Newly added manipulators will be applied before existing ones. :Parameters: - `manipulator`: the manipulator to add """ # We override add_son_manipulator to unwrap the AutoReference's # database attribute. if isinstance(manipulator, pymongo.son_manipulator.AutoReference): db = manipulator.database db_class = create_class_with_framework( AgnosticDatabase, self._framework, self.__module__) if isinstance(db, db_class): # db is a MotorDatabase; get the PyMongo Database instance. manipulator.database = db.delegate self.delegate.add_son_manipulator(manipulator) def get_io_loop(self): return self._client.get_io_loop() class AgnosticCollection(AgnosticBaseProperties): __motor_class_name__ = 'MotorCollection' __delegate_class__ = Collection bulk_write = AsyncCommand(doc=bulk_write_doc) count = AsyncRead() create_index = AsyncCommand() create_indexes = AsyncCommand(doc=create_indexes_doc) delete_many = AsyncCommand(doc=delete_many_doc) delete_one = AsyncCommand(doc=delete_one_doc) distinct = AsyncRead() drop = AsyncCommand(doc=drop_doc) drop_index = AsyncCommand() drop_indexes = AsyncCommand() ensure_index = AsyncCommand() find_and_modify = AsyncCommand() find_one = AsyncRead(doc=find_one_doc) find_one_and_delete = AsyncCommand(doc=find_one_and_delete_doc) find_one_and_replace = AsyncCommand(doc=find_one_and_replace_doc) find_one_and_update = AsyncCommand(doc=find_one_and_update_doc) full_name = ReadOnlyProperty() group = AsyncRead() index_information = AsyncRead(doc=index_information_doc) inline_map_reduce = AsyncRead() insert = AsyncWrite() insert_many = AsyncWrite(doc=insert_many_doc) insert_one = AsyncCommand(doc=insert_one_doc) map_reduce = AsyncCommand(doc=mr_doc).wrap(Collection) name = ReadOnlyProperty() options = AsyncRead() reindex = AsyncCommand() remove = AsyncWrite() rename = AsyncCommand() replace_one = AsyncCommand(doc=replace_one_doc) save = AsyncWrite() update = AsyncWrite(doc=update_doc) update_many = AsyncCommand(doc=update_many_doc) update_one = AsyncCommand(doc=update_one_doc) with_options = DelegateMethod().wrap(Collection) _async_aggregate = AsyncRead(attr_name='aggregate') _async_list_indexes = AsyncRead(attr_name='list_indexes') __parallel_scan = AsyncRead(attr_name='parallel_scan') def __init__(self, database, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None, _delegate=None): db_class = create_class_with_framework( AgnosticDatabase, self._framework, self.__module__) if not isinstance(database, db_class): raise TypeError("First argument to MotorCollection must be " "MotorDatabase, not %r" % database) delegate = _delegate or Collection( database.delegate, name, codec_options=codec_options, read_preference=read_preference, write_concern=write_concern, read_concern=read_concern) super(self.__class__, self).__init__(delegate) self.database = database def __getattr__(self, name): # Dotted collection name, like "foo.bar". if name.startswith('_'): full_name = "%s.%s" % (self.name, name) raise AttributeError( "%s has no attribute %r. To access the %s" " collection, use database['%s']." % ( self.__class__.__name__, name, full_name, full_name)) return self[name] def __getitem__(self, name): collection_class = create_class_with_framework( AgnosticCollection, self._framework, self.__module__) return collection_class(self.database, self.name + '.' + name) def __call__(self, *args, **kwargs): raise TypeError( "MotorCollection object is not callable. If you meant to " "call the '%s' method on a MotorCollection object it is " "failing because no such method exists." % self.delegate.name) def find(self, *args, **kwargs): """Create a :class:`MotorCursor`. Same parameters as for PyMongo's :meth:`~pymongo.collection.Collection.find`. Note that ``find`` does not take a `callback` parameter, nor does it return a Future, because ``find`` merely creates a :class:`MotorCursor` without performing any operations on the server. ``MotorCursor`` methods such as :meth:`~MotorCursor.to_list` or :meth:`~MotorCursor.count` perform actual operations. """ if 'callback' in kwargs: raise pymongo.errors.InvalidOperation( "Pass a callback to each, to_list, or count, not to find.") cursor = self.delegate.find(*args, **kwargs) cursor_class = create_class_with_framework( AgnosticCursor, self._framework, self.__module__) return cursor_class(cursor, self) def aggregate(self, pipeline, **kwargs): """Execute an aggregation pipeline on this collection. The aggregation can be run on a secondary if the client is connected to a replica set and its ``read_preference`` is not :attr:`PRIMARY`. :Parameters: - `pipeline`: a single command or list of aggregation commands - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs`: send arbitrary parameters to the aggregate command Returns a :class:`MotorCommandCursor` that can be iterated like a cursor from :meth:`find`:: pipeline = [{'$project': {'name': {'$toUpper': '$name'}}}] cursor = collection.aggregate(pipeline) while (yield cursor.fetch_next): doc = cursor.next_object() print(doc) In Python 3.5 and newer, aggregation cursors can be iterated elegantly in native coroutines with `async for`:: async def f(): async for doc in collection.aggregate(pipeline): print(doc) .. versionchanged:: 1.0 :meth:`aggregate` now **always** returns a cursor. .. versionchanged:: 0.5 :meth:`aggregate` now returns a cursor by default, and the cursor is returned immediately without a ``yield``. See :ref:`aggregation changes in Motor 0.5 `. .. versionchanged:: 0.2 Added cursor support. .. _aggregate command: http://docs.mongodb.org/manual/applications/aggregation """ if kwargs.get('cursor') is False: kwargs.pop('cursor') # One-shot aggregation, no cursor. Send command now, return Future. return self._async_aggregate(pipeline, **kwargs) else: if 'callback' in kwargs: raise pymongo.errors.InvalidOperation( "Pass a callback to to_list or each, not to aggregate.") kwargs.setdefault('cursor', {}) cursor_class = create_class_with_framework( AgnosticLatentCommandCursor, self._framework, self.__module__) # Latent cursor that will send initial command on first "async for". return cursor_class(self, self._async_aggregate, pipeline, **kwargs) def watch(self, pipeline=None, full_document='default', resume_after=None, max_await_time_ms=None, batch_size=None, collation=None, session=None): """Watch changes on this collection. Returns a :class:`~MotorChangeStream` cursor which iterates over changes on this collection. Introduced in MongoDB 3.6. A change stream continues waiting indefinitely for matching change events. Code like the following allows a program to cancel the change stream and exit. .. code-block:: python3 change_stream = None async def watch_collection(): global change_stream # Using the change stream in an "async with" block # ensures it is canceled promptly if your code breaks # from the loop or throws an exception. async with db.collection.watch() as change_stream: async for change in stream: print(change) # Tornado from tornado.ioloop import IOLoop def main(): loop = IOLoop.current() # Start watching collection for changes. loop.add_callback(watch_collection) try: loop.start() except KeyboardInterrupt: pass finally: if change_stream is not None: change_stream.close() # asyncio from asyncio import get_event_loop def main(): loop = get_event_loop() task = loop.create_task(watch_collection) try: loop.run_forever() except KeyboardInterrupt: pass finally: if change_stream is not None: change_stream.close() # Prevent "Task was destroyed but it is pending!" loop.run_until_complete(task) The :class:`~MotorChangeStream` async iterable blocks until the next change document is returned or an error is raised. If the :meth:`~MotorChangeStream.next` method encounters a network error when retrieving a batch from the server, it will automatically attempt to recreate the cursor such that no change events are missed. Any error encountered during the resume attempt indicates there may be an outage and will be raised. .. code-block:: python3 try: pipeline = [{'$match': {'operationType': 'insert'}}] async with db.collection.watch(pipeline) as stream: async for change in stream: print(change) except pymongo.errors.PyMongoError: # The ChangeStream encountered an unrecoverable error or the # resume attempt failed to recreate the cursor. logging.error('...') For a precise description of the resume process see the `change streams specification`_. :Parameters: - `pipeline` (optional): A list of aggregation pipeline stages to append to an initial ``$changeStream`` stage. Not all pipeline stages are valid after a ``$changeStream`` stage, see the MongoDB documentation on change streams for the supported stages. - `full_document` (optional): The fullDocument option to pass to the ``$changeStream`` stage. Allowed values: 'default', 'updateLookup'. Defaults to 'default'. When set to 'updateLookup', the change notification for partial updates will include both a delta describing the changes to the document, as well as a copy of the entire document that was changed from some time after the change occurred. - `resume_after` (optional): The logical starting point for this change stream. - `max_await_time_ms` (optional): The maximum time in milliseconds for the server to wait for changes before responding to a getMore operation. - `batch_size` (optional): The maximum number of documents to return per batch. - `collation` (optional): The :class:`~pymongo.collation.Collation` to use for the aggregation. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. :Returns: A :class:`~MotorChangeStream`. See the :ref:`tornado_change_stream_example`. .. versionadded:: 1.2 .. mongodoc:: changeStreams .. _change streams specification: https://github.com/mongodb/specifications/blob/master/source/change-streams.rst """ cursor_class = create_class_with_framework( AgnosticChangeStream, self._framework, self.__module__) # Latent cursor that will send initial command on first "async for". return cursor_class(self, pipeline, full_document, resume_after, max_await_time_ms, batch_size, collation, session) def list_indexes(self, session=None): """Get a cursor over the index documents for this collection. :: async def print_indexes(): for index in await db.test.list_indexes(): print(index) If the only index is the default index on ``_id``, this might print:: SON([('v', 1), ('key', SON([('_id', 1)])), ('name', '_id_'), ('ns', 'test.test')]) """ cursor_class = create_class_with_framework( AgnosticLatentCommandCursor, self._framework, self.__module__) # Latent cursor that will send initial command on first "async for". return cursor_class(self, self._async_list_indexes, session=session) def parallel_scan(self, num_cursors, **kwargs): """Scan this entire collection in parallel. Returns a list of up to ``num_cursors`` cursors that can be iterated concurrently. As long as the collection is not modified during scanning, each document appears once in one of the cursors' result sets. For example, to process each document in a collection using some function ``process_document()``:: @gen.coroutine def process_cursor(cursor): while (yield cursor.fetch_next): process_document(cursor.next_object()) # Get up to 4 cursors. cursors = yield collection.parallel_scan(4) yield [process_cursor(cursor) for cursor in cursors] # All documents have now been processed. If ``process_document()`` is a coroutine, do ``yield process_document(document)``. With a replica set, pass `read_preference` of :attr:`~pymongo.read_preference.ReadPreference.SECONDARY_PREFERRED` to scan a secondary. :Parameters: - `num_cursors`: the number of cursors to return - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. .. note:: Requires server version **>= 2.5.5**. """ io_loop = self.get_io_loop() original_future = self._framework.get_future(io_loop) # Return a future, or if user passed a callback chain it to the future. callback = kwargs.pop('callback', None) retval = self._framework.future_or_callback(original_future, callback, io_loop) # Once we have PyMongo Cursors, wrap in MotorCursors and resolve the # future with them, or pass them to the callback. self._framework.add_future( io_loop, self.__parallel_scan(num_cursors, **kwargs), self._scan_callback, original_future) return retval def _scan_callback(self, original_future, future): try: command_cursors = future.result() except Exception as exc: original_future.set_exception(exc) else: command_cursor_class = create_class_with_framework( AgnosticCommandCursor, self._framework, self.__module__) motor_command_cursors = [ command_cursor_class(cursor, self) for cursor in command_cursors] original_future.set_result(motor_command_cursors) def initialize_unordered_bulk_op(self, bypass_document_validation=False): """Initialize an unordered batch of write operations. Operations will be performed on the server in arbitrary order, possibly in parallel. All operations will be attempted. :Parameters: - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. Returns a :class:`~motor.MotorBulkOperationBuilder` instance. See :ref:`unordered_bulk` for examples. .. versionchanged:: 1.0 Added bypass_document_validation support .. versionadded:: 0.2 """ bob_class = create_class_with_framework( AgnosticBulkOperationBuilder, self._framework, self.__module__) return bob_class(self, ordered=False, bypass_document_validation=bypass_document_validation) def initialize_ordered_bulk_op(self, bypass_document_validation=False): """Initialize an ordered batch of write operations. Operations will be performed on the server serially, in the order provided. If an error occurs all remaining operations are aborted. :Parameters: - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. Returns a :class:`~motor.MotorBulkOperationBuilder` instance. See :ref:`ordered_bulk` for examples. .. versionchanged:: 1.0 Added bypass_document_validation support .. versionadded:: 0.2 """ bob_class = create_class_with_framework( AgnosticBulkOperationBuilder, self._framework, self.__module__) return bob_class(self, ordered=True, bypass_document_validation=bypass_document_validation) def wrap(self, obj): if obj.__class__ is Collection: # Replace pymongo.collection.Collection with MotorCollection. return self.__class__(self.database, obj.name, _delegate=obj) elif obj.__class__ is Cursor: return AgnosticCursor(obj, self) elif obj.__class__ is CommandCursor: command_cursor_class = create_class_with_framework( AgnosticCommandCursor, self._framework, self.__module__) return command_cursor_class(obj, self) elif obj.__class__ is ChangeStream: change_stream_class = create_class_with_framework( AgnosticChangeStream, self._framework, self.__module__) return change_stream_class(obj, self) else: return obj def get_io_loop(self): return self.database.get_io_loop() class AgnosticBaseCursor(AgnosticBase): """Base class for AgnosticCursor and AgnosticCommandCursor""" _refresh = AsyncRead() address = ReadOnlyProperty() cursor_id = ReadOnlyProperty() alive = ReadOnlyProperty() batch_size = MotorCursorChainingMethod() session = ReadOnlyProperty() def __init__(self, cursor, collection): """Don't construct a cursor yourself, but acquire one from methods like :meth:`MotorCollection.find` or :meth:`MotorCollection.aggregate`. .. note:: There is no need to manually close cursors; they are closed by the server after being fully iterated with :meth:`to_list`, :meth:`each`, or :attr:`fetch_next`, or automatically closed by the client when the :class:`MotorCursor` is cleaned up by the garbage collector. """ # 'cursor' is a PyMongo Cursor, CommandCursor, or a _LatentCursor. super(AgnosticBaseCursor, self).__init__(delegate=cursor) self.collection = collection self.started = False self.closed = False # python.org/dev/peps/pep-0492/#api-design-and-implementation-revisions if PY35: exec(textwrap.dedent(""" def __aiter__(self): return self async def __anext__(self): # An optimization: skip the "await" if possible. if self._buffer_size() or await self.fetch_next: return self.next_object() raise StopAsyncIteration() """), globals(), locals()) def _get_more(self): """Initial query or getMore. Returns a Future.""" if not self.alive: raise pymongo.errors.InvalidOperation( "Can't call get_more() on a MotorCursor that has been" " exhausted or killed.") self.started = True return self._refresh() @property @coroutine_annotation def fetch_next(self): """A Future used with `gen.coroutine`_ to asynchronously retrieve the next document in the result set, fetching a batch of documents from the server if necessary. Resolves to ``False`` if there are no more documents, otherwise :meth:`next_object` is guaranteed to return a document. .. _`gen.coroutine`: http://tornadoweb.org/en/stable/gen.html .. testsetup:: fetch_next MongoClient().test.test_collection.delete_many({}) collection = MotorClient().test.test_collection .. doctest:: fetch_next >>> @gen.coroutine ... def f(): ... yield collection.insert_many([{'_id': i} for i in range(5)]) ... cursor = collection.find().sort([('_id', 1)]) ... while (yield cursor.fetch_next): ... doc = cursor.next_object() ... sys.stdout.write(str(doc['_id']) + ', ') ... print('done') ... >>> IOLoop.current().run_sync(f) 0, 1, 2, 3, 4, done While it appears that fetch_next retrieves each document from the server individually, the cursor actually fetches documents efficiently in `large batches`_. In Python 3.5 and newer, cursors can be iterated elegantly and very efficiently in native coroutines with `async for`: .. doctest:: fetch_next >>> async def f(): ... async for doc in collection.find(): ... sys.stdout.write(str(doc['_id']) + ', ') ... print('done') ... >>> IOLoop.current().run_sync(f) 0, 1, 2, 3, 4, done .. _`large batches`: https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/#cursor-batches """ if not self._buffer_size() and self.alive: # Return the Future, which resolves to number of docs fetched or 0. return self._get_more() elif self._buffer_size(): future = self._framework.get_future(self.get_io_loop()) future.set_result(True) return future else: # Dead future = self._framework.get_future(self.get_io_loop()) future.set_result(False) return future def next_object(self): """Get a document from the most recently fetched batch, or ``None``. See :attr:`fetch_next`. """ if not self._buffer_size(): return None return next(self.delegate) def each(self, callback): """Iterates over all the documents for this cursor. :meth:`each` returns immediately, and `callback` is executed asynchronously for each document. `callback` is passed ``(None, None)`` when iteration is complete. Cancel iteration early by returning ``False`` from the callback. (Only ``False`` cancels iteration: returning ``None`` or 0 does not.) .. testsetup:: each from tornado.ioloop import IOLoop MongoClient().test.test_collection.delete_many({}) collection = MotorClient().test.test_collection .. doctest:: each >>> def inserted(result, error): ... if error: ... raise error ... cursor = collection.find().sort([('_id', 1)]) ... cursor.each(callback=each) ... >>> def each(result, error): ... if error: ... raise error ... elif result: ... sys.stdout.write(str(result['_id']) + ', ') ... else: ... # Iteration complete ... IOLoop.current().stop() ... print('done') ... >>> collection.insert_many( ... [{'_id': i} for i in range(5)], callback=inserted) >>> IOLoop.current().start() 0, 1, 2, 3, 4, done .. note:: Unlike other Motor methods, ``each`` requires a callback and does not return a Future, so it cannot be used in a coroutine. ``async for``, :meth:`to_list`, :attr:`fetch_next` are much easier to use. :Parameters: - `callback`: function taking (document, error) """ if not callable(callback): raise callback_type_error self._each_got_more(callback, None) def _each_got_more(self, callback, future): if future: try: future.result() except Exception as error: callback(None, error) return while self._buffer_size() > 0: doc = next(self.delegate) # decrements self.buffer_size # Quit if callback returns exactly False (not None). Note we # don't close the cursor: user may want to resume iteration. if callback(doc, None) is False: return # The callback closed this cursor? if self.closed: return if self.alive and (self.cursor_id or not self.started): self._framework.add_future( self.get_io_loop(), self._get_more(), self._each_got_more, callback) else: # Complete self._framework.call_soon( self.get_io_loop(), functools.partial(callback, None, None)) @coroutine_annotation def to_list(self, length, callback=None): """Get a list of documents. .. testsetup:: to_list MongoClient().test.test_collection.delete_many({}) from tornado import ioloop .. doctest:: to_list >>> from motor.motor_tornado import MotorClient >>> collection = MotorClient().test.test_collection >>> >>> @gen.coroutine ... def f(): ... yield collection.insert_many([{'_id': i} for i in range(4)]) ... cursor = collection.find().sort([('_id', 1)]) ... docs = yield cursor.to_list(length=2) ... while docs: ... print(docs) ... docs = yield cursor.to_list(length=2) ... ... print('done') ... >>> ioloop.IOLoop.current().run_sync(f) [{'_id': 0}, {'_id': 1}] [{'_id': 2}, {'_id': 3}] done :Parameters: - `length`: maximum number of documents to return for this call, or None - `callback` (optional): function taking (documents, error) If a callback is passed, returns None, else returns a Future. .. versionchanged:: 0.2 `callback` must be passed as a keyword argument, like ``to_list(10, callback=callback)``, and the `length` parameter is no longer optional. """ if length is not None: if not isinstance(length, int): raise TypeError('length must be an int, not %r' % length) elif length < 0: raise ValueError('length must be non-negative') if self._query_flags() & _QUERY_OPTIONS['tailable_cursor']: raise pymongo.errors.InvalidOperation( "Can't call to_list on tailable cursor") to_list_future = self._framework.get_future(self.get_io_loop()) # Run future_or_callback's type checking before we change anything. retval = self._framework.future_or_callback(to_list_future, callback, self.get_io_loop()) if not self.alive: to_list_future.set_result([]) else: the_list = [] self._framework.add_future( self.get_io_loop(), self._get_more(), self._to_list, length, the_list, to_list_future) return retval def _to_list(self, length, the_list, to_list_future, get_more_result): # get_more_result is the result of self._get_more(). # to_list_future will be the result of the user's to_list() call. try: result = get_more_result.result() collection = self.collection fix_outgoing = collection.database.delegate._fix_outgoing if length is None: n = result else: n = min(length, result) for _ in range(n): the_list.append(fix_outgoing(self._data().popleft(), collection)) reached_length = (length is not None and len(the_list) >= length) if reached_length or not self.alive: to_list_future.set_result(the_list) else: self._framework.add_future( self.get_io_loop(), self._get_more(), self._to_list, length, the_list, to_list_future) except Exception as exc: to_list_future.set_exception(exc) def get_io_loop(self): return self.collection.get_io_loop() @motor_coroutine def close(self): """Explicitly kill this cursor on the server. Call like (in Tornado): .. code-block:: python yield cursor.close() """ if not self.closed: self.closed = True yield self._framework.yieldable(self._close()) def _buffer_size(self): return len(self._data()) # Paper over some differences between PyMongo Cursor and CommandCursor. def _query_flags(self): raise NotImplementedError def _data(self): raise NotImplementedError def _clear_cursor_id(self): raise NotImplementedError def _close_exhaust_cursor(self): raise NotImplementedError def _killed(self): raise NotImplementedError @motor_coroutine def _close(self): raise NotImplementedError() class AgnosticCursor(AgnosticBaseCursor): __motor_class_name__ = 'MotorCursor' __delegate_class__ = Cursor address = ReadOnlyProperty() count = AsyncRead() collation = MotorCursorChainingMethod() distinct = AsyncRead() explain = AsyncRead() add_option = MotorCursorChainingMethod() remove_option = MotorCursorChainingMethod() limit = MotorCursorChainingMethod() skip = MotorCursorChainingMethod() max_scan = MotorCursorChainingMethod() sort = MotorCursorChainingMethod(doc=cursor_sort_doc) hint = MotorCursorChainingMethod() where = MotorCursorChainingMethod() max_await_time_ms = MotorCursorChainingMethod() max_time_ms = MotorCursorChainingMethod() min = MotorCursorChainingMethod() max = MotorCursorChainingMethod() comment = MotorCursorChainingMethod() _Cursor__die = AsyncRead() def rewind(self): """Rewind this cursor to its unevaluated state.""" self.delegate.rewind() self.started = False return self def clone(self): """Get a clone of this cursor.""" return self.__class__(self.delegate.clone(), self.collection) def __copy__(self): return self.__class__(self.delegate.__copy__(), self.collection) def __deepcopy__(self, memo): return self.__class__(self.delegate.__deepcopy__(memo), self.collection) def _query_flags(self): return self.delegate._Cursor__query_flags def _data(self): return self.delegate._Cursor__data def _clear_cursor_id(self): self.delegate._Cursor__id = 0 def _close_exhaust_cursor(self): # If an exhaust cursor is dying without fully iterating its results, # it must close the socket. PyMongo's Cursor does this, but we've # disabled its cleanup so we must do it ourselves. if self.delegate._Cursor__exhaust: manager = self.delegate._Cursor__exhaust_mgr if manager.sock: manager.sock.close() manager.close() def _killed(self): return self.delegate._Cursor__killed @motor_coroutine def _close(self): yield self._framework.yieldable(self._Cursor__die()) class AgnosticCommandCursor(AgnosticBaseCursor): __motor_class_name__ = 'MotorCommandCursor' __delegate_class__ = CommandCursor _CommandCursor__die = AsyncRead() def _query_flags(self): return 0 def _data(self): return self.delegate._CommandCursor__data def _clear_cursor_id(self): self.delegate._CommandCursor__id = 0 def _close_exhaust_cursor(self): # MongoDB doesn't have exhaust command cursors yet. pass def _killed(self): return self.delegate._CommandCursor__killed @motor_coroutine def _close(self): yield self._framework.yieldable(self._CommandCursor__die()) class _LatentCursor(object): """Take the place of a PyMongo CommandCursor until aggregate() begins.""" alive = True _CommandCursor__data = [] _CommandCursor__id = None _CommandCursor__killed = False cursor_id = None def clone(self): return _LatentCursor() def rewind(self): pass class AgnosticLatentCommandCursor(AgnosticCommandCursor): __motor_class_name__ = 'MotorLatentCommandCursor' def __init__(self, collection, start, *args, **kwargs): # We're being constructed without yield or await, like: # # cursor = collection.aggregate(pipeline) # # ... so we can't send the "aggregate" command to the server and get # a PyMongo CommandCursor back yet. Set self.delegate to a latent # cursor until the first yield or await triggers _get_more(), which # will execute the callback "start", which gets a PyMongo CommandCursor. super(self.__class__, self).__init__(_LatentCursor(), collection) self.start = start self.args = args self.kwargs = kwargs def _get_more(self): if not self.started: self.started = True original_future = self._framework.get_future(self.get_io_loop()) future = self.start( *self.args, **self.kwargs) self.start = self.args = self.kwargs = None self._framework.add_future( self.get_io_loop(), future, self._on_get_more, original_future) return original_future return super(self.__class__, self)._get_more() def _on_get_more(self, original_future, future): try: # "result" is a CommandCursor from PyMongo's aggregate(). self.delegate = future.result() except Exception as exc: original_future.set_exception(exc) else: # _get_more is complete. original_future.set_result(len(self.delegate._CommandCursor__data)) class AgnosticBulkOperationBuilder(AgnosticBase): __motor_class_name__ = 'MotorBulkOperationBuilder' __delegate_class__ = BulkOperationBuilder find = DelegateMethod() insert = DelegateMethod() execute = AsyncCommand() def __init__(self, collection, ordered, bypass_document_validation): self.io_loop = collection.get_io_loop() delegate = BulkOperationBuilder(collection.delegate, ordered, bypass_document_validation) super(self.__class__, self).__init__(delegate) def get_io_loop(self): return self.io_loop class AgnosticChangeStream(AgnosticBase): """A change stream cursor. Should not be called directly by application developers. See :meth:`~MotorCollection.watch` for example usage. .. versionadded: 1.2 .. mongodoc:: changeStreams """ __delegate_class__ = ChangeStream __motor_class_name__ = 'MotorChangeStream' _close = AsyncCommand(attr_name='close') def __init__(self, collection, pipeline, full_document, resume_after, max_await_time_ms, batch_size, collation, session): super(self.__class__, self).__init__(delegate=None) self._collection = collection self._kwargs = {'pipeline': pipeline, 'full_document': full_document, 'resume_after': resume_after, 'max_await_time_ms': max_await_time_ms, 'batch_size': batch_size, 'collation': collation, 'session': session} def _next(self): # This method is run on a thread. try: if not self.delegate: self.delegate = self._collection.delegate.watch(**self._kwargs) return self.delegate.next() except StopIteration: raise StopAsyncIteration() @coroutine_annotation(callback=False) def next(self): """Advance the cursor. This method blocks until the next change document is returned or an unrecoverable error is raised. Raises :exc:`StopAsyncIteration` if this change stream is closed. You can iterate the change stream by calling ``await change_stream.next()`` repeatedly, or with an "async for" loop: .. code-block:: python3 async for change in db.collection.watch(): print(change) """ loop = self.get_io_loop() return self._framework.run_on_executor(loop, self._next) @coroutine_annotation(callback=False) def close(self): """Close this change stream. Stops any "async for" loops using this change stream. """ if self.delegate: return self._close() # Never started. future = self._framework.get_future(self.get_io_loop()) future.set_result(None) return future if PY35: exec(textwrap.dedent(""" async def __aiter__(self): return self __anext__ = next async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): if self.delegate: self.delegate.close() """), globals(), locals()) def get_io_loop(self): return self._collection.get_io_loop() def __enter__(self): raise RuntimeError('Use a change stream in "async with", not "with"') def __exit__(self, exc_type, exc_val, exc_tb): pass motor-1.2.1/motor/docstrings.py000066400000000000000000001015431323007662700165560ustar00rootroot00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals get_database_doc = """ Get a :class:`MotorDatabase` with the given name and options. Useful for creating a :class:`MotorDatabase` with different codec options, read preference, and/or write concern from this :class:`MotorClient`. >>> from pymongo import ReadPreference >>> client.read_preference == ReadPreference.PRIMARY True >>> db1 = client.test >>> db1.read_preference == ReadPreference.PRIMARY True >>> db2 = client.get_database( ... 'test', read_preference=ReadPreference.SECONDARY) >>> db2.read_preference == ReadPreference.SECONDARY True :Parameters: - `name`: The name of the database - a string. - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`MongoClient` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`MongoClient` is used. See :mod:`~pymongo.read_preferences` for options. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`MongoClient` is used. """ bulk_write_doc = """Send a batch of write operations to the server. Requests are passed as a list of write operation instances imported from :mod:`pymongo`: :class:`~pymongo.operations.InsertOne`, :class:`~pymongo.operations.UpdateOne`, :class:`~pymongo.operations.UpdateMany`, :class:`~pymongo.operations.ReplaceOne`, :class:`~pymongo.operations.DeleteOne`, or :class:`~pymongo.operations.DeleteMany`). For example, say we have these documents:: {'x': 1, '_id': ObjectId('54f62e60fba5226811f634ef')} {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} We can insert a document, delete one, and replace one like so:: # DeleteMany, UpdateOne, and UpdateMany are also available. from pymongo import InsertOne, DeleteOne, ReplaceOne async def modify_data(): requests = [InsertOne({'y': 1}), DeleteOne({'x': 1}), ReplaceOne({'w': 1}, {'z': 1}, upsert=True)] result = await db.test.bulk_write(requests) print("inserted %d, deleted %d, modified %d" % ( result.inserted_count, result.deleted_count, result.modified_count)) print("upserted_ids: %s" % result.upserted_ids) print("collection:") async for doc in db.test.find(): print(doc) This will print something like:: inserted 1, deleted 1, modified 0 upserted_ids: {2: ObjectId('54f62ee28891e756a6e1abd5')} collection: {'x': 1, '_id': ObjectId('54f62e60fba5226811f634f0')} {'y': 1, '_id': ObjectId('54f62ee2fba5226811f634f1')} {'z': 1, '_id': ObjectId('54f62ee28891e756a6e1abd5')} :Parameters: - `requests`: A list of write operations (see examples above). - `ordered` (optional): If ``True`` (the default) requests will be performed on the server serially, in the order provided. If an error occurs all remaining operations are aborted. If ``False`` requests will be performed on the server in arbitrary order, possibly in parallel, and all operations will be attempted. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: An instance of :class:`~pymongo.results.BulkWriteResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 1.2 Added session parameter. """ create_indexes_doc = """Create one or more indexes on this collection:: from pymongo import IndexModel, ASCENDING, DESCENDING async def create_two_indexes(): index1 = IndexModel([("hello", DESCENDING), ("world", ASCENDING)], name="hello_world") index2 = IndexModel([("goodbye", DESCENDING)]) print(await db.test.create_indexes([index1, index2])) This prints:: ['hello_world', 'goodbye_-1'] :Parameters: - `indexes`: A list of :class:`~pymongo.operations.IndexModel` instances. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs` (optional): optional arguments to the createIndexes command (like maxTimeMS) can be passed as keyword arguments. The :attr:`~pymongo.collection.Collection.write_concern` of this collection is automatically applied to this operation when using MongoDB >= 3.4. .. versionchanged:: 1.2 Added session parameter. """ cmd_doc = """Issue a MongoDB command. Send command ``command`` to the database and return the response. If ``command`` is a string then the command ``{command: value}`` will be sent. Otherwise, ``command`` must be a :class:`dict` and will be sent as-is. Additional keyword arguments are added to the final command document before it is sent. For example, a command like ``{buildinfo: 1}`` can be sent using:: result = yield db.command("buildinfo") For a command where the value matters, like ``{collstats: collection_name}`` we can do:: result = yield db.command("collstats", collection_name) For commands that take additional arguments we can use kwargs. So ``{filemd5: object_id, root: file_root}`` becomes:: result = yield db.command("filemd5", object_id, root=file_root) :Parameters: - `command`: document representing the command to be issued, or the name of the command (for simple commands only). .. note:: the order of keys in the `command` document is significant (the "verb" must come first), so commands which require multiple keys (e.g. `findandmodify`) should use an instance of :class:`~bson.son.SON` or a string and kwargs instead of a Python :class:`dict`. - `value` (optional): value to use for the command verb when `command` is passed as a string - `check` (optional): check the response for errors, raising :class:`~pymongo.errors.OperationFailure` if there are any - `allowable_errors`: if `check` is ``True``, error messages in this list will be ignored by error-checking - `read_preference`: The read preference for this operation. See :mod:`~pymongo.read_preferences` for options. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs` (optional): additional keyword arguments will be added to the command document before it is sent .. versionchanged:: 1.2 Added session parameter. .. mongodoc:: commands """ delete_many_doc = """Delete one or more documents matching the filter. If we have a collection with 3 documents like ``{'x': 1}``, then:: async def clear_collection(): result = await db.test.delete_many({'x': 1}) print(result.deleted_count) This deletes all matching documents and prints "3". :Parameters: - `filter`: A query that matches the documents to delete. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: - An instance of :class:`~pymongo.results.DeleteResult`. .. versionchanged:: 1.2 Added session parameter. """ delete_one_doc = """Delete a single document matching the filter. If we have a collection with 3 documents like ``{'x': 1}``, then:: async def clear_collection(): result = await db.test.delete_one({'x': 1}) print(result.deleted_count) This deletes one matching document and prints "1". :Parameters: - `filter`: A query that matches the document to delete. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: - An instance of :class:`~pymongo.results.DeleteResult`. .. versionchanged:: 1.2 Added session parameter. """ drop_doc = """Alias for ``drop_collection``. The following two calls are equivalent:: await db.foo.drop() await db.drop_collection("foo") """ exists_doc = """Check if a file exists. The file to check for can be specified by the value of its ``_id`` key, or by passing in a query document. A query document can be passed in as dictionary, or by using keyword arguments. Thus, the following three calls are equivalent:: await fs.exists(file_id) await fs.exists({"_id": file_id}) await fs.exists(_id=file_id) As are the following two calls:: await fs.exists({"filename": "mike.txt"}) await fs.exists(filename="mike.txt") And the following two:: await fs.exists({"foo": {"$gt": 12}}) await fs.exists(foo={"$gt": 12}) Returns ``True`` if a matching file exists, ``False`` otherwise. Calls to :meth:`exists` will not automatically create appropriate indexes; application developers should be sure to create indexes if needed and as appropriate. :Parameters: - `document_or_id` (optional): query document, or _id of the document to check for - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs` (optional): keyword arguments are used as a query document, if they're present. .. versionchanged:: 1.2 Added session parameter. """ find_one_doc = """Get a single document from the database. All arguments to :meth:`find` are also valid arguments for :meth:`find_one`, although any `limit` argument will be ignored. Returns a single document, or ``None`` if no matching document is found. The :meth:`find_one` method obeys the :attr:`read_preference` of this Motor collection instance. :Parameters: - `filter` (optional): a dictionary specifying the query to be performed OR any other type to be used as the value for a query for ``"_id"``. - `*args` (optional): any additional positional arguments are the same as the arguments to :meth:`find`. - `**kwargs` (optional): any additional keyword arguments are the same as the arguments to :meth:`find`. - `max_time_ms` (optional): a value for max_time_ms may be specified as part of `**kwargs`, e.g.: .. code-block:: python3 await collection.find_one(max_time_ms=100) .. versionchanged:: 1.2 Added session parameter. """ find_one_and_delete_doc = """Finds a single document and deletes it, returning the document. If we have a collection with 2 documents like ``{'x': 1}``, then this code retrieves and deletes one of them:: async def delete_one_document(): print(await db.test.count({'x': 1})) doc = await db.test.find_one_and_delete({'x': 1}) print(doc) print(await db.test.count({'x': 1})) This outputs something like:: 2 {'x': 1, '_id': ObjectId('54f4e12bfba5220aa4d6dee8')} 1 If multiple documents match *filter*, a *sort* can be applied. Say we have 3 documents like:: {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} This code retrieves and deletes the document with the largest ``_id``:: async def delete_with_largest_id(): doc = await db.test.find_one_and_delete( {'x': 1}, sort=[('_id', pymongo.DESCENDING)]) This deletes one document and prints it:: {'x': 1, '_id': 2} The *projection* option can be used to limit the fields returned:: async def delete_and_return_x(): db.test.find_one_and_delete({'x': 1}, projection={'_id': False}) This prints:: {'x': 1} :Parameters: - `filter`: A query that matches the document to delete. - `projection` (optional): a list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a mapping to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is deleted. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). This command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionchanged:: 1.2 Added session parameter. """ find_one_and_replace_doc = """Finds a single document and replaces it, returning either the original or the replaced document. The :meth:`find_one_and_replace` method differs from :meth:`find_one_and_update` by replacing the document matched by *filter*, rather than modifying the existing document. Say we have 3 documents like:: {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} Replace one of them like so:: async def replace_one_doc(): original_doc = await db.test.find_one_and_replace({'x': 1}, {'y': 1}) print("original: %s" % original_doc) print("collection:") async for doc in db.test.find(): print(doc) This will print:: original: {'x': 1, '_id': 0} collection: {'y': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The replacement document. - `projection` (optional): A list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a mapping to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is replaced. - `upsert` (optional): When ``True``, inserts a new document if no document matches the query. Defaults to ``False``. - `return_document`: If :attr:`ReturnDocument.BEFORE` (the default), returns the original document before it was replaced, or ``None`` if no document matches. If :attr:`ReturnDocument.AFTER`, returns the replaced or inserted document. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). This command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionchanged:: 1.2 Added session parameter. """ find_one_and_update_doc = """Finds a single document and updates it, returning either the original or the updated document. By default :meth:`find_one_and_update` returns the original version of the document before the update was applied:: async def set_done(): print(await db.test.find_one_and_update( {'_id': 665}, {'$inc': {'count': 1}, '$set': {'done': True}})) This outputs:: {'_id': 665, 'done': False, 'count': 25}} To return the updated version of the document instead, use the *return_document* option. :: from pymongo import ReturnDocument async def increment_by_userid(): print(await db.example.find_one_and_update( {'_id': 'userid'}, {'$inc': {'seq': 1}}, return_document=ReturnDocument.AFTER)) This prints:: {'_id': 'userid', 'seq': 1} You can limit the fields returned with the *projection* option. :: async def increment_by_userid(): print(await db.example.find_one_and_update( {'_id': 'userid'}, {'$inc': {'seq': 1}}, projection={'seq': True, '_id': False}, return_document=ReturnDocument.AFTER)) This results in:: {'seq': 2} The *upsert* option can be used to create the document if it doesn't already exist. :: async def increment_by_userid(): print(await db.example.find_one_and_update( {'_id': 'userid'}, {'$inc': {'seq': 1}}, projection={'seq': True, '_id': False}, upsert=True, return_document=ReturnDocument.AFTER)) The result:: {'seq': 1} If multiple documents match *filter*, a *sort* can be applied. Say we have these two documents:: {'_id': 665, 'done': True, 'result': {'count': 26}} {'_id': 701, 'done': True, 'result': {'count': 17}} Then to update the one with the great ``_id``:: async def set_done(): print(await db.test.find_one_and_update( {'done': True}, {'$set': {'final': True}}, sort=[('_id', pymongo.DESCENDING)])) This would print:: {'_id': 701, 'done': True, 'result': {'count': 17}} :Parameters: - `filter`: A query that matches the document to update. - `update`: The update operations to apply. - `projection` (optional): A list of field names that should be returned in the result document or a mapping specifying the fields to include or exclude. If `projection` is a list "_id" will always be returned. Use a dict to exclude fields from the result (e.g. projection={'_id': False}). - `sort` (optional): a list of (key, direction) pairs specifying the sort order for the query. If multiple documents match the query, they are sorted and the first is updated. - `upsert` (optional): When ``True``, inserts a new document if no document matches the query. Defaults to ``False``. - `return_document`: If :attr:`ReturnDocument.BEFORE` (the default), returns the original document before it was updated, or ``None`` if no document matches. If :attr:`ReturnDocument.AFTER`, returns the updated or inserted document. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `**kwargs` (optional): additional command arguments can be passed as keyword arguments (for example maxTimeMS can be used with recent server versions). This command uses the :class:`~pymongo.write_concern.WriteConcern` of this :class:`~pymongo.collection.Collection` when connected to MongoDB >= 3.2. Note that using an elevated write concern with this command may be slower compared to using the default write concern. .. versionchanged:: 1.2 Added array_filters and session parameters. """ index_information_doc = """Get information on this collection's indexes. Returns a dictionary where the keys are index names (as returned by create_index()) and the values are dictionaries containing information about each index. The dictionary is guaranteed to contain at least a single key, ``"key"`` which is a list of (key, direction) pairs specifying the index (as passed to create_index()). It will also contain any other metadata about the indexes, except for the ``"ns"`` and ``"name"`` keys, which are cleaned. For example:: async def create_x_index(): print(await db.test.ensure_index("x", unique=True)) print(await db.test.index_information()) This prints:: 'x_1' {'_id_': {'key': [('_id', 1)]}, 'x_1': {'unique': True, 'key': [('x', 1)]}} .. versionchanged:: 1.2 Added session parameter. """ insert_many_doc = """Insert an iterable of documents. :: async def insert_2_docs(): result = db.test.insert_many([{'x': i} for i in range(2)]) result.inserted_ids This prints something like:: [ObjectId('54f113fffba522406c9cc20e'), ObjectId('54f113fffba522406c9cc20f')] :Parameters: - `documents`: A iterable of documents to insert. - `ordered` (optional): If ``True`` (the default) documents will be inserted on the server serially, in the order provided. If an error occurs all remaining inserts are aborted. If ``False``, documents will be inserted on the server in arbitrary order, possibly in parallel, and all document inserts will be attempted. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: An instance of :class:`~pymongo.results.InsertManyResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 1.2 Added session parameter. """ insert_one_doc = """Insert a single document. :: async def insert_x(): result = await db.test.insert_one({'x': 1}) print(result.inserted_id) This code outputs the new document's ``_id``:: ObjectId('54f112defba522406c9cc208') :Parameters: - `document`: The document to insert. Must be a mutable mapping type. If the document does not have an _id field one will be added automatically. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: - An instance of :class:`~pymongo.results.InsertOneResult`. .. seealso:: :ref:`writes-and-ids` .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 1.2 Added session parameter. """ mr_doc = """Perform a map/reduce operation on this collection. If `full_response` is ``False`` (default) returns a :class:`MotorCollection` instance containing the results of the operation. Otherwise, returns the full response from the server to the `map reduce command`_. :Parameters: - `map`: map function (as a JavaScript string) - `reduce`: reduce function (as a JavaScript string) - `out`: output collection name or `out object` (dict). See the `map reduce command`_ documentation for available options. Note: `out` options are order sensitive. :class:`~bson.son.SON` can be used to specify multiple options. e.g. SON([('replace', ), ('db', )]) - `full_response` (optional): if ``True``, return full response to this command - otherwise just return the result collection - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. - `callback` (optional): function taking (result, error), executed when operation completes. - `**kwargs` (optional): additional arguments to the `map reduce command`_ may be passed as keyword arguments to this helper method, e.g.:: result = yield db.test.map_reduce(map, reduce, "myresults", limit=2) If a callback is passed, returns None, else returns a Future. .. note:: The :meth:`map_reduce` method does **not** obey the :attr:`read_preference` of this :class:`MotorCollection`. To run mapReduce on a secondary use the :meth:`inline_map_reduce` method instead. .. _map reduce command: http://docs.mongodb.org/manual/reference/command/mapReduce/ .. mongodoc:: mapreduce .. versionchanged:: 1.2 Added session parameter. """ replace_one_doc = """Replace a single document matching the filter. Say our collection has one document:: {'x': 1, '_id': ObjectId('54f4c5befba5220aa4d6dee7')} Then to replace it with another:: async def_replace_x_with_y(): result = await db.test.replace_one({'x': 1}, {'y': 1}) print('matched %d, modified %d' % (result.matched_count, result.modified_count)) print('collection:') async for doc in db.test.find(): print(doc) This prints:: matched 1, modified 1 collection: {'y': 1, '_id': ObjectId('54f4c5befba5220aa4d6dee7')} The *upsert* option can be used to insert a new document if a matching document does not exist:: async def_replace_or_upsert(): result = await db.test.replace_one({'x': 1}, {'x': 1}, True) print('matched %d, modified %d, upserted_id %r' % (result.matched_count, result.modified_count, result.upserted_id)) print('collection:') async for doc in db.test.find(): print(doc) This prints:: matched 1, modified 1, upserted_id ObjectId('54f11e5c8891e756a6e1abd4') collection: {'y': 1, '_id': ObjectId('54f4c5befba5220aa4d6dee7')} :Parameters: - `filter`: A query that matches the document to replace. - `replacement`: The new document. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 1.2 Added session parameter. """ update_doc = """Update a document(s) in this collection. **DEPRECATED** - Use :meth:`replace_one`, :meth:`update_one`, or :meth:`update_many` instead.""" update_many_doc = """Update one or more documents that match the filter. Say our collection has 3 documents:: {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} We can add 3 to each "x" field:: async def add_3_to_x(): result = await db.test.update_many({'x': 1}, {'$inc': {'x': 3}}) print('matched %d, modified %d' % (result.matched_count, result.modified_count)) print('collection:') async for doc in db.test.find(): print(doc) This prints:: matched 3, modified 3 collection: {'x': 4, '_id': 0} {'x': 4, '_id': 1} {'x': 4, '_id': 2} :Parameters: - `filter`: A query that matches the documents to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation` (optional): If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 1.2 Added array_filters and session parameters. """ update_one_doc = """Update a single document matching the filter. Say our collection has 3 documents:: {'x': 1, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} We can add 3 to the "x" field of one of the documents:: async def add_3_to_x(): result = await db.test.update_one({'x': 1}, {'$inc': {'x': 3}}) print('matched %d, modified %d' % (result.matched_count, result.modified_count)) print('collection:') async for doc in db.test.find(): print(doc) This prints:: matched 1, modified 1 collection: {'x': 4, '_id': 0} {'x': 1, '_id': 1} {'x': 1, '_id': 2} :Parameters: - `filter`: A query that matches the document to update. - `update`: The modifications to apply. - `upsert` (optional): If ``True``, perform an insert if no documents match the filter. - `bypass_document_validation`: (optional) If ``True``, allows the write to opt-out of document level validation. Default is ``False``. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. This option is only supported on MongoDB 3.4 and above. - `array_filters` (optional): A list of filters specifying which array elements an update should apply. Requires MongoDB 3.6+. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. :Returns: - An instance of :class:`~pymongo.results.UpdateResult`. .. note:: `bypass_document_validation` requires server version **>= 3.2** .. versionchanged:: 1.2 Added array_filters and session parameters. """ cursor_sort_doc = """Sorts this cursor's results. Pass a field name and a direction, either :data:`~pymongo.ASCENDING` or :data:`~pymongo.DESCENDING`: .. testsetup:: sort MongoClient().test.test_collection.drop() MongoClient().test.test_collection.insert_many([ {'_id': i, 'field1': i % 2, 'field2': i} for i in range(5)]) collection = MotorClient().test.test_collection .. doctest:: sort >>> @gen.coroutine ... def f(): ... cursor = collection.find().sort('_id', pymongo.DESCENDING) ... docs = yield cursor.to_list(None) ... print([d['_id'] for d in docs]) ... >>> IOLoop.current().run_sync(f) [4, 3, 2, 1, 0] To sort by multiple fields, pass a list of (key, direction) pairs: .. doctest:: sort >>> @gen.coroutine ... def f(): ... cursor = collection.find().sort([ ... ('field1', pymongo.ASCENDING), ... ('field2', pymongo.DESCENDING)]) ... ... docs = yield cursor.to_list(None) ... print([(d['field1'], d['field2']) for d in docs]) ... >>> IOLoop.current().run_sync(f) [(0, 4), (0, 2), (0, 0), (1, 3), (1, 1)] Beginning with MongoDB version 2.6, text search results can be sorted by relevance: .. testsetup:: sort_text MongoClient().test.test_collection.drop() MongoClient().test.test_collection.insert_many([ {'field': 'words'}, {'field': 'words about some words'}]) MongoClient().test.test_collection.create_index([('field', 'text')]) collection = MotorClient().test.test_collection .. doctest:: sort_text >>> @gen.coroutine ... def f(): ... cursor = collection.find({ ... '$text': {'$search': 'some words'}}, ... {'score': {'$meta': 'textScore'}}) ... ... # Sort by 'score' field. ... cursor.sort([('score', {'$meta': 'textScore'})]) ... docs = yield cursor.to_list(None) ... for doc in docs: ... print('%.1f %s' % (doc['score'], doc['field'])) ... >>> IOLoop.current().run_sync(f) 1.5 words about some words 1.0 words Raises :class:`~pymongo.errors.InvalidOperation` if this cursor has already been used. Only the last :meth:`sort` applied to this cursor has any effect. :Parameters: - `key_or_list`: a single key or a list of (key, direction) pairs specifying the keys to sort on - `direction` (optional): only used if `key_or_list` is a single key, if not given :data:`~pymongo.ASCENDING` is assumed """ start_session_doc = """ Start a logical session. This method takes the same parameters as PyMongo's :class:`~pymongo.client_session.SessionOptions`. See the :mod:`~pymongo.client_session` module for details. .. code-block:: python3 async def coro(): collection = client.db.collection with (await client.start_session()) as s: doc = {'_id': ObjectId(), 'x': 1} await collection.insert_one(doc, session=s) secondary = collection.with_options( read_preference=ReadPreference.SECONDARY) # Sessions are causally consistent by default, we can read the doc # we just inserted, even reading from a secondary. async for doc in secondary.find(session=s): print(doc) Do **not** use the same session for multiple operations concurrently. Requires MongoDB 3.6. It is an error to call :meth:`start_session` if this client has been authenticated to multiple databases using the deprecated method :meth:`~pymongo.database.Database.authenticate`. A :class:`~pymongo.client_session.ClientSession` may only be used with the MongoClient that started it. :Returns: An instance of :class:`~pymongo.client_session.ClientSession`. .. versionadded:: 1.2 """ motor-1.2.1/motor/frameworks/000077500000000000000000000000001323007662700162015ustar00rootroot00000000000000motor-1.2.1/motor/frameworks/__init__.py000066400000000000000000000000001323007662700203000ustar00rootroot00000000000000motor-1.2.1/motor/frameworks/asyncio/000077500000000000000000000000001323007662700176465ustar00rootroot00000000000000motor-1.2.1/motor/frameworks/asyncio/__init__.py000066400000000000000000000116371323007662700217670ustar00rootroot00000000000000# Copyright 2014-2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """asyncio compatibility layer for Motor, an asynchronous MongoDB driver. See "Frameworks" in the Developer Guide. """ import asyncio import asyncio.tasks import os import functools import multiprocessing from concurrent.futures import ThreadPoolExecutor try: from asyncio import ensure_future except ImportError: from asyncio import async as ensure_future CLASS_PREFIX = 'AsyncIO' def get_event_loop(): return asyncio.get_event_loop() def is_event_loop(loop): return isinstance(loop, asyncio.AbstractEventLoop) def check_event_loop(loop): if not is_event_loop(loop): raise TypeError( "io_loop must be instance of asyncio-compatible event loop," "not %r" % loop) def get_future(loop): return asyncio.Future(loop=loop) if 'MOTOR_MAX_WORKERS' in os.environ: max_workers = int(os.environ['MOTOR_MAX_WORKERS']) else: max_workers = multiprocessing.cpu_count() * 5 _EXECUTOR = ThreadPoolExecutor(max_workers=max_workers) def run_on_executor(loop, fn, *args, **kwargs): # Adapted from asyncio's wrap_future and _chain_future. Ensure the wrapped # future is resolved on the main thread when the executor's future is # resolved on a worker thread. asyncio's wrap_future does the same, but # throws an error if the loop is stopped. We want to avoid errors if a # background task completes after the loop stops, e.g. ChangeStream.next() # returns while the program is shutting down. def _set_state(): if dest.cancelled(): return if source.cancelled(): dest.cancel() else: exception = source.exception() if exception is not None: dest.set_exception(exception) else: result = source.result() dest.set_result(result) def _call_check_cancel(_): if dest.cancelled(): source.cancel() def _call_set_state(_): if loop.is_closed(): return loop.call_soon_threadsafe(_set_state) source = _EXECUTOR.submit(functools.partial(fn, *args, **kwargs)) dest = asyncio.Future(loop=loop) dest.add_done_callback(_call_check_cancel) source.add_done_callback(_call_set_state) return dest _DEFAULT = object() def future_or_callback(future, callback, loop, return_value=_DEFAULT): """Compatible way to return a value in all Pythons. PEP 479, raise StopIteration(value) from a coroutine won't work forever, but "return value" doesn't work in Python 2. Instead, Motor methods that return values either execute a callback with the value or resolve a Future with it, and are implemented with callbacks rather than a coroutine internally. """ if callback: raise NotImplementedError("Motor with asyncio prohibits callbacks") if return_value is _DEFAULT: return future chained = asyncio.Future(loop=loop) def done_callback(_future): try: result = _future.result() chained.set_result(result if return_value is _DEFAULT else return_value) except Exception as exc: chained.set_exception(exc) future.add_done_callback(functools.partial(loop.call_soon_threadsafe, done_callback)) return chained def is_future(f): return isinstance(f, asyncio.Future) def call_soon(loop, callback, *args, **kwargs): if kwargs: loop.call_soon(functools.partial(callback, *args, **kwargs)) else: loop.call_soon(callback, *args) def add_future(loop, future, callback, *args): future.add_done_callback( functools.partial(loop.call_soon_threadsafe, callback, *args)) coroutine = asyncio.coroutine def pymongo_class_wrapper(f, pymongo_class): """Executes the coroutine f and wraps its result in a Motor class. See WrapAsync. """ @functools.wraps(f) @asyncio.coroutine def _wrapper(self, *args, **kwargs): result = yield from f(self, *args, **kwargs) # Don't call isinstance(), not checking subclasses. if result.__class__ == pymongo_class: # Delegate to the current object to wrap the result. return self.wrap(result) else: return result return _wrapper def yieldable(future): # TODO: really explain. return next(iter(future)) motor-1.2.1/motor/frameworks/tornado/000077500000000000000000000000001323007662700176475ustar00rootroot00000000000000motor-1.2.1/motor/frameworks/tornado/__init__.py000066400000000000000000000123671323007662700217710ustar00rootroot00000000000000# Copyright 2014-2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import, unicode_literals """Tornado compatibility layer for Motor, an asynchronous MongoDB driver. See "Frameworks" in the Developer Guide. """ import functools import os from concurrent.futures import ThreadPoolExecutor import tornado.process from tornado import concurrent, gen, ioloop from motor.motor_common import callback_type_error CLASS_PREFIX = '' def get_event_loop(): return ioloop.IOLoop.current() def is_event_loop(loop): return isinstance(loop, ioloop.IOLoop) def check_event_loop(loop): if not is_event_loop(loop): raise TypeError( "io_loop must be instance of IOLoop, not %r" % loop) def get_future(loop): return concurrent.Future() if 'MOTOR_MAX_WORKERS' in os.environ: max_workers = int(os.environ['MOTOR_MAX_WORKERS']) else: max_workers = tornado.process.cpu_count() * 5 _EXECUTOR = ThreadPoolExecutor(max_workers=max_workers) def run_on_executor(loop, fn, *args, **kwargs): # Need a Tornado Future for "await" expressions. exec_fut is resolved on a # worker thread, loop.add_future ensures "future" is resolved on main. future = concurrent.Future() exec_fut = _EXECUTOR.submit(fn, *args, **kwargs) def copy(_): if future.done(): return if exec_fut.exception() is not None: future.set_exception(exec_fut.exception()) else: future.set_result(exec_fut.result()) # Ensure copy runs on main thread. loop.add_future(exec_fut, copy) return future _DEFAULT = object() def future_or_callback(future, callback, io_loop, return_value=_DEFAULT): """Compatible way to return a value in all Pythons. PEP 479, raise StopIteration(value) from a coroutine won't work forever, but "return value" doesn't work in Python 2. Instead, Motor methods that return values either execute a callback with the value or resolve a Future with it, and are implemented with callbacks rather than a coroutine internally. """ if callback: if not callable(callback): raise callback_type_error # Motor's callback convention is "callback(result, error)". def done_callback(_future): try: result = _future.result() callback(result if return_value is _DEFAULT else return_value, None) except Exception as exc: callback(None, exc) io_loop.add_future(future, done_callback) elif return_value is not _DEFAULT: chained = concurrent.Future() def done_callback(_future): try: _future.result() except Exception as exc: chained.set_exception(exc) else: chained.set_result(return_value) io_loop.add_future(future, done_callback) return chained else: return future def is_future(f): return isinstance(f, concurrent.Future) def call_soon(loop, callback, *args, **kwargs): if args or kwargs: loop.add_callback(functools.partial(callback, *args, **kwargs)) else: loop.add_callback(callback) def add_future(loop, future, callback, *args): loop.add_future(future, functools.partial(callback, *args)) def coroutine(f): """A coroutine that accepts an optional callback. Given a callback, the function returns None, and the callback is run with (result, error). Without a callback the function returns a Future. """ coro = gen.coroutine(f) @functools.wraps(f) def wrapper(*args, **kwargs): callback = kwargs.pop('callback', None) if callback and not callable(callback): raise callback_type_error future = coro(*args, **kwargs) if callback: def _callback(_future): try: result = _future.result() callback(result, None) except Exception as e: callback(None, e) future.add_done_callback(_callback) else: return future return wrapper def pymongo_class_wrapper(f, pymongo_class): """Executes the coroutine f and wraps its result in a Motor class. See WrapAsync. """ @functools.wraps(f) @coroutine def _wrapper(self, *args, **kwargs): result = yield f(self, *args, **kwargs) # Don't call isinstance(), not checking subclasses. if result.__class__ == pymongo_class: # Delegate to the current object to wrap the result. raise gen.Return(self.wrap(result)) else: raise gen.Return(result) return _wrapper def yieldable(future): # TODO: really explain. return future motor-1.2.1/motor/metaprogramming.py000066400000000000000000000306201323007662700175650ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import """Dynamic class-creation for Motor.""" import inspect import functools from pymongo.cursor import Cursor from . import motor_py3_compat _class_cache = {} def asynchronize(framework, sync_method, doc=None): """Decorate `sync_method` so it accepts a callback or returns a Future. The method runs on a thread and calls the callback or resolves the Future when the thread completes. :Parameters: - `motor_class`: Motor class being created, e.g. MotorClient. - `framework`: An asynchronous framework - `sync_method`: Unbound method of pymongo Collection, Database, MongoClient, etc. - `doc`: Optionally override sync_method's docstring """ @functools.wraps(sync_method) def method(self, *args, **kwargs): loop = self.get_io_loop() callback = kwargs.pop('callback', None) future = framework.run_on_executor(loop, sync_method, self.delegate, *args, **kwargs) return framework.future_or_callback(future, callback, loop) # This is for the benefit of motor_extensions.py, which needs this info to # generate documentation with Sphinx. method.is_async_method = True name = sync_method.__name__ method.pymongo_method_name = name if doc is not None: method.__doc__ = doc return method _coro_token = object() def motor_coroutine(f): """Used by Motor classes to mark functions as coroutines. create_class_with_framework will decorate the function with a framework- specific coroutine decorator, like asyncio.coroutine or Tornado's gen.coroutine. You cannot return a value from a motor_coroutine, the syntax differences between Tornado on Python 2 and asyncio with Python 3.5 are impossible to bridge. """ f._is_motor_coroutine = _coro_token return f def coroutine_annotation(callback): """In docs, annotate a function that returns a Future with 'coroutine'. Unlike @motor_coroutine, this doesn't affect behavior. """ if isinstance(callback, bool): # Like: # @coroutine_annotation(callback=False) # def method(self): # def decorator(f): f.coroutine_annotation = True f.coroutine_has_callback = callback return f return decorator # Like: # @coroutine_annotation # def method(self): # f = callback f.coroutine_annotation = True f.coroutine_has_callback = True return f class MotorAttributeFactory(object): """Used by Motor classes to mark attributes that delegate in some way to PyMongo. At module import time, create_class_with_framework calls create_attribute() for each attr to create the final class attribute. """ def __init__(self, doc=None): self.doc = doc def create_attribute(self, cls, attr_name): raise NotImplementedError class Async(MotorAttributeFactory): def __init__(self, attr_name, doc=None): """A descriptor that wraps a PyMongo method, such as insert or remove, and returns an asynchronous version of the method, which accepts a callback or returns a Future. :Parameters: - `attr_name`: The name of the attribute on the PyMongo class, if different from attribute on the Motor class """ super(Async, self).__init__(doc) self.attr_name = attr_name def create_attribute(self, cls, attr_name): name = self.attr_name or attr_name method = getattr(cls.__delegate_class__, name) return asynchronize(framework=cls._framework, sync_method=method, doc=self.doc) def wrap(self, original_class): return WrapAsync(self, original_class) def unwrap(self, class_name): return Unwrap(self, class_name) class WrapBase(MotorAttributeFactory): def __init__(self, prop, doc=None): super(WrapBase, self).__init__(doc) self.property = prop class Wrap(WrapBase): def __init__(self, prop, original_class, doc=None): """Calls a synchronous method and wraps the PyMongo class instance it returns in a Motor class instance. :Parameters: - `prop`: A DelegateMethod, the method to call before wrapping its result in a Motor class. - `original_class`: A PyMongo class to be wrapped. """ super(Wrap, self).__init__(prop, doc=doc) self.original_class = original_class def create_attribute(self, cls, attr_name): method = getattr(cls.__delegate_class__, attr_name) original_class = self.original_class @functools.wraps(method) def wrapper(self_, *args, **kwargs): result = method(self_.delegate, *args, **kwargs) # Don't call isinstance(), not checking subclasses. if result.__class__ == original_class: # Delegate to the current object to wrap the result. return self_.wrap(result) else: return result if self.doc: wrapper.__doc__ = self.doc wrapper.is_wrap_method = True # For Synchro. return wrapper class WrapAsync(WrapBase): def __init__(self, prop, original_class): """Like Async, but before it executes the callback or resolves the Future, checks if result is a PyMongo class and wraps it in a Motor class. E.g., Motor's map_reduce should pass a MotorCollection instead of a PyMongo Collection to the Future. Uses the wrap() method on the owner object to do the actual wrapping. E.g., Database.create_collection returns a Collection, so MotorDatabase has: create_collection = AsyncCommand().wrap(Collection) Once Database.create_collection is done, Motor calls MotorDatabase.wrap() on its result, transforming the result from Collection to MotorCollection, which is passed to the callback or Future. :Parameters: - `prop`: An Async, the async method to call before wrapping its result in a Motor class. - `original_class`: A PyMongo class to be wrapped. """ super(WrapAsync, self).__init__(prop) self.original_class = original_class def create_attribute(self, cls, attr_name): async_method = self.property.create_attribute(cls, attr_name) original_class = self.original_class wrapper = cls._framework.pymongo_class_wrapper(async_method, original_class) if self.doc: wrapper.__doc__ = self.doc return wrapper class Unwrap(WrapBase): def __init__(self, prop, motor_class_name, doc=None): """A descriptor that checks if arguments are Motor classes and unwraps them. E.g., Motor's drop_database takes a MotorDatabase, unwraps it, and passes a PyMongo Database instead. :Parameters: - `prop`: An Async or DelegateMethod, the method to call with unwrapped arguments. - `motor_class_name`: Like 'MotorDatabase' or 'MotorCollection'. """ super(Unwrap, self).__init__(prop, doc=doc) assert isinstance(motor_class_name, motor_py3_compat.text_type) self.motor_class_name = motor_class_name def create_attribute(self, cls, attr_name): f = self.property.create_attribute(cls, attr_name) name = self.motor_class_name @functools.wraps(f) def _f(self, *args, **kwargs): # Don't call isinstance(), not checking subclasses. unwrapped_args = [ obj.delegate if obj.__class__.__name__.endswith(name) else obj for obj in args] unwrapped_kwargs = dict([ (key, obj.delegate if obj.__class__.__name__ == name else obj) for key, obj in kwargs.items()]) return f(self, *unwrapped_args, **unwrapped_kwargs) if self.doc: _f.__doc__ = self.doc _f.is_unwrap_method = True # For Synchro. return _f class AsyncRead(Async): def __init__(self, attr_name=None, doc=None): """A descriptor that wraps a PyMongo read method like find_one() that returns a Future. """ Async.__init__(self, attr_name=attr_name, doc=doc) class AsyncWrite(Async): def __init__(self, attr_name=None, doc=None): """A descriptor that wraps a PyMongo write method like update() that accepts getLastError options and returns a Future. """ Async.__init__(self, attr_name=attr_name, doc=doc) class AsyncCommand(Async): def __init__(self, attr_name=None, doc=None): """A descriptor that wraps a PyMongo command like copy_database() that returns a Future and does not accept getLastError options. """ Async.__init__(self, attr_name=attr_name, doc=doc) class ReadOnlyProperty(MotorAttributeFactory): """Creates a readonly attribute on the wrapped PyMongo object.""" def create_attribute(self, cls, attr_name): def fget(obj): return getattr(obj.delegate, attr_name) if self.doc: doc = self.doc else: doc = getattr(cls.__delegate_class__, attr_name).__doc__ if doc: return property(fget=fget, doc=doc) else: return property(fget=fget) class DelegateMethod(ReadOnlyProperty): """A method on the wrapped PyMongo object that does no I/O and can be called synchronously""" def wrap(self, original_class): return Wrap(self, original_class, doc=self.doc) def unwrap(self, class_name): return Unwrap(self, class_name, doc=self.doc) class MotorCursorChainingMethod(MotorAttributeFactory): def create_attribute(self, cls, attr_name): cursor_method = getattr(Cursor, attr_name) @functools.wraps(cursor_method) def return_clone(self, *args, **kwargs): cursor_method(self.delegate, *args, **kwargs) return self # This is for the benefit of Synchro, and motor_extensions.py return_clone.is_motorcursor_chaining_method = True return_clone.pymongo_method_name = attr_name if self.doc: return_clone.__doc__ = self.doc return return_clone def create_class_with_framework(cls, framework, module_name): motor_class_name = framework.CLASS_PREFIX + cls.__motor_class_name__ cache_key = (cls, motor_class_name, framework) cached_class = _class_cache.get(cache_key) if cached_class: return cached_class new_class = type(str(motor_class_name), cls.__bases__, cls.__dict__.copy()) new_class.__module__ = module_name new_class._framework = framework assert hasattr(new_class, '__delegate_class__') # If we're constructing MotorClient from AgnosticClient, for example, # the method resolution order is (AgnosticClient, AgnosticBase, object). # Iterate over bases looking for attributes and coroutines that must be # replaced with framework-specific ones. for base in reversed(inspect.getmro(cls)): # Turn attribute factories into real methods or descriptors. for name, attr in base.__dict__.items(): if isinstance(attr, MotorAttributeFactory): new_class_attr = attr.create_attribute(new_class, name) setattr(new_class, name, new_class_attr) elif getattr(attr, '_is_motor_coroutine', None) is _coro_token: coro = framework.coroutine(attr) del coro._is_motor_coroutine coro.coroutine_annotation = True setattr(new_class, name, coro) _class_cache[cache_key] = new_class return new_class motor-1.2.1/motor/motor_asyncio.py000066400000000000000000000037241323007662700172660ustar00rootroot00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Asyncio support for Motor, an asynchronous driver for MongoDB.""" from . import core, motor_gridfs from .frameworks import asyncio as asyncio_framework from .metaprogramming import create_class_with_framework __all__ = ['AsyncIOMotorClient'] def create_asyncio_class(cls): return create_class_with_framework(cls, asyncio_framework, 'motor_asyncio') AsyncIOMotorClient = create_asyncio_class(core.AgnosticClient) AsyncIOMotorDatabase = create_asyncio_class( core.AgnosticDatabase) AsyncIOMotorCollection = create_asyncio_class( core.AgnosticCollection) AsyncIOMotorCursor = create_asyncio_class( core.AgnosticCursor) AsyncIOMotorCommandCursor = create_asyncio_class( core.AgnosticCommandCursor) AsyncIOMotorLatentCommandCursor = create_asyncio_class( core.AgnosticLatentCommandCursor) AsyncIOMotorChangeStream = create_asyncio_class( core.AgnosticChangeStream) AsyncIOMotorBulkOperationBuilder = create_asyncio_class( core.AgnosticBulkOperationBuilder) AsyncIOMotorGridFS = create_asyncio_class( motor_gridfs.AgnosticGridFS) AsyncIOMotorGridFSBucket = create_asyncio_class( motor_gridfs.AgnosticGridFSBucket) AsyncIOMotorGridIn = create_asyncio_class( motor_gridfs.AgnosticGridIn) AsyncIOMotorGridOut = create_asyncio_class( motor_gridfs.AgnosticGridOut) AsyncIOMotorGridOutCursor = create_asyncio_class( motor_gridfs.AgnosticGridOutCursor) motor-1.2.1/motor/motor_common.py000066400000000000000000000013601323007662700171030ustar00rootroot00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import """Common code to support all async frameworks.""" callback_type_error = TypeError("callback must be a callable") motor-1.2.1/motor/motor_gridfs.py000066400000000000000000000556341323007662700171060ustar00rootroot00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import """GridFS implementation for Motor, an asynchronous driver for MongoDB.""" import textwrap import gridfs import pymongo import pymongo.errors from gridfs import grid_file from motor.core import (AgnosticBaseCursor, AgnosticCollection, AgnosticDatabase, PY35) from motor.docstrings import * from motor.metaprogramming import (AsyncCommand, AsyncRead, coroutine_annotation, create_class_with_framework, DelegateMethod, motor_coroutine, MotorCursorChainingMethod, ReadOnlyProperty) class AgnosticGridOutCursor(AgnosticBaseCursor): __motor_class_name__ = 'MotorGridOutCursor' __delegate_class__ = gridfs.GridOutCursor add_option = MotorCursorChainingMethod() address = ReadOnlyProperty() collation = ReadOnlyProperty() comment = MotorCursorChainingMethod() count = AsyncRead() distinct = AsyncRead() explain = AsyncRead() hint = MotorCursorChainingMethod() limit = MotorCursorChainingMethod() max = MotorCursorChainingMethod() max_await_time_ms = MotorCursorChainingMethod() max_scan = MotorCursorChainingMethod() max_time_ms = MotorCursorChainingMethod() min = MotorCursorChainingMethod() remove_option = MotorCursorChainingMethod() skip = MotorCursorChainingMethod() sort = MotorCursorChainingMethod(doc=cursor_sort_doc) where = MotorCursorChainingMethod() # PyMongo's GridOutCursor inherits __die from Cursor. _Cursor__die = AsyncCommand() def clone(self): """Get a clone of this cursor.""" return self.__class__(self.delegate.clone(), self.collection) def next_object(self): """Get next GridOut object from cursor.""" grid_out = super(self.__class__, self).next_object() if grid_out: grid_out_class = create_class_with_framework( AgnosticGridOut, self._framework, self.__module__) return grid_out_class(self.collection, delegate=grid_out) else: # Exhausted. return None def rewind(self): """Rewind this cursor to its unevaluated state.""" self.delegate.rewind() self.started = False return self def _empty(self): return self.delegate._Cursor__empty def _query_flags(self): return self.delegate._Cursor__query_flags def _data(self): return self.delegate._Cursor__data def _clear_cursor_id(self): self.delegate._Cursor__id = 0 def _close_exhaust_cursor(self): # Exhaust MotorGridOutCursors are prohibited. pass def _killed(self): return self.delegate._Cursor__killed @motor_coroutine def _close(self): yield self._framework.yieldable(self._Cursor__die()) class MotorGridOutProperty(ReadOnlyProperty): """Creates a readonly attribute on the wrapped PyMongo GridOut.""" def create_attribute(self, cls, attr_name): def fget(obj): if not obj.delegate._file: raise pymongo.errors.InvalidOperation( "You must call MotorGridOut.open() before accessing " "the %s property" % attr_name) return getattr(obj.delegate, attr_name) doc = getattr(cls.__delegate_class__, attr_name).__doc__ return property(fget=fget, doc=doc) class AgnosticGridOut(object): """Class to read data out of GridFS. MotorGridOut supports the same attributes as PyMongo's :class:`~gridfs.grid_file.GridOut`, such as ``_id``, ``content_type``, etc. You don't need to instantiate this class directly - use the methods provided by :class:`~motor.MotorGridFSBucket`. If it **is** instantiated directly, call :meth:`open`, :meth:`read`, or :meth:`readline` before accessing its attributes. """ __motor_class_name__ = 'MotorGridOut' __delegate_class__ = gridfs.GridOut _ensure_file = AsyncCommand() _id = MotorGridOutProperty() aliases = MotorGridOutProperty() chunk_size = MotorGridOutProperty() close = MotorGridOutProperty() content_type = MotorGridOutProperty() filename = MotorGridOutProperty() length = MotorGridOutProperty() md5 = MotorGridOutProperty() metadata = MotorGridOutProperty() name = MotorGridOutProperty() read = AsyncRead() readchunk = AsyncRead() readline = AsyncRead() seek = DelegateMethod() tell = DelegateMethod() upload_date = MotorGridOutProperty() def __init__( self, root_collection, file_id=None, file_document=None, delegate=None, ): collection_class = create_class_with_framework( AgnosticCollection, self._framework, self.__module__) if not isinstance(root_collection, collection_class): raise TypeError( "First argument to MotorGridOut must be " "MotorCollection, not %r" % root_collection) if delegate: self.delegate = delegate else: self.delegate = self.__delegate_class__( root_collection.delegate, file_id, file_document) self.io_loop = root_collection.get_io_loop() # python.org/dev/peps/pep-0492/#api-design-and-implementation-revisions if PY35: exec(textwrap.dedent(""" def __aiter__(self): return self async def __anext__(self): chunk = await self.readchunk() if chunk: return chunk raise StopAsyncIteration() """), globals(), locals()) def __getattr__(self, item): if not self.delegate._file: raise pymongo.errors.InvalidOperation( "You must call MotorGridOut.open() before accessing " "the %s property" % item) return getattr(self.delegate, item) @coroutine_annotation def open(self, callback=None): """Retrieve this file's attributes from the server. Takes an optional callback, or returns a Future. :Parameters: - `callback`: Optional function taking parameters (self, error) .. versionchanged:: 0.2 :class:`~motor.MotorGridOut` now opens itself on demand, calling ``open`` explicitly is rarely needed. """ return self._framework.future_or_callback(self._ensure_file(), callback, self.get_io_loop(), self) def get_io_loop(self): return self.io_loop @motor_coroutine def stream_to_handler(self, request_handler): """Write the contents of this file to a :class:`tornado.web.RequestHandler`. This method calls :meth:`~tornado.web.RequestHandler.flush` on the RequestHandler, so ensure all headers have already been set. For a more complete example see the implementation of :class:`~motor.web.GridFSHandler`. .. code-block:: python class FileHandler(tornado.web.RequestHandler): @tornado.web.asynchronous @gen.coroutine def get(self, filename): db = self.settings['db'] fs = yield motor.MotorGridFSBucket(db()) try: gridout = yield fs.open_download_stream_by_name(filename) except gridfs.NoFile: raise tornado.web.HTTPError(404) self.set_header("Content-Type", gridout.content_type) self.set_header("Content-Length", gridout.length) yield gridout.stream_to_handler(self) self.finish() .. seealso:: Tornado `RequestHandler `_ """ written = 0 while written < self.length: # Reading chunk_size at a time minimizes buffering. f = self._framework.yieldable(self.read(self.chunk_size)) yield f chunk = f.result() # write() simply appends the output to a list; flush() sends it # over the network and minimizes buffering in the handler. request_handler.write(chunk) request_handler.flush() written += len(chunk) class AgnosticGridIn(object): __motor_class_name__ = 'MotorGridIn' __delegate_class__ = gridfs.GridIn __getattr__ = DelegateMethod() abort = AsyncCommand() closed = ReadOnlyProperty() close = AsyncCommand() write = AsyncCommand().unwrap('MotorGridOut') writelines = AsyncCommand().unwrap('MotorGridOut') _id = ReadOnlyProperty() md5 = ReadOnlyProperty() filename = ReadOnlyProperty() name = ReadOnlyProperty() content_type = ReadOnlyProperty() length = ReadOnlyProperty() chunk_size = ReadOnlyProperty() upload_date = ReadOnlyProperty() set = AsyncCommand(attr_name='__setattr__', doc=""" Set an arbitrary metadata attribute on the file. Stores value on the server as a key-value pair within the file document once the file is closed. If the file is already closed, calling :meth:`set` will immediately update the file document on the server. Metadata set on the file appears as attributes on a :class:`~motor.MotorGridOut` object created from the file. :Parameters: - `name`: Name of the attribute, will be stored as a key in the file document on the server - `value`: Value of the attribute """) def __init__(self, root_collection, delegate=None, **kwargs): """ Class to write data to GridFS. Application developers should not generally need to instantiate this class - see :meth:`~motor.MotorGridFSBucket.open_upload_stream`. Any of the file level options specified in the `GridFS Spec `_ may be passed as keyword arguments. Any additional keyword arguments will be set as additional fields on the file document. Valid keyword arguments include: - ``"_id"``: unique ID for this file (default: :class:`~bson.objectid.ObjectId`) - this ``"_id"`` must not have already been used for another file - ``"filename"``: human name for the file - ``"contentType"`` or ``"content_type"``: valid mime-type for the file - ``"chunkSize"`` or ``"chunk_size"``: size of each of the chunks, in bytes (default: 256 kb) - ``"encoding"``: encoding used for this file. In Python 2, any :class:`unicode` that is written to the file will be converted to a :class:`str`. In Python 3, any :class:`str` that is written to the file will be converted to :class:`bytes`. :Parameters: - `root_collection`: A :class:`~motor.MotorCollection`, the root collection to write to - `**kwargs` (optional): file level options (see above) .. versionchanged:: 0.2 ``open`` method removed, no longer needed. """ collection_class = create_class_with_framework( AgnosticCollection, self._framework, self.__module__) if not isinstance(root_collection, collection_class): raise TypeError( "First argument to MotorGridIn must be " "MotorCollection, not %r" % root_collection) self.io_loop = root_collection.get_io_loop() if delegate: # Short cut. self.delegate = delegate else: self.delegate = self.__delegate_class__( root_collection.delegate, **kwargs) if PY35: # Support "async with fs.new_file() as f:" exec(textwrap.dedent(""" async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close() """), globals(), locals()) def get_io_loop(self): return self.io_loop class _GFSBase(object): __delegate_class__ = None def __init__(self, database, collection="fs"): db_class = create_class_with_framework( AgnosticDatabase, self._framework, self.__module__) if not isinstance(database, db_class): raise TypeError( "First argument to %s must be MotorDatabase, not %r" % ( self.__class__, database)) self.io_loop = database.get_io_loop() self.collection = database[collection] self.delegate = self.__delegate_class__( database.delegate, collection) def get_io_loop(self): return self.io_loop def wrap(self, obj): if obj.__class__ is grid_file.GridIn: grid_in_class = create_class_with_framework( AgnosticGridIn, self._framework, self.__module__) return grid_in_class( root_collection=self.collection, delegate=obj) elif obj.__class__ is grid_file.GridOut: grid_out_class = create_class_with_framework( AgnosticGridOut, self._framework, self.__module__) return grid_out_class( root_collection=self.collection, delegate=obj) elif obj.__class__ is gridfs.GridOutCursor: grid_out_class = create_class_with_framework( AgnosticGridOutCursor, self._framework, self.__module__) return grid_out_class( cursor=obj, collection=self.collection) class AgnosticGridFS(_GFSBase): __motor_class_name__ = 'MotorGridFS' __delegate_class__ = gridfs.GridFS find_one = AsyncRead().wrap(grid_file.GridOut) new_file = AsyncRead().wrap(grid_file.GridIn) get = AsyncRead().wrap(grid_file.GridOut) get_version = AsyncRead().wrap(grid_file.GridOut) get_last_version = AsyncRead().wrap(grid_file.GridOut) list = AsyncRead() exists = AsyncRead(doc=exists_doc) delete = AsyncCommand() put = AsyncCommand() def __init__(self, database, collection="fs"): """**DEPRECATED**: Use :class:`MotorGridFSBucket` or :class:`AsyncIOMotorGridFSBucket`. An instance of GridFS on top of a single Database. :Parameters: - `database`: a :class:`~motor.MotorDatabase` - `collection` (optional): A string, name of root collection to use, such as "fs" or "my_files" .. mongodoc:: gridfs .. versionchanged:: 0.2 ``open`` method removed; no longer needed. """ super(self.__class__, self).__init__(database, collection) def find(self, *args, **kwargs): """Query GridFS for files. Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: cursor = fs.find({"filename": "lisa.txt"}, no_cursor_timeout=True) while (yield cursor.fetch_next): grid_out = cursor.next_object() data = yield grid_out.read() This iterates through all versions of "lisa.txt" stored in GridFS. Note that setting no_cursor_timeout may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) would return a cursor to the three most recently uploaded files in GridFS. :meth:`~motor.MotorGridFS.find` follows a similar interface to :meth:`~motor.MotorCollection.find` in :class:`~motor.MotorCollection`. :Parameters: - `filter` (optional): a SON object specifying elements which must be present for a document to be included in the result set - `skip` (optional): the number of files to omit (from the start of the result set) when returning the results - `limit` (optional): the maximum number of results to return - `no_cursor_timeout` (optional): if False (the default), any returned cursor is closed by the server after 10 minutes of inactivity. If set to True, the returned cursor will never time out on the server. Care should be taken to ensure that cursors with no_cursor_timeout turned on are properly closed. - `sort` (optional): a list of (key, direction) pairs specifying the sort order for this query. See :meth:`~pymongo.cursor.Cursor.sort` for details. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. Raises :class:`TypeError` if any of the arguments are of improper type. Returns an instance of :class:`~gridfs.grid_file.GridOutCursor` corresponding to this query. If a :class:`~pymongo.client_session.ClientSession` is passed to :meth:`find`, all returned :class:`~MotorGridOut` instances are associated with that session. .. versionchanged:: 1.2 Added session parameter. .. versionchanged:: 1.0 Removed the read_preference, tag_sets, and secondary_acceptable_latency_ms options. .. versionadded:: 0.2 .. mongodoc:: find """ cursor = self.delegate.find(*args, **kwargs) grid_out_cursor = create_class_with_framework( AgnosticGridOutCursor, self._framework, self.__module__) return grid_out_cursor(cursor, self.collection) class AgnosticGridFSBucket(_GFSBase): __motor_class_name__ = 'MotorGridFSBucket' __delegate_class__ = gridfs.GridFSBucket delete = AsyncCommand() download_to_stream = AsyncCommand() download_to_stream_by_name = AsyncCommand() open_download_stream = AsyncCommand().wrap(gridfs.GridOut) open_download_stream_by_name = AsyncCommand().wrap(gridfs.GridOut) open_upload_stream = DelegateMethod().wrap(gridfs.GridIn) open_upload_stream_with_id = DelegateMethod().wrap(gridfs.GridIn) rename = AsyncCommand() upload_from_stream = AsyncCommand() upload_from_stream_with_id = AsyncCommand() def __init__(self, database, collection="fs"): """Create a handle to a GridFS bucket. Raises :exc:`~pymongo.errors.ConfigurationError` if `write_concern` is not acknowledged. This class is a replacement for :class:`.MotorGridFS`; it conforms to the `GridFS API Spec `_ for MongoDB drivers. :Parameters: - `database`: database to use. - `bucket_name` (optional): The name of the bucket. Defaults to 'fs'. - `chunk_size_bytes` (optional): The chunk size in bytes. Defaults to 255KB. - `write_concern` (optional): The :class:`~pymongo.write_concern.WriteConcern` to use. If ``None`` (the default) db.write_concern is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) db.read_preference is used. .. versionadded:: 1.0 .. mongodoc:: gridfs """ super(self.__class__, self).__init__(database, collection) def find(self, *args, **kwargs): """Find and return the files collection documents that match ``filter``. Returns a cursor that iterates across files matching arbitrary queries on the files collection. Can be combined with other modifiers for additional control. For example:: cursor = bucket.find({"filename": "lisa.txt"}, no_cursor_timeout=True) while (yield cursor.fetch_next): grid_out = cursor.next_object() data = yield grid_out.read() This iterates through all versions of "lisa.txt" stored in GridFS. Note that setting no_cursor_timeout to True may be important to prevent the cursor from timing out during long multi-file processing work. As another example, the call:: most_recent_three = fs.find().sort("uploadDate", -1).limit(3) would return a cursor to the three most recently uploaded files in GridFS. Follows a similar interface to :meth:`~motor.MotorCollection.find` in :class:`~motor.MotorCollection`. :Parameters: - `filter`: Search query. - `batch_size` (optional): The number of documents to return per batch. - `limit` (optional): The maximum number of documents to return. - `no_cursor_timeout` (optional): The server normally times out idle cursors after an inactivity period (10 minutes) to prevent excess memory use. Set this option to True prevent that. - `skip` (optional): The number of documents to skip before returning. - `sort` (optional): The order by which to sort results. Defaults to None. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`, created with :meth:`~MotorClient.start_session`. If a :class:`~pymongo.client_session.ClientSession` is passed to :meth:`find`, all returned :class:`MotorGridOut` instances are associated with that session. .. versionchanged:: 1.2 Added session parameter. """ cursor = self.delegate.find(*args, **kwargs) grid_out_cursor = create_class_with_framework( AgnosticGridOutCursor, self._framework, self.__module__) return grid_out_cursor(cursor, self.collection) motor-1.2.1/motor/motor_py3_compat.py000066400000000000000000000020621323007662700176710ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import """Python 2/3 compatibility utilities for Motor.""" import sys PY3 = False if sys.version_info[0] >= 3: PY3 = True if PY3: string_types = str, integer_types = int, text_type = str from io import BytesIO as StringIO else: string_types = basestring, integer_types = (int, long) text_type = unicode try: from cStringIO import StringIO except ImportError: from StringIO import StringIO motor-1.2.1/motor/motor_tornado.py000066400000000000000000000035241323007662700172650ustar00rootroot00000000000000# Copyright 2011-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tornado support for Motor, an asynchronous driver for MongoDB.""" from __future__ import unicode_literals, absolute_import from . import core, motor_gridfs from .frameworks import tornado as tornado_framework from .metaprogramming import create_class_with_framework __all__ = ['MotorClient'] def create_motor_class(cls): return create_class_with_framework(cls, tornado_framework, 'motor_tornado') MotorClient = create_motor_class(core.AgnosticClient) MotorDatabase = create_motor_class(core.AgnosticDatabase) MotorCollection = create_motor_class(core.AgnosticCollection) MotorCursor = create_motor_class(core.AgnosticCursor) MotorCommandCursor = create_motor_class(core.AgnosticCommandCursor) MotorLatentCommandCursor = create_motor_class(core.AgnosticLatentCommandCursor) MotorChangeStream = create_motor_class(core.AgnosticChangeStream) MotorBulkOperationBuilder = create_motor_class(core.AgnosticBulkOperationBuilder) MotorGridFS = create_motor_class(motor_gridfs.AgnosticGridFS) MotorGridFSBucket = create_motor_class(motor_gridfs.AgnosticGridFSBucket) MotorGridIn = create_motor_class(motor_gridfs.AgnosticGridIn) MotorGridOut = create_motor_class(motor_gridfs.AgnosticGridOut) MotorGridOutCursor = create_motor_class(motor_gridfs.AgnosticGridOutCursor) motor-1.2.1/motor/web.py000066400000000000000000000154741323007662700151630ustar00rootroot00000000000000# Copyright 2011-2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Utilities for using Motor with Tornado web applications.""" import datetime import email.utils import mimetypes import time import tornado.web from tornado import gen import gridfs import motor # TODO: this class is not a drop-in replacement for StaticFileHandler. # StaticFileHandler provides class method make_static_url, which appends # an MD5 of the static file's contents. Templates thus can do # {{ static_url('image.png') }} and get "/static/image.png?v=1234abcdef", # which is cached forever. Problem is, it calculates the MD5 synchronously. # Two options: keep a synchronous GridFS available to get each grid file's # MD5 synchronously for every static_url call, or find some other idiom. class GridFSHandler(tornado.web.RequestHandler): """A handler that can serve content from GridFS, very similar to :class:`tornado.web.StaticFileHandler`. .. code-block:: python db = motor.MotorClient().my_database application = web.Application([ (r"/static/(.*)", web.GridFSHandler, {"database": db}), ]) By default, requests' If-Modified-Since headers are honored, but no specific cache-control timeout is sent to clients. Thus each request for a GridFS file requires a quick check of the file's ``uploadDate`` in MongoDB. Override :meth:`get_cache_time` in a subclass to customize this. """ def initialize(self, database, root_collection='fs'): self.database = database self.root_collection = root_collection def get_gridfs_file(self, bucket, filename, request): """Overridable method to choose a GridFS file to serve at a URL. By default, if a URL pattern like ``"/static/(.*)"`` is mapped to this ``GridFSHandler``, then the trailing portion of the URL is used as the filename, so a request for "/static/image.png" results in a call to :meth:`MotorGridFSBucket.open_download_stream_by_name` with "image.png" as the ``filename`` argument. To customize the mapping of path to GridFS file, override ``get_gridfs_file`` and return a Future :class:`~motor.MotorGridOut` from it. For example, to retrieve the file by ``_id`` instead of filename:: class CustomGridFSHandler(motor.web.GridFSHandler): def get_gridfs_file(self, bucket, filename, request): # Path is interpreted as _id instead of name. # Return a Future MotorGridOut. return fs.open_download_stream(file_id=ObjectId(path)) :Parameters: - `bucket`: A :class:`~motor.motor_tornado.MotorGridFSBucket` - `filename`: A string, the matched group of the URL pattern - `request`: An :class:`tornado.httputil.HTTPServerRequest` .. versionchanged:: 1.0 **BREAKING CHANGE**: Now takes a :class:`~motor.motor_tornado.MotorGridFSBucket`, not a :class:`~motor.motor_tornado.MotorGridFS`. Also takes an additional ``request`` parameter. .. versionchanged:: 0.2 ``get_gridfs_file`` no longer accepts a callback, instead returns a Future. """ return bucket.open_download_stream_by_name(filename) @gen.coroutine def get(self, path, include_body=True): fs = motor.MotorGridFSBucket(self.database, self.root_collection) try: gridout = yield self.get_gridfs_file(fs, path, self.request) except gridfs.NoFile: raise tornado.web.HTTPError(404) # If-Modified-Since header is only good to the second. modified = gridout.upload_date.replace(microsecond=0) self.set_header("Last-Modified", modified) # MD5 is calculated on the MongoDB server when GridFS file is created self.set_header("Etag", '"%s"' % gridout.md5) mime_type = gridout.content_type # If content type is not defined, try to check it with mimetypes if mime_type is None: mime_type, encoding = mimetypes.guess_type(path) # Starting from here, largely a copy of StaticFileHandler if mime_type: self.set_header("Content-Type", mime_type) cache_time = self.get_cache_time(path, modified, mime_type) if cache_time > 0: self.set_header("Expires", datetime.datetime.utcnow() + datetime.timedelta(seconds=cache_time)) self.set_header("Cache-Control", "max-age=" + str(cache_time)) else: self.set_header("Cache-Control", "public") self.set_extra_headers(path, gridout) # Check the If-Modified-Since, and don't send the result if the # content has not been modified ims_value = self.request.headers.get("If-Modified-Since") if ims_value is not None: date_tuple = email.utils.parsedate(ims_value) # If our MotorClient is tz-aware, assume the naive ims_value is in # its time zone. if_since = datetime.datetime.fromtimestamp( time.mktime(date_tuple) ).replace(tzinfo=modified.tzinfo) if if_since >= modified: self.set_status(304) return # Same for Etag etag = self.request.headers.get("If-None-Match") if etag is not None and etag.strip('"') == gridout.md5: self.set_status(304) return self.set_header("Content-Length", gridout.length) if include_body: yield gridout.stream_to_handler(self) # Needed until fix for Tornado bug 751 is released, see # https://github.com/facebook/tornado/issues/751 and # https://github.com/facebook/tornado/commit/5491685 self.finish() def head(self, path): # get() is a coroutine. Return its Future. return self.get(path, include_body=False) def get_cache_time(self, path, modified, mime_type): """Override to customize cache control behavior. Return a positive number of seconds to trigger aggressive caching or 0 to mark resource as cacheable, only. 0 is the default. """ return 0 def set_extra_headers(self, path, gridout): """For subclass to add extra headers to the response""" pass motor-1.2.1/setup.cfg000066400000000000000000000000321323007662700144750ustar00rootroot00000000000000[bdist_wheel] universal=1 motor-1.2.1/setup.py000066400000000000000000000140771323007662700144040ustar00rootroot00000000000000import sys from distutils.cmd import Command from distutils.errors import DistutilsOptionError try: from setuptools import setup except ImportError: from ez_setup import use_setuptools use_setuptools('28.0') from setuptools import setup classifiers = """\ Intended Audience :: Developers License :: OSI Approved :: Apache Software License Development Status :: 4 - Beta Natural Language :: English Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Operating System :: MacOS :: MacOS X Operating System :: Unix Programming Language :: Python Programming Language :: Python :: Implementation :: CPython """ description = 'Non-blocking MongoDB driver for Tornado or asyncio' long_description = open("README.rst").read() install_requires = ['pymongo>=3.4,<4'] tests_require = ['mockupdb>=1.2.1'] if sys.version_info[0] < 3: # Need concurrent.futures backport in Python 2 for MotorMockServerTest. tests_require.append('futures') install_requires.append('futures') class test(Command): description = "run the tests" user_options = [ ("test-module=", "m", "Discover tests in specified module"), ("test-suite=", "s", "Test suite to run (e.g. 'some_module.test_suite')"), ("failfast", "f", "Stop running tests on first failure or error"), ("tornado-warnings", "w", "Let Tornado log warnings"), ("xunit-output=", "x", "Generate a results directory with XUnit XML format")] def initialize_options(self): self.test_module = None self.test_suite = None self.failfast = False self.tornado_warnings = False self.xunit_output = None def finalize_options(self): if self.test_suite is None and self.test_module is None: self.test_module = 'test' elif self.test_module is not None and self.test_suite is not None: raise DistutilsOptionError( "You may specify a module or suite, but not both") def run(self): # Installing required packages, running egg_info and build_ext are # part of normal operation for setuptools.command.test.test. Motor # has no extensions so build_ext is a no-op. if self.distribution.install_requires: self.distribution.fetch_build_eggs( self.distribution.install_requires) if self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) if self.xunit_output: self.distribution.fetch_build_eggs( ["unittest-xml-reporting>=1.14.0,<2.0.0a0"]) self.run_command('egg_info') build_ext_cmd = self.reinitialize_command('build_ext') build_ext_cmd.inplace = 1 self.run_command('build_ext') from test import (env, suppress_tornado_warnings, MotorTestLoader, test_environment as testenv) loader = MotorTestLoader() loader.avoid('high_availability', 'Runs separately') if not (testenv.HAVE_ASYNCIO or testenv.HAVE_TORNADO): raise ImportError("No tornado nor asyncio") elif not testenv.HAVE_TORNADO: loader.avoid('tornado_tests', reason='no tornado') elif not testenv.HAVE_ASYNCIO: loader.avoid('asyncio_tests', reason='no asyncio') if not testenv.HAVE_AIOHTTP: loader.avoid('asyncio_tests.test_aiohttp_gridfs', reason='no aiohttp') if sys.version_info[:2] < (3, 5): loader.avoid('asyncio_tests.test_asyncio_await', reason='python < 3.5') loader.avoid('asyncio_tests.test_asyncio_change_stream', reason='python < 3.5') # Decide if we can run async / await tests with Tornado. test_motor_await = 'tornado_tests.test_motor_await' if not testenv.HAVE_TORNADO: loader.avoid(test_motor_await, reason='no tornado') elif sys.version_info[:2] < (3, 5): loader.avoid(test_motor_await, reason='python < 3.5') loader.avoid('tornado_tests.test_motor_change_stream', reason='python < 3.5') if self.test_suite is None: suite = loader.discover(self.test_module) else: suite = loader.loadTestsFromName(self.test_suite) runner_kwargs = dict(verbosity=2, failfast=self.failfast) if self.xunit_output: runner_kwargs['output'] = self.xunit_output from xmlrunner import XMLTestRunner runner_class = XMLTestRunner else: import unittest runner_class = unittest.TextTestRunner runner = runner_class(**runner_kwargs) env.setup() if not self.tornado_warnings: suppress_tornado_warnings() result = runner.run(suite) sys.exit(not result.wasSuccessful()) packages = ['motor', 'motor.frameworks', 'motor.frameworks.tornado'] if sys.version_info[0] >= 3: # Trying to install and byte-compile motor/frameworks/asyncio/__init__.py # causes SyntaxError in Python 2. packages.append('motor.frameworks.asyncio') # Install aiohttp integration - aiohttp itself need not be installed. packages.append('motor.aiohttp') setup(name='motor', version='1.2.1', packages=packages, description=description, long_description=long_description, author='A. Jesse Jiryu Davis', author_email='jesse@mongodb.com', url='https://github.com/mongodb/motor/', install_requires=install_requires, license='http://www.apache.org/licenses/LICENSE-2.0', classifiers=[c for c in classifiers.split('\n') if c], keywords=[ "mongo", "mongodb", "pymongo", "gridfs", "bson", "motor", "tornado", "asyncio", ], tests_require=tests_require, test_suite='test', zip_safe=False, cmdclass={'test': test}) motor-1.2.1/synchro/000077500000000000000000000000001323007662700143465ustar00rootroot00000000000000motor-1.2.1/synchro/__init__.py000066400000000000000000000533671323007662700164750ustar00rootroot00000000000000# Copyright 2012-2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Synchro, a fake synchronous PyMongo implementation built on top of Motor, for the sole purpose of checking that Motor passes the same unittests as PyMongo. DO NOT USE THIS MODULE. """ import functools import inspect from tornado.ioloop import IOLoop import motor import motor.frameworks.tornado import motor.motor_tornado from motor.metaprogramming import MotorAttributeFactory # Make e.g. "from pymongo.errors import AutoReconnect" work. Note that # importing * won't pick up underscore-prefixed attrs. from gridfs.errors import * from pymongo import * from pymongo import (collation, change_stream, errors, monotonic, operations, server_selectors, server_type, son_manipulator, ssl_match_hostname, ssl_support, write_concern) from pymongo.auth import _build_credentials_tuple from pymongo.helpers import _check_command_response from pymongo.collation import * from pymongo.common import * from pymongo.common import _UUID_REPRESENTATIONS, _MAX_END_SESSIONS from pymongo.cursor import * from pymongo.cursor import _QUERY_OPTIONS from pymongo.errors import * from pymongo.message import (_COMMAND_OVERHEAD, _CursorAddress, _gen_find_command, _maybe_add_read_preference) from pymongo.monitor import * from pymongo.monitoring import * from pymongo.monitoring import _LISTENERS, _Listeners, _SENSITIVE_COMMANDS from pymongo.monotonic import time from pymongo.operations import * from pymongo.pool import * from pymongo.pool import _METADATA from pymongo.periodic_executor import * from pymongo.periodic_executor import _EXECUTORS from pymongo.read_concern import * from pymongo.read_preferences import * from pymongo.read_preferences import _ServerMode from pymongo.results import * from pymongo.results import _WriteResult from pymongo.server import * from pymongo.server_selectors import * from pymongo.settings import * from pymongo.ssl_support import * from pymongo.son_manipulator import * from pymongo.topology import * from pymongo.topology_description import * from pymongo.uri_parser import * from pymongo.uri_parser import _partition, _rpartition, _HAVE_DNSPYTHON from pymongo.write_concern import * from pymongo import auth from pymongo.auth import * from pymongo.auth import _password_digest from gridfs.grid_file import DEFAULT_CHUNK_SIZE, _SEEK_CUR, _SEEK_END from pymongo import GEOSPHERE, HASHED from pymongo.pool import SocketInfo, Pool def unwrap_synchro(fn): """If first argument to decorated function is a Synchro object, pass the wrapped Motor object into the function. """ @functools.wraps(fn) def _unwrap_synchro(*args, **kwargs): def _unwrap_obj(obj): if isinstance(obj, Synchro): return obj.delegate else: return obj args = [_unwrap_obj(arg) for arg in args] kwargs = dict([ (key, _unwrap_obj(value)) for key, value in kwargs.items()]) return fn(*args, **kwargs) return _unwrap_synchro def wrap_synchro(fn): """If decorated Synchro function returns a Motor object, wrap in a Synchro object. """ @functools.wraps(fn) def _wrap_synchro(*args, **kwargs): motor_obj = fn(*args, **kwargs) # Not all Motor classes appear here, only those we need to return # from methods like map_reduce() or create_collection() if isinstance(motor_obj, motor.MotorCollection): client = MongoClient(delegate=motor_obj.database.client) database = Database(client, motor_obj.database.name) return Collection(database, motor_obj.name, delegate=motor_obj) if isinstance(motor_obj, motor.MotorDatabase): client = MongoClient(delegate=motor_obj.client) return Database(client, motor_obj.name, delegate=motor_obj) if isinstance(motor_obj, motor.motor_tornado.MotorLatentCommandCursor): return CommandCursor(motor_obj) if isinstance(motor_obj, motor.motor_tornado.MotorCommandCursor): return CommandCursor(motor_obj) if isinstance(motor_obj, motor.motor_tornado.MotorCursor): return Cursor(motor_obj) if isinstance(motor_obj, motor.MotorBulkOperationBuilder): return BulkOperationBuilder(motor_obj) if isinstance(motor_obj, motor.MotorGridFS): return GridFS(motor_obj) if isinstance(motor_obj, motor.MotorGridIn): return GridIn(None, delegate=motor_obj) if isinstance(motor_obj, motor.MotorGridOut): return GridOut(None, delegate=motor_obj) if isinstance(motor_obj, motor.motor_tornado.MotorGridOutCursor): return GridOutCursor(motor_obj) else: return motor_obj return _wrap_synchro class Sync(object): def __init__(self, name): self.name = name def __get__(self, obj, objtype): async_method = getattr(obj.delegate, self.name) return wrap_synchro(unwrap_synchro(obj.synchronize(async_method))) class WrapOutgoing(object): def __get__(self, obj, objtype): # self.name is set by SynchroMeta. name = self.name def synchro_method(*args, **kwargs): motor_method = getattr(obj.delegate, name) return wrap_synchro(motor_method)(*args, **kwargs) return synchro_method class SynchroProperty(object): """Used to fake private properties like MongoClient.__member - don't use for real properties like write_concern or you'll mask missing features in Motor! """ def __init__(self): self.name = None def __get__(self, obj, objtype): # self.name is set by SynchroMeta. return getattr(obj.delegate.delegate, self.name) def __set__(self, obj, val): # self.name is set by SynchroMeta. return setattr(obj.delegate.delegate, self.name, val) def wrap_outgoing(delegate_attr): for decoration in ('is_motorcursor_chaining_method', 'is_wrap_method'): if getattr(delegate_attr, decoration, False): return True return False class SynchroMeta(type): """This metaclass customizes creation of Synchro's MongoClient, Database, etc., classes: - All asynchronized methods of Motor classes, such as MotorDatabase.command(), are re-synchronized. - Properties delegated from Motor's classes to PyMongo's, such as ``name`` or ``host``, are delegated **again** from Synchro's class to Motor's. - Motor methods which return Motor class instances are wrapped to return Synchro class instances. - Certain internals accessed by PyMongo's unittests, such as _Cursor__data, are delegated from Synchro directly to PyMongo. """ def __new__(cls, name, bases, attrs): # Create the class, e.g. the Synchro MongoClient or Database class. new_class = type.__new__(cls, name, bases, attrs) # delegate_class is a Motor class like MotorClient. delegate_class = new_class.__delegate_class__ if delegate_class: delegated_attrs = {} for klass in reversed(inspect.getmro(delegate_class)): delegated_attrs.update(klass.__dict__) for attrname, delegate_attr in delegated_attrs.items(): # If attrname is in attrs, it means Synchro has overridden # this attribute, e.g. Database.add_son_manipulator which is # special-cased. Ignore such attrs. if attrname in attrs: continue if getattr(delegate_attr, 'is_async_method', False): # Re-synchronize the method. setattr(new_class, attrname, Sync(attrname)) elif wrap_outgoing(delegate_attr): # Wrap MotorCursors in Synchro Cursors. wrapper = WrapOutgoing() wrapper.name = attrname setattr(new_class, attrname, wrapper) elif isinstance(delegate_attr, property): # Delegate the property from Synchro to Motor. setattr(new_class, attrname, delegate_attr) # Set DelegateProperties' and SynchroProperties' names. for name, attr in attrs.items(): if isinstance(attr, (MotorAttributeFactory, SynchroProperty, WrapOutgoing)): attr.name = name return new_class class Synchro(object): """ Wraps a MotorClient, MotorDatabase, MotorCollection, etc. and makes it act like the synchronous pymongo equivalent """ __metaclass__ = SynchroMeta __delegate_class__ = None def __cmp__(self, other): return cmp(self.delegate, other.delegate) def synchronize(self, async_method): """ @param async_method: Bound method of a MotorClient, MotorDatabase, etc. @return: A synchronous wrapper around the method """ @functools.wraps(async_method) def synchronized_method(*args, **kwargs): @functools.wraps(async_method) def partial(): return async_method(*args, **kwargs) return IOLoop.current().run_sync(partial) return synchronized_method class MongoClient(Synchro): __delegate_class__ = motor.MotorClient HOST = 'localhost' PORT = 27017 _cache_credentials = SynchroProperty() get_database = WrapOutgoing() get_default_database = WrapOutgoing() max_pool_size = SynchroProperty() max_write_batch_size = SynchroProperty() def __init__(self, host=None, port=None, *args, **kwargs): # So that TestClient.test_constants and test_types work. host = host if host is not None else MongoClient.HOST port = port if port is not None else MongoClient.PORT self.delegate = kwargs.pop('delegate', None) # Motor passes connect=False by default. kwargs.setdefault('connect', True) if not self.delegate: self.delegate = self.__delegate_class__(host, port, *args, **kwargs) @property def is_locked(self): # MotorClient doesn't support the is_locked property. # Synchro has already synchronized current_op; use it. result = self.admin.current_op() return bool(result.get('fsyncLock', None)) def __enter__(self): return self def __exit__(self, *args): self.delegate.close() def __getattr__(self, name): return Database(self, name, delegate=getattr(self.delegate, name)) def __getitem__(self, name): return Database(self, name, delegate=self.delegate[name]) _MongoClient__options = SynchroProperty() _get_topology = SynchroProperty() _kill_cursors_executor = SynchroProperty() class Database(Synchro): __delegate_class__ = motor.MotorDatabase get_collection = WrapOutgoing() def __init__(self, client, name, **kwargs): assert isinstance(client, MongoClient), ( "Expected MongoClient, got %s" % repr(client)) self._client = client self.delegate = kwargs.get('delegate') or motor.MotorDatabase( client.delegate, name, **kwargs) assert isinstance(self.delegate, motor.MotorDatabase), ( "synchro.Database delegate must be MotorDatabase, not " " %s" % repr(self.delegate)) def add_son_manipulator(self, manipulator): if isinstance(manipulator, son_manipulator.AutoReference): db = manipulator.database if isinstance(db, Database): manipulator.database = db.delegate.delegate self.delegate.add_son_manipulator(manipulator) @property def client(self): return self._client def __getattr__(self, name): return Collection(self, name, delegate=getattr(self.delegate, name)) def __getitem__(self, name): return Collection(self, name, delegate=self.delegate[name]) class Collection(Synchro): __delegate_class__ = motor.MotorCollection find = WrapOutgoing() initialize_unordered_bulk_op = WrapOutgoing() initialize_ordered_bulk_op = WrapOutgoing() list_indexes = WrapOutgoing() def __init__(self, database, name, **kwargs): if not isinstance(database, Database): raise TypeError( "First argument to synchro Collection must be synchro " "Database, not %s" % repr(database)) self.database = database self.delegate = kwargs.get('delegate') or motor.MotorCollection( self.database.delegate, name, **kwargs) if not isinstance(self.delegate, motor.MotorCollection): raise TypeError( "Expected to get synchro Collection from Database," " got %s" % repr(self.delegate)) def aggregate(self, *args, **kwargs): # Motor does no I/O initially in aggregate() but PyMongo does. cursor = wrap_synchro(self.delegate.aggregate)(*args, **kwargs) self.synchronize(cursor.delegate._get_more)() return cursor def parallel_scan(self, *args, **kwargs): # Motor does no I/O initially in parallel_scan() but PyMongo does. return self.synchronize(self.delegate.parallel_scan)(*args, **kwargs) def __getattr__(self, name): # Access to collections with dotted names, like db.test.mike fullname = self.name + '.' + name return Collection(self.database, fullname, delegate=getattr(self.delegate, name)) def __getitem__(self, name): # Access to collections with dotted names, like db.test['mike'] fullname = self.name + '.' + name return Collection(self.database, fullname, delegate=self.delegate[name]) class Cursor(Synchro): __delegate_class__ = motor.motor_tornado.MotorCursor rewind = WrapOutgoing() clone = WrapOutgoing() close = Sync('close') def __init__(self, motor_cursor): self.delegate = motor_cursor def __iter__(self): return self # These are special cases, they need to be accessed on the class, not just # on instances. @wrap_synchro def __copy__(self): return self.delegate.__copy__() @wrap_synchro def __deepcopy__(self, memo): return self.delegate.__deepcopy__(memo) def next(self): cursor = self.delegate if cursor._buffer_size(): return cursor.next_object() elif cursor.alive: self.synchronize(cursor._get_more)() if cursor._buffer_size(): return cursor.next_object() raise StopIteration def __getitem__(self, index): if isinstance(index, slice): return Cursor(self.delegate[index]) else: to_list = self.synchronize(self.delegate[index].to_list) return to_list(length=10000)[0] @property @wrap_synchro def collection(self): return self.delegate.collection def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() # Don't suppress exceptions. return False # For PyMongo tests that access cursor internals. _Cursor__data = SynchroProperty() _Cursor__exhaust = SynchroProperty() _Cursor__max_await_time_ms = SynchroProperty() _Cursor__max_time_ms = SynchroProperty() _Cursor__query_flags = SynchroProperty() _Cursor__query_spec = SynchroProperty() _Cursor__read_preference = SynchroProperty() _Cursor__retrieved = SynchroProperty() _Cursor__spec = SynchroProperty() class CommandCursor(Cursor): __delegate_class__ = motor.motor_tornado.MotorCommandCursor class GridOutCursor(Cursor): __delegate_class__ = motor.motor_tornado.MotorGridOutCursor def __init__(self, delegate): if not isinstance(delegate, motor.motor_tornado.MotorGridOutCursor): raise TypeError( "Expected MotorGridOutCursor, got %r" % delegate) self.delegate = delegate def next(self): motor_grid_out = super(GridOutCursor, self).next() if motor_grid_out: return GridOut(self.collection, delegate=motor_grid_out) __next__ = next class CursorManager(object): # Motor doesn't support cursor managers, just avoid ImportError. pass class BulkOperationBuilder(Synchro): __delegate_class__ = motor.MotorBulkOperationBuilder def __init__(self, motor_bob): if not isinstance(motor_bob, motor.MotorBulkOperationBuilder): raise TypeError( "Expected MotorBulkOperationBuilder, got %r" % motor_bob) self.delegate = motor_bob class GridFS(Synchro): __delegate_class__ = motor.MotorGridFS def __init__(self, database, collection='fs'): if not isinstance(database, Database): raise TypeError( "Expected Database, got %s" % repr(database)) self.delegate = motor.MotorGridFS(database.delegate, collection) def put(self, *args, **kwargs): return self.synchronize(self.delegate.put)(*args, **kwargs) def find(self, *args, **kwargs): motor_method = self.delegate.find unwrapping_method = wrap_synchro(unwrap_synchro(motor_method)) return unwrapping_method(*args, **kwargs) class GridFSBucket(Synchro): __delegate_class__ = motor.MotorGridFSBucket def __init__(self, database, bucket_name='fs'): if not isinstance(database, Database): raise TypeError( "Expected Database, got %s" % repr(database)) self.delegate = motor.MotorGridFSBucket(database.delegate, bucket_name) def find(self, *args, **kwargs): motor_method = self.delegate.find unwrapping_method = wrap_synchro(unwrap_synchro(motor_method)) return unwrapping_method(*args, **kwargs) class GridIn(Synchro): __delegate_class__ = motor.MotorGridIn def __init__(self, collection, **kwargs): """Can be created with collection and kwargs like a PyMongo GridIn, or with a 'delegate' keyword arg, where delegate is a MotorGridIn. """ delegate = kwargs.pop('delegate', None) if delegate: self.delegate = delegate else: if not isinstance(collection, Collection): raise TypeError( "Expected Collection, got %s" % repr(collection)) self.delegate = motor.MotorGridIn(collection.delegate, **kwargs) def __getattr__(self, item): return getattr(self.delegate, item) class SynchroGridOutProperty(object): def __init__(self, name): self.name = name def __get__(self, obj, objtype): obj.synchronize(obj.delegate.open)() return getattr(obj.delegate, self.name) class GridOut(Synchro): __delegate_class__ = motor.MotorGridOut _id = SynchroGridOutProperty('_id') aliases = SynchroGridOutProperty('aliases') chunk_size = SynchroGridOutProperty('chunk_size') close = SynchroGridOutProperty('close') content_type = SynchroGridOutProperty('content_type') filename = SynchroGridOutProperty('filename') length = SynchroGridOutProperty('length') md5 = SynchroGridOutProperty('md5') metadata = SynchroGridOutProperty('metadata') name = SynchroGridOutProperty('name') upload_date = SynchroGridOutProperty('upload_date') def __init__( self, root_collection, file_id=None, file_document=None, delegate=None): """Can be created with collection and kwargs like a PyMongo GridOut, or with a 'delegate' keyword arg, where delegate is a MotorGridOut. """ if delegate: self.delegate = delegate else: if not isinstance(root_collection, Collection): raise TypeError( "Expected Collection, got %s" % repr(root_collection)) self.delegate = motor.MotorGridOut( root_collection.delegate, file_id, file_document) def __getattr__(self, item): self.synchronize(self.delegate.open)() return getattr(self.delegate, item) def __setattr__(self, key, value): # PyMongo's GridOut prohibits setting these values; do the same # to make PyMongo's assertRaises tests pass. if key in ( "_id", "name", "content_type", "length", "chunk_size", "upload_date", "aliases", "metadata", "md5"): raise AttributeError() super(GridOut, self).__setattr__(key, value) class TimeModule(object): """Fake time module so time.sleep() lets other tasks run on the IOLoop. See e.g. test_schedule_refresh() in test_replica_set_client.py. """ def __getattr__(self, item): def sleep(seconds): loop = IOLoop.current() loop.add_timeout(time.time() + seconds, loop.stop) loop.start() if item == 'sleep': return sleep else: return getattr(time, item) motor-1.2.1/synchro/synchrotest.py000066400000000000000000000217231323007662700173120ustar00rootroot00000000000000# Copyright 2012-2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor by testing that Synchro, a fake PyMongo implementation built on top of Motor, passes the same unittests as PyMongo. This program monkey-patches sys.modules, so run it alone, rather than as part of a larger test suite. """ import sys import nose from nose.config import Config from nose.plugins import Plugin from nose.plugins.manager import PluginManager from nose.plugins.skip import Skip from nose.plugins.xunit import Xunit from nose.selector import Selector import synchro from motor.motor_py3_compat import PY3 excluded_modules = [ # Exclude some PyMongo tests that can't be applied to Synchro. 'test.test_cursor_manager', 'test.test_threads', 'test.test_pooling', 'test.test_legacy_api', # Complex PyMongo-specific mocking. 'test.test_replica_set_reconfig', # Accesses PyMongo internals. Tested directly in Motor. 'test.test_session', ] excluded_tests = [ # Motor's reprs aren't the same as PyMongo's. '*.test_repr', 'TestClient.test_unix_socket', # Lazy-connection tests require multithreading; we test concurrent # lazy connection directly. 'TestClientLazyConnect.*', # Motor doesn't support forking or threading. '*.test_interrupt_signal', 'TestGridfs.test_threaded_reads', 'TestGridfs.test_threaded_writes', 'TestGSSAPI.test_gssapi_threaded', 'TestCursor.test_concurrent_close', # Relies on threads; tested directly. 'TestCollection.test_parallel_scan', # Motor's aggregate API is different, always sends "cursor={}" by default. 'TestCollection.test_aggregate', 'TestCollection.test_aggregate_raw_bson', 'TestAllScenarios.test_read_aggregate_Aggregate_with_multiple_stages', 'TestSingleSlaveOk.test_reads_from_secondary', # Can't do MotorCollection(name, create=True), Motor constructors do no I/O. 'TestCollection.test_create', 'TestCollection.test_reindex', # Motor doesn't support PyMongo's syntax, db.system_js['my_func'] = "code", # users should just use system.js as a regular collection. 'TestDatabase.test_system_js', 'TestDatabase.test_system_js_list', # Requires indexing / slicing cursors, which Motor doesn't do, see MOTOR-84. 'TestCollection.test_min_query', 'TestCursor.test_clone', 'TestCursor.test_count_with_limit_and_skip', 'TestCursor.test_getitem_numeric_index', 'TestCursor.test_getitem_slice_index', 'TestCursor.test_tailable', # Raw batches aren't implemented yet, MOTOR-172. 'TestRawBatchCursor.*', 'TestRawBatchCommandCursor.*', # No context-manager protocol for MotorCursor. 'TestCursor.test_with_statement', # Can't iterate a GridOut in Motor. 'TestGridFile.test_iterator', 'TestGridfs.test_missing_length_iter', # No context-manager protocol for MotorGridIn, and can't set attrs. 'TestGridFile.test_context_manager', 'TestGridFile.test_grid_in_default_opts', 'TestGridFile.test_set_after_close', # GridFS always connects lazily in Motor. 'TestGridFile.test_grid_out_lazy_connect', 'TestGridfs.test_gridfs_lazy_connect', # Complex PyMongo-specific mocking. '*.test_wire_version', 'TestClient.test_heartbeat_frequency_ms', 'TestExhaustCursor.*', 'TestHeartbeatMonitoring.*', 'TestMongoClientFailover.*', 'TestMongosLoadBalancing.*', 'TestReplicaSetClientInternalIPs.*', 'TestReplicaSetClientMaxWriteBatchSize.*', 'TestSSL.test_system_certs_config_error', # Motor is correct here, it's just unreliable on slow CI servers. 'TestReplicaSetClient.test_timeout_does_not_mark_member_down', # Accesses PyMongo internals. 'TestClient.test_close_kills_cursors', 'TestClient.test_stale_getmore', 'TestCollection.test_aggregation_cursor', 'TestCommandAndReadPreference.*', 'TestCommandMonitoring.test_get_more_failure', 'TestCommandMonitoring.test_sensitive_commands', 'TestCursor.test_close_kills_cursor_synchronously', 'TestGridFile.test_grid_out_cursor_options', 'TestGridfsReplicaSet.test_gridfs_replica_set', 'TestMaxStaleness.test_last_write_date_absent', 'TestMonitor.test_atexit_hook', 'TestReplicaSetClient.test_kill_cursor_explicit_primary', 'TestReplicaSetClient.test_kill_cursor_explicit_secondary', 'TestSelections.test_bool', 'TestMaxStaleness.test_last_write_date', ] excluded_modules_matched = set() excluded_tests_matched = set() class SynchroNosePlugin(Plugin): name = 'synchro' def __init__(self, *args, **kwargs): # We need a standard Nose selector in order to filter out methods that # don't match TestSuite.test_* self.selector = Selector(config=None) super(SynchroNosePlugin, self).__init__(*args, **kwargs) def configure(self, options, conf): super(SynchroNosePlugin, self).configure(options, conf) self.enabled = True def wantModule(self, module): # Depending on PYTHONPATH, Motor's direct tests may be imported - don't # run them now. if module.__name__.startswith('test.test_motor_'): return False for module_name in excluded_modules: if module_name.endswith('*'): if module.__name__.startswith(module_name.rstrip('*')): # E.g., test_motor_cursor matches "test_motor_*". excluded_modules_matched.add(module_name) return False elif module.__name__ == module_name: excluded_modules_matched.add(module_name) return False return True def wantFunction(self, fn): # PyMongo's test generators run at import time; tell Nose not to run # them as unittests. if fn.__name__ in ('test_cases', 'create_test', 'create_selection_tests'): return False def wantMethod(self, method): # Run standard Nose checks on name, like "does it start with test_"? if not self.selector.matches(method.__name__): return False for excluded_name in excluded_tests: if PY3: classname = method.__self__.__class__.__name__ else: classname = method.im_class.__name__ # Should we exclude this method's whole TestCase? suite_name, method_name = excluded_name.split('.') suite_matches = (suite_name == classname or suite_name == '*') # Should we exclude this particular method? method_matches = ( method.__name__ == method_name or method_name == '*') if suite_matches and method_matches: excluded_tests_matched.add(excluded_name) return False return True # So that e.g. 'from pymongo.mongo_client import MongoClient' gets the # Synchro MongoClient, not the real one. class SynchroModuleFinder(object): def find_module(self, fullname, path=None): parts = fullname.split('.') if parts[-1] in ('gridfs', 'pymongo'): return SynchroModuleLoader(path) elif len(parts) >= 2 and parts[-2] in ('gridfs', 'pymongo'): return SynchroModuleLoader(path) # Let regular module search continue. return None class SynchroModuleLoader(object): def __init__(self, path): self.path = path def load_module(self, fullname): return synchro if __name__ == '__main__': # Monkey-patch all pymongo's unittests so they think Synchro is the # real PyMongo. sys.meta_path[0:0] = [SynchroModuleFinder()] # Ensure time.sleep() acts as PyMongo's tests expect: background tasks # can run to completion while foreground pauses. sys.modules['time'] = synchro.TimeModule() if '--check-exclude-patterns' in sys.argv: check_exclude_patterns = True sys.argv.remove('--check-exclude-patterns') else: check_exclude_patterns = False nose.main( config=Config(plugins=PluginManager()), addplugins=[SynchroNosePlugin(), Skip(), Xunit()], exit=False) if check_exclude_patterns: unused_module_pats = set(excluded_modules) - excluded_modules_matched assert not unused_module_pats, "Unused module patterns: %s" % ( unused_module_pats, ) unused_test_pats = set(excluded_tests) - excluded_tests_matched assert not unused_test_pats, "Unused test patterns: %s" % ( unused_test_pats, ) motor-1.2.1/test/000077500000000000000000000000001323007662700136405ustar00rootroot00000000000000motor-1.2.1/test/__init__.py000066400000000000000000000043631323007662700157570ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import logging import unittest from unittest import SkipTest from test.test_environment import env, db_user, CLIENT_PEM def suppress_tornado_warnings(): for name in [ 'tornado.general', 'tornado.access']: logger = logging.getLogger(name) logger.setLevel(logging.ERROR) class SkippedModule(object): def __init__(self, name, reason): def runTest(self): raise SkipTest(str(reason)) self.test_case = type( str(name), (unittest.TestCase, ), {'runTest': runTest}) class MotorTestLoader(unittest.TestLoader): def __init__(self, avoid=None, reason=None): super(MotorTestLoader, self).__init__() self._avoid = [] def avoid(self, prefix, reason): """Skip a module. The usual "raise SkipTest" from a module doesn't work if the module won't even parse in Python 2, so prevent TestLoader from importing modules with the given prefix. "prefix" is a path prefix like "asyncio_tests". """ self._avoid.append((prefix, reason)) def _get_module_from_name(self, name): for prefix, reason in self._avoid: if name.startswith(prefix): return SkippedModule(name, reason) return super(MotorTestLoader, self)._get_module_from_name(name) class MockRequestHandler(object): """For testing MotorGridOut.stream_to_handler.""" def __init__(self): self.n_written = 0 def write(self, data): self.n_written += len(data) def flush(self): pass motor-1.2.1/test/assert_logs_backport.py000066400000000000000000000051601323007662700204260ustar00rootroot00000000000000"""Backport assertLogs from Python 3.4.""" import collections import logging _LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) class _BaseTestCaseContext(object): def __init__(self, test_case): self.test_case = test_case def _raiseFailure(self, standardMsg): msg = self.test_case._formatMessage(self.msg, standardMsg) raise self.test_case.failureException(msg) class _CapturingHandler(logging.Handler): """Handler capturing all (raw and formatted) logging output.""" def __init__(self): logging.Handler.__init__(self) self.watcher = _LoggingWatcher([], []) def flush(self): pass def emit(self, record): self.watcher.records.append(record) msg = self.format(record) self.watcher.output.append(msg) class _AssertLogsContext(_BaseTestCaseContext): """A context manager used to implement TestCase.assertLogs().""" LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" def __init__(self, test_case, logger_name, level): assert isinstance(level, int) _BaseTestCaseContext.__init__(self, test_case) self.logger_name = logger_name self.level = level or logging.INFO self.level_name = logging.getLevelName(level) self.msg = None def __enter__(self): if isinstance(self.logger_name, logging.Logger): logger = self.logger = self.logger_name else: logger = self.logger = logging.getLogger(self.logger_name) formatter = logging.Formatter(self.LOGGING_FORMAT) handler = _CapturingHandler() handler.setFormatter(formatter) self.watcher = handler.watcher self.old_handlers = logger.handlers[:] self.old_level = logger.level self.old_propagate = logger.propagate logger.handlers = [handler] logger.setLevel(self.level) logger.propagate = False return handler.watcher def __exit__(self, exc_type, exc_value, tb): self.logger.handlers = self.old_handlers self.logger.propagate = self.old_propagate self.logger.setLevel(self.old_level) if exc_type is not None: # let unexpected exceptions pass through return False if len(self.watcher.records) == 0: self._raiseFailure( "no logs of level {0} or higher triggered on {1}" .format(self.level_name, self.logger.name)) class AssertLogsMixin(object): def assertLogs(self, logger=None, level=None): return _AssertLogsContext(self, logger, level) motor-1.2.1/test/asyncio_tests/000077500000000000000000000000001323007662700165275ustar00rootroot00000000000000motor-1.2.1/test/asyncio_tests/__init__.py000066400000000000000000000215451323007662700206470ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for testing Motor with asyncio.""" import asyncio import functools import gc import inspect import os import unittest from unittest import SkipTest try: from asyncio import ensure_future except ImportError: from asyncio import async as ensure_future from mockupdb import MockupDB from motor import motor_asyncio from test.assert_logs_backport import AssertLogsMixin from test.test_environment import env, CA_PEM, CLIENT_PEM class _TestMethodWrapper(object): """Wraps a test method to raise an error if it returns a value. This is mainly used to detect undecorated generators (if a test method yields it must use a decorator to consume the generator), but will also detect other kinds of return values (these are not necessarily errors, but we alert anyway since there is no good reason to return a value from a test). Adapted from Tornado's test framework. """ def __init__(self, orig_method): self.orig_method = orig_method def __call__(self): result = self.orig_method() if inspect.isgenerator(result): raise TypeError("Generator test methods should be decorated with " "@asyncio_test") elif result is not None: raise ValueError("Return value from test method ignored: %r" % result) def __getattr__(self, name): """Proxy all unknown attributes to the original method. This is important for some of the decorators in the `unittest` module, such as `unittest.skipIf`. """ return getattr(self.orig_method, name) class AsyncIOTestCase(AssertLogsMixin, unittest.TestCase): longMessage = True # Used by unittest.TestCase ssl = False # If True, connect with SSL, skip if mongod isn't SSL def __init__(self, methodName='runTest'): super().__init__(methodName) # It's easy to forget the @asyncio_test decorator, but if you do # the test will silently be ignored because nothing will consume # the generator. Replace the test method with a wrapper that will # make sure it's not an undecorated generator. # (Adapted from Tornado's AsyncTestCase.) setattr(self, methodName, _TestMethodWrapper( getattr(self, methodName))) def setUp(self): super(AsyncIOTestCase, self).setUp() # Ensure that the event loop is passed explicitly in Motor. asyncio.set_event_loop(None) self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) if self.ssl and not env.mongod_started_with_ssl: raise SkipTest("mongod doesn't support SSL, or is down") self.cx = self.asyncio_client() self.db = self.cx.motor_test self.collection = self.db.test_collection self.loop.run_until_complete(self.collection.drop()) def get_client_kwargs(self, **kwargs): if env.mongod_started_with_ssl: kwargs.setdefault('ssl_ca_certs', CA_PEM) kwargs.setdefault('ssl_certfile', CLIENT_PEM) kwargs.setdefault('ssl', env.mongod_started_with_ssl) kwargs.setdefault('io_loop', self.loop) return kwargs def asyncio_client(self, uri=None, *args, **kwargs): """Get an AsyncIOMotorClient. Ignores self.ssl, you must pass 'ssl' argument. """ return motor_asyncio.AsyncIOMotorClient( uri or env.uri, *args, **self.get_client_kwargs(**kwargs)) def asyncio_rsc(self, uri=None, *args, **kwargs): """Get an open MotorClient for replica set. Ignores self.ssl, you must pass 'ssl' argument. """ return motor_asyncio.AsyncIOMotorClient( uri or env.rs_uri, *args, **self.get_client_kwargs(**kwargs)) @asyncio.coroutine def make_test_data(self): yield from self.collection.delete_many({}) yield from self.collection.insert_many([{'_id': i} for i in range(200)]) make_test_data.__test__ = False def tearDown(self): self.cx.close() self.loop.stop() self.loop.run_forever() self.loop.close() gc.collect() class AsyncIOMockServerTestCase(AsyncIOTestCase): def server(self, *args, **kwargs): server = MockupDB(*args, **kwargs) server.run() self.addCleanup(server.stop) return server def client_server(self, *args, **kwargs): server = self.server(*args, **kwargs) client = motor_asyncio.AsyncIOMotorClient(server.uri, io_loop=self.loop) self.addCleanup(client.close) return client, server def run_thread(self, fn, *args, **kwargs): return self.loop.run_in_executor(None, functools.partial(fn, *args, **kwargs)) def ensure_future(self, coro): return ensure_future(coro, loop=self.loop) def fetch_next(self, cursor): @asyncio.coroutine def fetch_next(): return (yield from cursor.fetch_next) return self.ensure_future(fetch_next()) def get_async_test_timeout(default=5): """Get the global timeout setting for async tests. Returns a float, the timeout in seconds. """ try: timeout = float(os.environ.get('ASYNC_TEST_TIMEOUT')) return max(timeout, default) except (ValueError, TypeError): return default # TODO: Spin off to a PyPI package. def asyncio_test(func=None, timeout=None): """Decorator for coroutine methods of AsyncIOTestCase:: class MyTestCase(AsyncIOTestCase): @asyncio_test def test(self): # Your test code here.... pass Default timeout is 5 seconds. Override like:: class MyTestCase(AsyncIOTestCase): @asyncio_test(timeout=10) def test(self): # Your test code here.... pass You can also set the ASYNC_TEST_TIMEOUT environment variable to a number of seconds. The final timeout is the ASYNC_TEST_TIMEOUT or the timeout in the test (5 seconds or the passed-in timeout), whichever is longest. """ def wrap(f): @functools.wraps(f) def wrapped(self, *args, **kwargs): if timeout is None: actual_timeout = get_async_test_timeout() else: actual_timeout = get_async_test_timeout(timeout) coro_exc = None def exc_handler(loop, context): nonlocal coro_exc coro_exc = context['exception'] # Raise CancelledError from run_until_complete below. task.cancel() self.loop.set_exception_handler(exc_handler) coro = asyncio.coroutine(f)(self, *args, **kwargs) coro = asyncio.wait_for(coro, actual_timeout, loop=self.loop) task = ensure_future(coro, loop=self.loop) try: self.loop.run_until_complete(task) except: if coro_exc: # Raise the error thrown in on_timeout, with only the # traceback from the coroutine itself, not from # run_until_complete. raise coro_exc from None raise return wrapped if func is not None: # Used like: # @gen_test # def f(self): # pass if not inspect.isfunction(func): msg = ("%r is not a test method. Pass a timeout as" " a keyword argument, like @asyncio_test(timeout=7)") raise TypeError(msg % func) return wrap(func) else: # Used like @gen_test(timeout=10) return wrap @asyncio.coroutine def get_command_line(client): command_line = yield from client.admin.command('getCmdLineOpts') assert command_line['ok'] == 1, "getCmdLineOpts() failed" return command_line @asyncio.coroutine def server_is_mongos(client): ismaster_response = yield from client.admin.command('ismaster') return ismaster_response.get('msg') == 'isdbgrid' @asyncio.coroutine def skip_if_mongos(client): is_mongos = yield from server_is_mongos(client) if is_mongos: raise unittest.SkipTest("connected to mongos") @asyncio.coroutine def remove_all_users(db): yield from db.command({"dropAllUsersFromDatabase": 1}) motor-1.2.1/test/asyncio_tests/test_aiohttp_gridfs.py000066400000000000000000000262431323007662700231550ustar00rootroot00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test Motor's AIOHTTPGridFSHandler.""" import asyncio import datetime import email import hashlib import logging import time import aiohttp import aiohttp.web import gridfs from motor.aiohttp import AIOHTTPGridFS import test from test.asyncio_tests import AsyncIOTestCase, asyncio_test def format_date(d): return time.strftime("%a, %d %b %Y %H:%M:%S GMT", d.utctimetuple()) def parse_date(d): date_tuple = email.utils.parsedate(d) return datetime.datetime.fromtimestamp(time.mktime(date_tuple)) def expires(response): return parse_date(response.headers['Expires']) class AIOHTTPGridFSHandlerTestBase(AsyncIOTestCase): fs = None file_id = None def tearDown(self): self.loop.run_until_complete(self.stop()) super().tearDown() @classmethod def setUpClass(cls): super().setUpClass() logging.getLogger('aiohttp.web').setLevel(logging.CRITICAL) cls.fs = gridfs.GridFS(test.env.sync_cx.motor_test) # Make a 500k file in GridFS with filename 'foo' cls.contents = b'Jesse' * 100 * 1024 cls.contents_hash = hashlib.md5(cls.contents).hexdigest() # Record when we created the file, to check the Last-Modified header cls.put_start = datetime.datetime.utcnow().replace(microsecond=0) cls.file_id = 'id' cls.fs.delete(cls.file_id) cls.fs.put(cls.contents, _id='id', filename='foo', content_type='my type') cls.put_end = datetime.datetime.utcnow().replace(microsecond=0) cls.app = cls.srv = cls.app_handler = None @classmethod def tearDownClass(cls): cls.fs.delete(cls.file_id) super().tearDownClass() @asyncio.coroutine def start_app(self, http_gridfs=None, extra_routes=None): self.app = aiohttp.web.Application() resource = self.app.router.add_resource('/fs/{filename}') handler = http_gridfs or AIOHTTPGridFS(self.db) resource.add_route('GET', handler) resource.add_route('HEAD', handler) if extra_routes: for route, handler in extra_routes.items(): resource = self.app.router.add_resource(route) resource.add_route('GET', handler) self.app_handler = self.app.make_handler() server = self.loop.create_server(self.app_handler, host='localhost', port=8088) self.srv, _ = yield from asyncio.gather(server, self.app.startup(), loop=self.loop) @asyncio.coroutine def request(self, method, path, if_modified_since=None, headers=None): headers = headers or {} if if_modified_since: headers['If-Modified-Since'] = format_date(if_modified_since) session = aiohttp.ClientSession() try: method = getattr(session, method) resp = yield from method('http://localhost:8088%s' % path, headers=headers) yield from resp.read() return resp finally: yield from session.close() def get(self, path, **kwargs): return self.request('get', path, **kwargs) def head(self, path, **kwargs): return self.request('head', path, **kwargs) @asyncio.coroutine def stop(self): # aiohttp.rtfd.io/en/stable/web.html#aiohttp-web-graceful-shutdown self.srv.close() yield from self.srv.wait_closed() yield from self.app.shutdown() yield from self.app_handler.shutdown(timeout=1) yield from self.app.cleanup() class AIOHTTPGridFSHandlerTest(AIOHTTPGridFSHandlerTestBase): @asyncio_test def test_basic(self): yield from self.start_app() # First request response = yield from self.get('/fs/foo') self.assertEqual(200, response.status) self.assertEqual(self.contents, (yield from response.read())) self.assertEqual( len(self.contents), int(response.headers['Content-Length'])) self.assertEqual('my type', response.headers['Content-Type']) self.assertEqual('public', response.headers['Cache-Control']) self.assertTrue('Expires' not in response.headers) etag = response.headers['Etag'] last_mod_dt = parse_date(response.headers['Last-Modified']) self.assertEqual(self.contents_hash, etag.strip('"')) self.assertTrue(self.put_start <= last_mod_dt <= self.put_end) # Now check we get 304 NOT MODIFIED responses as appropriate for ims_value in ( last_mod_dt, last_mod_dt + datetime.timedelta(seconds=1) ): response = yield from self.get('/fs/foo', if_modified_since=ims_value) self.assertEqual(304, response.status) self.assertEqual(b'', (yield from response.read())) # If-Modified-Since in the past, get whole response back response = yield from self.get( '/fs/foo', if_modified_since=last_mod_dt - datetime.timedelta(seconds=1)) self.assertEqual(200, response.status) self.assertEqual(self.contents, (yield from response.read())) # Matching Etag response = yield from self.get('/fs/foo', headers={'If-None-Match': etag}) self.assertEqual(304, response.status) self.assertEqual(b'', (yield from response.read())) # Mismatched Etag response = yield from self.get('/fs/foo', headers={'If-None-Match': etag + 'a'}) self.assertEqual(200, response.status) self.assertEqual(self.contents, (yield from response.read())) @asyncio_test def test_404(self): yield from self.start_app() response = yield from self.get('/fs/bar') self.assertEqual(404, response.status) @asyncio_test def test_head(self): yield from self.start_app() response = yield from self.head('/fs/foo') etag = response.headers['Etag'] last_mod_dt = parse_date(response.headers['Last-Modified']) self.assertEqual(200, response.status) # Empty body for HEAD request. self.assertEqual(b'', (yield from response.read())) self.assertEqual( len(self.contents), int(response.headers['Content-Length'])) self.assertEqual('my type', response.headers['Content-Type']) self.assertEqual(self.contents_hash, etag.strip('"')) self.assertTrue(self.put_start <= last_mod_dt <= self.put_end) self.assertEqual('public', response.headers['Cache-Control']) @asyncio_test def test_bad_route(self): handler = AIOHTTPGridFS(self.db) yield from self.start_app(extra_routes={'/x/{wrongname}': handler}) response = yield from self.get('/x/foo') self.assertEqual(500, response.status) msg = 'Bad AIOHTTPGridFS route "/x/{wrongname}"' self.assertIn(msg, (yield from response.text())) @asyncio_test def test_content_type(self): yield from self.start_app() # Check that GridFSHandler uses file extension to guess Content-Type # if not provided for filename, expected_type in [ ('bar', 'octet-stream'), ('bar.png', 'png'), ('ht.html', 'html'), ('jscr.js', 'javascript'), ]: # 'fs' is PyMongo's blocking GridFS _id = self.fs.put(b'', filename=filename) self.addCleanup(self.fs.delete, _id) for method in self.get, self.head: response = yield from method('/fs/' + filename) self.assertEqual(200, response.status) # mimetypes are platform-defined, be fuzzy self.assertIn( expected_type, response.headers['Content-Type'].lower()) @asyncio_test def test_post(self): # Only allow GET and HEAD, even if a POST route is added. handler = AIOHTTPGridFS(self.db) yield from self.start_app(extra_routes={'/fs/{filename}': handler}) result = yield from self.request('post', '/fs/foo') self.assertEqual(405, result.status) class AIOHTTPTZAwareGridFSHandlerTest(AIOHTTPGridFSHandlerTestBase): @asyncio_test def test_tz_aware(self): client = self.asyncio_client(tz_aware=True) yield from self.start_app(AIOHTTPGridFS(client.motor_test)) now = datetime.datetime.utcnow() ago = now - datetime.timedelta(minutes=10) hence = now + datetime.timedelta(minutes=10) response = yield from self.get('/fs/foo', if_modified_since=ago) self.assertEqual(200, response.status) response = yield from self.get('/fs/foo', if_modified_since=hence) self.assertEqual(304, response.status) class AIOHTTPCustomHTTPGridFSTest(AIOHTTPGridFSHandlerTestBase): @asyncio_test def test_get_gridfs_file(self): def getter(bucket, filename, request): # Test overriding the get_gridfs_file() method, path is # interpreted as file_id instead of filename. return bucket.open_download_stream(file_id=filename) def cache_time(path, modified, mime_type): return 10 def extras(response, gridout): response.headers['quux'] = 'fizzledy' yield from self.start_app(AIOHTTPGridFS(self.db, get_gridfs_file=getter, get_cache_time=cache_time, set_extra_headers=extras)) # We overrode get_gridfs_file so we expect getting by filename *not* to # work now; we'll get a 404. We have to get by file_id now. response = yield from self.get('/fs/foo') self.assertEqual(404, response.status) response = yield from self.get('/fs/' + str(self.file_id)) self.assertEqual(200, response.status) self.assertEqual(self.contents, (yield from response.read())) cache_control = response.headers['Cache-Control'] self.assertRegex(cache_control, r'max-age=\d+') self.assertEqual(10, int(cache_control.split('=')[1])) expiration = parse_date(response.headers['Expires']) now = datetime.datetime.utcnow() # It should expire about 10 seconds from now self.assertTrue( datetime.timedelta(seconds=8) < expiration - now < datetime.timedelta(seconds=12)) self.assertEqual('fizzledy', response.headers['quux']) motor-1.2.1/test/asyncio_tests/test_asyncio_await.py000066400000000000000000000120421323007662700227710ustar00rootroot00000000000000# Copyright 2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import import warnings from motor.motor_asyncio import AsyncIOMotorGridFS import test from test.asyncio_tests import asyncio_test, AsyncIOTestCase class TestAsyncIOAwait(AsyncIOTestCase): @asyncio_test async def test_to_list(self): collection = self.collection await collection.delete_many({}) results = await collection.find().sort('_id').to_list(length=None) self.assertEqual([], results) docs = [{'_id': 1}, {'_id': 2}] await collection.insert_many(docs) cursor = collection.find().sort('_id') results = await cursor.to_list(length=None) self.assertEqual(docs, results) results = await cursor.to_list(length=None) self.assertEqual([], results) @asyncio_test async def test_iter_cursor(self): collection = self.collection await collection.delete_many({}) for n_docs in 0, 1, 2, 10: if n_docs: docs = [{'_id': i} for i in range(n_docs)] await collection.insert_many(docs) # Force extra batches to test iteration. j = 0 async for doc in collection.find().sort('_id').batch_size(3): self.assertEqual(j, doc['_id']) j += 1 self.assertEqual(j, n_docs) await collection.delete_many({}) @asyncio_test async def test_iter_aggregate(self): collection = self.collection await collection.delete_many({}) pipeline = [{'$sort': {'_id': 1}}] # Empty iterator. async for _ in collection.aggregate(pipeline): self.fail() for n_docs in 1, 2, 10: if n_docs: docs = [{'_id': i} for i in range(n_docs)] await collection.insert_many(docs) # Force extra batches to test iteration. j = 0 async for doc in collection.aggregate(pipeline, cursor={'batchSize': 3}): self.assertEqual(j, doc['_id']) j += 1 self.assertEqual(j, n_docs) await collection.delete_many({}) @asyncio_test async def test_iter_gridfs(self): gfs = AsyncIOMotorGridFS(self.db) async def cleanup(): await self.db.fs.files.delete_many({}) await self.db.fs.chunks.delete_many({}) await cleanup() # Empty iterator. async for _ in gfs.find({'_id': 1}): self.fail() data = b'data' for n_files in 1, 2, 10: for i in range(n_files): await gfs.put(data, filename='filename') # Force extra batches to test iteration. j = 0 async for _ in gfs.find({'filename': 'filename'}).batch_size(3): j += 1 self.assertEqual(j, n_files) await cleanup() async with await gfs.new_file(_id=1, chunk_size=1) as f: await f.write(data) gout = await gfs.find_one({'_id': 1}) chunks = [] async for chunk in gout: chunks.append(chunk) self.assertEqual(len(chunks), len(data)) self.assertEqual(b''.join(chunks), data) @asyncio_test async def test_stream_to_handler(self): # Sort of Tornado-specific, but it does work with asyncio. fs = AsyncIOMotorGridFS(self.db) content_length = 1000 await fs.delete(1) self.assertEqual(1, await fs.put(b'a' * content_length, _id=1)) gridout = await fs.get(1) handler = test.MockRequestHandler() await gridout.stream_to_handler(handler) self.assertEqual(content_length, handler.n_written) await fs.delete(1) @asyncio_test async def test_cursor_iter(self): # Have we handled the async iterator change in Python 3.5.2?: # python.org/dev/peps/pep-0492/#api-design-and-implementation-revisions with warnings.catch_warnings(record=True) as w: async for _ in self.collection.find(): pass if w: self.fail(w[0].message) @asyncio_test async def test_list_indexes(self): await self.collection.drop() await self.collection.create_index([('x', 1)]) await self.collection.create_index([('y', -1)]) keys = set() async for info in self.collection.list_indexes(): keys.add(info['name']) self.assertEqual(keys, {'_id_', 'x_1', 'y_-1'}) motor-1.2.1/test/asyncio_tests/test_asyncio_basic.py000066400000000000000000000111651323007662700227520ustar00rootroot00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import from unittest import SkipTest import pymongo from pymongo import WriteConcern from pymongo.errors import ConfigurationError from pymongo.read_preferences import ReadPreference, Secondary, Nearest from motor import motor_asyncio import test from test.asyncio_tests import asyncio_test, AsyncIOTestCase from test.utils import ignore_deprecations class AIOMotorTestBasic(AsyncIOTestCase): def test_repr(self): self.assertTrue(repr(self.cx).startswith('AsyncIOMotorClient')) self.assertTrue(repr(self.db).startswith('AsyncIOMotorDatabase')) self.assertTrue(repr(self.collection).startswith( 'AsyncIOMotorCollection')) cursor = self.collection.find() self.assertTrue(repr(cursor).startswith('AsyncIOMotorCursor')) @asyncio_test def test_write_concern(self): # Default empty dict means "w=1" self.assertEqual(WriteConcern(), self.cx.write_concern) yield from self.collection.delete_many({}) yield from self.collection.insert_one({'_id': 0}) for gle_options in [ {}, {'w': 0}, {'w': 1}, {'wtimeout': 1000}, ]: cx = self.asyncio_client(test.env.uri, **gle_options) wc = WriteConcern(**gle_options) self.assertEqual(wc, cx.write_concern) db = cx.motor_test self.assertEqual(wc, db.write_concern) collection = db.test_collection self.assertEqual(wc, collection.write_concern) if wc.acknowledged: with self.assertRaises(pymongo.errors.DuplicateKeyError): yield from collection.insert_one({'_id': 0}) else: yield from collection.insert_one({'_id': 0}) # No error # No error c = collection.with_options(write_concern=WriteConcern(w=0)) yield from c.insert_one({'_id': 0}) cx.close() @ignore_deprecations @asyncio_test def test_read_preference(self): # Check the default cx = motor_asyncio.AsyncIOMotorClient(test.env.uri, io_loop=self.loop) self.assertEqual(ReadPreference.PRIMARY, cx.read_preference) # We can set mode, tags, and latency. cx = self.asyncio_client( read_preference=Secondary(tag_sets=[{'foo': 'bar'}]), localThresholdMS=42) self.assertEqual(ReadPreference.SECONDARY.mode, cx.read_preference.mode) self.assertEqual([{'foo': 'bar'}], cx.read_preference.tag_sets) self.assertEqual(42, cx.local_threshold_ms) # Make a MotorCursor and get its PyMongo Cursor collection = cx.motor_test.test_collection.with_options( read_preference=Nearest(tag_sets=[{'yay': 'jesse'}])) motor_cursor = collection.find() cursor = motor_cursor.delegate self.assertEqual(Nearest(tag_sets=[{'yay': 'jesse'}]), cursor._Cursor__read_preference) cx.close() def test_underscore(self): self.assertIsInstance(self.cx['_db'], motor_asyncio.AsyncIOMotorDatabase) self.assertIsInstance(self.db['_collection'], motor_asyncio.AsyncIOMotorCollection) self.assertIsInstance(self.collection['_collection'], motor_asyncio.AsyncIOMotorCollection) with self.assertRaises(AttributeError): self.cx._db with self.assertRaises(AttributeError): self.db._collection with self.assertRaises(AttributeError): self.collection._collection def test_abc(self): try: from abc import ABC except ImportError: # Python < 3.4. raise SkipTest() class C(ABC): db = self.db collection = self.collection subcollection = self.collection.subcollection # MOTOR-104, TypeError: Can't instantiate abstract class C with abstract # methods collection, db, subcollection. C() motor-1.2.1/test/asyncio_tests/test_asyncio_bulk.py000066400000000000000000000063341323007662700226300ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test Motor's bulk API with asyncio.""" import unittest from pymongo.errors import BulkWriteError from motor.motor_asyncio import AsyncIOMotorBulkOperationBuilder from test.asyncio_tests import asyncio_test, AsyncIOTestCase class TestAsyncIOBulk(AsyncIOTestCase): # Little testing is needed: Most of the logic is in PyMongo, and Motor's # bulk operations are lightly tested with Tornado already. @asyncio_test(timeout=30) def test_multiple_error_ordered_batch(self): yield from self.collection.delete_many({}) yield from self.collection.create_index('a', unique=True) try: bulk = self.collection.initialize_ordered_bulk_op() self.assertTrue(isinstance(bulk, AsyncIOMotorBulkOperationBuilder)) bulk.insert({'b': 1, 'a': 1}) bulk.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) bulk.find({'b': 3}).upsert().update_one({'$set': {'a': 2}}) bulk.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) bulk.insert({'b': 4, 'a': 3}) bulk.insert({'b': 5, 'a': 1}) try: yield from bulk.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(1, result['nInserted']) self.assertEqual(1, len(result['writeErrors'])) cursor = self.collection.find({}, {'_id': False}) docs = yield from cursor.to_list(None) self.assertEqual([{'a': 1, 'b': 1}], docs) finally: yield from self.collection.drop() @asyncio_test def test_single_unordered_batch(self): yield from self.collection.delete_many({}) bulk = self.collection.initialize_unordered_bulk_op() self.assertTrue(isinstance(bulk, AsyncIOMotorBulkOperationBuilder)) bulk.insert({'a': 1}) bulk.find({'a': 1}).update_one({'$set': {'b': 1}}) bulk.find({'a': 2}).upsert().update_one({'$set': {'b': 2}}) bulk.insert({'a': 3}) bulk.find({'a': 3}).remove() result = yield from bulk.execute() self.assertEqual(0, len(result['writeErrors'])) upserts = result['upserted'] self.assertEqual(1, len(upserts)) self.assertEqual(2, upserts[0]['index']) self.assertTrue(upserts[0].get('_id')) a_values = yield from self.collection.distinct('a') self.assertEqual( set([1, 2]), set(a_values)) if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_change_stream.py000066400000000000000000000110161323007662700244640ustar00rootroot00000000000000# Copyright 2017-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorChangeStream.""" import os import threading import time from pymongo.errors import InvalidOperation, OperationFailure from test import SkipTest, env from test.asyncio_tests import asyncio_test, AsyncIOTestCase class TestAsyncIOChangeStream(AsyncIOTestCase): @classmethod @env.require_version_min(3, 6) def setUpClass(cls): super(TestAsyncIOChangeStream, cls).setUpClass() if env.is_standalone: raise SkipTest("Standalone") # Ensure the collection exists. env.sync_cx.motor_test.test_collection.delete_many({}) env.sync_cx.motor_test.test_collection.insert_one({'_id': 1}) def wait_and_insert(self, change_stream, n=1): # The start time of the change stream is nondeterministic. Wait # to ensure this insert comes after the change stream starts. def target(): start = time.time() timeout = float(os.environ.get('ASYNC_TEST_TIMEOUT', 5)) while not change_stream.delegate: if time.time() - start > timeout: print("MotorChangeStream never created ChangeStream") return time.sleep(0.1) self.loop.call_soon_threadsafe(self.collection.insert_many, [{} for _ in range(n)]) t = threading.Thread(target=target) t.daemon = True t.start() @asyncio_test async def test_async_for(self): change_stream = self.collection.watch() self.wait_and_insert(change_stream, 2) i = 0 async for _ in change_stream: i += 1 if i == 2: break self.assertEqual(i, 2) @asyncio_test async def test_watch(self): coll = self.collection with self.assertRaises(TypeError): # pipeline must be a list. async for _ in coll.watch(pipeline={}): pass change_stream = coll.watch() self.wait_and_insert(change_stream, 1) change = await change_stream.next() # New change stream with resume token. await coll.insert_one({'_id': 23}) change = await coll.watch(resume_after=change['_id']).next() self.assertEqual(change['fullDocument'], {'_id': 23}) @asyncio_test async def test_close(self): coll = self.collection change_stream = coll.watch() future = change_stream.next() self.wait_and_insert(change_stream, 1) await future await change_stream.close() with self.assertRaises(StopAsyncIteration): await change_stream.next() async for _ in change_stream: pass @asyncio_test async def test_missing_id(self): coll = self.collection change_stream = coll.watch([{'$project': {'_id': 0}}]) future = change_stream.next() self.wait_and_insert(change_stream) with self.assertRaises(InvalidOperation): await future # The cursor should now be closed. with self.assertRaises(StopAsyncIteration): await change_stream.next() @asyncio_test async def test_unknown_full_document(self): coll = self.collection change_stream = coll.watch(full_document="unknownFullDocOption") future = change_stream.next() self.wait_and_insert(change_stream, 1) with self.assertRaises(OperationFailure): await future @asyncio_test async def test_async_with(self): async with self.collection.watch() as change_stream: self.wait_and_insert(change_stream, 1) async for _ in change_stream: self.assertTrue(change_stream.delegate._cursor.alive) break self.assertFalse(change_stream.delegate._cursor.alive) @asyncio_test async def test_with_statement(self): with self.assertRaises(RuntimeError): with self.collection.watch(): pass motor-1.2.1/test/asyncio_tests/test_asyncio_client.py000066400000000000000000000244721323007662700231540ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorClient.""" import asyncio import os import unittest from unittest import SkipTest import pymongo from bson import CodecOptions from pymongo import ReadPreference, WriteConcern from pymongo.errors import ConnectionFailure, OperationFailure from motor import motor_asyncio import test from test.asyncio_tests import (asyncio_test, AsyncIOTestCase, AsyncIOMockServerTestCase, remove_all_users) from test.test_environment import db_user, db_password, env from test.utils import get_primary_pool class TestAsyncIOClient(AsyncIOTestCase): @asyncio_test def test_client_lazy_connect(self): yield from self.db.test_client_lazy_connect.delete_many({}) # Create client without connecting; connect on demand. cx = self.asyncio_client() collection = cx.motor_test.test_client_lazy_connect future0 = collection.insert_one({'foo': 'bar'}) future1 = collection.insert_one({'foo': 'bar'}) yield from asyncio.gather(future0, future1, loop=self.loop) resp = yield from collection.find({'foo': 'bar'}).count() self.assertEqual(2, resp) cx.close() @asyncio_test def test_close(self): cx = self.asyncio_client() cx.close() self.assertEqual(None, get_primary_pool(cx)) @asyncio_test def test_unix_socket(self): if env.mongod_started_with_ssl: raise SkipTest("Server started with SSL") mongodb_socket = '/tmp/mongodb-%d.sock' % env.port if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessible") encoded_socket = '%2Ftmp%2Fmongodb-' + str(env.port) + '.sock' uri = 'mongodb://%s' % encoded_socket client = self.asyncio_client(uri) collection = client.motor_test.test if test.env.auth: yield from client.admin.authenticate(db_user, db_password) yield from collection.insert_one({"dummy": "object"}) # Confirm it fails with a missing socket. client = motor_asyncio.AsyncIOMotorClient( "mongodb://%2Ftmp%2Fnon-existent.sock", io_loop=self.loop, serverSelectionTimeoutMS=100) with self.assertRaises(ConnectionFailure): yield from client.admin.command('ismaster') client.close() def test_database_named_delegate(self): self.assertTrue( isinstance(self.cx.delegate, pymongo.mongo_client.MongoClient)) self.assertTrue(isinstance(self.cx['delegate'], motor_asyncio.AsyncIOMotorDatabase)) @asyncio_test def test_reconnect_in_case_connection_closed_by_mongo(self): cx = self.asyncio_client(maxPoolSize=1) yield from cx.admin.command('ping') # close motor_socket, we imitate that connection to mongo server # lost, as result we should have AutoReconnect instead of # IncompleteReadError pool = get_primary_pool(cx) socket = pool.sockets.pop() socket.sock.close() pool.sockets.add(socket) with self.assertRaises(pymongo.errors.AutoReconnect): yield from cx.motor_test.test_collection.find_one() @asyncio_test def test_connection_failure(self): # Assuming there isn't anything actually running on this port client = motor_asyncio.AsyncIOMotorClient('localhost', 8765, serverSelectionTimeoutMS=10, io_loop=self.loop) with self.assertRaises(ConnectionFailure): yield from client.admin.command('ismaster') @asyncio_test(timeout=30) def test_connection_timeout(self): # Motor merely tries to time out a connection attempt within the # specified duration; DNS lookup in particular isn't charged against # the timeout. So don't measure how long this takes. client = motor_asyncio.AsyncIOMotorClient( 'example.com', port=12345, serverSelectionTimeoutMS=1, io_loop=self.loop) with self.assertRaises(ConnectionFailure): yield from client.admin.command('ismaster') @asyncio_test def test_max_pool_size_validation(self): with self.assertRaises(ValueError): motor_asyncio.AsyncIOMotorClient(maxPoolSize=-1, io_loop=self.loop) with self.assertRaises(ValueError): motor_asyncio.AsyncIOMotorClient(maxPoolSize='foo', io_loop=self.loop) cx = self.asyncio_client(maxPoolSize=100) self.assertEqual(cx.max_pool_size, 100) cx.close() @asyncio_test(timeout=60) def test_high_concurrency(self): yield from self.make_test_data() concurrency = 25 cx = self.asyncio_client(maxPoolSize=concurrency) expected_finds = 200 * concurrency n_inserts = 25 collection = cx.motor_test.test_collection insert_collection = cx.motor_test.insert_collection yield from insert_collection.delete_many({}) ndocs = 0 insert_future = asyncio.Future(loop=self.loop) @asyncio.coroutine def find(): nonlocal ndocs cursor = collection.find() while (yield from cursor.fetch_next): cursor.next_object() ndocs += 1 # Half-way through, start an insert loop if ndocs == expected_finds / 2: asyncio.Task(insert(), loop=self.loop) @asyncio.coroutine def insert(): for i in range(n_inserts): yield from insert_collection.insert_one({'s': hex(i)}) insert_future.set_result(None) # Finished yield from asyncio.gather(*[find() for _ in range(concurrency)], loop=self.loop) yield from insert_future self.assertEqual(expected_finds, ndocs) self.assertEqual(n_inserts, (yield from insert_collection.count())) yield from collection.delete_many({}) @asyncio_test(timeout=30) def test_drop_database(self): # Make sure we can pass an AsyncIOMotorDatabase instance # to drop_database db = self.cx.test_drop_database yield from db.test_collection.insert_one({}) names = yield from self.cx.database_names() self.assertTrue('test_drop_database' in names) yield from self.cx.drop_database(db) names = yield from self.cx.database_names() self.assertFalse('test_drop_database' in names) @asyncio_test def test_auth_from_uri(self): if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # self.db is logged in as root. yield from remove_all_users(self.db) db = self.db try: yield from db.add_user( 'mike', 'password', roles=['userAdmin', 'readWrite']) client = self.asyncio_client( 'mongodb://u:pass@%s:%d' % (env.host, env.port)) with self.assertRaises(OperationFailure): yield from client.db.collection.find_one() client = self.asyncio_client( 'mongodb://mike:password@%s:%d/%s' % (env.host, env.port, db.name)) yield from client[db.name].collection.find_one() finally: yield from db.remove_user('mike') @asyncio_test def test_socketKeepAlive(self): # Connect. yield from self.cx.server_info() ka = get_primary_pool(self.cx).opts.socket_keepalive self.assertTrue(ka) client = self.asyncio_client(socketKeepAlive=False) yield from client.server_info() ka = get_primary_pool(client).opts.socket_keepalive self.assertFalse(ka) def test_get_database(self): codec_options = CodecOptions(tz_aware=True) write_concern = WriteConcern(w=2, j=True) db = self.cx.get_database( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) assert isinstance(db, motor_asyncio.AsyncIOMotorDatabase) self.assertEqual('foo', db.name) self.assertEqual(codec_options, db.codec_options) self.assertEqual(ReadPreference.SECONDARY, db.read_preference) self.assertEqual(write_concern, db.write_concern) @asyncio_test def test_list_databases(self): yield from self.collection.insert_one({}) cursor = yield from self.cx.list_databases() self.assertIsInstance(cursor, motor_asyncio.AsyncIOMotorCommandCursor) while (yield from cursor.fetch_next): info = cursor.next_object() if info['name'] == self.collection.database.name: break else: self.fail("'%s' database not found" % self.collection.database.name) @asyncio_test def test_list_database_names(self): yield from self.collection.insert_one({}) names = yield from self.cx.list_database_names() self.assertIsInstance(names, list) self.assertIn(self.collection.database.name, names) class TestAsyncIOClientTimeout(AsyncIOMockServerTestCase): @asyncio_test def test_timeout(self): server = self.server(auto_ismaster=True) client = motor_asyncio.AsyncIOMotorClient(server.uri, socketTimeoutMS=100, io_loop=self.loop) with self.assertRaises(pymongo.errors.AutoReconnect) as context: yield from client.motor_test.test_collection.find_one() self.assertIn('timed out', str(context.exception)) client.close() if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_collection.py000066400000000000000000000356621323007662700240340ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorCollection.""" import asyncio import sys import traceback import unittest import bson from bson import CodecOptions from bson.binary import JAVA_LEGACY from bson.objectid import ObjectId from pymongo import ReadPreference, WriteConcern from pymongo.errors import DuplicateKeyError, OperationFailure from pymongo.read_preferences import Secondary from motor import motor_asyncio from motor.motor_asyncio import (AsyncIOMotorCollection, AsyncIOMotorCommandCursor) import test from test.asyncio_tests import (asyncio_test, AsyncIOTestCase, skip_if_mongos) from test.utils import ignore_deprecations class TestAsyncIOCollection(AsyncIOTestCase): @asyncio_test def test_collection(self): # Test that we can create a collection directly, not just from # database accessors. collection = AsyncIOMotorCollection(self.db, 'test_collection') # Make sure we got the right collection and it can do an operation self.assertEqual('test_collection', collection.name) yield from collection.delete_many({}) yield from collection.insert_one({'_id': 1}) doc = yield from collection.find_one({'_id': 1}) self.assertEqual(1, doc['_id']) # If you pass kwargs to PyMongo's Collection(), it calls # db.create_collection(). Motor can't do I/O in a constructor # so this is prohibited. self.assertRaises( TypeError, AsyncIOMotorCollection, self.db, 'test_collection', capped=True) @asyncio_test def test_dotted_collection_name(self): # Ensure that remove, insert, and find work on collections with dots # in their names. for coll in ( self.db.foo.bar, self.db.foo.bar.baz): yield from coll.delete_many({}) result = yield from coll.insert_one({'_id': 'xyzzy'}) self.assertEqual('xyzzy', result.inserted_id) result = yield from coll.find_one({'_id': 'xyzzy'}) self.assertEqual(result['_id'], 'xyzzy') yield from coll.delete_many({}) resp = yield from coll.find_one({'_id': 'xyzzy'}) self.assertEqual(None, resp) def test_call(self): # Prevents user error with nice message. try: self.db.foo() except TypeError as e: self.assertTrue('no such method exists' in str(e)) else: self.fail('Expected TypeError') @ignore_deprecations @asyncio_test def test_update(self): yield from self.collection.insert_one({'_id': 1}) result = yield from self.collection.update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual(1, result['ok']) self.assertEqual(True, result['updatedExisting']) self.assertEqual(1, result['n']) self.assertEqual(None, result.get('err')) @ignore_deprecations @asyncio_test def test_update_bad(self): # Violate a unique index, make sure we handle error well coll = self.db.unique_collection yield from coll.create_index('s', unique=True) try: yield from coll.insert_many([{'s': 1}, {'s': 2}]) with self.assertRaises(DuplicateKeyError): yield from coll.update({'s': 2}, {'$set': {'s': 1}}) finally: yield from coll.drop() @asyncio_test def test_insert_one(self): collection = self.collection result = yield from collection.insert_one({'_id': 201}) self.assertEqual(201, result.inserted_id) @ignore_deprecations @asyncio_test def test_insert_many_one_bad(self): collection = self.collection yield from collection.insert_one({'_id': 2}) # Violate a unique index in one of many updates, handle error. with self.assertRaises(DuplicateKeyError): yield from collection.insert([ {'_id': 1}, {'_id': 2}, # Already exists {'_id': 3}]) # First insert should have succeeded, but not second or third. self.assertEqual( set([1, 2]), set((yield from collection.distinct('_id')))) @ignore_deprecations @asyncio_test def test_save_with_id(self): # save() returns the _id, in this case 5. self.assertEqual( 5, (yield from self.collection.save({'_id': 5}))) @ignore_deprecations @asyncio_test def test_save_without_id(self): collection = self.collection result = yield from collection.save({'fiddle': 'faddle'}) # save() returns the new _id self.assertTrue(isinstance(result, ObjectId)) @ignore_deprecations @asyncio_test def test_save_bad(self): coll = self.db.unique_collection yield from coll.ensure_index('s', unique=True) yield from coll.save({'s': 1}) try: with self.assertRaises(DuplicateKeyError): yield from coll.save({'s': 1}) finally: yield from coll.drop() @asyncio_test def test_delete_one(self): # Remove a document twice, check that we get a success responses # and n = 0 for the second time. yield from self.collection.insert_one({'_id': 1}) result = yield from self.collection.delete_one({'_id': 1}) # First time we remove, n = 1 self.assertEqual(1, result.raw_result['n']) self.assertEqual(1, result.raw_result['ok']) self.assertEqual(None, result.raw_result.get('err')) result = yield from self.collection.delete_one({'_id': 1}) # Second time, document is already gone, n = 0 self.assertEqual(0, result.raw_result['n']) self.assertEqual(1, result.raw_result['ok']) self.assertEqual(None, result.raw_result.get('err')) @ignore_deprecations @asyncio_test def test_unacknowledged_insert(self): coll = self.db.test_unacknowledged_insert coll.with_options(write_concern=WriteConcern(0)).insert_one({'_id': 1}) # The insert is eventually executed. while not (yield from coll.count()): yield from asyncio.sleep(0.1, loop=self.loop) # DuplicateKeyError not raised. future = coll.insert({'_id': 1}) yield from coll.insert({'_id': 1}, w=0) with self.assertRaises(DuplicateKeyError): yield from future @ignore_deprecations @asyncio_test def test_unacknowledged_save(self): # Test that unsafe saves with no callback still work collection_name = 'test_unacknowledged_save' coll = self.db[collection_name] future = coll.save({'_id': 201}, w=0) while not (yield from coll.find_one({'_id': 201})): yield from asyncio.sleep(0.1, loop=self.loop) # DuplicateKeyError not raised coll.save({'_id': 201}) yield from coll.save({'_id': 201}, w=0) # Clean up. yield from future coll.database.client.close() @ignore_deprecations @asyncio_test def test_unacknowledged_update(self): # Test that unsafe updates with no callback still work coll = self.collection yield from coll.insert_one({'_id': 1}) coll.update({'_id': 1}, {'$set': {'a': 1}}, w=0) while not (yield from coll.find_one({'a': 1})): yield from asyncio.sleep(0.1, loop=self.loop) coll.database.client.close() @asyncio_test(timeout=30) def test_nested_callbacks(self): results = [0] future = asyncio.Future(loop=self.loop) yield from self.collection.delete_many({}) yield from self.collection.insert_one({'_id': 1}) def callback(result, error): if error: future.set_exception(error) elif result: results[0] += 1 if results[0] < 1000: self.collection.find({'_id': 1}).each(callback) else: future.set_result(None) self.collection.find({'_id': 1}).each(callback) yield from future self.assertEqual(1000, results[0]) @asyncio_test def test_map_reduce(self): # Count number of documents with even and odd _id yield from self.make_test_data() expected_result = [{'_id': 0, 'value': 100}, {'_id': 1, 'value': 100}] map_fn = bson.Code('function map() { emit(this._id % 2, 1); }') reduce_fn = bson.Code(''' function reduce(key, values) { r = 0; values.forEach(function(value) { r += value; }); return r; }''') yield from self.db.tmp_mr.drop() # First do a standard mapreduce, should return AsyncIOMotorCollection collection = self.collection tmp_mr = yield from collection.map_reduce(map_fn, reduce_fn, 'tmp_mr') self.assertTrue( isinstance(tmp_mr, motor_asyncio.AsyncIOMotorCollection), 'map_reduce should return AsyncIOMotorCollection, not %s' % tmp_mr) result = yield from tmp_mr.find().sort([('_id', 1)]).to_list( length=1000) self.assertEqual(expected_result, result) # Standard mapreduce with full response yield from self.db.tmp_mr.drop() response = yield from collection.map_reduce( map_fn, reduce_fn, 'tmp_mr', full_response=True) self.assertTrue( isinstance(response, dict), 'map_reduce should return dict, not %s' % response) self.assertEqual('tmp_mr', response['result']) result = yield from tmp_mr.find().sort([('_id', 1)]).to_list( length=1000) self.assertEqual(expected_result, result) # Inline mapreduce yield from self.db.tmp_mr.drop() result = yield from collection.inline_map_reduce( map_fn, reduce_fn) result.sort(key=lambda doc: doc['_id']) self.assertEqual(expected_result, result) @ignore_deprecations @asyncio_test def test_indexes(self): test_collection = self.collection # Create an index idx_name = yield from test_collection.create_index([('foo', 1)]) index_info = yield from test_collection.index_information() self.assertEqual([('foo', 1)], index_info[idx_name]['key']) # Ensure the same index, test that callback is executed result = yield from test_collection.ensure_index([('foo', 1)]) self.assertEqual('foo_1', result) result2 = yield from test_collection.ensure_index([('foo', 1)]) self.assertEqual(None, result2) # Ensure an index that doesn't exist, test it's created yield from test_collection.ensure_index([('bar', 1)]) index_info = yield from test_collection.index_information() self.assertTrue(any([ info['key'] == [('bar', 1)] for info in index_info.values()])) # Don't test drop_index or drop_indexes -- Synchro tests them @asyncio.coroutine def _make_test_data(self, n): yield from self.db.drop_collection("test") yield from self.db.test.insert_many([{'_id': i} for i in range(n)]) expected_sum = sum(range(n)) return expected_sum pipeline = [{'$project': {'_id': '$_id'}}] @asyncio_test(timeout=30) def test_aggregation_cursor(self): db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): expected_sum = yield from self._make_test_data(collection_size) cursor = db.test.aggregate(self.pipeline) docs = yield from cursor.to_list(collection_size) self.assertEqual(expected_sum, sum(doc['_id'] for doc in docs)) @asyncio_test def test_aggregation_cursor_exc_info(self): yield from self._make_test_data(200) cursor = self.db.test.aggregate(self.pipeline) yield from cursor.to_list(length=10) yield from self.db.test.drop() try: yield from cursor.to_list(length=None) except OperationFailure: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_unpack_response' in formatted or '_check_command_response' in formatted) @asyncio_test(timeout=30) def test_parallel_scan(self): yield from skip_if_mongos(self.cx) collection = self.collection.with_options( write_concern=WriteConcern(test.env.w)) # Enough documents that each cursor requires multiple batches. yield from collection.delete_many({}) yield from collection.insert_many(({'_id': i} for i in range(8000))) if test.env.is_replica_set: # Test that getMore messages are sent to the right server. client = self.asyncio_rsc(read_preference=Secondary()) collection = client.motor_test.test_collection docs = [] @asyncio.coroutine def f(cursor): self.assertTrue(isinstance(cursor, AsyncIOMotorCommandCursor)) while (yield from cursor.fetch_next): docs.append(cursor.next_object()) cursors = yield from collection.parallel_scan(3) yield from asyncio.wait( [f(cursor) for cursor in cursors], loop=self.loop) self.assertEqual(len(docs), (yield from collection.count())) def test_with_options(self): coll = self.db.test codec_options = CodecOptions( tz_aware=True, uuid_representation=JAVA_LEGACY) write_concern = WriteConcern(w=2, j=True) coll2 = coll.with_options( codec_options, ReadPreference.SECONDARY, write_concern) self.assertTrue(isinstance(coll2, AsyncIOMotorCollection)) self.assertEqual(codec_options, coll2.codec_options) self.assertEqual(Secondary(), coll2.read_preference) self.assertEqual(write_concern, coll2.write_concern) pref = Secondary([{"dc": "sf"}]) coll2 = coll.with_options(read_preference=pref) self.assertEqual(pref, coll2.read_preference) self.assertEqual(coll.codec_options, coll2.codec_options) self.assertEqual(coll.write_concern, coll2.write_concern) if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_cursor.py000066400000000000000000000505201323007662700232040ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorCursor.""" import asyncio import sys import traceback import unittest import warnings from unittest import SkipTest from pymongo import CursorType from pymongo.errors import InvalidOperation, ExecutionTimeout from pymongo.errors import OperationFailure from motor import motor_asyncio from test.utils import one, safe_get, get_primary_pool from test.asyncio_tests import (asyncio_test, AsyncIOTestCase, AsyncIOMockServerTestCase, server_is_mongos, get_command_line) class TestAsyncIOCursor(AsyncIOMockServerTestCase): def test_cursor(self): cursor = self.collection.find() self.assertTrue(isinstance(cursor, motor_asyncio.AsyncIOMotorCursor)) self.assertFalse(cursor.started, "Cursor shouldn't start immediately") @asyncio_test def test_count(self): yield from self.make_test_data() coll = self.collection self.assertEqual(200, (yield from coll.find().count())) self.assertEqual( 100, (yield from coll.find({'_id': {'$gt': 99}}).count())) where = 'this._id % 2 == 0 && this._id >= 50' self.assertEqual(75, (yield from coll.find().where(where).count())) @asyncio_test def test_fetch_next(self): yield from self.make_test_data() coll = self.collection # 200 results, only including _id field, sorted by _id. cursor = coll.find({}, {'_id': 1}).sort('_id').batch_size(75) self.assertEqual(None, cursor.cursor_id) self.assertEqual(None, cursor.next_object()) # Haven't fetched yet. i = 0 while (yield from cursor.fetch_next): self.assertEqual({'_id': i}, cursor.next_object()) i += 1 # With batch_size 75 and 200 results, cursor should be exhausted on # the server by third fetch. if i <= 150: self.assertNotEqual(0, cursor.cursor_id) else: self.assertEqual(0, cursor.cursor_id) self.assertEqual(False, (yield from cursor.fetch_next)) self.assertEqual(None, cursor.next_object()) self.assertEqual(0, cursor.cursor_id) self.assertEqual(200, i) @unittest.skipUnless(sys.version_info >= (3, 4), "Python 3.4 required") @unittest.skipIf('PyPy' in sys.version, "PyPy") @asyncio_test def test_fetch_next_delete(self): client, server = self.client_server(auto_ismaster=True) cursor = client.test.coll.find() self.fetch_next(cursor) request = yield from self.run_thread(server.receives, "find", "coll") request.replies({"cursor": { "id": 123, "ns": "db.coll", "firstBatch": [{"_id": 1}]}}) # Decref the cursor and clear from the event loop. del cursor yield request = yield from self.run_thread( server.receives, "killCursors", "coll") request.ok() @asyncio_test def test_fetch_next_without_results(self): coll = self.collection # Nothing matches this query. cursor = coll.find({'foo': 'bar'}) self.assertEqual(None, cursor.next_object()) self.assertEqual(False, (yield from cursor.fetch_next)) self.assertEqual(None, cursor.next_object()) # Now cursor knows it's exhausted. self.assertEqual(0, cursor.cursor_id) @asyncio_test def test_fetch_next_is_idempotent(self): # Subsequent calls to fetch_next don't do anything yield from self.make_test_data() coll = self.collection cursor = coll.find() self.assertEqual(None, cursor.cursor_id) yield from cursor.fetch_next self.assertTrue(cursor.cursor_id) self.assertEqual(101, cursor._buffer_size()) yield from cursor.fetch_next # Does nothing self.assertEqual(101, cursor._buffer_size()) yield from cursor.close() @asyncio_test def test_fetch_next_exception(self): coll = self.collection cursor = coll.find() cursor.delegate._Cursor__id = 1234 # Not valid on server. with self.assertRaises(OperationFailure): yield from cursor.fetch_next # Avoid the cursor trying to close itself when it goes out of scope. cursor.delegate._Cursor__id = None @asyncio_test(timeout=30) def test_each(self): yield from self.make_test_data() cursor = self.collection.find({}, {'_id': 1}).sort('_id') future = asyncio.Future(loop=self.loop) results = [] def callback(result, error): if error: raise error if result is not None: results.append(result) else: # Done iterating. future.set_result(True) cursor.each(callback) yield from future expected = [{'_id': i} for i in range(200)] self.assertEqual(expected, results) @asyncio_test def test_to_list_argument_checking(self): # We need more than 10 documents so the cursor stays alive. yield from self.make_test_data() coll = self.collection cursor = coll.find() with self.assertRaises(ValueError): yield from cursor.to_list(-1) with self.assertRaises(TypeError): yield from cursor.to_list('foo') @asyncio_test def test_to_list_with_length(self): yield from self.make_test_data() coll = self.collection cursor = coll.find().sort('_id') def expected(start, stop): return [{'_id': i} for i in range(start, stop)] self.assertEqual(expected(0, 10), (yield from cursor.to_list(10))) self.assertEqual(expected(10, 100), (yield from cursor.to_list(90))) # Test particularly rigorously around the 101-doc mark, since this is # where the first batch ends self.assertEqual(expected(100, 101), (yield from cursor.to_list(1))) self.assertEqual(expected(101, 102), (yield from cursor.to_list(1))) self.assertEqual(expected(102, 103), (yield from cursor.to_list(1))) self.assertEqual([], (yield from cursor.to_list(0))) self.assertEqual(expected(103, 105), (yield from cursor.to_list(2))) # Only 95 docs left, make sure length=100 doesn't error or hang self.assertEqual(expected(105, 200), (yield from cursor.to_list(100))) self.assertEqual(0, cursor.cursor_id) # Nothing left. self.assertEqual([], (yield from cursor.to_list(100))) yield from cursor.close() @asyncio_test def test_to_list_exc_info(self): yield from self.make_test_data() coll = self.collection cursor = coll.find() yield from cursor.to_list(length=10) yield from self.collection.drop() try: yield from cursor.to_list(length=None) except OperationFailure: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_unpack_response' in formatted or '_check_command_response' in formatted) @asyncio_test def test_to_list_with_length_of_none(self): yield from self.make_test_data() collection = self.collection cursor = collection.find() docs = yield from cursor.to_list(None) # Unlimited. count = yield from collection.count() self.assertEqual(count, len(docs)) @asyncio_test def test_to_list_tailable(self): coll = self.collection cursor = coll.find(cursor_type=CursorType.TAILABLE) # Can't call to_list on tailable cursor. with self.assertRaises(InvalidOperation): yield from cursor.to_list(10) @asyncio_test def test_cursor_explicit_close(self): client, server = self.client_server(auto_ismaster=True) collection = client.test.coll cursor = collection.find() future = self.fetch_next(cursor) self.assertTrue(cursor.alive) request = yield from self.run_thread(server.receives, "find", "coll") request.replies({"cursor": { "id": 123, "ns": "db.coll", "firstBatch": [{"_id": 1}]}}) self.assertTrue((yield from future)) self.assertEqual(123, cursor.cursor_id) future = self.ensure_future(cursor.close()) # No reply to OP_KILLCURSORS. request = yield from self.run_thread( server.receives, "killCursors", "coll") request.ok() yield from future # Cursor reports it's alive because it has buffered data, even though # it's killed on the server. self.assertTrue(cursor.alive) self.assertEqual({'_id': 1}, cursor.next_object()) self.assertFalse((yield from cursor.fetch_next)) self.assertFalse(cursor.alive) @asyncio_test def test_each_cancel(self): yield from self.make_test_data() loop = self.loop collection = self.collection results = [] future = asyncio.Future(loop=self.loop) def cancel(result, error): if error: future.set_exception(error) else: results.append(result) loop.call_soon(canceled) return False # Cancel iteration. def canceled(): try: self.assertFalse(cursor.delegate._Cursor__killed) self.assertTrue(cursor.alive) # Resume iteration cursor.each(each) except Exception as e: future.set_exception(e) def each(result, error): if error: future.set_exception(error) elif result: pass results.append(result) else: # Complete future.set_result(None) cursor = collection.find() cursor.each(cancel) yield from future self.assertEqual((yield from collection.count()), len(results)) @asyncio_test def test_rewind(self): yield from self.collection.insert_many([{}, {}, {}]) cursor = self.collection.find().limit(2) count = 0 while (yield from cursor.fetch_next): cursor.next_object() count += 1 self.assertEqual(2, count) cursor.rewind() count = 0 while (yield from cursor.fetch_next): cursor.next_object() count += 1 self.assertEqual(2, count) cursor.rewind() count = 0 while (yield from cursor.fetch_next): cursor.next_object() break cursor.rewind() while (yield from cursor.fetch_next): cursor.next_object() count += 1 self.assertEqual(2, count) self.assertEqual(cursor, cursor.rewind()) @unittest.skipUnless(sys.version_info >= (3, 4), "Python 3.4 required") @unittest.skipIf("PyPy" in sys.version, "PyPy") @asyncio_test def test_cursor_del(self): client, server = self.client_server(auto_ismaster=True) cursor = client.test.coll.find() future = self.fetch_next(cursor) request = yield from self.run_thread(server.receives, "find", "coll") request.replies({"cursor": { "id": 123, "ns": "db.coll", "firstBatch": [{"_id": 1}]}}) yield from future # Complete the first fetch. # Dereference the cursor. del cursor # Let the event loop iterate once more to clear its references to # callbacks, allowing the cursor to be freed. yield from asyncio.sleep(0, loop=self.loop) request = yield from self.run_thread( server.receives, "killCursors", "coll") request.ok() @unittest.skipUnless(sys.version_info >= (3, 4), "Python 3.4 required") @asyncio_test def test_exhaust(self): if (yield from server_is_mongos(self.cx)): self.assertRaises(InvalidOperation, self.db.test.find, cursor_type=CursorType.EXHAUST) return self.assertRaises(ValueError, self.db.test.find, cursor_type=5) cur = self.db.test.find(cursor_type=CursorType.EXHAUST) self.assertRaises(InvalidOperation, cur.limit, 5) cur = self.db.test.find(limit=5) self.assertRaises(InvalidOperation, cur.add_option, 64) cur = self.db.test.find() cur.add_option(64) self.assertRaises(InvalidOperation, cur.limit, 5) yield from self.db.drop_collection("test") # Insert enough documents to require more than one batch. yield from self.db.test.insert_many([{} for _ in range(150)]) client = self.asyncio_client(maxPoolSize=1) # Ensure a pool. yield from client.db.collection.find_one() socks = get_primary_pool(client).sockets # Make sure the socket is returned after exhaustion. cur = client[self.db.name].test.find(cursor_type=CursorType.EXHAUST) has_next = yield from cur.fetch_next self.assertTrue(has_next) self.assertEqual(0, len(socks)) while (yield from cur.fetch_next): cur.next_object() self.assertEqual(1, len(socks)) # Same as previous but with to_list instead of next_object. docs = yield from client[self.db.name].test.find( cursor_type=CursorType.EXHAUST).to_list( None) self.assertEqual(1, len(socks)) self.assertEqual( (yield from self.db.test.count()), len(docs)) # If the Cursor instance is discarded before being # completely iterated we have to close and # discard the socket. sock = one(socks) cur = client[self.db.name].test.find( cursor_type=CursorType.EXHAUST).batch_size(1) has_next = yield from cur.fetch_next self.assertTrue(has_next) self.assertEqual(0, len(socks)) if 'PyPy' in sys.version: # Don't wait for GC or use gc.collect(), it's unreliable. yield from cur.close() del cur yield from asyncio.sleep(0.1, loop=self.loop) # The exhaust cursor's socket was discarded, although another may # already have been opened to send OP_KILLCURSORS. self.assertNotIn(sock, socks) self.assertTrue(sock.closed) @asyncio_test def test_close_with_docs_in_batch(self): # MOTOR-67 Killed cursor with docs batched is "alive", don't kill again. yield from self.make_test_data() # Ensure multiple batches. cursor = self.collection.find() yield from cursor.fetch_next yield from cursor.close() # Killed but still "alive": has a batch. self.cx.close() with warnings.catch_warnings(record=True) as w: del cursor # No-op, no error. self.assertEqual(0, len(w)) class TestAsyncIOCursorMaxTimeMS(AsyncIOTestCase): def setUp(self): super(TestAsyncIOCursorMaxTimeMS, self).setUp() self.loop.run_until_complete(self.maybe_skip()) def tearDown(self): self.loop.run_until_complete(self.disable_timeout()) super(TestAsyncIOCursorMaxTimeMS, self).tearDown() @asyncio.coroutine def maybe_skip(self): if (yield from server_is_mongos(self.cx)): raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point") cmdline = yield from get_command_line(self.cx) if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'): if 'enableTestCommands=1' not in cmdline['argv']: raise SkipTest("testing maxTimeMS requires failpoints") @asyncio.coroutine def enable_timeout(self): yield from self.cx.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") @asyncio.coroutine def disable_timeout(self): yield from self.cx.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") @asyncio_test def test_max_time_ms_query(self): # Cursor parses server timeout error in response to initial query. yield from self.enable_timeout() cursor = self.collection.find().max_time_ms(100000) with self.assertRaises(ExecutionTimeout): yield from cursor.fetch_next cursor = self.collection.find().max_time_ms(100000) with self.assertRaises(ExecutionTimeout): yield from cursor.to_list(10) with self.assertRaises(ExecutionTimeout): yield from self.collection.find_one(max_time_ms=100000) @asyncio_test(timeout=60) def test_max_time_ms_getmore(self): # Cursor handles server timeout during getmore, also. yield from self.collection.insert_many({} for _ in range(200)) try: # Send initial query. cursor = self.collection.find().max_time_ms(100000) yield from cursor.fetch_next cursor.next_object() # Test getmore timeout. yield from self.enable_timeout() with self.assertRaises(ExecutionTimeout): while (yield from cursor.fetch_next): cursor.next_object() yield from cursor.close() # Send another initial query. yield from self.disable_timeout() cursor = self.collection.find().max_time_ms(100000) yield from cursor.fetch_next cursor.next_object() # Test getmore timeout. yield from self.enable_timeout() with self.assertRaises(ExecutionTimeout): yield from cursor.to_list(None) # Avoid 'IOLoop is closing' warning. yield from cursor.close() finally: # Cleanup. yield from self.disable_timeout() yield from self.collection.delete_many({}) @asyncio_test def test_max_time_ms_each_query(self): # Cursor.each() handles server timeout during initial query. yield from self.enable_timeout() cursor = self.collection.find().max_time_ms(100000) future = asyncio.Future(loop=self.loop) def callback(result, error): if error: future.set_exception(error) elif not result: # Done. future.set_result(None) with self.assertRaises(ExecutionTimeout): cursor.each(callback) yield from future @asyncio_test(timeout=30) def test_max_time_ms_each_getmore(self): # Cursor.each() handles server timeout during getmore. yield from self.collection.insert_many({} for _ in range(200)) try: # Send initial query. cursor = self.collection.find().max_time_ms(100000) yield from cursor.fetch_next cursor.next_object() future = asyncio.Future(loop=self.loop) def callback(result, error): if error: future.set_exception(error) elif not result: # Done. future.set_result(None) yield from self.enable_timeout() with self.assertRaises(ExecutionTimeout): cursor.each(callback) yield from future yield from cursor.close() finally: # Cleanup. yield from self.disable_timeout() yield from self.collection.delete_many({}) def test_iter(self): # Iteration should be prohibited. with self.assertRaises(TypeError): for _ in self.db.test.find(): pass if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_database.py000066400000000000000000000173461323007662700234440ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorDatabase.""" import asyncio import unittest from unittest import SkipTest import pymongo.database from bson import CodecOptions from bson.binary import JAVA_LEGACY from pymongo import ReadPreference, WriteConcern from pymongo.errors import CollectionInvalid, OperationFailure from pymongo.read_preferences import Secondary from pymongo.son_manipulator import NamespaceInjector, AutoReference from motor.motor_asyncio import (AsyncIOMotorDatabase, AsyncIOMotorClient, AsyncIOMotorCollection) import test from test import env from test.asyncio_tests import (asyncio_test, AsyncIOTestCase, remove_all_users) from test.utils import ignore_deprecations class TestAsyncIODatabase(AsyncIOTestCase): @asyncio_test def test_database(self): # Test that we can create a db directly, not just get on from # AsyncIOMotorClient. db = AsyncIOMotorDatabase(self.cx, 'motor_test') # Make sure we got the right DB and it can do an operation. self.assertEqual('motor_test', db.name) yield from db.test_collection.delete_many({}) yield from db.test_collection.insert_one({'_id': 1}) doc = yield from db.test_collection.find_one({'_id': 1}) self.assertEqual(1, doc['_id']) def test_collection_named_delegate(self): db = self.db self.assertTrue(isinstance(db.delegate, pymongo.database.Database)) self.assertTrue(isinstance(db['delegate'], AsyncIOMotorCollection)) db.client.close() def test_call(self): # Prevents user error with nice message. try: self.cx.foo() except TypeError as e: self.assertTrue('no such method exists' in str(e)) else: self.fail('Expected TypeError') @asyncio_test def test_command(self): result = yield from self.cx.admin.command("buildinfo") # Make sure we got some sane result or other. self.assertEqual(1, result['ok']) @asyncio_test def test_create_collection(self): # Test creating collection, return val is wrapped in # AsyncIOMotorCollection, creating it again raises CollectionInvalid. db = self.db yield from db.drop_collection('test_collection2') collection = yield from db.create_collection('test_collection2') self.assertTrue(isinstance(collection, AsyncIOMotorCollection)) self.assertTrue( 'test_collection2' in (yield from db.collection_names())) with self.assertRaises(CollectionInvalid): yield from db.create_collection('test_collection2') @asyncio_test def test_drop_collection(self): # Make sure we can pass an AsyncIOMotorCollection instance to # drop_collection. db = self.db collection = db.test_drop_collection yield from collection.insert_one({}) names = yield from db.collection_names() self.assertTrue('test_drop_collection' in names) yield from db.drop_collection(collection) names = yield from db.collection_names() self.assertFalse('test_drop_collection' in names) @ignore_deprecations @asyncio_test def test_auto_ref_and_deref(self): # Test same functionality as in PyMongo's test_database.py; the # implementation for Motor for async is a little complex so we test # that it works here, and we don't just rely on synchrotest # to cover it. db = self.db # We test a special hack where add_son_manipulator corrects our mistake # if we pass an AsyncIOMotorDatabase, instead of Database, to # AutoReference. db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) a = {"hello": "world"} b = {"test": a} c = {"another test": b} yield from db.a.delete_many({}) yield from db.b.delete_many({}) yield from db.c.delete_many({}) yield from db.a.save(a) yield from db.b.save(b) yield from db.c.save(c) a["hello"] = "jesse" yield from db.a.save(a) result_a = yield from db.a.find_one() result_b = yield from db.b.find_one() result_c = yield from db.c.find_one() self.assertEqual(a, result_a) self.assertEqual(a, result_b["test"]) self.assertEqual(a, result_c["another test"]["test"]) self.assertEqual(b, result_b) self.assertEqual(b, result_c["another test"]) self.assertEqual(c, result_c) @asyncio_test def test_authenticate(self): if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # self.db is logged in as root. yield from self.db.add_user("jesse", "password") db = AsyncIOMotorClient(env.host, env.port, **self.get_client_kwargs()).motor_test try: # Authenticate many times at once to test concurrency. yield from asyncio.wait( [db.authenticate("jesse", "password") for _ in range(10)], loop=self.loop) # Just make sure there are no exceptions here. yield from db.remove_user("jesse") yield from db.logout() info = yield from self.db.command("usersInfo", "jesse") users = info.get('users', []) self.assertFalse("jesse" in [u['user'] for u in users]) finally: yield from remove_all_users(self.db) test.env.sync_cx.close() @asyncio_test def test_validate_collection(self): db = self.db with self.assertRaises(TypeError): yield from db.validate_collection(5) with self.assertRaises(TypeError): yield from db.validate_collection(None) with self.assertRaises(OperationFailure): yield from db.validate_collection("test.doesnotexist") with self.assertRaises(OperationFailure): yield from db.validate_collection(db.test.doesnotexist) yield from db.test.insert_one({"dummy": "object"}) self.assertTrue((yield from db.validate_collection("test"))) self.assertTrue((yield from db.validate_collection(db.test))) def test_get_collection(self): codec_options = CodecOptions( tz_aware=True, uuid_representation=JAVA_LEGACY) write_concern = WriteConcern(w=2, j=True) coll = self.db.get_collection( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) self.assertTrue(isinstance(coll, AsyncIOMotorCollection)) self.assertEqual('foo', coll.name) self.assertEqual(codec_options, coll.codec_options) self.assertEqual(ReadPreference.SECONDARY, coll.read_preference) self.assertEqual(write_concern, coll.write_concern) pref = Secondary([{"dc": "sf"}]) coll = self.db.get_collection('foo', read_preference=pref) self.assertEqual(pref, coll.read_preference) self.assertEqual(self.db.codec_options, coll.codec_options) self.assertEqual(self.db.write_concern, coll.write_concern) if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_gridfs.py000066400000000000000000000312431323007662700231460ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorGridFS.""" import asyncio import datetime import sys import traceback import unittest from pymongo.errors import InvalidOperation, ConfigurationError import test from gridfs.errors import FileExists, NoFile from motor.motor_asyncio import (AsyncIOMotorGridFS, AsyncIOMotorGridIn, AsyncIOMotorGridOut) from motor.motor_py3_compat import StringIO from test.asyncio_tests import asyncio_test, AsyncIOTestCase class TestAsyncIOGridFile(AsyncIOTestCase): @asyncio.coroutine def _reset(self): yield from self.db.drop_collection("fs.files") yield from self.db.drop_collection("fs.chunks") yield from self.db.drop_collection("alt.files") yield from self.db.drop_collection("alt.chunks") def tearDown(self): self.loop.run_until_complete(self._reset()) super(TestAsyncIOGridFile, self).tearDown() @asyncio_test def test_attributes(self): f = AsyncIOMotorGridIn( self.db.fs, filename="test", foo="bar", content_type="text") yield from f.close() g = AsyncIOMotorGridOut(self.db.fs, f._id) attr_names = ( '_id', 'filename', 'name', 'name', 'content_type', 'length', 'chunk_size', 'upload_date', 'aliases', 'metadata', 'md5') for attr_name in attr_names: self.assertRaises(InvalidOperation, getattr, g, attr_name) yield from g.open() for attr_name in attr_names: getattr(g, attr_name) @asyncio_test def test_gridout_open_exc_info(self): g = AsyncIOMotorGridOut(self.db.fs, "_id that doesn't exist") try: yield from g.open() except NoFile: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_ensure_file' in formatted) @asyncio_test def test_alternate_collection(self): yield from self.db.alt.files.delete_many({}) yield from self.db.alt.chunks.delete_many({}) f = AsyncIOMotorGridIn(self.db.alt) yield from f.write(b"hello world") yield from f.close() self.assertEqual(1, (yield from self.db.alt.files.find().count())) self.assertEqual(1, (yield from self.db.alt.chunks.find().count())) g = AsyncIOMotorGridOut(self.db.alt, f._id) self.assertEqual(b"hello world", (yield from g.read())) # test that md5 still works... self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5) @asyncio_test def test_grid_in_custom_opts(self): self.assertRaises(TypeError, AsyncIOMotorGridIn, "foo") a = AsyncIOMotorGridIn( self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") self.assertEqual(5, a._id) self.assertEqual("my_file", a.filename) self.assertEqual("text/html", a.content_type) self.assertEqual(1000, a.chunk_size) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1, "bar": 2}, a.metadata) self.assertEqual(3, a.bar) self.assertEqual("hello", a.baz) self.assertRaises(AttributeError, getattr, a, "mike") b = AsyncIOMotorGridIn( self.db.fs, content_type="text/html", chunk_size=1000, baz=100) self.assertEqual("text/html", b.content_type) self.assertEqual(1000, b.chunk_size) self.assertEqual(100, b.baz) @asyncio_test def test_grid_out_default_opts(self): self.assertRaises(TypeError, AsyncIOMotorGridOut, "foo") gout = AsyncIOMotorGridOut(self.db.fs, 5) with self.assertRaises(NoFile): yield from gout.open() a = AsyncIOMotorGridIn(self.db.fs) yield from a.close() b = yield from AsyncIOMotorGridOut(self.db.fs, a._id).open() self.assertEqual(a._id, b._id) self.assertEqual(0, b.length) self.assertEqual(None, b.content_type) self.assertEqual(255 * 1024, b.chunk_size) self.assertTrue(isinstance(b.upload_date, datetime.datetime)) self.assertEqual(None, b.aliases) self.assertEqual(None, b.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5) @asyncio_test def test_grid_out_custom_opts(self): one = AsyncIOMotorGridIn( self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") yield from one.write(b"hello world") yield from one.close() two = yield from AsyncIOMotorGridOut(self.db.fs, 5).open() self.assertEqual(5, two._id) self.assertEqual(11, two.length) self.assertEqual("text/html", two.content_type) self.assertEqual(1000, two.chunk_size) self.assertTrue(isinstance(two.upload_date, datetime.datetime)) self.assertEqual(["foo"], two.aliases) self.assertEqual({"foo": 1, "bar": 2}, two.metadata) self.assertEqual(3, two.bar) self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5) @asyncio_test def test_grid_out_file_document(self): one = AsyncIOMotorGridIn(self.db.fs) yield from one.write(b"foo bar") yield from one.close() file_document = yield from self.db.fs.files.find_one() two = AsyncIOMotorGridOut( self.db.fs, file_document=file_document) self.assertEqual(b"foo bar", (yield from two.read())) file_document = yield from self.db.fs.files.find_one() three = AsyncIOMotorGridOut(self.db.fs, 5, file_document) self.assertEqual(b"foo bar", (yield from three.read())) gridout = AsyncIOMotorGridOut(self.db.fs, file_document={}) with self.assertRaises(NoFile): yield from gridout.open() @asyncio_test def test_write_file_like(self): one = AsyncIOMotorGridIn(self.db.fs) yield from one.write(b"hello world") yield from one.close() two = AsyncIOMotorGridOut(self.db.fs, one._id) three = AsyncIOMotorGridIn(self.db.fs) yield from three.write(two) yield from three.close() four = AsyncIOMotorGridOut(self.db.fs, three._id) self.assertEqual(b"hello world", (yield from four.read())) @asyncio_test def test_set_after_close(self): f = AsyncIOMotorGridIn(self.db.fs, _id="foo", bar="baz") self.assertEqual("foo", f._id) self.assertEqual("baz", f.bar) self.assertRaises(AttributeError, getattr, f, "baz") self.assertRaises(AttributeError, getattr, f, "uploadDate") self.assertRaises(AttributeError, setattr, f, "_id", 5) f.bar = "foo" f.baz = 5 self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertRaises(AttributeError, getattr, f, "uploadDate") yield from f.close() self.assertEqual("foo", f._id) self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertTrue(f.uploadDate) self.assertRaises(AttributeError, setattr, f, "_id", 5) yield from f.set("bar", "a") yield from f.set("baz", "b") self.assertRaises(AttributeError, setattr, f, "upload_date", 5) g = yield from AsyncIOMotorGridOut(self.db.fs, f._id).open() self.assertEqual("a", g.bar) self.assertEqual("b", g.baz) @asyncio_test def test_stream_to_handler(self): fs = AsyncIOMotorGridFS(self.db) for content_length in (0, 1, 100, 100 * 1000): _id = yield from fs.put(b'a' * content_length) gridout = yield from fs.get(_id) handler = test.MockRequestHandler() yield from gridout.stream_to_handler(handler) self.assertEqual(content_length, handler.n_written) yield from fs.delete(_id) class TestAsyncIOGridFS(AsyncIOTestCase): @asyncio.coroutine def _reset(self): yield from self.db.drop_collection("fs.files") yield from self.db.drop_collection("fs.chunks") yield from self.db.drop_collection("alt.files") yield from self.db.drop_collection("alt.chunks") def setUp(self): super().setUp() self.loop.run_until_complete(self._reset()) self.fs = AsyncIOMotorGridFS(self.db) def tearDown(self): self.loop.run_until_complete(self._reset()) super().tearDown() @asyncio_test def test_get_version(self): # new_file creates a MotorGridIn. gin = yield from self.fs.new_file(_id=1, filename='foo', field=0) yield from gin.write(b'a') yield from gin.close() yield from self.fs.put(b'', filename='foo', field=1) yield from self.fs.put(b'', filename='foo', field=2) gout = yield from self.fs.get_version('foo') self.assertEqual(2, gout.field) gout = yield from self.fs.get_version('foo', -3) self.assertEqual(0, gout.field) gout = yield from self.fs.get_last_version('foo') self.assertEqual(2, gout.field) @asyncio_test def test_basic(self): oid = yield from self.fs.put(b"hello world") out = yield from self.fs.get(oid) self.assertEqual(b"hello world", (yield from out.read())) self.assertEqual(1, (yield from self.db.fs.files.count())) self.assertEqual(1, (yield from self.db.fs.chunks.count())) yield from self.fs.delete(oid) with self.assertRaises(NoFile): yield from self.fs.get(oid) self.assertEqual(0, (yield from self.db.fs.files.count())) self.assertEqual(0, (yield from self.db.fs.chunks.count())) with self.assertRaises(NoFile): yield from self.fs.get("foo") self.assertEqual( "foo", (yield from self.fs.put(b"hello world", _id="foo"))) gridout = yield from self.fs.get("foo") self.assertEqual(b"hello world", (yield from gridout.read())) @asyncio_test def test_list(self): self.assertEqual([], (yield from self.fs.list())) yield from self.fs.put(b"hello world") self.assertEqual([], (yield from self.fs.list())) yield from self.fs.put(b"", filename="mike") yield from self.fs.put(b"foo", filename="test") yield from self.fs.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set((yield from self.fs.list()))) @asyncio_test(timeout=30) def test_put_filelike(self): oid = yield from self.fs.put(StringIO(b"hello world"), chunk_size=1) self.assertEqual(11, (yield from self.cx.motor_test.fs.chunks.count())) gridout = yield from self.fs.get(oid) self.assertEqual(b"hello world", (yield from gridout.read())) @asyncio_test def test_put_duplicate(self): oid = yield from self.fs.put(b"hello") with self.assertRaises(FileExists): yield from self.fs.put(b"world", _id=oid) @asyncio_test def test_put_unacknowledged(self): client = self.asyncio_client(w=0) with self.assertRaises(ConfigurationError): AsyncIOMotorGridFS(client.motor_test) client.close() @asyncio_test def test_gridfs_find(self): yield from self.fs.put(b"test2", filename="two") yield from self.fs.put(b"test2+", filename="two") yield from self.fs.put(b"test1", filename="one") yield from self.fs.put(b"test2++", filename="two") cursor = self.fs.find().sort("_id", -1).skip(1).limit(2) self.assertTrue((yield from cursor.fetch_next)) grid_out = cursor.next_object() self.assertTrue(isinstance(grid_out, AsyncIOMotorGridOut)) self.assertEqual(b"test1", (yield from grid_out.read())) self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) if __name__ == "__main__": unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_gridfsbucket.py000066400000000000000000000044501323007662700243440ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorGridFSBucket.""" import asyncio from io import BytesIO from gridfs.errors import NoFile from motor.motor_asyncio import AsyncIOMotorGridFSBucket from test.asyncio_tests import AsyncIOTestCase, asyncio_test class TestAsyncIOGridFSBucket(AsyncIOTestCase): @asyncio.coroutine def _reset(self): yield from self.db.drop_collection("fs.files") yield from self.db.drop_collection("fs.chunks") yield from self.db.drop_collection("alt.files") yield from self.db.drop_collection("alt.chunks") def setUp(self): super(TestAsyncIOGridFSBucket, self).setUp() self.loop.run_until_complete(self._reset()) self.bucket = AsyncIOMotorGridFSBucket(self.db) def tearDown(self): self.loop.run_until_complete(self._reset()) super(TestAsyncIOGridFSBucket, self).tearDown() @asyncio_test def test_basic(self): oid = yield from self.bucket.upload_from_stream("test_filename", b"hello world") gout = yield from self.bucket.open_download_stream(oid) self.assertEqual(b"hello world", (yield from gout.read())) self.assertEqual(1, (yield from self.db.fs.files.count())) self.assertEqual(1, (yield from self.db.fs.chunks.count())) dst = BytesIO() yield from self.bucket.download_to_stream(gout._id, dst) self.assertEqual(b"hello world", dst.getvalue()) yield from self.bucket.delete(oid) with self.assertRaises(NoFile): yield from self.bucket.open_download_stream(oid) self.assertEqual(0, (yield from self.db.fs.files.count())) self.assertEqual(0, (yield from self.db.fs.chunks.count())) motor-1.2.1/test/asyncio_tests/test_asyncio_ipv6.py000066400000000000000000000037471323007662700225640ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorClient with IPv6.""" import unittest from pymongo import MongoClient from pymongo.errors import ConnectionFailure import test from test import SkipTest from test.asyncio_tests import asyncio_test, AsyncIOTestCase from test.test_environment import db_user, db_password, connected, env class MotorIPv6Test(AsyncIOTestCase): @asyncio_test def test_ipv6(self): assert env.host in ('localhost', '127.0.0.1'), ( "This unittest isn't written to test IPv6 " "with host %s" % repr(env.host)) try: connected(MongoClient("[::1]", username=db_user, password=db_password, serverSelectionTimeoutMS=100)) except ConnectionFailure: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest("No IPV6") if test.env.auth: cx_string = 'mongodb://%s:%s@[::1]:%d' % ( db_user, db_password, env.port) else: cx_string = 'mongodb://[::1]:%d' % env.port cx = self.asyncio_client(uri=cx_string) collection = cx.motor_test.test_collection yield from collection.insert_one({"dummy": "object"}) self.assertTrue((yield from collection.find_one({"dummy": "object"}))) if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_replica_set.py000066400000000000000000000044511323007662700241630ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test replica set AsyncIOClient.""" import unittest import pymongo import pymongo.errors import pymongo.mongo_replica_set_client import test from motor import motor_asyncio from test import env, SkipTest from test.asyncio_tests import AsyncIOTestCase, asyncio_test from test.test_environment import env class TestAsyncIOReplicaSet(AsyncIOTestCase): def setUp(self): if not test.env.is_replica_set: raise SkipTest('Not connected to a replica set') super().setUp() @asyncio_test def test_connection_failure(self): # Assuming there isn't anything actually running on this port. client = motor_asyncio.AsyncIOMotorClient( 'localhost:8765', replicaSet='rs', io_loop=self.loop, serverSelectionTimeoutMS=10) with self.assertRaises(pymongo.errors.ConnectionFailure): yield from client.admin.command('ismaster') class TestReplicaSetClientAgainstStandalone(AsyncIOTestCase): """This is a funny beast -- we want to run tests for a replica set AsyncIOMotorClient but only if the database at DB_IP and DB_PORT is a standalone. """ def setUp(self): super(TestReplicaSetClientAgainstStandalone, self).setUp() if test.env.is_replica_set: raise SkipTest( "Connected to a replica set, not a standalone mongod") @asyncio_test def test_connect(self): client = motor_asyncio.AsyncIOMotorClient( '%s:%s' % (env.host, env.port), replicaSet='anything', serverSelectionTimeoutMS=10, io_loop=self.loop) with self.assertRaises(pymongo.errors.ServerSelectionTimeoutError): yield from client.test.test.find_one() if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_session.py000066400000000000000000000165731323007662700233640ustar00rootroot00000000000000# Copyright 2017-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test ClientSession support with asyncio.""" import asyncio import copy import sys import unittest from pymongo import InsertOne, IndexModel from pymongo.errors import InvalidOperation from test import SkipTest from test.asyncio_tests import AsyncIOTestCase, asyncio_test from test.test_environment import env from test.utils import SessionTestListener, session_ids class TestAsyncIOSession(AsyncIOTestCase): @classmethod def setUpClass(cls): super(TestAsyncIOSession, cls).setUpClass() if not env.sessions_enabled: raise SkipTest("Sessions not supported") @asyncio.coroutine def _test_ops(self, client, *ops): listener = client.event_listeners()[0][0] for f, args, kw in ops: with (yield from client.start_session()) as s: listener.results.clear() # In case "f" modifies its inputs. args2 = copy.copy(args) kw2 = copy.copy(kw) kw2['session'] = s yield from f(*args2, **kw2) for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( f.__name__, event.command_name)) self.assertEqual( s.session_id, event.command['lsid'], "%s sent wrong lsid with %s" % ( f.__name__, event.command_name)) self.assertFalse(s.has_ended) with self.assertRaisesRegex(InvalidOperation, "ended session"): yield from f(*args2, **kw2) # No explicit session. for f, args, kw in ops: listener.results.clear() yield from f(*args, **kw) self.assertGreaterEqual(len(listener.results['started']), 1) lsids = [] for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( f.__name__, event.command_name)) lsids.append(event.command['lsid']) if 'PyPy' not in sys.version: # Server session was returned to pool. Ignore interpreters with # non-deterministic GC. for lsid in lsids: self.assertIn( lsid, session_ids(client), "%s did not return implicit session to pool" % ( f.__name__,)) @asyncio_test def test_database(self): listener = SessionTestListener() client = self.asyncio_client(event_listeners=[listener]) db = client.pymongo_test ops = [ (db.command, ['ping'], {}), (db.drop_collection, ['collection'], {}), (db.create_collection, ['collection'], {}), (db.collection_names, [], {}), ] yield from self._test_ops(client, *ops) @asyncio_test(timeout=30) def test_collection(self): listener = SessionTestListener() client = self.asyncio_client(event_listeners=[listener]) yield from client.drop_database('motor_test') coll = client.motor_test.test_collection @asyncio.coroutine def list_indexes(session=None): yield from coll.list_indexes(session=session).to_list(length=None) @asyncio.coroutine def aggregate(session=None): yield from coll.aggregate([], session=session).to_list(length=None) # Test some collection methods - the rest are in test_cursor. yield from self._test_ops( client, (coll.drop, [], {}), (coll.bulk_write, [[InsertOne({})]], {}), (coll.insert_one, [{}], {}), (coll.insert_many, [[{}, {}]], {}), (coll.replace_one, [{}, {}], {}), (coll.update_one, [{}, {'$set': {'a': 1}}], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_one, [{}], {}), (coll.delete_many, [{}], {}), (coll.find_one_and_replace, [{}, {}], {}), (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}), (coll.find_one_and_delete, [{}, {}], {}), (coll.rename, ['collection2'], {}), # Drop collection2 between tests of "rename", above. (client.motor_test.drop_collection, ['collection2'], {}), (coll.distinct, ['a'], {}), (coll.find_one, [], {}), (coll.count, [], {}), (coll.create_indexes, [[IndexModel('a')]], {}), (coll.create_index, ['a'], {}), (coll.drop_index, ['a_1'], {}), (coll.drop_indexes, [], {}), (coll.reindex, [], {}), (list_indexes, [], {}), (coll.index_information, [], {}), (coll.options, [], {}), (aggregate, [], {})) @asyncio_test def test_cursor(self): listener = SessionTestListener() client = self.asyncio_client(event_listeners=[listener]) yield from self.make_test_data() coll = client.motor_test.test_collection with (yield from client.start_session()) as s: listener.results.clear() cursor = coll.find(session=s) yield from cursor.to_list(length=None) self.assertEqual(len(listener.results['started']), 2) for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "find sent no lsid with %s" % (event.command_name,)) self.assertEqual( s.session_id, event.command['lsid'], "find sent wrong lsid with %s" % (event.command_name,)) with self.assertRaisesRegex(InvalidOperation, "ended session"): yield from coll.find(session=s).to_list(length=None) # No explicit session. listener.results.clear() cursor = coll.find() yield from cursor.to_list(length=None) self.assertEqual(len(listener.results['started']), 2) event0 = listener.first_command_started() self.assertTrue( 'lsid' in event0.command, "find sent no lsid with %s" % (event0.command_name,)) lsid = event0.command['lsid'] for event in listener.results['started'][1:]: self.assertTrue( 'lsid' in event.command, "find sent no lsid with %s" % (event.command_name,)) self.assertEqual( lsid, event.command['lsid'], "find sent wrong lsid with %s" % (event.command_name,)) if __name__ == '__main__': unittest.main() motor-1.2.1/test/asyncio_tests/test_asyncio_son_manipulator.py000066400000000000000000000061501323007662700251010ustar00rootroot00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pymongo.son_manipulator from test.asyncio_tests import AsyncIOTestCase, asyncio_test from test.utils import ignore_deprecations class CustomSONManipulator(pymongo.son_manipulator.SONManipulator): """A pymongo outgoing SON Manipulator that adds ``{'added_field' : 42}`` """ def will_copy(self): return False def transform_outgoing(self, son, collection): assert 'added_field' not in son son['added_field'] = 42 return son class SONManipulatorTest(AsyncIOTestCase): def setUp(self): super(SONManipulatorTest, self).setUp() def tearDown(self): remove_coro = self.db.son_manipulator_test_collection.delete_many({}) self.loop.run_until_complete(remove_coro) super(SONManipulatorTest, self).tearDown() @ignore_deprecations @asyncio_test def test_with_find_one(self): coll = self.cx.motor_test.son_manipulator_test_collection result = yield from coll.insert_one({'foo': 'bar'}) expected = {'_id': result.inserted_id, 'foo': 'bar'} self.assertEqual(expected, (yield from coll.find_one())) # Add SONManipulator and test again. coll.database.add_son_manipulator(CustomSONManipulator()) expected = {'_id': result.inserted_id, 'foo': 'bar', 'added_field': 42} self.assertEqual(expected, (yield from coll.find_one())) @ignore_deprecations @asyncio_test def test_with_fetch_next(self): coll = self.cx.motor_test.son_manipulator_test_collection coll.database.add_son_manipulator(CustomSONManipulator()) result = yield from coll.insert_one({'foo': 'bar'}) cursor = coll.find() self.assertTrue((yield from cursor.fetch_next)) expected = {'_id': result.inserted_id, 'foo': 'bar', 'added_field': 42} self.assertEqual(expected, cursor.next_object()) @ignore_deprecations @asyncio_test def test_with_to_list(self): coll = self.cx.motor_test.son_manipulator_test_collection _id1, _id2 = (yield from coll.insert_many([{}, {}])).inserted_ids found = yield from coll.find().sort([('_id', 1)]).to_list(length=2) self.assertEqual([{'_id': _id1}, {'_id': _id2}], found) coll.database.add_son_manipulator(CustomSONManipulator()) expected = [ {'_id': _id1, 'added_field': 42}, {'_id': _id2, 'added_field': 42}] cursor = coll.find().sort([('_id', 1)]) found = yield from cursor.to_list(length=2) self.assertEqual(expected, found) yield from cursor.close() motor-1.2.1/test/asyncio_tests/test_asyncio_ssl.py000066400000000000000000000224711323007662700224740ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test AsyncIOMotorClient with SSL.""" import asyncio import gc import os import ssl import unittest from unittest import SkipTest from urllib.parse import quote_plus # The 'parse' submodule is Python 3. from pymongo.errors import (ConfigurationError, ConnectionFailure, OperationFailure) from motor.motor_asyncio import AsyncIOMotorClient import test from test.asyncio_tests import asyncio_test, remove_all_users from test.test_environment import (CA_PEM, CLIENT_PEM, env, MONGODB_X509_USERNAME) # Start a mongod instance like: # # mongod \ # --sslOnNormalPorts \ # --sslPEMKeyFile test/certificates/server.pem \ # --sslCAFile test/certificates/ca.pem # # Also, make sure you have 'server' as an alias for localhost in /etc/hosts class TestAsyncIOSSL(unittest.TestCase): def setUp(self): if not test.env.server_is_resolvable: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") asyncio.set_event_loop(None) self.loop = asyncio.new_event_loop() def tearDown(self): self.loop.stop() self.loop.run_forever() self.loop.close() gc.collect() def test_config_ssl(self): # This test doesn't require a running mongod. self.assertRaises(ValueError, AsyncIOMotorClient, io_loop=self.loop, ssl='foo') self.assertRaises(ConfigurationError, AsyncIOMotorClient, io_loop=self.loop, ssl=False, ssl_certfile=CLIENT_PEM) self.assertRaises(IOError, AsyncIOMotorClient, io_loop=self.loop, ssl_certfile="NoFile") self.assertRaises(TypeError, AsyncIOMotorClient, io_loop=self.loop, ssl_certfile=True) self.assertRaises(IOError, AsyncIOMotorClient, io_loop=self.loop, ssl_keyfile="NoFile") self.assertRaises(TypeError, AsyncIOMotorClient, io_loop=self.loop, ssl_keyfile=True) @asyncio_test def test_cert_ssl(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("Can't test with auth") client = AsyncIOMotorClient(env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from client.db.collection.find_one() response = yield from client.admin.command('ismaster') if 'setName' in response: client = AsyncIOMotorClient( env.host, env.port, ssl=True, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, replicaSet=response['setName'], io_loop=self.loop) yield from client.db.collection.find_one() @asyncio_test def test_cert_ssl_validation(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("Can't test with auth") client = AsyncIOMotorClient(env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from client.db.collection.find_one() response = yield from client.admin.command('ismaster') if 'setName' in response: client = AsyncIOMotorClient( env.host, env.port, replicaSet=response['setName'], ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from client.db.collection.find_one() @asyncio_test def test_cert_ssl_validation_none(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("Can't test with auth") client = AsyncIOMotorClient(test.env.fake_hostname_uri, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_NONE, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from client.admin.command('ismaster') @asyncio_test def test_cert_ssl_validation_hostname_fail(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("Can't test with auth") client = AsyncIOMotorClient(env.host, env.port, ssl=True, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.loop) response = yield from client.admin.command('ismaster') with self.assertRaises(ConnectionFailure): # Create client with hostname 'server', not 'localhost', # which is what the server cert presents. client = AsyncIOMotorClient(test.env.fake_hostname_uri, serverSelectionTimeoutMS=1000, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from client.db.collection.find_one() if 'setName' in response: with self.assertRaises(ConnectionFailure): client = AsyncIOMotorClient( test.env.fake_hostname_uri, serverSelectionTimeoutMS=1000, replicaSet=response['setName'], ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from client.db.collection.find_one() @asyncio_test @unittest.skipIf('EVERGREEN' in os.environ, "TODO: fix on Evergreen") def test_mongodb_x509_auth(self): # Expects the server to be running with SSL config described above, # and with "--auth". if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") # self.env.uri includes username and password. authenticated_client = AsyncIOMotorClient(test.env.uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.loop) if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield from authenticated_client['$external'].add_user( MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) # Not authenticated. client = AsyncIOMotorClient(env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.loop) collection = client.motor_test.test with self.assertRaises(OperationFailure): yield from collection.count() yield from client.admin.authenticate( MONGODB_X509_USERNAME, mechanism='MONGODB-X509') yield from collection.delete_many({}) uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), env.host, env.port)) # SSL options aren't supported in the URI.... auth_uri_client = AsyncIOMotorClient(uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.loop) yield from auth_uri_client.db.collection.find_one() # Cleanup. yield from remove_all_users(authenticated_client['$external']) yield from authenticated_client['$external'].logout() motor-1.2.1/test/asyncio_tests/test_asyncio_tests.py000066400000000000000000000142741323007662700230370ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Test Motor's asyncio test utilities.""" import asyncio import contextlib import io import os import unittest import concurrent.futures from test.asyncio_tests import AsyncIOTestCase, asyncio_test def run_test_case(case, suppress_output=True): suite = unittest.defaultTestLoader.loadTestsFromTestCase(case) if suppress_output: stream = io.StringIO() else: stream = None runner = unittest.TextTestRunner(stream=stream) return runner.run(suite) @contextlib.contextmanager def set_environ(name, value): old_value = os.environ.get(name) os.environ[name] = value try: yield finally: if old_value is None: del os.environ[name] else: os.environ[name] = old_value class TestAsyncIOTests(unittest.TestCase): def test_basic(self): class Test(AsyncIOTestCase): @asyncio_test def test(self): pass result = run_test_case(Test) self.assertEqual(1, result.testsRun) self.assertEqual(0, len(result.errors)) def test_decorator_with_no_args(self): class TestPasses(AsyncIOTestCase): @asyncio_test() def test_decorated_with_no_args(self): pass result = run_test_case(TestPasses) self.assertEqual(0, len(result.errors)) class TestFails(AsyncIOTestCase): @asyncio_test() def test_decorated_with_no_args(self): assert False result = run_test_case(TestFails) self.assertEqual(1, len(result.failures)) def test_timeout_passed_as_positional(self): with self.assertRaises(TypeError): class _(AsyncIOTestCase): # Should be "timeout=10". @asyncio_test(10) def test_decorated_with_no_args(self): pass def test_timeout(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.addCleanup(self.loop.close) self.addCleanup(setattr, self, 'loop', None) class Test(AsyncIOTestCase): @asyncio_test(timeout=0.01) def test_that_is_too_slow(self): yield from self.middle() @asyncio.coroutine def middle(self): yield from self.inner() @asyncio.coroutine def inner(self): yield from asyncio.sleep(1, loop=self.loop) with set_environ('ASYNC_TEST_TIMEOUT', '0'): result = run_test_case(Test) self.assertEqual(1, len(result.errors)) case, text = result.errors[0] self.assertTrue('TimeoutError' in text) def test_timeout_environment_variable(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.addCleanup(self.loop.close) self.addCleanup(setattr, self, 'loop', None) @asyncio_test def default_timeout(self): yield from asyncio.sleep(0.1, loop=self.loop) with set_environ('ASYNC_TEST_TIMEOUT', '0.2'): # No error, sleeps for 0.1 seconds and the timeout is 0.2 seconds. default_timeout(self) @asyncio_test(timeout=0.1) def custom_timeout(self): yield from asyncio.sleep(0.2, loop=self.loop) with set_environ('ASYNC_TEST_TIMEOUT', '0'): # No error, default timeout of 5 seconds overrides '0'. default_timeout(self) with set_environ('ASYNC_TEST_TIMEOUT', '0'): with self.assertRaises(concurrent.futures.TimeoutError): custom_timeout(self) with set_environ('ASYNC_TEST_TIMEOUT', '1'): # No error, 1-second timeout from environment overrides custom # timeout of 0.1 seconds. custom_timeout(self) def test_failure(self): class Test(AsyncIOTestCase): @asyncio_test def test_that_fails(self): yield from self.middle() @asyncio.coroutine def middle(self): yield from self.inner() @asyncio.coroutine def inner(self): assert False, 'expected error' result = run_test_case(Test) self.assertEqual(1, len(result.failures)) case, text = result.failures[0] self.assertFalse('CancelledError' in text) self.assertTrue('AssertionError' in text) self.assertTrue('expected error' in text) # The traceback shows where the coroutine raised. self.assertTrue('test_that_fails' in text) self.assertTrue('middle' in text) self.assertTrue('inner' in text) def test_undecorated(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.addCleanup(self.loop.close) self.addCleanup(setattr, self, 'loop', None) class Test(AsyncIOTestCase): def test_that_should_be_decorated(self): yield from asyncio.sleep(0.01, loop=self.loop) result = run_test_case(Test) self.assertEqual(1, len(result.errors)) case, text = result.errors[0] self.assertFalse('CancelledError' in text) self.assertTrue('TypeError' in text) self.assertTrue('should be decorated with @asyncio_test' in text) def test_other_return(self): class Test(AsyncIOTestCase): def test_other_return(self): return 42 result = run_test_case(Test) self.assertEqual(len(result.errors), 1) case, text = result.errors[0] self.assertIn('Return value from test method ignored', text) if __name__ == '__main__': unittest.main() motor-1.2.1/test/certificates/000077500000000000000000000000001323007662700163055ustar00rootroot00000000000000motor-1.2.1/test/certificates/ca.pem000066400000000000000000000055631323007662700174040ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDczCCAlugAwIBAgIBATANBgkqhkiG9w0BAQUFADB0MRcwFQYDVQQDEw5LZXJu ZWwgVGVzdCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RCMRYw FAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkGA1UE BhMCVVMwHhcNMTQwNzE3MTYwMDAwWhcNMjAwNzE3MTYwMDAwWjB0MRcwFQYDVQQD Ew5LZXJuZWwgVGVzdCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25n b0RCMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazEL MAkGA1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCBxSXj qA5y2EMQkcmvLDNikE88Og3+spJ3ex60HWVPk8EeXN68jyfbKLYsoCcBE2rBAE/N shVBJa8irh0o/UTh1XNW4iGCsfMvYamXiHnaOjmGVKjfBoj6pzQH0uK0X5olm3Sa zZPkLLCR81yxsK6woJZMFTvrlEjxj/SmDZ9tVXW692bC4i6nGvOCSpgv9kms85xO Ed2xbuCLXFDXKafXZd5AK+iegkDs3ah7VXMEE8sbqGnlqC1nsy5bpCnb7aC+3af7 SV2XEFlSQT5kwTmk9CvTDzM9O78SO8nNhEOFBLQEdGDGd3BShE8dCdh2JTy3zKsb WeE+mxy0mEwxNfGfAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF BQADggEBAANwbvhM5K/Jcl6yzUSqAawvyAypT5aWBob7rt9KFq/aemjMN0gY2nsS 8WTGd9jiXlxGc/TzrK6MOsJ904UAFE1L9uR//G1gIBa9dNbYoiii2Fc8b1xDVJEP b23rl/+GAT6UTSY+YgEjeA4Jk6H9zotO07lSw06rbCQam5SdA5UiMvuLHWCo3BHY 8WzqLiW/uHlb4K5prF9yuTUBEIgkRvvvyOKXlRvm1Ed5UopT2hmwA86mffAfgJc2 vSbm9/8Q00fYwO7mluB6mbEcnbquaqRLoB83k+WbwUAZ2yjWHXuXVMPwyaysazcp nOjaLwQJQgKejY62PiNcw7xC/nIxBeI= -----END CERTIFICATE----- -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAgcUl46gOcthDEJHJrywzYpBPPDoN/rKSd3setB1lT5PBHlze vI8n2yi2LKAnARNqwQBPzbIVQSWvIq4dKP1E4dVzVuIhgrHzL2Gpl4h52jo5hlSo 3waI+qc0B9LitF+aJZt0ms2T5CywkfNcsbCusKCWTBU765RI8Y/0pg2fbVV1uvdm wuIupxrzgkqYL/ZJrPOcThHdsW7gi1xQ1ymn12XeQCvonoJA7N2oe1VzBBPLG6hp 5agtZ7MuW6Qp2+2gvt2n+0ldlxBZUkE+ZME5pPQr0w8zPTu/EjvJzYRDhQS0BHRg xndwUoRPHQnYdiU8t8yrG1nhPpsctJhMMTXxnwIDAQABAoIBAD5iGOnM800wO2Uu wGbOd9FNEFoiinHDRHfdnw/1BavwmqjO+mBo7T8E3jarsrRosiwfyz1V+7O6uuuQ CgKXZlKuOuksgfGDPCWt7EolWHaZAOhbsGaujJD6ah/MuCD/yGmFxtNYOl05QpSX Cht9lSzhtf7TQl/og/xkOLbO27JB540ck/OCSOczXg9Z/O8AmIUyDn7AKb6G1Zhk 2IN//HQoAvDUMZLWrzy+L7YGbA8pBR3yiPsYBH0rX2Oc9INpiGA+B9Nf1HDDsxeZ /o+5xLbRDDfIDtlYO0cekJ053W0zUQLrMEIn9991EpG2O/fPgs10NlKJtaFH8CmT ExgVA9ECgYEA+6AjtUdxZ0BL3Wk773nmhesNH5/5unWFaGgWpMEaEM7Ou7i6QApL KAbzOYItV3NNCbkcrejq7jsDGEmiwUOdXeQx6XN7/Gb2Byc/wezy5ALi0kcUwaur 6s9+Ah+T4vcU2AjfuCWXIpe46KLEbwORmCRQGwkCBCwRhHGt5sGGxTkCgYEAhAaw voHI6Cb+4z3PNAKRnf2rExBYRyCz1KF16ksuwJyQSLzFleXRyRWFUEwLuVRL0+EZ JXhMbtrILrc23dJGEsB8kOCFehSH/IuL5eB0QfKpDFA+e6pimsbVeggx/rZhcERB WkcV3jN4O82gSL3EnIgvAT1/nwhmbmjvDhFJhZcCgYBaW4E3IbaZaz9S/O0m69Fa GbQWvS3CRV1oxqgK9cTUcE9Qnd9UC949O3GwHw0FMERjz3N7B/8FGW/dEuQ9Hniu NLmvqWbGlnqWywNcMihutJKbDCdp/Km5olUPkiNbB3sWsOkViXoiU/V0pK6BZvir d67EZpGwydpogyH9kVVCEQKBgGHXc3Q7SmCBRbOyQrQQk0m6i+V8328W1S5m2bPg M62aWXMOMn976ZRT1pBDSwz1Y5yJ3NDf7gTZLjEwpgCNrFCJRcc4HLL0NDL8V5js VjvpUU5GyYdsJdb+M4ZUPHi/QEaqzqPQumwJSLlJEdfWirZWVj9dDA8XcpGwQjjy psHRAoGBAJUTgeJYhjK7k5sgfh+PRqiRJP0msIH8FK7SenBGRUkelWrW6td2Riey EcOCMFkRWBeDgnZN5xDyWLBgrzpw9iHQQIUyyBaFknQcRUYKHkCx+k+fr0KHHCUb X2Kvf0rbeMucb4y/h7950HkBBq83AYKMAoI8Ql3cx7pKmyOLXRov -----END RSA PRIVATE KEY-----motor-1.2.1/test/certificates/client.pem000066400000000000000000000055331323007662700202740ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDXTCCAkWgAwIBAgIBAzANBgkqhkiG9w0BAQUFADB0MRcwFQYDVQQDEw5LZXJu ZWwgVGVzdCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RCMRYw FAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkGA1UE BhMCVVMwHhcNMTQwNzE3MTYwMDAwWhcNMjAwNzE3MTYwMDAwWjBwMQ8wDQYDVQQD EwZjbGllbnQxEzARBgNVBAsTCktlcm5lbFVzZXIxEDAOBgNVBAoTB01vbmdvREIx FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD VQQGEwJVUzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJIFboAk9Fdi DY5Xld2iw36vB3IpHEfgWIimd+l1HX4jyp35i6xoqkZZHJUL/NMbUFJ6+44EfFJ5 biB1y1Twr6GqpYp/3R30jKQU4PowO7DSal38MR34yiRFYPG4ZPPXXfwPSuwKrSNo bjqa0/DRJRVQlnGwzJkPsWxIgCjc8KNO/dSHv/CGymc9TjiFAI0VVOhMok1CBNvc ifwWjGBg5V1s3ItMw9x5qk+b9ff5hiOAGxPiCrr8R0C7RoeXg7ZG8K/TqXbsOZEG AOQPRGcrmqG3t4RNBJpZugarPWW6lr11zMpiPLFTrbq3ZNYB9akdsps4R43TKI4J AOtGMJmK430CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAA+nPgVT4addi13yB6mjW +UhdUkFwtb1Wcg0sLtnNucopHZLlCj5FfDdp1RQxe3CyMonxyHTKkrWtQmVtUyvf C/fjpIKt9A9kAmveMHBiu9FTNTc0sbiXcrEBeHF5cD7N+Uwfoc/4rJm0WjEGNkAd pYLCCLVZXPVr3bnc3ZLY1dFZPsJrdH3nJGMjLgUmoNsKnaGozcjiKiXqm6doFzkg 0Le5yD4C/QTaie2ycFa1X5bJfrgoMP7NqKko05h4l0B0+DnjpoTJN+zRreNTMKvE ETGvpUu0IYGxe8ZVAFnlEO/lUeMrPFvH+nDmJYsxO1Sjpds2hi1M1JoeyrTQPwXj 2Q== -----END CERTIFICATE----- -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAkgVugCT0V2INjleV3aLDfq8HcikcR+BYiKZ36XUdfiPKnfmL rGiqRlkclQv80xtQUnr7jgR8UnluIHXLVPCvoaqlin/dHfSMpBTg+jA7sNJqXfwx HfjKJEVg8bhk89dd/A9K7AqtI2huOprT8NElFVCWcbDMmQ+xbEiAKNzwo0791Ie/ 8IbKZz1OOIUAjRVU6EyiTUIE29yJ/BaMYGDlXWzci0zD3HmqT5v19/mGI4AbE+IK uvxHQLtGh5eDtkbwr9Opduw5kQYA5A9EZyuaobe3hE0Emlm6Bqs9ZbqWvXXMymI8 sVOturdk1gH1qR2ymzhHjdMojgkA60YwmYrjfQIDAQABAoIBAB249VEoNIRE9TVw JpVCuEBlKELYk2UeCWdnWykuKZ6vcmLNlNy3QVGoeeTs172w5ZykY+f4icXP6da5 o3XauCVUMvYKKNwcFzSe+1xxzPSlH/mZh/Xt2left6f8PLBVuk/AXSPG2I9Ihodv VIzERaQdD0J9FmhhhV/hMhUfQ+w5rTCaDpq1KVGU61ks+JAtlQ46g+cvPF9c80cI TEC875n2LqWKmLRN43JUnctV3uGTmolIqCRMHPAs/egl+lG2RXJjqXSQ2uFLOvC/ PXtBb597yadSs2BWPnTu/r7LbLGBAExzlQK1uFsTvuKsBPb3qrvUux0L68qwPuiv W24N8BECgYEAydtAvVB7OymQEX3mck2j7ixDN01wc1ZaCLBDvYPYS/Pvzq4MBiAD lHRtbIa6HPGA5jskbccPqQn8WGnJWCaYvCQryvgaA+BBgo1UTLfQJUo/7N5517vv KvbUa6NF0nj3VwfDV1vvy+amoWi9NOVn6qOh0K84PF4gwagb1EVy9MsCgYEAuTAt KCWdZ/aNcKgJc4NCUqBpLPF7EQypX14teixrbF/IRNS1YC9S20hpkG25HMBXjpBe tVg/MJe8R8CKzYjCt3z5Ff1bUQ2bzivbAtgjcaO0Groo8WWjnamQlrIQcvWM7vBf dnIflQ0slxbHfCi3XEe8tj2T69R7wJZ8L7PxR9cCgYEACgwNtt6Qo6s37obzt3DB 3hL57YC/Ph5oMNKFLKOpWm5z2zeyhYOGahc5cxNppBMpNUxwTb6AuwsyMjxhty+E nqi2PU4IDXVWDWd3cLIdfB2r/OA99Ez4ZI0QmaLw0L8QoJZUVL7QurdqR9JsyHs6 puUqIrb195s/yiPR7sjeJe0CgYEAuJviKEd3JxCN52RcJ58OGrh2oKsJ9/EbV0rX Ixfs7th9GMDDHuOOQbNqKOR4yMSlhCU/hKA4PgTFWPIEbOiM08XtuZIb2i0qyNjH N4qnqr166bny3tJnzOAgl1ljNHa8y+UsBTO3cCr17Jh0vL0KLSAGa9XvBAWKaG6b 1iIXwXkCgYAVz+DA1yy0qfXdS1pgPiCJGlGZXpbBcFnqvbpGSclKWyUG4obYCbrb p5VKVfoK7uU0ly60w9+PNIRsX/VN/6SVcoOzKx40qQBMuYfJ72DQrsPjPYvNg/Nb 4SK94Qhp9TlAyXbqKJ02DjtuDim44sGZ8g7b+k3FfoK4OtzNsqdVdQ== -----END RSA PRIVATE KEY-----motor-1.2.1/test/certificates/crl.pem000066400000000000000000000037661323007662700176040ustar00rootroot00000000000000Certificate Revocation List (CRL): Version 2 (0x1) Signature Algorithm: sha256WithRSAEncryption Issuer: /CN=Kernel Test CA/OU=Kernel/O=MongoDB/L=New York City/ST=New York/C=US Last Update: Aug 21 13:56:28 2014 GMT Next Update: Aug 18 13:56:28 2024 GMT CRL extensions: X509v3 CRL Number: 4096 No Revoked Certificates. Signature Algorithm: sha256WithRSAEncryption 48:1b:0b:b1:89:f5:6f:af:3c:dd:2a:a0:e5:55:04:80:16:b4: 23:98:39:bb:9f:16:c9:25:73:72:c6:a6:73:21:1d:1a:b6:99: fc:47:5e:bc:af:64:29:02:9c:a5:db:15:8a:65:48:3c:4f:a6: cd:35:47:aa:c6:c0:39:f5:a6:88:8f:1b:6c:26:61:4e:10:d7: e2:b0:20:3a:64:92:c1:d3:2a:11:3e:03:e2:50:fd:4e:3c:de: e2:e5:78:dc:8e:07:a5:69:55:13:2b:8f:ae:21:00:42:85:ff: b6:b1:2b:69:08:40:5a:25:8c:fe:57:7f:b1:06:b0:72:ff:61: de:21:59:05:a8:1b:9e:c7:8a:08:ab:f5:bc:51:b3:36:68:0f: 54:65:3c:8d:b7:80:d0:27:01:3e:43:97:89:19:89:0e:c5:01: 2c:55:9f:b6:e4:c8:0b:35:f8:52:45:d3:b4:09:ce:df:73:98: f5:4c:e4:5a:06:ac:63:4c:f8:4d:9c:af:88:fc:19:f7:77:ea: ee:56:18:49:16:ce:62:66:d1:1b:8d:66:33:b5:dc:b1:25:b3: 6c:81:e9:d0:8a:1d:83:61:49:0e:d9:94:6a:46:80:41:d6:b6: 59:a9:30:55:3d:5b:d3:5b:f1:37:ec:2b:76:d0:3a:ac:b2:c8: 7c:77:04:78 -----BEGIN X509 CRL----- MIIBzjCBtwIBATANBgkqhkiG9w0BAQsFADB0MRcwFQYDVQQDEw5LZXJuZWwgVGVz dCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RCMRYwFAYDVQQH Ew1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkGA1UEBhMCVVMX DTE0MDgyMTEzNTYyOFoXDTI0MDgxODEzNTYyOFqgDzANMAsGA1UdFAQEAgIQADAN BgkqhkiG9w0BAQsFAAOCAQEASBsLsYn1b6883Sqg5VUEgBa0I5g5u58WySVzcsam cyEdGraZ/EdevK9kKQKcpdsVimVIPE+mzTVHqsbAOfWmiI8bbCZhThDX4rAgOmSS wdMqET4D4lD9Tjze4uV43I4HpWlVEyuPriEAQoX/trEraQhAWiWM/ld/sQawcv9h 3iFZBagbnseKCKv1vFGzNmgPVGU8jbeA0CcBPkOXiRmJDsUBLFWftuTICzX4UkXT tAnO33OY9UzkWgasY0z4TZyviPwZ93fq7lYYSRbOYmbRG41mM7XcsSWzbIHp0Iod g2FJDtmUakaAQda2WakwVT1b01vxN+wrdtA6rLLIfHcEeA== -----END X509 CRL----- motor-1.2.1/test/certificates/server.pem000066400000000000000000000056071323007662700203260ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDfjCCAmagAwIBAgIBBzANBgkqhkiG9w0BAQUFADB0MRcwFQYDVQQDEw5LZXJu ZWwgVGVzdCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RCMRYw FAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkGA1UE BhMCVVMwHhcNMTQwNzE3MTYwMDAwWhcNMjAwNzE3MTYwMDAwWjBsMQ8wDQYDVQQD EwZzZXJ2ZXIxDzANBgNVBAsTBktlcm5lbDEQMA4GA1UEChMHTW9uZ29EQjEWMBQG A1UEBxMNTmV3IFlvcmsgQ2l0eTERMA8GA1UECBMITmV3IFlvcmsxCzAJBgNVBAYT AlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp76KJeDczBqjSPJj 5f8DHdtrWpQDK9AWNDlslWpi6+pL8hMqwbX0D7hC2r3kAgccMyFoNIudPqIXfXVd 1LOh6vyY+jveRvqjKW/UZVzZeiL4Gy4bhke6R8JRC3O5aMKIAbaiQUAI1Nd8LxIt LGvH+ia/DFza1whgB8ym/uzVQB6igOifJ1qHWJbTtIhDKaW8gvjOhv5R3jzjfLEb R9r5Q0ZyE0lrO27kTkqgBnHKPmu54GSzU/r0HM3B+Sc/6UN+xNhNbuR+LZ+EvJHm r4de8jhW8wivmjTIvte33jlLibQ5nYIHrlpDLEwlzvDGaIio+OfWcgs2WuPk98MU tht0IQIDAQABoyMwITAfBgNVHREEGDAWgglsb2NhbGhvc3SCCTEyNy4wLjAuMTAN BgkqhkiG9w0BAQUFAAOCAQEANoYxvVFsIol09BQA0fwryAye/Z4dYItvKhmwB9VS t99DsmJcyx0P5meB3Ed8SnwkD0NGCm5TkUY/YLacPP9uJ4SkbPkNZ1fRISyShCCn SGgQUJWHbCbcIEj+vssFb91c5RFJbvnenDkQokRvD2VJWspwioeLzuwtARUoMH3Y qg0k0Mn7Bx1bW1Y6xQJHeVlnZtzxfeueoFO55ZRkZ0ceAD/q7q1ohTXi0vMydYgu 1CB6VkDuibGlv56NdjbttPJm2iQoPaez8tZGpBo76N/Z1ydan0ow2pVjDXVOR84Y 2HSZgbHOGBiycNw2W3vfw7uK0OmiPRTFpJCmewDjYwZ/6w== -----END CERTIFICATE----- -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAp76KJeDczBqjSPJj5f8DHdtrWpQDK9AWNDlslWpi6+pL8hMq wbX0D7hC2r3kAgccMyFoNIudPqIXfXVd1LOh6vyY+jveRvqjKW/UZVzZeiL4Gy4b hke6R8JRC3O5aMKIAbaiQUAI1Nd8LxItLGvH+ia/DFza1whgB8ym/uzVQB6igOif J1qHWJbTtIhDKaW8gvjOhv5R3jzjfLEbR9r5Q0ZyE0lrO27kTkqgBnHKPmu54GSz U/r0HM3B+Sc/6UN+xNhNbuR+LZ+EvJHmr4de8jhW8wivmjTIvte33jlLibQ5nYIH rlpDLEwlzvDGaIio+OfWcgs2WuPk98MUtht0IQIDAQABAoIBACgi1ilECXCouwMc RDzm7Jb7Rk+Q9MVJ79YlG08Q+oRaNjvAzE03PSN5wj1WjDTUALJXPvi7oy82V4qE R6Q6Kvbv46aUJpYzKFEk2dw7ACpSLa1LNfjGNtMusnecA/QF/8bxLReRu8s5mBQn NDnZvCqllLbfjNlAvsF+/UIn5sqFZpAZPMtPwkTAeh5ge8H9JvrG8y8aXsiFGAhV Z7tMZyn8wPCUrRi14NLvVB4hxM66G/tuTp8r9AmeTU+PV+qbCnKXd+v0IS52hvX9 z75OPfAc66nm4bbPCapb6Yx7WaewPXXU0HDxeaT0BeQ/YfoNa5OT+ZOX1KndSfHa VhtmEsECgYEA3m86yYMsNOo+dkhqctNVRw2N+8gTO28GmWxNV9AC+fy1epW9+FNR yTQXpBkRrR7qrd5mF7WBc7vAIiSfVs021RMofzn5B1x7jzkH34VZtlviNdE3TZhx lPinqo0Yy3UEksgsCBJFIofuCmeTLk4ZtqoiZnXr35RYibaZoQdUT4kCgYEAwQ6Y xsKFYFks1+HYl29kR0qUkXFlVbKOhQIlj/dPm0JjZ0xYkUxmzoXD68HrOWgz7hc2 hZaQTgWf+8cRaZNfh7oL+Iglczc2UXuwuUYguYssD/G6/ZPY15PhItgCghaU5Ewy hMwIJ81NENY2EQTgk/Z1KZitXdVJfHl/IPMQgdkCgYASdqkqkPjaa5dDuj8byO8L NtTSUYlHJbAmjBbfcyTMG230/vkF4+SmDuznci1FcYuJYyyWSzqzoKISM3gGfIJQ rYZvCSDiu4qGGPXOWANaX8YnMXalukGzW/CO96dXPB9lD7iX8uxKMX5Q3sgYz+LS hszUNHWf2XB//ehCtZkKAQKBgQCxL2luepeZHx82H9T+38BkYgHLHw0HQzLkxlyd LjlE4QCEjSB4cmukvkZbuYXfEVEgAvQKVW6p/SWhGkpT4Gt8EXftKV9dyF21GVXQ JZnhUOcm1xBsrWYGLXYi2agrpvgONBTlprERfq5tdnz2z8giZL+RZswu45Nnh8bz AcKzuQKBgQCGOQvKvNL5XKKmws/KRkfJbXgsyRT2ubO6pVL9jGQG5wntkeIRaEpT oxFtWMdPx3b3cxtgSP2ojllEiISk87SFIN1zEhHZy/JpTF0GlU1qg3VIaA78M1p2 ZdpUsuqJzYmc3dDbQMepIaqdW4xMoTtZFyenUJyoezz6eWy/NlZ/XQ== -----END RSA PRIVATE KEY-----motor-1.2.1/test/test_environment.py000066400000000000000000000234551323007662700176260ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Discover environment and server configuration, initialize PyMongo client.""" import os import socket import sys import warnings from functools import wraps import pymongo.errors from test import SkipTest from test.version import Version HAVE_SSL = True try: import ssl except ImportError: HAVE_SSL = False ssl = None HAVE_TORNADO = True try: import tornado except ImportError: HAVE_TORNADO = False tornado = None HAVE_ASYNCIO = True try: import asyncio except ImportError: HAVE_ASYNCIO = False asyncio = None HAVE_AIOHTTP = True try: import aiohttp except ImportError: HAVE_AIOHTTP = False aiohttp = None # Copied from PyMongo. def partition_node(node): """Split a host:port string into (host, int(port)) pair.""" host = node port = 27017 idx = node.rfind(':') if idx != -1: host, port = node[:idx], int(node[idx + 1:]) if host.startswith('['): host = host[1:-1] return host, port def connected(client): """Convenience, wait for a new PyMongo MongoClient to connect.""" with warnings.catch_warnings(): # Ignore warning that "ismaster" is always routed to primary even # if client's read preference isn't PRIMARY. warnings.simplefilter("ignore", UserWarning) client.admin.command('ismaster') # Force connection. return client # If these are set to the empty string, substitute None. db_user = os.environ.get("DB_USER") or None db_password = os.environ.get("DB_PASSWORD") or None CERT_PATH = os.environ.get( 'CERT_DIR', os.path.join(os.path.dirname(os.path.realpath(__file__)), 'certificates')) CLIENT_PEM = os.path.join(CERT_PATH, 'client.pem') CA_PEM = os.path.join(CERT_PATH, 'ca.pem') MONGODB_X509_USERNAME = \ "CN=client,OU=kerneluser,O=10Gen,L=New York City,ST=New York,C=US" def is_server_resolvable(): """Returns True if 'server' is resolvable.""" socket_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(1) try: socket.gethostbyname('server') return True except socket.error: return False finally: socket.setdefaulttimeout(socket_timeout) class TestEnvironment(object): def __init__(self): self.initialized = False self.host = None self.port = None self.mongod_started_with_ssl = False self.mongod_validates_client_cert = False self.server_is_resolvable = is_server_resolvable() self.sync_cx = None self.is_standalone = False self.is_mongos = False self.is_replica_set = False self.rs_name = None self.w = 1 self.hosts = None self.arbiters = None self.primary = None self.secondaries = None self.v8 = False self.auth = False self.uri = None self.rs_uri = None self.version = None self.sessions_enabled = False self.fake_hostname_uri = None def setup(self): assert not self.initialized self.setup_sync_cx() self.setup_auth_and_uri() self.setup_version() self.setup_v8() self.initialized = True def setup_sync_cx(self): """Get a synchronous PyMongo MongoClient and determine SSL config.""" host = os.environ.get("DB_IP", "localhost") port = int(os.environ.get("DB_PORT", 27017)) connectTimeoutMS = 100 serverSelectionTimeoutMS = 100 socketTimeoutMS = 10000 try: client = connected(pymongo.MongoClient( host, port, username=db_user, password=db_password, connectTimeoutMS=connectTimeoutMS, socketTimeoutMS=socketTimeoutMS, serverSelectionTimeoutMS=serverSelectionTimeoutMS, ssl_ca_certs=CA_PEM, ssl=True)) self.mongod_started_with_ssl = True except pymongo.errors.ServerSelectionTimeoutError: try: client = connected(pymongo.MongoClient( host, port, username=db_user, password=db_password, connectTimeoutMS=connectTimeoutMS, socketTimeoutMS=socketTimeoutMS, serverSelectionTimeoutMS=serverSelectionTimeoutMS, ssl_ca_certs=CA_PEM, ssl_certfile=CLIENT_PEM)) self.mongod_started_with_ssl = True self.mongod_validates_client_cert = True except pymongo.errors.ServerSelectionTimeoutError: client = connected(pymongo.MongoClient( host, port, username=db_user, password=db_password, connectTimeoutMS=connectTimeoutMS, socketTimeoutMS=socketTimeoutMS, serverSelectionTimeoutMS=serverSelectionTimeoutMS)) response = client.admin.command('ismaster') self.sessions_enabled = 'logicalSessionTimeoutMinutes' in response self.is_mongos = response.get('msg') == 'isdbgrid' if 'setName' in response: self.is_replica_set = True self.rs_name = str(response['setName']) self.w = len(response['hosts']) self.hosts = set([partition_node(h) for h in response["hosts"]]) host, port = self.primary = partition_node(response['primary']) self.arbiters = set([ partition_node(h) for h in response.get("arbiters", [])]) self.secondaries = [ partition_node(m) for m in response['hosts'] if m != self.primary and m not in self.arbiters] elif not self.is_mongos: self.is_standalone = True # Reconnect to found primary, without short timeouts. if self.mongod_started_with_ssl: client = connected(pymongo.MongoClient(host, port, username=db_user, password=db_password, ssl_ca_certs=CA_PEM, ssl_certfile=CLIENT_PEM)) else: client = connected(pymongo.MongoClient(host, port, username=db_user, password=db_password, ssl=False)) self.sync_cx = client self.host = host self.port = port def setup_auth_and_uri(self): """Set self.auth and self.uri.""" if db_user or db_password: if not (db_user and db_password): sys.stderr.write( "You msut set both DB_USER and DB_PASSWORD, or neither\n") sys.exit(1) self.auth = True uri_template = 'mongodb://%s:%s@%s:%s/admin' self.uri = uri_template % (db_user, db_password, self.host, self.port) # If the hostname 'server' is resolvable, this URI lets us use it # to test SSL hostname validation with auth. self.fake_hostname_uri = uri_template % ( db_user, db_password, 'server', self.port) else: self.uri = 'mongodb://%s:%s/admin' % ( self.host, self.port) self.fake_hostname_uri = 'mongodb://%s:%s/admin' % ( 'server', self.port) if self.rs_name: self.rs_uri = self.uri + '?replicaSet=' + self.rs_name def setup_version(self): """Set self.version to the server's version.""" self.version = Version.from_client(self.sync_cx) def setup_v8(self): """Determine if server is running SpiderMonkey or V8.""" if self.sync_cx.server_info().get('javascriptEngine') == 'V8': self.v8 = True def _require(self, condition, msg, func=None): def make_wrapper(f): @wraps(f) def wrap(*args, **kwargs): if condition(): return f(*args, **kwargs) raise SkipTest(msg) return wrap if func is None: def decorate(f): return make_wrapper(f) return decorate return make_wrapper(func) def require_version_min(self, *ver): """Run a test only if the server version is at least ``version``.""" other_version = Version(*ver) return self._require(lambda: self.version >= other_version, "Server version must be at least %s" % str(other_version)) def require_version_max(self, *ver): """Run a test only if the server version is at most ``version``.""" other_version = Version(*ver) return self._require(lambda: self.version <= other_version, "Server version must be at most %s" % str(other_version)) def require_replica_set(self, func): """Run a test only if the client is connected to a replica set.""" return self._require(lambda: self.is_replica_set, "Not connected to a replica set", func=func) env = TestEnvironment() motor-1.2.1/test/tornado_tests/000077500000000000000000000000001323007662700165305ustar00rootroot00000000000000motor-1.2.1/test/tornado_tests/__init__.py000066400000000000000000000107401323007662700206430ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Utilities for testing Motor with Tornado.""" import concurrent.futures from unittest import SkipTest from mockupdb import MockupDB from tornado import gen, testing import motor from test.test_environment import env, CA_PEM, CLIENT_PEM @gen.coroutine def get_command_line(client): command_line = yield client.admin.command('getCmdLineOpts') assert command_line['ok'] == 1, "getCmdLineOpts() failed" raise gen.Return(command_line) @gen.coroutine def server_is_mongos(client): ismaster_response = yield client.admin.command('ismaster') raise gen.Return(ismaster_response.get('msg') == 'isdbgrid') @gen.coroutine def skip_if_mongos(client): is_mongos = yield server_is_mongos(client) if is_mongos: raise SkipTest("connected to mongos") @gen.coroutine def remove_all_users(db): yield db.command({"dropAllUsersFromDatabase": 1}) @gen.coroutine def skip_if_mongos(client): is_mongos = yield server_is_mongos(client) if is_mongos: raise SkipTest("connected to mongos") @gen.coroutine def remove_all_users(db): yield db.command({"dropAllUsersFromDatabase": 1}) class MotorTest(testing.AsyncTestCase): longMessage = True # Used by unittest.TestCase ssl = False # If True, connect with SSL, skip if mongod isn't SSL def setUp(self): super(MotorTest, self).setUp() if self.ssl and not env.mongod_started_with_ssl: raise SkipTest("mongod doesn't support SSL, or is down") self.cx = self.motor_client() self.db = self.cx.motor_test self.collection = self.db.test_collection @gen.coroutine def make_test_data(self): yield self.collection.delete_many({}) yield self.collection.insert_many([{'_id': i} for i in range(200)]) make_test_data.__test__ = False def get_client_kwargs(self, **kwargs): if env.mongod_started_with_ssl: kwargs.setdefault('ssl_certfile', CLIENT_PEM) kwargs.setdefault('ssl_ca_certs', CA_PEM) kwargs.setdefault('ssl', env.mongod_started_with_ssl) kwargs.setdefault('io_loop', self.io_loop) return kwargs def motor_client(self, uri=None, *args, **kwargs): """Get a MotorClient. Ignores self.ssl, you must pass 'ssl' argument. You'll probably need to close the client to avoid file-descriptor problems after AsyncTestCase calls self.io_loop.close(all_fds=True). """ return motor.MotorClient( uri or env.uri, *args, **self.get_client_kwargs(**kwargs)) def motor_rsc(self, uri=None, *args, **kwargs): """Get an open MotorClient for replica set. Ignores self.ssl, you must pass 'ssl' argument. """ return motor.MotorClient( uri or env.rs_uri, *args, **self.get_client_kwargs(**kwargs)) def tearDown(self): env.sync_cx.motor_test.test_collection.delete_many({}) self.cx.close() super(MotorTest, self).tearDown() class MotorReplicaSetTestBase(MotorTest): def setUp(self): super(MotorReplicaSetTestBase, self).setUp() if not env.is_replica_set: raise SkipTest("Not connected to a replica set") self.rsc = self.motor_rsc() class MotorMockServerTest(MotorTest): executor = concurrent.futures.ThreadPoolExecutor(1) def server(self, *args, **kwargs): server = MockupDB(*args, **kwargs) server.run() self.addCleanup(server.stop) return server def client_server(self, *args, **kwargs): server = self.server(*args, **kwargs) client = motor.motor_tornado.MotorClient(server.uri, io_loop=self.io_loop) self.addCleanup(client.close) return client, server def run_thread(self, fn, *args, **kwargs): return self.executor.submit(fn, *args, **kwargs) motor-1.2.1/test/tornado_tests/test_motor_await.py000066400000000000000000000112271323007662700224710ustar00rootroot00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import import warnings """Test Motor, an asynchronous driver for MongoDB and Tornado.""" from tornado.testing import gen_test from motor import MotorGridFS import test from test import SkipTest from test.tornado_tests import MotorTest class MotorTestAwait(MotorTest): @gen_test async def test_to_list(self): collection = self.collection await collection.delete_many({}) results = await collection.find().sort('_id').to_list(length=None) self.assertEqual([], results) docs = [{'_id': 1}, {'_id': 2}] await collection.insert_many(docs) cursor = collection.find().sort('_id') results = await cursor.to_list(length=None) self.assertEqual(docs, results) results = await cursor.to_list(length=None) self.assertEqual([], results) @gen_test async def test_iter_cursor(self): collection = self.collection await collection.delete_many({}) for n_docs in 0, 1, 2, 10: if n_docs: docs = [{'_id': i} for i in range(n_docs)] await collection.insert_many(docs) # Force extra batches to test iteration. j = 0 async for doc in collection.find().sort('_id').batch_size(3): self.assertEqual(j, doc['_id']) j += 1 self.assertEqual(j, n_docs) await collection.delete_many({}) @gen_test async def test_iter_aggregate(self): collection = self.collection await collection.delete_many({}) pipeline = [{'$sort': {'_id': 1}}] # Empty iterator. async for _ in collection.aggregate(pipeline): self.fail() for n_docs in 1, 2, 10: if n_docs: docs = [{'_id': i} for i in range(n_docs)] await collection.insert_many(docs) # Force extra batches to test iteration. j = 0 async for doc in collection.aggregate(pipeline, cursor={'batchSize': 3}): self.assertEqual(j, doc['_id']) j += 1 self.assertEqual(j, n_docs) await collection.delete_many({}) @gen_test async def test_iter_gridfs(self): gfs = MotorGridFS(self.db) async def cleanup(): await self.db.fs.files.delete_many({}) await self.db.fs.chunks.delete_many({}) await cleanup() # Empty iterator. async for _ in gfs.find({'_id': 1}): self.fail() data = b'data' for n_files in 1, 2, 10: for i in range(n_files): await gfs.put(data, filename='filename') # Force extra batches to test iteration. j = 0 async for _ in gfs.find({'filename': 'filename'}).batch_size(3): j += 1 self.assertEqual(j, n_files) await cleanup() async with await gfs.new_file(_id=1, chunk_size=1) as f: await f.write(data) gout = await gfs.find_one({'_id': 1}) chunks = [] async for chunk in gout: chunks.append(chunk) self.assertEqual(len(chunks), len(data)) self.assertEqual(b''.join(chunks), data) @gen_test async def test_stream_to_handler(self): fs = MotorGridFS(self.db) content_length = 1000 await fs.delete(1) self.assertEqual(1, await fs.put(b'a' * content_length, _id=1)) gridout = await fs.get(1) handler = test.MockRequestHandler() await gridout.stream_to_handler(handler) self.assertEqual(content_length, handler.n_written) await fs.delete(1) @gen_test async def test_cursor_iter(self): # Have we handled the async iterator change in Python 3.5.2?: # python.org/dev/peps/pep-0492/#api-design-and-implementation-revisions with warnings.catch_warnings(record=True) as w: async for _ in self.collection.find(): pass if w: self.fail(w[0].message) motor-1.2.1/test/tornado_tests/test_motor_basic.py000066400000000000000000000110401323007662700224360ustar00rootroot00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals, absolute_import """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import pymongo from pymongo import WriteConcern from pymongo.errors import ConfigurationError from pymongo.read_preferences import ReadPreference, Secondary, Nearest from tornado.testing import gen_test import motor import test from test import SkipTest from test.tornado_tests import MotorTest from test.utils import ignore_deprecations class MotorTestBasic(MotorTest): def test_repr(self): self.assertTrue(repr(self.cx).startswith('MotorClient')) self.assertTrue(repr(self.db).startswith('MotorDatabase')) self.assertTrue(repr(self.collection).startswith('MotorCollection')) cursor = self.collection.find() self.assertTrue(repr(cursor).startswith('MotorCursor')) @gen_test def test_write_concern(self): # Default empty dict means "w=1" self.assertEqual(WriteConcern(), self.cx.write_concern) yield self.collection.delete_many({}) yield self.collection.insert_one({'_id': 0}) for gle_options in [ {}, {'w': 0}, {'w': 1}, {'wtimeout': 1000}, ]: cx = self.motor_client(test.env.uri, **gle_options) wc = WriteConcern(**gle_options) self.assertEqual(wc, cx.write_concern) db = cx.motor_test self.assertEqual(wc, db.write_concern) collection = db.test_collection self.assertEqual(wc, collection.write_concern) if wc.acknowledged: with self.assertRaises(pymongo.errors.DuplicateKeyError): yield collection.insert_one({'_id': 0}) else: yield collection.insert_one({'_id': 0}) # No error # No error c = collection.with_options(write_concern=WriteConcern(w=0)) yield c.insert_one({'_id': 0}) cx.close() @ignore_deprecations @gen_test def test_read_preference(self): # Check the default cx = motor.MotorClient(test.env.uri, io_loop=self.io_loop) self.assertEqual(ReadPreference.PRIMARY, cx.read_preference) # We can set mode, tags, and latency. cx = self.motor_client( read_preference=Secondary(tag_sets=[{'foo': 'bar'}]), localThresholdMS=42) self.assertEqual(ReadPreference.SECONDARY.mode, cx.read_preference.mode) self.assertEqual([{'foo': 'bar'}], cx.read_preference.tag_sets) self.assertEqual(42, cx.local_threshold_ms) # Make a MotorCursor and get its PyMongo Cursor collection = cx.motor_test.test_collection.with_options( read_preference=Nearest(tag_sets=[{'yay': 'jesse'}])) motor_cursor = collection.find() cursor = motor_cursor.delegate self.assertEqual(Nearest(tag_sets=[{'yay': 'jesse'}]), cursor._Cursor__read_preference) cx.close() def test_underscore(self): self.assertIsInstance(self.cx['_db'], motor.MotorDatabase) self.assertIsInstance(self.db['_collection'], motor.MotorCollection) self.assertIsInstance(self.collection['_collection'], motor.MotorCollection) with self.assertRaises(AttributeError): self.cx._db with self.assertRaises(AttributeError): self.db._collection with self.assertRaises(AttributeError): self.collection._collection def test_abc(self): try: from abc import ABC except ImportError: # Python < 3.4. raise SkipTest() class C(ABC): db = self.db collection = self.collection subcollection = self.collection.subcollection # MOTOR-104, TypeError: Can't instantiate abstract class C with abstract # methods collection, db, subcollection. C() motor-1.2.1/test/tornado_tests/test_motor_bulk.py000066400000000000000000000067261323007662700223310ustar00rootroot00000000000000# Copyright 2014 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor's bulk API.""" import unittest from pymongo.errors import BulkWriteError from tornado.testing import gen_test import motor import motor.motor_tornado from test.tornado_tests import MotorTest class MotorBulkTest(MotorTest): # This is just a smattering of tests, since the logic is all in PyMongo. @gen_test(timeout=30) def test_multiple_error_ordered_batch(self): yield self.collection.delete_many({}) yield self.collection.create_index('a', unique=True) try: bulk = self.collection.initialize_ordered_bulk_op() self.assertTrue(isinstance( bulk, motor.motor_tornado.MotorBulkOperationBuilder)) bulk.insert({'b': 1, 'a': 1}) bulk.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) bulk.find({'b': 3}).upsert().update_one({'$set': {'a': 2}}) bulk.find({'b': 2}).upsert().update_one({'$set': {'a': 1}}) bulk.insert({'b': 4, 'a': 3}) bulk.insert({'b': 5, 'a': 1}) try: yield bulk.execute() except BulkWriteError as exc: result = exc.details self.assertEqual(exc.code, 65) else: self.fail("Error not raised") self.assertEqual(1, result['nInserted']) self.assertEqual(1, len(result['writeErrors'])) error = result['writeErrors'][0] self.assertEqual(1, error['index']) failed = error['op'] self.assertEqual(2, failed['q']['b']) self.assertEqual(1, failed['u']['$set']['a']) self.assertFalse(failed['multi']) self.assertTrue(failed['upsert']) cursor = self.collection.find({}, {'_id': False}) docs = yield cursor.to_list(None) self.assertEqual([{'a': 1, 'b': 1}], docs) finally: yield self.collection.drop() @gen_test def test_single_unordered_batch(self): yield self.collection.delete_many({}) bulk = self.collection.initialize_unordered_bulk_op() self.assertTrue(isinstance( bulk, motor.motor_tornado.MotorBulkOperationBuilder)) bulk.insert({'a': 1}) bulk.find({'a': 1}).update_one({'$set': {'b': 1}}) bulk.find({'a': 2}).upsert().update_one({'$set': {'b': 2}}) bulk.insert({'a': 3}) bulk.find({'a': 3}).remove() result = yield bulk.execute() self.assertEqual(0, len(result['writeErrors'])) upserts = result['upserted'] self.assertEqual(1, len(upserts)) self.assertEqual(2, upserts[0]['index']) self.assertTrue(upserts[0].get('_id')) a_values = yield self.collection.distinct('a') self.assertEqual( set([1, 2]), set(a_values)) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_change_stream.py000066400000000000000000000110421323007662700241570ustar00rootroot00000000000000# Copyright 2017-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import, unicode_literals """Test MotorChangeStream.""" import os import threading import time from pymongo.errors import InvalidOperation, OperationFailure from tornado.testing import gen_test from test import SkipTest, env from test.tornado_tests import MotorTest class MotorChangeStreamTest(MotorTest): @classmethod @env.require_version_min(3, 6) def setUpClass(cls): super(MotorChangeStreamTest, cls).setUpClass() if env.is_standalone: raise SkipTest("Standalone") # Ensure the collection exists. env.sync_cx.motor_test.test_collection.delete_many({}) env.sync_cx.motor_test.test_collection.insert_one({'_id': 1}) def wait_and_insert(self, change_stream, n=1): # The start time of the change stream is nondeterministic. Wait # to ensure this insert comes after the change stream starts. def target(): start = time.time() timeout = float(os.environ.get('ASYNC_TEST_TIMEOUT', 5)) while not change_stream.delegate: if time.time() - start > timeout: print("MotorChangeStream never created ChangeStream") return time.sleep(0.1) self.io_loop.add_callback(self.collection.insert_many, [{} for _ in range(n)]) t = threading.Thread(target=target) t.daemon = True t.start() @gen_test async def test_async_for(self): change_stream = self.collection.watch() self.wait_and_insert(change_stream, 2) i = 0 async for _ in change_stream: i += 1 if i == 2: break self.assertEqual(i, 2) @gen_test async def test_watch(self): coll = self.collection with self.assertRaises(TypeError): # pipeline must be a list. async for _ in coll.watch(pipeline={}): pass change_stream = coll.watch() self.wait_and_insert(change_stream, 1) change = await change_stream.next() # New change stream with resume token. await coll.insert_one({'_id': 23}) change = await coll.watch(resume_after=change['_id']).next() self.assertEqual(change['fullDocument'], {'_id': 23}) @gen_test async def test_close(self): coll = self.collection change_stream = coll.watch() future = change_stream.next() self.wait_and_insert(change_stream, 1) await future await change_stream.close() with self.assertRaises(StopAsyncIteration): await change_stream.next() async for _ in change_stream: pass @gen_test async def test_missing_id(self): coll = self.collection change_stream = coll.watch([{'$project': {'_id': 0}}]) future = change_stream.next() self.wait_and_insert(change_stream) with self.assertRaises(InvalidOperation): await future # The cursor should now be closed. with self.assertRaises(StopAsyncIteration): await change_stream.next() @gen_test async def test_unknown_full_document(self): coll = self.collection change_stream = coll.watch(full_document="unknownFullDocOption") future = change_stream.next() self.wait_and_insert(change_stream, 1) with self.assertRaises(OperationFailure): await future @gen_test async def test_async_with(self): async with self.collection.watch() as change_stream: self.wait_and_insert(change_stream, 1) async for _ in change_stream: self.assertTrue(change_stream.delegate._cursor.alive) break self.assertFalse(change_stream.delegate._cursor.alive) @gen_test async def test_with_statement(self): with self.assertRaises(RuntimeError): with self.collection.watch(): pass motor-1.2.1/test/tornado_tests/test_motor_client.py000066400000000000000000000275041323007662700226470ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import os import unittest import pymongo from pymongo import CursorType import pymongo.mongo_client from bson import CodecOptions from mockupdb import OpQuery from pymongo import ReadPreference, WriteConcern from pymongo.errors import ConfigurationError, OperationFailure from pymongo.errors import ConnectionFailure from tornado import gen from tornado.concurrent import Future from tornado.testing import gen_test import motor import test from test import SkipTest from test.test_environment import db_user, db_password, env from test.tornado_tests import (MotorMockServerTest, MotorTest, remove_all_users) from test.utils import one, get_primary_pool class MotorClientTest(MotorTest): @gen_test def test_client_lazy_connect(self): yield self.db.test_client_lazy_connect.delete_many({}) # Create client without connecting; connect on demand. cx = self.motor_client() collection = cx.motor_test.test_client_lazy_connect future0 = collection.insert_one({'foo': 'bar'}) future1 = collection.insert_one({'foo': 'bar'}) yield [future0, future1] self.assertEqual(2, (yield collection.find({'foo': 'bar'}).count())) cx.close() @gen_test def test_unix_socket(self): if env.mongod_started_with_ssl: raise SkipTest("Server started with SSL") mongodb_socket = '/tmp/mongodb-%d.sock' % env.port if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessible") encoded_socket = '%2Ftmp%2Fmongodb-' + str(env.port) + '.sock' uri = 'mongodb://%s' % encoded_socket client = self.motor_client(uri) if test.env.auth: yield client.admin.authenticate(db_user, db_password) yield client.motor_test.test.insert_one({"dummy": "object"}) # Confirm it fails with a missing socket. client = motor.MotorClient( "mongodb://%2Ftmp%2Fnon-existent.sock", io_loop=self.io_loop, serverSelectionTimeoutMS=100) with self.assertRaises(ConnectionFailure): yield client.admin.command('ismaster') def test_io_loop(self): with self.assertRaises(TypeError): motor.MotorClient(test.env.uri, io_loop='foo') def test_database_named_delegate(self): self.assertTrue( isinstance(self.cx.delegate, pymongo.mongo_client.MongoClient)) self.assertTrue(isinstance(self.cx['delegate'], motor.MotorDatabase)) @gen_test def test_connection_failure(self): # Assuming there isn't anything actually running on this port client = motor.MotorClient('localhost', 8765, io_loop=self.io_loop, serverSelectionTimeoutMS=10) # Test the Future interface. with self.assertRaises(ConnectionFailure): yield client.admin.command('ismaster') # Test with a callback. (result, error), _ = yield gen.Task(client.admin.command, 'ismaster') self.assertEqual(None, result) self.assertTrue(isinstance(error, ConnectionFailure)) @gen_test(timeout=30) def test_connection_timeout(self): # Motor merely tries to time out a connection attempt within the # specified duration; DNS lookup in particular isn't charged against # the timeout. So don't measure how long this takes. client = motor.MotorClient( 'example.com', port=12345, serverSelectionTimeoutMS=1, io_loop=self.io_loop) with self.assertRaises(ConnectionFailure): yield client.admin.command('ismaster') @gen_test def test_max_pool_size_validation(self): with self.assertRaises(ValueError): motor.MotorClient(maxPoolSize=-1) with self.assertRaises(ValueError): motor.MotorClient(maxPoolSize='foo') cx = self.motor_client(maxPoolSize=100) self.assertEqual(cx.max_pool_size, 100) cx.close() @gen_test(timeout=60) def test_high_concurrency(self): yield self.make_test_data() concurrency = 25 cx = self.motor_client(maxPoolSize=concurrency) expected_finds = 200 * concurrency n_inserts = 25 collection = cx.motor_test.test_collection insert_collection = cx.motor_test.insert_collection yield insert_collection.delete_many({}) ndocs = [0] insert_future = Future() @gen.coroutine def find(): cursor = collection.find() while (yield cursor.fetch_next): cursor.next_object() ndocs[0] += 1 # Half-way through, start an insert loop if ndocs[0] == expected_finds / 2: insert() @gen.coroutine def insert(): for i in range(n_inserts): yield insert_collection.insert_one({'s': hex(i)}) insert_future.set_result(None) # Finished yield [find() for _ in range(concurrency)] yield insert_future self.assertEqual(expected_finds, ndocs[0]) self.assertEqual(n_inserts, (yield insert_collection.count())) yield collection.delete_many({}) @gen_test(timeout=30) def test_drop_database(self): # Make sure we can pass a MotorDatabase instance to drop_database db = self.cx.test_drop_database yield db.test_collection.insert_one({}) names = yield self.cx.database_names() self.assertTrue('test_drop_database' in names) yield self.cx.drop_database(db) names = yield self.cx.database_names() self.assertFalse('test_drop_database' in names) @gen_test def test_auth_from_uri(self): if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # self.db is logged in as root. yield remove_all_users(self.db) db = self.db try: yield db.add_user( 'mike', 'password', roles=['userAdmin', 'readWrite']) client = self.motor_client( 'mongodb://u:pass@%s:%d' % (env.host, env.port)) with self.assertRaises(OperationFailure): yield client.db.collection.find_one() client = self.motor_client( 'mongodb://mike:password@%s:%d/%s' % (env.host, env.port, db.name)) yield client[db.name].collection.find_one() finally: yield db.remove_user('mike') def test_get_database(self): codec_options = CodecOptions(tz_aware=True) write_concern = WriteConcern(w=2, j=True) db = self.cx.get_database( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) self.assertTrue(isinstance(db, motor.MotorDatabase)) self.assertEqual('foo', db.name) self.assertEqual(codec_options, db.codec_options) self.assertEqual(ReadPreference.SECONDARY, db.read_preference) self.assertEqual(write_concern, db.write_concern) @gen_test def test_list_databases(self): yield self.collection.insert_one({}) cursor = yield self.cx.list_databases() self.assertIsInstance(cursor, motor.motor_tornado.MotorCommandCursor) # Make sure the cursor works, by searching for "local" database. while (yield cursor.fetch_next): info = cursor.next_object() if info['name'] == self.collection.database.name: break else: self.fail("'%s' database not found" % self.collection.database.name) @gen_test def test_list_database_names(self): yield self.collection.insert_one({}) names = yield self.cx.list_database_names() self.assertIsInstance(names, list) self.assertIn(self.collection.database.name, names) class MotorClientTimeoutTest(MotorMockServerTest): @gen_test def test_timeout(self): server = self.server(auto_ismaster=True) client = motor.MotorClient(server.uri, socketTimeoutMS=100) with self.assertRaises(pymongo.errors.AutoReconnect) as context: yield client.motor_test.test_collection.find_one() self.assertIn('timed out', str(context.exception)) client.close() class MotorClientExhaustCursorTest(MotorMockServerTest): def primary_server(self): primary = self.server() hosts = [primary.address_string] primary.autoresponds( 'ismaster', ismaster=True, setName='rs', hosts=hosts, maxWireVersion=6) return primary def primary_or_standalone(self, rs): if rs: return self.primary_server() else: return self.server(auto_ismaster=True) @gen.coroutine def _test_exhaust_query_server_error(self, rs): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid counter leak. server = self.primary_or_standalone(rs=rs) client = motor.MotorClient(server.uri, maxPoolSize=1) yield client.admin.command('ismaster') pool = get_primary_pool(client) sock_info = one(pool.sockets) cursor = client.db.collection.find(cursor_type=CursorType.EXHAUST) # With Tornado, simply accessing fetch_next starts the fetch. fetch_next = cursor.fetch_next request = yield self.run_thread(server.receives, OpQuery) request.fail() with self.assertRaises(pymongo.errors.OperationFailure): yield fetch_next self.assertFalse(sock_info.closed) self.assertEqual(sock_info, one(pool.sockets)) @gen_test def test_exhaust_query_server_error_standalone(self): yield self._test_exhaust_query_server_error(rs=False) @gen_test def test_exhaust_query_server_error_rs(self): yield self._test_exhaust_query_server_error(rs=True) @gen.coroutine def _test_exhaust_query_network_error(self, rs): # When doing an exhaust query, the socket stays checked out on success # but must be checked in on error to avoid counter leak. server = self.primary_or_standalone(rs=rs) client = motor.MotorClient(server.uri, maxPoolSize=1) yield client.admin.command('ismaster') pool = get_primary_pool(client) pool._check_interval_seconds = None # Never check. sock_info = one(pool.sockets) cursor = client.db.collection.find(cursor_type=CursorType.EXHAUST) # With Tornado, simply accessing fetch_next starts the fetch. fetch_next = cursor.fetch_next request = yield self.run_thread(server.receives, OpQuery) request.hangs_up() with self.assertRaises(pymongo.errors.ConnectionFailure): yield fetch_next self.assertTrue(sock_info.closed) del cursor self.assertNotIn(sock_info, pool.sockets) @gen_test def test_exhaust_query_network_error_standalone(self): yield self._test_exhaust_query_network_error(rs=False) @gen_test def test_exhaust_query_network_error_rs(self): yield self._test_exhaust_query_network_error(rs=True) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_collection.py000066400000000000000000000367571323007662700235360ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import sys import traceback import unittest import bson from bson import CodecOptions from bson.binary import JAVA_LEGACY from bson.objectid import ObjectId from pymongo import ReadPreference, WriteConcern from pymongo.read_preferences import Secondary from pymongo.errors import DuplicateKeyError, OperationFailure from tornado import gen from tornado.concurrent import Future from tornado.testing import gen_test import motor import motor.motor_tornado import test from test import SkipTest from test.tornado_tests import MotorTest, skip_if_mongos from test.utils import ignore_deprecations class MotorCollectionTest(MotorTest): @gen_test def test_collection(self): # Test that we can create a collection directly, not just from # MotorClient's accessors collection = motor.MotorCollection(self.db, 'test_collection') # Make sure we got the right collection and it can do an operation self.assertEqual('test_collection', collection.name) yield collection.insert_one({'_id': 1}) doc = yield collection.find_one({'_id': 1}) self.assertEqual(1, doc['_id']) # If you pass kwargs to PyMongo's Collection(), it calls # db.create_collection(). Motor can't do I/O in a constructor # so this is prohibited. self.assertRaises( TypeError, motor.MotorCollection, self.db, 'test_collection', capped=True) @gen_test def test_dotted_collection_name(self): # Ensure that remove, insert, and find work on collections with dots # in their names. for coll in ( self.db.foo.bar, self.db.foo.bar.baz): yield coll.delete_many({}) result = yield coll.insert_one({'_id': 'xyzzy'}) self.assertEqual('xyzzy', result.inserted_id) result = yield coll.find_one({'_id': 'xyzzy'}) self.assertEqual(result['_id'], 'xyzzy') yield coll.delete_many({}) self.assertEqual(None, (yield coll.find_one({'_id': 'xyzzy'}))) def test_call(self): # Prevents user error with nice message. try: self.db.foo() except TypeError as e: self.assertTrue('no such method exists' in str(e)) else: self.fail('Expected TypeError') @ignore_deprecations @gen_test def test_update(self): yield self.collection.insert_one({'_id': 1}) result = yield self.collection.update( {'_id': 1}, {'$set': {'foo': 'bar'}}) self.assertEqual(1, result['ok']) self.assertEqual(True, result['updatedExisting']) self.assertEqual(1, result['n']) self.assertEqual(None, result.get('err')) @ignore_deprecations @gen_test def test_update_bad(self): # Violate a unique index, make sure we handle error well coll = self.db.unique_collection yield coll.create_index('s', unique=True) try: yield coll.insert_many([{'s': 1}, {'s': 2}]) with self.assertRaises(DuplicateKeyError): yield coll.update({'s': 2}, {'$set': {'s': 1}}) finally: yield coll.drop() @gen_test def test_insert_one(self): collection = self.collection result = yield collection.insert_one({'_id': 201}) self.assertEqual(201, result.inserted_id) @ignore_deprecations @gen_test def test_insert_many_one_bad(self): collection = self.collection yield collection.insert_one({'_id': 2}) # Violate a unique index in one of many updates, handle error. with self.assertRaises(DuplicateKeyError): yield collection.insert([ {'_id': 1}, {'_id': 2}, # Already exists {'_id': 3}]) # First insert should have succeeded, but not second or third. self.assertEqual( set([1, 2]), set((yield collection.distinct('_id')))) @ignore_deprecations @gen_test def test_save_callback(self): yield self.collection.save({}, callback=None) # Should not raise (result, error), _ = yield gen.Task(self.collection.save, {}) if error: raise error @ignore_deprecations @gen_test def test_save_with_id(self): # save() returns the _id, in this case 5. self.assertEqual( 5, (yield self.collection.save({'_id': 5}))) @ignore_deprecations @gen_test def test_save_without_id(self): collection = self.collection result = yield collection.save({'fiddle': 'faddle'}) # save() returns the new _id self.assertTrue(isinstance(result, ObjectId)) @ignore_deprecations @gen_test def test_save_bad(self): coll = self.db.unique_collection yield coll.ensure_index('s', unique=True) yield coll.save({'s': 1}) try: with self.assertRaises(DuplicateKeyError): yield coll.save({'s': 1}) finally: yield coll.drop() @gen_test def test_delete_one(self): # Remove a document twice, check that we get a success responses # and n = 0 for the second time. yield self.collection.insert_one({'_id': 1}) result = yield self.collection.delete_one({'_id': 1}) # First time we remove, n = 1 self.assertEqual(1, result.raw_result['n']) self.assertEqual(1, result.raw_result['ok']) self.assertEqual(None, result.raw_result.get('err')) result = yield self.collection.delete_one({'_id': 1}) # Second time, document is already gone, n = 0 self.assertEqual(0, result.raw_result['n']) self.assertEqual(1, result.raw_result['ok']) self.assertEqual(None, result.raw_result.get('err')) @ignore_deprecations @gen_test def test_unacknowledged_insert(self): # Test that unsafe inserts with no callback still work # Insert id 1 without a callback or w=1. coll = self.db.test_unacknowledged_insert coll.with_options(write_concern=WriteConcern(0)).insert_one({'_id': 1}) # The insert is eventually executed. while not (yield coll.count()): yield gen.sleep(0.1) # DuplicateKeyError not raised. future = coll.insert({'_id': 1}) yield coll.insert({'_id': 1}, w=0) with self.assertRaises(DuplicateKeyError): yield future @ignore_deprecations @gen_test def test_unacknowledged_save(self): # Test that unsafe saves with no callback still work collection_name = 'test_unacknowledged_save' coll = self.db[collection_name] coll.save({'_id': 201}, w=0) while not (yield coll.find_one({'_id': 201})): yield gen.sleep(0.1) # DuplicateKeyError not raised coll.save({'_id': 201}) yield coll.save({'_id': 201}, w=0) coll.database.client.close() @ignore_deprecations @gen_test def test_unacknowledged_update(self): # Test that unsafe updates with no callback still work coll = self.collection yield coll.insert_one({'_id': 1}) coll.update({'_id': 1}, {'$set': {'a': 1}}, w=0) while not (yield coll.find_one({'a': 1})): yield gen.sleep(0.1) coll.database.client.close() @gen_test(timeout=30) def test_nested_callbacks(self): results = [0] future = Future() yield self.collection.delete_many({}) yield self.collection.insert_one({'_id': 1}) def callback(result, error): if error: future.set_exception(error) elif result: results[0] += 1 if results[0] < 1000: self.collection.find({'_id': 1}).each(callback) else: future.set_result(None) self.collection.find({'_id': 1}).each(callback) yield future self.assertEqual(1000, results[0]) @gen_test def test_map_reduce(self): # Count number of documents with even and odd _id yield self.make_test_data() expected_result = [{'_id': 0, 'value': 100}, {'_id': 1, 'value': 100}] map_fn = bson.Code('function map() { emit(this._id % 2, 1); }') reduce_fn = bson.Code(''' function reduce(key, values) { r = 0; values.forEach(function(value) { r += value; }); return r; }''') yield self.db.tmp_mr.drop() # First do a standard mapreduce, should return MotorCollection collection = self.collection tmp_mr = yield collection.map_reduce(map_fn, reduce_fn, 'tmp_mr') self.assertTrue( isinstance(tmp_mr, motor.MotorCollection), 'map_reduce should return MotorCollection, not %s' % tmp_mr) result = yield tmp_mr.find().sort([('_id', 1)]).to_list(length=1000) self.assertEqual(expected_result, result) # Standard mapreduce with full response yield self.db.tmp_mr.drop() response = yield collection.map_reduce( map_fn, reduce_fn, 'tmp_mr', full_response=True) self.assertTrue( isinstance(response, dict), 'map_reduce should return dict, not %s' % response) self.assertEqual('tmp_mr', response['result']) result = yield tmp_mr.find().sort([('_id', 1)]).to_list(length=1000) self.assertEqual(expected_result, result) # Inline mapreduce yield self.db.tmp_mr.drop() result = yield collection.inline_map_reduce( map_fn, reduce_fn) result.sort(key=lambda doc: doc['_id']) self.assertEqual(expected_result, result) @ignore_deprecations @gen_test def test_indexes(self): test_collection = self.collection # Create an index idx_name = yield test_collection.create_index([('foo', 1)]) index_info = yield test_collection.index_information() self.assertEqual([('foo', 1)], index_info[idx_name]['key']) # Ensure the same index, test that callback is executed result = yield test_collection.ensure_index([('foo', 1)]) self.assertEqual('foo_1', result) result2 = yield test_collection.ensure_index([('foo', 1)]) self.assertEqual(None, result2) # Ensure an index that doesn't exist, test it's created yield test_collection.ensure_index([('bar', 1)]) index_info = yield test_collection.index_information() self.assertTrue(any([ info['key'] == [('bar', 1)] for info in index_info.values()])) # Don't test drop_index or drop_indexes -- Synchro tests them @gen.coroutine def _make_test_data(self, n): yield self.db.drop_collection("test") yield self.db.test.insert_many([{'_id': i} for i in range(n)]) expected_sum = sum(range(n)) raise gen.Return(expected_sum) pipeline = [{'$project': {'_id': '$_id'}}] def assertAllDocs(self, expected_sum, docs): self.assertEqual(expected_sum, sum(doc['_id'] for doc in docs)) @gen_test(timeout=30) def test_aggregation_cursor(self): db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): expected_sum = yield self._make_test_data(collection_size) cursor = db.test.aggregate(self.pipeline) docs = yield cursor.to_list(collection_size) self.assertAllDocs(expected_sum, docs) @gen_test def test_aggregation_cursor_exc_info(self): if sys.version_info < (3,): raise SkipTest("Requires Python 3") yield self._make_test_data(200) cursor = self.db.test.aggregate(self.pipeline) yield cursor.to_list(length=10) yield self.db.test.drop() try: yield cursor.to_list(length=None) except OperationFailure: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_unpack_response' in formatted or '_check_command_response' in formatted) @gen_test(timeout=30) def test_aggregation_cursor_to_list_callback(self): db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): expected_sum = yield self._make_test_data(collection_size) cursor = db.test.aggregate(self.pipeline) future = Future() def cb(result, error): if error: future.set_exception(error) else: future.set_result(result) cursor.to_list(collection_size, callback=cb) docs = yield future self.assertAllDocs(expected_sum, docs) @gen_test(timeout=30) def test_parallel_scan(self): yield skip_if_mongos(self.cx) collection = self.collection.with_options( write_concern=WriteConcern(test.env.w)) # Enough documents that each cursor requires multiple batches. yield collection.delete_many({}) yield collection.insert_many(({'_id': i} for i in range(8000))) if test.env.is_replica_set: # Test that getMore messages are sent to the right server. client = self.motor_rsc(read_preference=Secondary()) collection = client.motor_test.test_collection docs = [] @gen.coroutine def f(cursor): self.assertTrue(isinstance(cursor, motor.motor_tornado.MotorCommandCursor)) while (yield cursor.fetch_next): docs.append(cursor.next_object()) cursors = yield collection.parallel_scan(3) yield [f(cursor) for cursor in cursors] self.assertEqual(len(docs), (yield collection.count())) def test_with_options(self): coll = self.db.test codec_options = CodecOptions( tz_aware=True, uuid_representation=JAVA_LEGACY) write_concern = WriteConcern(w=2, j=True) coll2 = coll.with_options( codec_options, ReadPreference.SECONDARY, write_concern) self.assertTrue(isinstance(coll2, motor.MotorCollection)) self.assertEqual(codec_options, coll2.codec_options) self.assertEqual(Secondary(), coll2.read_preference) self.assertEqual(write_concern, coll2.write_concern) pref = Secondary([{"dc": "sf"}]) coll2 = coll.with_options(read_preference=pref) self.assertEqual(pref, coll2.read_preference) self.assertEqual(coll.codec_options, coll2.codec_options) self.assertEqual(coll.write_concern, coll2.write_concern) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_core.py000066400000000000000000000111651323007662700223150ustar00rootroot00000000000000# Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Validate list of PyMongo attributes wrapped by Motor.""" from tornado.testing import gen_test from gridfs import GridFS, GridIn from motor import MotorGridFS, MotorGridIn, MotorGridOut from test import env from test.tornado_tests import MotorTest def attrs(klass): return set(a for a in dir(klass) if not a.startswith('_')) motor_only = set([ 'delegate', 'get_io_loop', 'io_loop', 'wrap']) pymongo_only = set(['next']) motor_client_only = motor_only.union(['open']) pymongo_client_only = set([ 'is_locked', 'set_cursor_manager']).union(pymongo_only) pymongo_database_only = set([ 'system_js']).union(pymongo_only) pymongo_collection_only = set([ 'aggregate_raw_batches', 'find_raw_batches']).union(pymongo_only) motor_cursor_only = set([ 'fetch_next', 'to_list', 'each', 'started', 'next_object', 'closed']).union(motor_only) pymongo_cursor_only = set(['retrieved']).union(pymongo_only) class MotorCoreTest(MotorTest): def test_client_attrs(self): self.assertEqual( attrs(env.sync_cx) - pymongo_client_only, attrs(self.cx) - motor_client_only) def test_database_attrs(self): self.assertEqual( attrs(env.sync_cx.test) - pymongo_database_only, attrs(self.cx.test) - motor_only) def test_collection_attrs(self): self.assertEqual( attrs(env.sync_cx.test.test) - pymongo_collection_only, attrs(self.cx.test.test) - motor_only) def test_cursor_attrs(self): self.assertEqual( attrs(env.sync_cx.test.test.find()) - pymongo_cursor_only, attrs(self.cx.test.test.find()) - motor_cursor_only) @env.require_replica_set @env.require_version_min(3, 6) def test_change_stream_attrs(self): # Ensure the database exists before creating a change stream. env.sync_cx.test.test.insert_one({}) self.assertEqual( attrs(env.sync_cx.test.test.watch()), attrs(self.cx.test.test.watch()) - motor_only) @gen_test def test_command_cursor_attrs(self): motor_agg_cursor_only = set([ 'collection', 'start', 'args', 'kwargs', 'pipeline' ]).union(motor_cursor_only) pymongo_cursor = env.sync_cx.test.test.aggregate([], cursor={}) motor_cursor = self.cx.test.test.aggregate([]) self.assertEqual( attrs(pymongo_cursor) - pymongo_cursor_only, attrs(motor_cursor) - motor_agg_cursor_only) class MotorCoreTestGridFS(MotorTest): def setUp(self): super(MotorCoreTestGridFS, self).setUp() self.sync_fs = GridFS(env.sync_cx.test) self.sync_fs.delete(file_id=1) self.sync_fs.put(b'', _id=1) def tearDown(self): self.sync_fs.delete(file_id=1) super(MotorCoreTestGridFS, self).tearDown() def test_gridfs_attrs(self): pymongo_gridfs_only = set([ # Obsolete PyMongo methods. 'open', 'remove']) motor_gridfs_only = set(['collection']).union(motor_only) self.assertEqual( attrs(GridFS(env.sync_cx.test)) - pymongo_gridfs_only, attrs(MotorGridFS(self.cx.test)) - motor_gridfs_only) def test_gridin_attrs(self): motor_gridin_only = set(['set']).union(motor_only) self.assertEqual( attrs(GridIn(env.sync_cx.test.fs)), attrs(MotorGridIn(self.cx.test.fs)) - motor_gridin_only) @gen_test def test_gridout_attrs(self): motor_gridout_only = set([ 'open', 'stream_to_handler' ]).union(motor_only) motor_gridout = yield MotorGridOut(self.cx.test.fs, file_id=1).open() self.assertEqual( attrs(self.sync_fs.get(1)), attrs(motor_gridout) - motor_gridout_only) def test_gridout_cursor_attrs(self): self.assertEqual( attrs(self.sync_fs.find()) - pymongo_cursor_only, attrs(MotorGridFS(self.cx.test).find()) - motor_cursor_only) motor-1.2.1/test/tornado_tests/test_motor_cursor.py000066400000000000000000000531401323007662700227010ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import sys import traceback import unittest import warnings import pymongo from tornado import gen from tornado.concurrent import Future from tornado.testing import gen_test from pymongo import CursorType from pymongo.collation import Collation from pymongo.errors import InvalidOperation, ExecutionTimeout from pymongo.errors import OperationFailure import motor import motor.motor_tornado from test import SkipTest, env from test.tornado_tests import (get_command_line, MotorTest, MotorMockServerTest, server_is_mongos) from test.utils import one, safe_get, get_primary_pool class MotorCursorTest(MotorMockServerTest): def test_cursor(self): cursor = self.collection.find() self.assertTrue(isinstance(cursor, motor.motor_tornado.MotorCursor)) self.assertFalse(cursor.started, "Cursor shouldn't start immediately") @gen_test def test_count(self): yield self.make_test_data() coll = self.collection self.assertEqual(200, (yield coll.find().count())) self.assertEqual(100, (yield coll.find({'_id': {'$gt': 99}}).count())) where = 'this._id % 2 == 0 && this._id >= 50' self.assertEqual(75, (yield coll.find({'$where': where}).count())) self.assertEqual(75, (yield coll.find().where(where).count())) self.assertEqual( 25, (yield coll.find({'_id': {'$lt': 100}}).where(where).count())) self.assertEqual( 25, (yield coll.find({'_id': {'$lt': 100}, '$where': where}).count())) @gen_test def test_fetch_next(self): yield self.make_test_data() coll = self.collection # 200 results, only including _id field, sorted by _id cursor = coll.find({}, {'_id': 1}).sort( [('_id', pymongo.ASCENDING)]).batch_size(75) self.assertEqual(None, cursor.cursor_id) self.assertEqual(None, cursor.next_object()) # Haven't fetched yet i = 0 while (yield cursor.fetch_next): self.assertEqual({'_id': i}, cursor.next_object()) i += 1 # With batch_size 75 and 200 results, cursor should be exhausted on # the server by third fetch if i <= 150: self.assertNotEqual(0, cursor.cursor_id) else: self.assertEqual(0, cursor.cursor_id) self.assertEqual(False, (yield cursor.fetch_next)) self.assertEqual(None, cursor.next_object()) self.assertEqual(0, cursor.cursor_id) self.assertEqual(200, i) @gen_test def test_fetch_next_delete(self): if sys.version_info < (3, 4): raise SkipTest("requires Python 3.4") if 'PyPy' in sys.version: raise SkipTest('PyPy') client, server = self.client_server(auto_ismaster=True) cursor = client.test.coll.find() # With Tornado, simply accessing fetch_next starts the fetch. cursor.fetch_next request = yield self.run_thread(server.receives, "find", "coll") request.replies({"cursor": { "id": 123, "ns": "db.coll", "firstBatch": [{"_id": 1}]}}) # Decref'ing the cursor eventually closes it on the server. del cursor # Clear Runner's reference. yield gen.moment request = yield self.run_thread(server.receives, "killCursors", "coll") request.ok() @gen_test def test_fetch_next_without_results(self): coll = self.collection # Nothing matches this query cursor = coll.find({'foo': 'bar'}) self.assertEqual(None, cursor.next_object()) self.assertEqual(False, (yield cursor.fetch_next)) self.assertEqual(None, cursor.next_object()) # Now cursor knows it's exhausted self.assertEqual(0, cursor.cursor_id) @gen_test def test_fetch_next_is_idempotent(self): # Subsequent calls to fetch_next don't do anything yield self.make_test_data() coll = self.collection cursor = coll.find() self.assertEqual(None, cursor.cursor_id) yield cursor.fetch_next self.assertTrue(cursor.cursor_id) self.assertEqual(101, cursor._buffer_size()) yield cursor.fetch_next # Does nothing self.assertEqual(101, cursor._buffer_size()) yield cursor.close() @gen_test def test_fetch_next_exception(self): coll = self.collection cursor = coll.find() cursor.delegate._Cursor__id = 1234 # Not valid on server with self.assertRaises(OperationFailure): yield cursor.fetch_next # Avoid the cursor trying to close itself when it goes out of scope cursor.delegate._Cursor__id = None @gen_test def test_each_callback(self): cursor = self.collection.find() self.assertRaises(TypeError, cursor.each, callback='foo') self.assertRaises(TypeError, cursor.each, callback=None) self.assertRaises(TypeError, cursor.each) # No callback. # Should not raise (result, error), _ = yield gen.Task(cursor.each) if error: raise error @gen_test(timeout=30) def test_each(self): yield self.make_test_data() cursor = self.collection.find({}, {'_id': 1}) cursor.sort([('_id', pymongo.ASCENDING)]) future = Future() results = [] def callback(result, error): if error: raise error if result is not None: results.append(result) else: # Done iterating. future.set_result(True) cursor.each(callback) yield future expected = [{'_id': i} for i in range(200)] self.assertEqual(expected, results) @gen_test def test_to_list_argument_checking(self): # We need more than 10 documents so the cursor stays alive. yield self.make_test_data() coll = self.collection cursor = coll.find() with self.assertRaises(ValueError): yield cursor.to_list(-1) with self.assertRaises(TypeError): yield cursor.to_list('foo') @gen_test def test_to_list_callback(self): yield self.make_test_data() cursor = self.collection.find({}, {'_id': 1}) cursor.sort([('_id', pymongo.ASCENDING)]) expected = [{'_id': i} for i in range(200)] (result, error), _ = yield gen.Task(cursor.to_list, length=1000) self.assertEqual(expected, result) cursor = self.collection.find().where('return foo') (result, error), _ = yield gen.Task(cursor.to_list, length=1000) self.assertEqual(None, result) self.assertTrue(isinstance(error, OperationFailure)) @gen_test def test_to_list_with_length(self): yield self.make_test_data() coll = self.collection cursor = coll.find().sort('_id') self.assertEqual([], (yield cursor.to_list(0))) def expected(start, stop): return [{'_id': i} for i in range(start, stop)] self.assertEqual(expected(0, 10), (yield cursor.to_list(10))) self.assertEqual(expected(10, 100), (yield cursor.to_list(90))) # Test particularly rigorously around the 101-doc mark, since this is # where the first batch ends self.assertEqual(expected(100, 101), (yield cursor.to_list(1))) self.assertEqual(expected(101, 102), (yield cursor.to_list(1))) self.assertEqual(expected(102, 103), (yield cursor.to_list(1))) self.assertEqual([], (yield cursor.to_list(0))) self.assertEqual(expected(103, 105), (yield cursor.to_list(2))) # Only 95 docs left, make sure length=100 doesn't error or hang self.assertEqual(expected(105, 200), (yield cursor.to_list(100))) self.assertEqual(0, cursor.cursor_id) # Nothing left. self.assertEqual([], (yield cursor.to_list(100))) @gen_test def test_to_list_exc_info(self): if sys.version_info < (3,): raise SkipTest("Requires Python 3") yield self.make_test_data() coll = self.collection cursor = coll.find() yield cursor.to_list(length=10) yield self.collection.drop() try: yield cursor.to_list(length=None) except OperationFailure: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_unpack_response' in formatted or '_check_command_response' in formatted) @gen_test def test_to_list_with_length_of_none(self): yield self.make_test_data() collection = self.collection cursor = collection.find() docs = yield cursor.to_list(None) # Unlimited. count = yield collection.count() self.assertEqual(count, len(docs)) @gen_test def test_to_list_tailable(self): coll = self.collection cursor = coll.find(cursor_type=CursorType.TAILABLE) # Can't call to_list on tailable cursor. with self.assertRaises(InvalidOperation): yield cursor.to_list(10) @env.require_version_min(3, 4) @gen_test def test_to_list_with_chained_collation(self): yield self.make_test_data() cursor = self.collection.find({}, {'_id': 1}) \ .sort([('_id', pymongo.ASCENDING)]) \ .collation(Collation("en")) expected = [{'_id': i} for i in range(200)] result = yield cursor.to_list(length=1000) self.assertEqual(expected, result) @gen_test def test_cursor_explicit_close(self): client, server = self.client_server(auto_ismaster=True) collection = client.test.coll cursor = collection.find() # With Tornado, simply accessing fetch_next starts the fetch. fetch_next = cursor.fetch_next request = yield self.run_thread(server.receives, "find", "coll") request.replies({"cursor": { "id": 123, "ns": "db.coll", "firstBatch": [{"_id": 1}]}}) self.assertTrue((yield fetch_next)) close_future = cursor.close() request = yield self.run_thread(server.receives, "killCursors", "coll") request.ok() yield close_future # Cursor reports it's alive because it has buffered data, even though # it's killed on the server. self.assertTrue(cursor.alive) self.assertEqual({'_id': 1}, cursor.next_object()) self.assertFalse((yield cursor.fetch_next)) self.assertFalse(cursor.alive) @gen_test def test_each_cancel(self): yield self.make_test_data() loop = self.io_loop collection = self.collection results = [] future = Future() def cancel(result, error): if error: future.set_exception(error) else: results.append(result) loop.add_callback(canceled) return False # Cancel iteration. def canceled(): try: self.assertFalse(cursor.delegate._Cursor__killed) self.assertTrue(cursor.alive) # Resume iteration cursor.each(each) except Exception as e: future.set_exception(e) def each(result, error): if error: future.set_exception(error) elif result: pass results.append(result) else: # Complete future.set_result(None) cursor = collection.find() cursor.each(cancel) yield future self.assertEqual((yield collection.count()), len(results)) @gen_test def test_rewind(self): yield self.collection.insert_many([{}, {}, {}]) cursor = self.collection.find().limit(2) count = 0 while (yield cursor.fetch_next): cursor.next_object() count += 1 self.assertEqual(2, count) cursor.rewind() count = 0 while (yield cursor.fetch_next): cursor.next_object() count += 1 self.assertEqual(2, count) cursor.rewind() count = 0 while (yield cursor.fetch_next): cursor.next_object() break cursor.rewind() while (yield cursor.fetch_next): cursor.next_object() count += 1 self.assertEqual(2, count) self.assertEqual(cursor, cursor.rewind()) @gen_test def test_cursor_del(self): if sys.version_info < (3, 4): raise SkipTest("requires Python 3.4") if 'PyPy' in sys.version: raise SkipTest("PyPy") client, server = self.client_server(auto_ismaster=True) cursor = client.test.coll.find() future = cursor.fetch_next request = yield self.run_thread(server.receives, "find", "coll") request.replies({"cursor": { "id": 123, "ns": "db.coll", "firstBatch": [{"_id": 1}]}}) yield future # Complete the first fetch. # Dereference the cursor. del cursor # Let the event loop iterate once more to clear its references to # callbacks, allowing the cursor to be freed. yield gen.sleep(0.1) request = yield self.run_thread(server.receives, "killCursors", "coll") request.ok() @gen_test def test_exhaust(self): if sys.version_info < (3, 4): raise SkipTest("requires Python 3.4") if (yield server_is_mongos(self.cx)): self.assertRaises(InvalidOperation, self.db.test.find, cursor_type=CursorType.EXHAUST) return cur = self.db.test.find(cursor_type=CursorType.EXHAUST) self.assertRaises(InvalidOperation, cur.limit, 5) cur = self.db.test.find(limit=5) self.assertRaises(InvalidOperation, cur.add_option, 64) cur = self.db.test.find() cur.add_option(64) self.assertRaises(InvalidOperation, cur.limit, 5) yield self.db.drop_collection("test") # Insert enough documents to require more than one batch. yield self.db.test.insert_many([{} for _ in range(150)]) client = self.motor_client(maxPoolSize=1) # Ensure a pool. yield client.db.collection.find_one() socks = get_primary_pool(client).sockets # Make sure the socket is returned after exhaustion. cur = client[self.db.name].test.find(cursor_type=CursorType.EXHAUST) has_next = yield cur.fetch_next self.assertTrue(has_next) self.assertEqual(0, len(socks)) while (yield cur.fetch_next): cur.next_object() self.assertEqual(1, len(socks)) # Same as previous but with to_list instead of next_object. docs = yield client[self.db.name].test.find( cursor_type=CursorType.EXHAUST).to_list(None) self.assertEqual(1, len(socks)) self.assertEqual( (yield self.db.test.count()), len(docs)) # If the Cursor instance is discarded before being # completely iterated we have to close and # discard the socket. sock = one(socks) cur = client[self.db.name].test.find( cursor_type=CursorType.EXHAUST).batch_size(1) has_next = yield cur.fetch_next self.assertTrue(has_next) self.assertEqual(0, len(socks)) if 'PyPy' in sys.version: # Don't wait for GC or use gc.collect(), it's unreliable. yield cur.close() del cur yield gen.sleep(0.1) # The exhaust cursor's socket was discarded, although another may # already have been opened to send OP_KILLCURSORS. self.assertNotIn(sock, socks) self.assertTrue(sock.closed) def test_iter(self): # Iteration should be prohibited. with self.assertRaises(TypeError): for _ in self.db.test.find(): pass @gen_test def test_close_with_docs_in_batch(self): # MOTOR-67 Killed cursor with docs batched is "alive", don't kill again. yield self.make_test_data() # Ensure multiple batches. cursor = self.collection.find() yield cursor.fetch_next yield cursor.close() # Killed but still "alive": has a batch. self.cx.close() with warnings.catch_warnings(record=True) as w: del cursor # No-op, no error. self.assertEqual(0, len(w)) class MotorCursorMaxTimeMSTest(MotorTest): def setUp(self): super(MotorCursorMaxTimeMSTest, self).setUp() self.io_loop.run_sync(self.maybe_skip) def tearDown(self): self.io_loop.run_sync(self.disable_timeout) super(MotorCursorMaxTimeMSTest, self).tearDown() @gen.coroutine def maybe_skip(self): if (yield server_is_mongos(self.cx)): raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point") cmdline = yield get_command_line(self.cx) if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'): if 'enableTestCommands=1' not in cmdline['argv']: raise SkipTest("testing maxTimeMS requires failpoints") @gen.coroutine def enable_timeout(self): yield self.cx.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="alwaysOn") @gen.coroutine def disable_timeout(self): yield self.cx.admin.command("configureFailPoint", "maxTimeAlwaysTimeOut", mode="off") @gen_test def test_max_time_ms_query(self): # Cursor parses server timeout error in response to initial query. yield self.enable_timeout() cursor = self.collection.find().max_time_ms(100000) with self.assertRaises(ExecutionTimeout): yield cursor.fetch_next cursor = self.collection.find().max_time_ms(100000) with self.assertRaises(ExecutionTimeout): yield cursor.to_list(10) with self.assertRaises(ExecutionTimeout): yield self.collection.find_one(max_time_ms=100000) @gen_test(timeout=60) def test_max_time_ms_getmore(self): # Cursor handles server timeout during getmore, also. yield self.collection.insert_many({} for _ in range(200)) try: # Send initial query. cursor = self.collection.find().max_time_ms(100000) yield cursor.fetch_next cursor.next_object() # Test getmore timeout. yield self.enable_timeout() with self.assertRaises(ExecutionTimeout): while (yield cursor.fetch_next): cursor.next_object() yield cursor.close() # Send another initial query. yield self.disable_timeout() cursor = self.collection.find().max_time_ms(100000) yield cursor.fetch_next cursor.next_object() # Test getmore timeout. yield self.enable_timeout() with self.assertRaises(ExecutionTimeout): yield cursor.to_list(None) # Avoid 'IOLoop is closing' warning. yield cursor.close() finally: # Cleanup. yield self.disable_timeout() yield self.collection.delete_many({}) @gen_test def test_max_time_ms_each_query(self): # Cursor.each() handles server timeout during initial query. yield self.enable_timeout() cursor = self.collection.find().max_time_ms(100000) future = Future() def callback(result, error): if error: future.set_exception(error) elif not result: # Done. future.set_result(None) with self.assertRaises(ExecutionTimeout): cursor.each(callback) yield future @gen_test(timeout=30) def test_max_time_ms_each_getmore(self): # Cursor.each() handles server timeout during getmore. yield self.collection.insert_many({} for _ in range(200)) try: # Send initial query. cursor = self.collection.find().max_time_ms(100000) yield cursor.fetch_next cursor.next_object() future = Future() def callback(result, error): if error: future.set_exception(error) elif not result: # Done. future.set_result(None) yield self.enable_timeout() with self.assertRaises(ExecutionTimeout): cursor.each(callback) yield future yield cursor.close() finally: # Cleanup. yield self.disable_timeout() yield self.collection.delete_many({}) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_database.py000066400000000000000000000203431323007662700231270ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import unittest import pymongo.database from bson import CodecOptions from bson.binary import JAVA_LEGACY from pymongo import ReadPreference, WriteConcern from pymongo.read_preferences import Secondary from pymongo.errors import OperationFailure, CollectionInvalid from pymongo.son_manipulator import AutoReference, NamespaceInjector from tornado import gen from tornado.testing import gen_test import motor import test from test.test_environment import env from test.tornado_tests import MotorTest, remove_all_users from test.utils import ignore_deprecations class MotorDatabaseTest(MotorTest): @gen_test def test_database(self): # Test that we can create a db directly, not just from MotorClient's # accessors db = motor.MotorDatabase(self.cx, 'motor_test') # Make sure we got the right DB and it can do an operation self.assertEqual('motor_test', db.name) yield db.test_collection.insert_one({'_id': 1}) doc = yield db.test_collection.find_one({'_id': 1}) self.assertEqual(1, doc['_id']) def test_collection_named_delegate(self): db = self.db self.assertTrue(isinstance(db.delegate, pymongo.database.Database)) self.assertTrue(isinstance(db['delegate'], motor.MotorCollection)) db.client.close() def test_call(self): # Prevents user error with nice message. try: self.cx.foo() except TypeError as e: self.assertTrue('no such method exists' in str(e)) else: self.fail('Expected TypeError') @gen_test def test_database_callbacks(self): db = self.db yield db.drop_collection('c') self.assertRaises(TypeError, db.create_collection, 'c', callback='foo') self.assertRaises(TypeError, db.create_collection, 'c', callback=1) # No error without callback db.create_collection('c', callback=None) # Wait for create_collection to complete for _ in range(10): yield gen.sleep(0.1) if 'c' in (yield db.collection_names()): break @gen_test def test_command(self): result = yield self.cx.admin.command("buildinfo") # Make sure we got some sane result or other. self.assertEqual(1, result['ok']) @gen_test def test_create_collection(self): # Test creating collection, return val is wrapped in MotorCollection, # creating it again raises CollectionInvalid. db = self.db yield db.drop_collection('test_collection2') collection = yield db.create_collection('test_collection2') self.assertTrue(isinstance(collection, motor.MotorCollection)) self.assertTrue( 'test_collection2' in (yield db.collection_names())) with self.assertRaises(CollectionInvalid): yield db.create_collection('test_collection2') yield db.drop_collection('test_collection2') # Test creating capped collection collection = yield db.create_collection( 'test_capped', capped=True, size=4096) self.assertTrue(isinstance(collection, motor.MotorCollection)) self.assertEqual( {"capped": True, 'size': 4096}, (yield db.test_capped.options())) yield db.drop_collection('test_capped') @gen_test def test_drop_collection(self): # Make sure we can pass a MotorCollection instance to drop_collection db = self.db collection = db.test_drop_collection yield collection.insert_one({}) names = yield db.collection_names() self.assertTrue('test_drop_collection' in names) yield db.drop_collection(collection) names = yield db.collection_names() self.assertFalse('test_drop_collection' in names) @ignore_deprecations @gen_test def test_auto_ref_and_deref(self): # Test same functionality as in PyMongo's test_database.py; the # implementation for Motor for async is a little complex so we test # that it works here, and we don't just rely on synchrotest # to cover it. db = self.db # We test a special hack where add_son_manipulator corrects our mistake # if we pass a MotorDatabase, instead of Database, to AutoReference. db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) a = {"hello": "world"} b = {"test": a} c = {"another test": b} yield db.a.delete_many({}) yield db.b.delete_many({}) yield db.c.delete_many({}) yield db.a.save(a) yield db.b.save(b) yield db.c.save(c) a["hello"] = "mike" yield db.a.save(a) result_a = yield db.a.find_one() result_b = yield db.b.find_one() result_c = yield db.c.find_one() self.assertEqual(a, result_a) self.assertEqual(a, result_b["test"]) self.assertEqual(a, result_c["another test"]["test"]) self.assertEqual(b, result_b) self.assertEqual(b, result_c["another test"]) self.assertEqual(c, result_c) # SCRAM-SHA-1 is slow, install backports.pbkdf2 for speed. @gen_test(timeout=30) def test_authenticate(self): # self.db is logged in as root. with ignore_deprecations(): yield self.db.add_user("mike", "password") client = motor.MotorClient(env.host, env.port, **self.get_client_kwargs()) db = client.motor_test try: # Authenticate many times at once to test concurrency. yield [db.authenticate("mike", "password") for _ in range(10)] # Just make sure there are no exceptions here. yield db.remove_user("mike") yield db.logout() info = yield self.db.command("usersInfo", "mike") users = info.get('users', []) self.assertFalse("mike" in [u['user'] for u in users]) finally: yield remove_all_users(self.db) test.env.sync_cx.close() @gen_test def test_validate_collection(self): db = self.db with self.assertRaises(TypeError): yield db.validate_collection(5) with self.assertRaises(TypeError): yield db.validate_collection(None) with self.assertRaises(OperationFailure): yield db.validate_collection("test.doesnotexist") with self.assertRaises(OperationFailure): yield db.validate_collection(db.test.doesnotexist) yield db.test.insert_one({"dummy": "object"}) self.assertTrue((yield db.validate_collection("test"))) self.assertTrue((yield db.validate_collection(db.test))) def test_get_collection(self): codec_options = CodecOptions( tz_aware=True, uuid_representation=JAVA_LEGACY) write_concern = WriteConcern(w=2, j=True) coll = self.db.get_collection( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) self.assertTrue(isinstance(coll, motor.MotorCollection)) self.assertEqual('foo', coll.name) self.assertEqual(codec_options, coll.codec_options) self.assertEqual(ReadPreference.SECONDARY, coll.read_preference) self.assertEqual(write_concern, coll.write_concern) pref = Secondary([{"dc": "sf"}]) coll = self.db.get_collection('foo', read_preference=pref) self.assertEqual(pref, coll.read_preference) self.assertEqual(self.db.codec_options, coll.codec_options) self.assertEqual(self.db.write_concern, coll.write_concern) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_grid_file.py000066400000000000000000000307521323007662700233140ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test GridFS with Motor, an asynchronous driver for MongoDB and Tornado.""" import datetime import sys import traceback import unittest from bson.objectid import ObjectId from gridfs.errors import NoFile from tornado import gen from tornado.testing import gen_test from pymongo.errors import InvalidOperation import motor from test import MockRequestHandler, SkipTest from test.tornado_tests import MotorTest class MotorGridFileTest(MotorTest): @gen.coroutine def _reset(self): yield self.db.drop_collection("fs.files") yield self.db.drop_collection("fs.chunks") yield self.db.drop_collection("alt.files") yield self.db.drop_collection("alt.chunks") def tearDown(self): self.io_loop.run_sync(self._reset) super(MotorGridFileTest, self).tearDown() @gen_test def test_attributes(self): f = motor.MotorGridIn( self.db.fs, filename="test", foo="bar", content_type="text") yield f.close() g = motor.MotorGridOut(self.db.fs, f._id) attr_names = ( '_id', 'filename', 'name', 'name', 'content_type', 'length', 'chunk_size', 'upload_date', 'aliases', 'metadata', 'md5') for attr_name in attr_names: self.assertRaises(InvalidOperation, getattr, g, attr_name) yield g.open() for attr_name in attr_names: getattr(g, attr_name) @gen_test def test_iteration(self): fs = motor.MotorGridFS(self.db) _id = yield fs.put(b'foo') g = motor.MotorGridOut(self.db.fs, _id) # Iteration is prohibited. self.assertRaises(TypeError, iter, g) @gen_test def test_basic(self): f = motor.MotorGridIn(self.db.fs, filename="test") yield f.write(b"hello world") yield f.close() self.assertEqual(1, (yield self.db.fs.files.find().count())) self.assertEqual(1, (yield self.db.fs.chunks.find().count())) g = motor.MotorGridOut(self.db.fs, f._id) self.assertEqual(b"hello world", (yield g.read())) f = motor.MotorGridIn(self.db.fs, filename="test") yield f.close() self.assertEqual(2, (yield self.db.fs.files.find().count())) self.assertEqual(1, (yield self.db.fs.chunks.find().count())) g = motor.MotorGridOut(self.db.fs, f._id) self.assertEqual(b"", (yield g.read())) @gen_test def test_readchunk(self): in_data = b'a' * 10 f = motor.MotorGridIn(self.db.fs, chunkSize=3) yield f.write(in_data) yield f.close() g = motor.MotorGridOut(self.db.fs, f._id) # This is starting to look like Lisp. self.assertEqual(3, len((yield g.readchunk()))) self.assertEqual(2, len((yield g.read(2)))) self.assertEqual(1, len((yield g.readchunk()))) self.assertEqual(3, len((yield g.read(3)))) self.assertEqual(1, len((yield g.readchunk()))) self.assertEqual(0, len((yield g.readchunk()))) @gen_test def test_gridout_open_exc_info(self): if sys.version_info < (3, ): raise SkipTest("Requires Python 3") g = motor.MotorGridOut(self.db.fs, "_id that doesn't exist") try: yield g.open() except NoFile: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_ensure_file' in formatted) @gen_test def test_alternate_collection(self): yield self.db.alt.files.delete_many({}) yield self.db.alt.chunks.delete_many({}) f = motor.MotorGridIn(self.db.alt) yield f.write(b"hello world") yield f.close() self.assertEqual(1, (yield self.db.alt.files.find().count())) self.assertEqual(1, (yield self.db.alt.chunks.find().count())) g = motor.MotorGridOut(self.db.alt, f._id) self.assertEqual(b"hello world", (yield g.read())) # test that md5 still works... self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", g.md5) @gen_test def test_grid_in_default_opts(self): self.assertRaises(TypeError, motor.MotorGridIn, "foo") a = motor.MotorGridIn(self.db.fs) self.assertTrue(isinstance(a._id, ObjectId)) self.assertRaises(AttributeError, setattr, a, "_id", 5) self.assertEqual(None, a.filename) # This raises AttributeError because you can't directly set properties # in Motor, have to use set() def setter(): a.filename = "my_file" self.assertRaises(AttributeError, setter) # This method of setting attributes works in Motor yield a.set("filename", "my_file") self.assertEqual("my_file", a.filename) self.assertEqual(None, a.content_type) yield a.set("content_type", "text/html") self.assertEqual("text/html", a.content_type) self.assertRaises(AttributeError, getattr, a, "length") self.assertRaises(AttributeError, setattr, a, "length", 5) self.assertEqual(255 * 1024, a.chunk_size) self.assertRaises(AttributeError, setattr, a, "chunk_size", 5) self.assertRaises(AttributeError, getattr, a, "upload_date") self.assertRaises(AttributeError, setattr, a, "upload_date", 5) self.assertRaises(AttributeError, getattr, a, "aliases") yield a.set("aliases", ["foo"]) self.assertEqual(["foo"], a.aliases) self.assertRaises(AttributeError, getattr, a, "metadata") yield a.set("metadata", {"foo": 1}) self.assertEqual({"foo": 1}, a.metadata) self.assertRaises(AttributeError, setattr, a, "md5", 5) yield a.close() self.assertTrue(isinstance(a._id, ObjectId)) self.assertRaises(AttributeError, setattr, a, "_id", 5) self.assertEqual("my_file", a.filename) self.assertEqual("text/html", a.content_type) self.assertEqual(0, a.length) self.assertRaises(AttributeError, setattr, a, "length", 5) self.assertEqual(255 * 1024, a.chunk_size) self.assertRaises(AttributeError, setattr, a, "chunk_size", 5) self.assertTrue(isinstance(a.upload_date, datetime.datetime)) self.assertRaises(AttributeError, setattr, a, "upload_date", 5) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1}, a.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", a.md5) self.assertRaises(AttributeError, setattr, a, "md5", 5) @gen_test def test_grid_in_custom_opts(self): self.assertRaises(TypeError, motor.MotorGridIn, "foo") a = motor.MotorGridIn( self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") self.assertEqual(5, a._id) self.assertEqual("my_file", a.filename) self.assertEqual("text/html", a.content_type) self.assertEqual(1000, a.chunk_size) self.assertEqual(["foo"], a.aliases) self.assertEqual({"foo": 1, "bar": 2}, a.metadata) self.assertEqual(3, a.bar) self.assertEqual("hello", a.baz) self.assertRaises(AttributeError, getattr, a, "mike") b = motor.MotorGridIn( self.db.fs, content_type="text/html", chunk_size=1000, baz=100) self.assertEqual("text/html", b.content_type) self.assertEqual(1000, b.chunk_size) self.assertEqual(100, b.baz) @gen_test def test_grid_out_default_opts(self): self.assertRaises(TypeError, motor.MotorGridOut, "foo") gout = motor.MotorGridOut(self.db.fs, 5) with self.assertRaises(NoFile): yield gout.open() a = motor.MotorGridIn(self.db.fs) yield a.close() b = yield motor.MotorGridOut(self.db.fs, a._id).open() self.assertEqual(a._id, b._id) self.assertEqual(0, b.length) self.assertEqual(None, b.content_type) self.assertEqual(255 * 1024, b.chunk_size) self.assertTrue(isinstance(b.upload_date, datetime.datetime)) self.assertEqual(None, b.aliases) self.assertEqual(None, b.metadata) self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", b.md5) @gen_test def test_grid_out_custom_opts(self): one = motor.MotorGridIn( self.db.fs, _id=5, filename="my_file", contentType="text/html", chunkSize=1000, aliases=["foo"], metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") yield one.write(b"hello world") yield one.close() two = yield motor.MotorGridOut(self.db.fs, 5).open() self.assertEqual(5, two._id) self.assertEqual(11, two.length) self.assertEqual("text/html", two.content_type) self.assertEqual(1000, two.chunk_size) self.assertTrue(isinstance(two.upload_date, datetime.datetime)) self.assertEqual(["foo"], two.aliases) self.assertEqual({"foo": 1, "bar": 2}, two.metadata) self.assertEqual(3, two.bar) self.assertEqual("5eb63bbbe01eeed093cb22bb8f5acdc3", two.md5) @gen_test def test_grid_out_file_document(self): one = motor.MotorGridIn(self.db.fs) yield one.write(b"foo bar") yield one.close() file_document = yield self.db.fs.files.find_one() two = motor.MotorGridOut( self.db.fs, file_document=file_document) self.assertEqual(b"foo bar", (yield two.read())) file_document = yield self.db.fs.files.find_one() three = motor.MotorGridOut(self.db.fs, 5, file_document) self.assertEqual(b"foo bar", (yield three.read())) gridout = motor.MotorGridOut(self.db.fs, file_document={}) with self.assertRaises(NoFile): yield gridout.open() @gen_test def test_write_file_like(self): one = motor.MotorGridIn(self.db.fs) yield one.write(b"hello world") yield one.close() two = motor.MotorGridOut(self.db.fs, one._id) three = motor.MotorGridIn(self.db.fs) yield three.write(two) yield three.close() four = motor.MotorGridOut(self.db.fs, three._id) self.assertEqual(b"hello world", (yield four.read())) @gen_test def test_set_after_close(self): f = motor.MotorGridIn(self.db.fs, _id="foo", bar="baz") self.assertEqual("foo", f._id) self.assertEqual("baz", f.bar) self.assertRaises(AttributeError, getattr, f, "baz") self.assertRaises(AttributeError, getattr, f, "uploadDate") self.assertRaises(AttributeError, setattr, f, "_id", 5) f.bar = "foo" f.baz = 5 self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertRaises(AttributeError, getattr, f, "uploadDate") yield f.close() self.assertEqual("foo", f._id) self.assertEqual("foo", f.bar) self.assertEqual(5, f.baz) self.assertTrue(f.uploadDate) self.assertRaises(AttributeError, setattr, f, "_id", 5) yield f.set("bar", "a") yield f.set("baz", "b") self.assertRaises(AttributeError, setattr, f, "upload_date", 5) g = yield motor.MotorGridOut(self.db.fs, f._id).open() self.assertEqual("a", g.bar) self.assertEqual("b", g.baz) @gen_test def test_stream_to_handler(self): fs = motor.MotorGridFS(self.db) for content_length in (0, 1, 100, 100 * 1000): _id = yield fs.put(b'a' * content_length) gridout = yield fs.get(_id) handler = MockRequestHandler() yield gridout.stream_to_handler(handler) self.assertEqual(content_length, handler.n_written) yield fs.delete(_id) if __name__ == "__main__": unittest.main() motor-1.2.1/test/tornado_tests/test_motor_gridfs.py000066400000000000000000000160611323007662700226430ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test GridFS with Motor, an asynchronous driver for MongoDB and Tornado.""" import unittest from functools import partial from bson import ObjectId from gridfs.errors import FileExists, NoFile from pymongo.errors import ConfigurationError from tornado import gen from tornado.testing import gen_test import motor from motor.motor_py3_compat import StringIO from test.tornado_tests import MotorTest class MotorGridfsTest(MotorTest): @gen.coroutine def _reset(self): yield self.db.drop_collection("fs.files") yield self.db.drop_collection("fs.chunks") yield self.db.drop_collection("alt.files") yield self.db.drop_collection("alt.chunks") def setUp(self): super(MotorGridfsTest, self).setUp() self.fs = motor.MotorGridFS(self.db) def tearDown(self): self.io_loop.run_sync(self._reset) super(MotorGridfsTest, self).tearDown() @gen_test def test_gridfs(self): self.assertRaises(TypeError, motor.MotorGridFS, "foo") self.assertRaises(TypeError, motor.MotorGridFS, 5) @gen_test def test_get_version(self): # new_file creates a MotorGridIn. gin = yield self.fs.new_file(_id=1, filename='foo', field=0) yield gin.write(b'a') yield gin.close() yield self.fs.put(b'', filename='foo', field=1) yield self.fs.put(b'', filename='foo', field=2) gout = yield self.fs.get_version('foo') self.assertEqual(2, gout.field) gout = yield self.fs.get_version('foo', -3) self.assertEqual(0, gout.field) gout = yield self.fs.get_last_version('foo') self.assertEqual(2, gout.field) @gen_test def test_basic(self): oid = yield self.fs.put(b"hello world") out = yield self.fs.get(oid) self.assertEqual(b"hello world", (yield out.read())) self.assertEqual(1, (yield self.db.fs.files.count())) self.assertEqual(1, (yield self.db.fs.chunks.count())) yield self.fs.delete(oid) with self.assertRaises(NoFile): yield self.fs.get(oid) self.assertEqual(0, (yield self.db.fs.files.count())) self.assertEqual(0, (yield self.db.fs.chunks.count())) with self.assertRaises(NoFile): yield self.fs.get("foo") self.assertEqual( "foo", (yield self.fs.put(b"hello world", _id="foo"))) gridout = yield self.fs.get("foo") self.assertEqual(b"hello world", (yield gridout.read())) @gen_test def test_list(self): self.assertEqual([], (yield self.fs.list())) yield self.fs.put(b"hello world") self.assertEqual([], (yield self.fs.list())) yield self.fs.put(b"", filename="mike") yield self.fs.put(b"foo", filename="test") yield self.fs.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set((yield self.fs.list()))) @gen_test def test_alt_collection(self): db = self.db alt = motor.MotorGridFS(db, 'alt') oid = yield alt.put(b"hello world") gridout = yield alt.get(oid) self.assertEqual(b"hello world", (yield gridout.read())) self.assertEqual(1, (yield self.db.alt.files.count())) self.assertEqual(1, (yield self.db.alt.chunks.count())) yield alt.delete(oid) with self.assertRaises(NoFile): yield alt.get(oid) self.assertEqual(0, (yield self.db.alt.files.count())) self.assertEqual(0, (yield self.db.alt.chunks.count())) with self.assertRaises(NoFile): yield alt.get("foo") oid = yield alt.put(b"hello world", _id="foo") self.assertEqual("foo", oid) gridout = yield alt.get("foo") self.assertEqual(b"hello world", (yield gridout.read())) yield alt.put(b"", filename="mike") yield alt.put(b"foo", filename="test") yield alt.put(b"", filename="hello world") self.assertEqual(set(["mike", "test", "hello world"]), set((yield alt.list()))) @gen_test(timeout=30) def test_put_filelike(self): oid = yield self.fs.put(StringIO(b"hello world"), chunk_size=1) self.assertEqual(11, (yield self.cx.motor_test.fs.chunks.count())) gridout = yield self.fs.get(oid) self.assertEqual(b"hello world", (yield gridout.read())) @gen_test def test_put_callback(self): (oid, error), _ = yield gen.Task(self.fs.put, b"hello") self.assertTrue(isinstance(oid, ObjectId)) self.assertEqual(None, error) (result, error), _ = yield gen.Task(self.fs.put, b"hello", _id=oid) self.assertEqual(None, result) self.assertTrue(isinstance(error, FileExists)) @gen_test def test_put_duplicate(self): oid = yield self.fs.put(b"hello") with self.assertRaises(FileExists): yield self.fs.put(b"world", _id=oid) @gen_test def test_put_kwargs(self): # 'w' is not special here. oid = yield self.fs.put(b"hello", foo='bar', w=0) gridout = yield self.fs.get(oid) self.assertEqual('bar', gridout.foo) self.assertEqual(0, gridout.w) @gen_test def test_put_unacknowledged(self): client = self.motor_client(w=0) with self.assertRaises(ConfigurationError): motor.MotorGridFS(client.motor_test) client.close() @gen_test def test_gridfs_find(self): yield self.fs.put(b"test2", filename="two") yield self.fs.put(b"test2+", filename="two") yield self.fs.put(b"test1", filename="one") yield self.fs.put(b"test2++", filename="two") cursor = self.fs.find().sort("_id", -1).skip(1).limit(2) self.assertTrue((yield cursor.fetch_next)) grid_out = cursor.next_object() self.assertTrue(isinstance(grid_out, motor.MotorGridOut)) self.assertEqual(b"test1", (yield grid_out.read())) cursor.rewind() self.assertTrue((yield cursor.fetch_next)) grid_out = cursor.next_object() self.assertEqual(b"test1", (yield grid_out.read())) self.assertTrue((yield cursor.fetch_next)) grid_out = cursor.next_object() self.assertEqual(b"test2+", (yield grid_out.read())) self.assertFalse((yield cursor.fetch_next)) self.assertEqual(None, cursor.next_object()) self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) if __name__ == "__main__": unittest.main() motor-1.2.1/test/tornado_tests/test_motor_gridfsbucket.py000066400000000000000000000045101323007662700240350ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Copyright 2016 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test MotorGridFSBucket.""" from io import BytesIO from gridfs.errors import NoFile from tornado import gen from tornado.testing import gen_test import motor from test.tornado_tests import MotorTest class MotorGridFSBucketTest(MotorTest): @gen.coroutine def _reset(self): yield self.db.drop_collection("fs.files") yield self.db.drop_collection("fs.chunks") yield self.db.drop_collection("alt.files") yield self.db.drop_collection("alt.chunks") def setUp(self): super(MotorGridFSBucketTest, self).setUp() self.io_loop.run_sync(self._reset) self.bucket = motor.MotorGridFSBucket(self.db) def tearDown(self): self.io_loop.run_sync(self._reset) super(MotorGridFSBucketTest, self).tearDown() @gen_test def test_basic(self): oid = yield self.bucket.upload_from_stream("test_filename", b"hello world") gout = yield self.bucket.open_download_stream(oid) self.assertEqual(b"hello world", (yield gout.read())) self.assertEqual(1, (yield self.db.fs.files.count())) self.assertEqual(1, (yield self.db.fs.chunks.count())) yield self.bucket.delete(oid) with self.assertRaises(NoFile): yield self.bucket.open_download_stream(oid) self.assertEqual(0, (yield self.db.fs.files.count())) self.assertEqual(0, (yield self.db.fs.chunks.count())) gin = self.bucket.open_upload_stream("test_filename") yield gin.write(b"hello world") yield gin.close() dst = BytesIO() yield self.bucket.download_to_stream(gin._id, dst) self.assertEqual(b"hello world", dst.getvalue()) motor-1.2.1/test/tornado_tests/test_motor_ipv6.py000066400000000000000000000041051323007662700222450ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import unittest from pymongo import MongoClient from pymongo.errors import ConnectionFailure from tornado.testing import gen_test import motor import test from test import SkipTest from test.test_environment import connected, db_user, db_password, env from test.tornado_tests import MotorTest class MotorIPv6Test(MotorTest): @gen_test def test_ipv6(self): assert env.host in ('localhost', '127.0.0.1'), ( "This unittest isn't written to test IPv6 with host %s" % repr(env.host)) try: connected(MongoClient("[::1]", username=db_user, password=db_password, serverSelectionTimeoutMS=100)) except ConnectionFailure: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest("No IPV6") if test.env.auth: cx_string = 'mongodb://%s:%s@[::1]:%d' % ( db_user, db_password, env.port) else: cx_string = 'mongodb://[::1]:%d' % env.port cx = motor.MotorClient(cx_string, io_loop=self.io_loop) collection = cx.motor_test.test_collection yield collection.insert_one({"dummy": "object"}) self.assertTrue((yield collection.find_one({"dummy": "object"}))) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_replica_set.py000066400000000000000000000112151323007662700236530ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test replica set MotorClient.""" import unittest import pymongo import pymongo.auth import pymongo.errors import pymongo.mongo_replica_set_client from tornado import gen from tornado.testing import gen_test import motor import motor.core import test from test import SkipTest from test.test_environment import db_user, db_password, env from test.tornado_tests import MotorReplicaSetTestBase, MotorTest from test.utils import one, get_primary_pool from motor.motor_py3_compat import text_type class MotorReplicaSetTest(MotorReplicaSetTestBase): def test_io_loop(self): with self.assertRaises(TypeError): motor.MotorClient(test.env.rs_uri, io_loop='foo') @gen_test def test_connection_failure(self): # Assuming there isn't anything actually running on this port client = motor.MotorClient( 'localhost:8765', replicaSet='rs', io_loop=self.io_loop, serverSelectionTimeoutMS=10) # Test the Future interface. with self.assertRaises(pymongo.errors.ConnectionFailure): yield client.admin.command('ismaster') # Test with a callback. (result, error), _ = yield gen.Task(client.admin.command, 'ismaster') self.assertEqual(None, result) self.assertTrue(isinstance(error, pymongo.errors.ConnectionFailure)) @gen_test def test_socketKeepAlive(self): # Connect. yield self.rsc.server_info() ka = get_primary_pool(self.rsc).opts.socket_keepalive self.assertTrue(ka) client = self.motor_rsc(socketKeepAlive=False) yield client.server_info() ka = get_primary_pool(client).opts.socket_keepalive self.assertFalse(ka) @gen_test def test_auth_network_error(self): if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Make sure there's no semaphore leak if we get a network error # when authenticating a new socket with cached credentials. # Get a client with one socket so we detect if it's leaked. c = self.motor_rsc(maxPoolSize=1, waitQueueTimeoutMS=1) yield c.admin.command('ismaster') # Simulate an authenticate() call on a different socket. credentials = pymongo.auth._build_credentials_tuple( 'DEFAULT', 'admin', text_type(db_user), text_type(db_password), {}) c.delegate._cache_credentials('test', credentials, connect=False) # Cause a network error on the actual socket. pool = get_primary_pool(c) socket_info = one(pool.sockets) socket_info.sock.close() # In __check_auth, the client authenticates its socket with the # new credential, but gets a socket.error. Should be reraised as # AutoReconnect. with self.assertRaises(pymongo.errors.AutoReconnect): yield c.test.collection.find_one() # No semaphore leak, the pool is allowed to make a new socket. yield c.test.collection.find_one() @gen_test def test_open_concurrent(self): # MOTOR-66: don't block on PyMongo's __monitor_lock, but also don't # spawn multiple monitors. c = self.motor_rsc() yield [c.db.collection.find_one(), c.db.collection.find_one()] class TestReplicaSetClientAgainstStandalone(MotorTest): """This is a funny beast -- we want to run tests for a replica set MotorClient but only if the database at DB_IP and DB_PORT is a standalone. """ def setUp(self): super(TestReplicaSetClientAgainstStandalone, self).setUp() if test.env.is_replica_set: raise SkipTest( "Connected to a replica set, not a standalone mongod") @gen_test def test_connect(self): with self.assertRaises(pymongo.errors.ServerSelectionTimeoutError): yield motor.MotorClient( '%s:%s' % (env.host, env.port), replicaSet='anything', io_loop=self.io_loop, serverSelectionTimeoutMS=10).test.test.find_one() if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_session.py000066400000000000000000000166301323007662700230520ustar00rootroot00000000000000# Copyright 2017-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import copy import sys import unittest from pymongo import InsertOne, IndexModel from tornado import gen from pymongo.errors import InvalidOperation from tornado.testing import gen_test from test import SkipTest from test.test_environment import env from test.tornado_tests import MotorTest from test.utils import SessionTestListener, session_ids class MotorSessionTest(MotorTest): @classmethod def setUpClass(cls): super(MotorSessionTest, cls).setUpClass() if not env.sessions_enabled: raise SkipTest("Sessions not supported") @gen.coroutine def _test_ops(self, client, *ops): listener = client.event_listeners()[0][0] for f, args, kw in ops: with (yield client.start_session()) as s: listener.results.clear() # In case "f" modifies its inputs. args2 = copy.copy(args) kw2 = copy.copy(kw) kw2['session'] = s yield f(*args2, **kw2) for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( f.__name__, event.command_name)) self.assertEqual( s.session_id, event.command['lsid'], "%s sent wrong lsid with %s" % ( f.__name__, event.command_name)) self.assertFalse(s.has_ended) with self.assertRaises(InvalidOperation) as ctx: yield f(*args2, **kw2) self.assertIn("ended session", str(ctx.exception)) # No explicit session. for f, args, kw in ops: listener.results.clear() yield f(*args, **kw) self.assertGreaterEqual(len(listener.results['started']), 1) lsids = [] for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "%s sent no lsid with %s" % ( f.__name__, event.command_name)) lsids.append(event.command['lsid']) if 'PyPy' not in sys.version: # Server session was returned to pool. Ignore interpreters with # non-deterministic GC. for lsid in lsids: self.assertIn( lsid, session_ids(client), "%s did not return implicit session to pool" % ( f.__name__,)) @gen_test def test_database(self): listener = SessionTestListener() client = self.motor_client(event_listeners=[listener]) db = client.pymongo_test ops = [ (db.command, ['ping'], {}), (db.drop_collection, ['collection'], {}), (db.create_collection, ['collection'], {}), (db.collection_names, [], {}), ] yield self._test_ops(client, *ops) @gen_test(timeout=30) def test_collection(self): listener = SessionTestListener() client = self.motor_client(event_listeners=[listener]) yield client.drop_database('motor_test') coll = client.motor_test.test_collection @gen.coroutine def list_indexes(session=None): yield coll.list_indexes(session=session).to_list(length=None) @gen.coroutine def aggregate(session=None): yield coll.aggregate([], session=session).to_list(length=None) # Test some collection methods - the rest are in test_cursor. yield self._test_ops( client, (coll.drop, [], {}), (coll.bulk_write, [[InsertOne({})]], {}), (coll.insert_one, [{}], {}), (coll.insert_many, [[{}, {}]], {}), (coll.replace_one, [{}, {}], {}), (coll.update_one, [{}, {'$set': {'a': 1}}], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_one, [{}], {}), (coll.delete_many, [{}], {}), (coll.find_one_and_replace, [{}, {}], {}), (coll.find_one_and_update, [{}, {'$set': {'a': 1}}], {}), (coll.find_one_and_delete, [{}, {}], {}), (coll.rename, ['collection2'], {}), # Drop collection2 between tests of "rename", above. (client.motor_test.drop_collection, ['collection2'], {}), (coll.distinct, ['a'], {}), (coll.find_one, [], {}), (coll.count, [], {}), (coll.create_indexes, [[IndexModel('a')]], {}), (coll.create_index, ['a'], {}), (coll.drop_index, ['a_1'], {}), (coll.drop_indexes, [], {}), (coll.reindex, [], {}), (list_indexes, [], {}), (coll.index_information, [], {}), (coll.options, [], {}), (aggregate, [], {})) @gen_test def test_cursor(self): listener = SessionTestListener() client = self.motor_client(event_listeners=[listener]) yield self.make_test_data() coll = client.motor_test.test_collection with (yield client.start_session()) as s: listener.results.clear() cursor = coll.find(session=s) yield cursor.to_list(length=None) self.assertEqual(len(listener.results['started']), 2) for event in listener.results['started']: self.assertTrue( 'lsid' in event.command, "find sent no lsid with %s" % (event.command_name,)) self.assertEqual( s.session_id, event.command['lsid'], "find sent wrong lsid with %s" % (event.command_name,)) with self.assertRaises(InvalidOperation) as ctx: yield coll.find(session=s).to_list(length=None) self.assertIn("ended session", str(ctx.exception)) # No explicit session. listener.results.clear() cursor = coll.find() yield cursor.to_list(length=None) self.assertEqual(len(listener.results['started']), 2) event0 = listener.first_command_started() self.assertTrue( 'lsid' in event0.command, "find sent no lsid with %s" % (event0.command_name,)) lsid = event0.command['lsid'] for event in listener.results['started'][1:]: self.assertTrue( 'lsid' in event.command, "find sent no lsid with %s" % (event.command_name,)) self.assertEqual( lsid, event.command['lsid'], "find sent wrong lsid with %s" % (event.command_name,)) if __name__ == '__main__': unittest.main() motor-1.2.1/test/tornado_tests/test_motor_son_manipulator.py000066400000000000000000000075641323007662700246070ustar00rootroot00000000000000# Copyright 2013-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import pymongo.son_manipulator from tornado.testing import gen_test from test import env, SkipTest from test.tornado_tests import MotorTest from test.utils import ignore_deprecations class CustomSONManipulator(pymongo.son_manipulator.SONManipulator): """A pymongo outgoing SON Manipulator that adds ``{'added_field' : 42}`` """ def will_copy(self): return False def transform_outgoing(self, son, collection): assert 'added_field' not in son son['added_field'] = 42 return son class SONManipulatorTest(MotorTest): def _clear_collection(self): env.sync_cx.motor_test.son_manipulator_test_collection.delete_many({}) def setUp(self): super(SONManipulatorTest, self).setUp() self._clear_collection() def tearDown(self): self._clear_collection() super(SONManipulatorTest, self).tearDown() @ignore_deprecations @gen_test def test_with_find_one(self): coll = self.cx.motor_test.son_manipulator_test_collection _id = yield coll.insert({'foo': 'bar'}) self.assertEqual( {'_id': _id, 'foo': 'bar'}, (yield coll.find_one())) # Add SONManipulator and test again. coll.database.add_son_manipulator(CustomSONManipulator()) self.assertEqual( {'_id': _id, 'foo': 'bar', 'added_field': 42}, (yield coll.find_one())) @ignore_deprecations @gen_test def test_with_fetch_next(self): coll = self.cx.motor_test.son_manipulator_test_collection coll.database.add_son_manipulator(CustomSONManipulator()) _id = yield coll.insert({'foo': 'bar'}) cursor = coll.find() self.assertTrue((yield cursor.fetch_next)) self.assertEqual( {'_id': _id, 'foo': 'bar', 'added_field': 42}, cursor.next_object()) yield cursor.close() @ignore_deprecations @gen_test def test_with_to_list(self): coll = self.cx.motor_test.son_manipulator_test_collection _id1, _id2 = yield coll.insert([{}, {}]) found = yield coll.find().sort([('_id', 1)]).to_list(length=2) self.assertEqual([{'_id': _id1}, {'_id': _id2}], found) coll.database.add_son_manipulator(CustomSONManipulator()) expected = [ {'_id': _id1, 'added_field': 42}, {'_id': _id2, 'added_field': 42}] cursor = coll.find().sort([('_id', 1)]) found = yield cursor.to_list(length=2) self.assertEqual(expected, found) yield cursor.close() @ignore_deprecations @gen_test def test_with_aggregate(self): coll = self.cx.motor_test.son_manipulator_test_collection _id = yield coll.insert({'foo': 'bar'}) coll.database.add_son_manipulator(CustomSONManipulator()) # Test aggregation cursor, both with fetch_next and to_list. cursor = coll.aggregate([]) assert (yield cursor.fetch_next) self.assertEqual( {'_id': _id, 'foo': 'bar', 'added_field': 42}, cursor.next_object()) cursor = coll.aggregate([]) self.assertEqual( [{'_id': _id, 'foo': 'bar', 'added_field': 42}], (yield cursor.to_list(length=None))) motor-1.2.1/test/tornado_tests/test_motor_ssl.py000066400000000000000000000176661323007662700222020ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test Motor, an asynchronous driver for MongoDB and Tornado.""" import os try: import ssl except ImportError: ssl = None try: # Python 2. from urllib import quote_plus except ImportError: # Python 3. from urllib.parse import quote_plus from pymongo.errors import (ConfigurationError, ConnectionFailure, OperationFailure) from tornado.testing import gen_test import motor import test from test import SkipTest from test.tornado_tests import MotorTest, remove_all_users from test.test_environment import (CA_PEM, CLIENT_PEM, env, MONGODB_X509_USERNAME) # Start a mongod instance like: # # mongod \ # --sslOnNormalPorts \ # --sslPEMKeyFile test/certificates/server.pem \ # --sslCAFile test/certificates/ca.pem # # Also, make sure you have 'server' as an alias for localhost in /etc/hosts class MotorSSLTest(MotorTest): ssl = True def setUp(self): if not test.env.server_is_resolvable: raise SkipTest("The hostname 'server' must be a localhost alias") super(MotorSSLTest, self).setUp() def test_config_ssl(self): self.assertRaises(ValueError, motor.MotorClient, ssl='foo') self.assertRaises(ConfigurationError, motor.MotorClient, ssl=False, ssl_certfile=CLIENT_PEM) self.assertRaises(IOError, motor.MotorClient, ssl_certfile="NoFile") self.assertRaises(TypeError, motor.MotorClient, ssl_certfile=True) self.assertRaises(IOError, motor.MotorClient, ssl_keyfile="NoFile") self.assertRaises(TypeError, motor.MotorClient, ssl_keyfile=True) @gen_test def test_cert_ssl(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("can't test with auth") client = motor.MotorClient(env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one() response = yield client.admin.command('ismaster') if 'setName' in response: client = self.motor_rsc(ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM) yield client.db.collection.find_one() @gen_test def test_cert_ssl_validation(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("can't test with auth") client = motor.MotorClient( env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one() response = yield client.admin.command('ismaster') if 'setName' in response: client = motor.MotorClient( env.host, env.port, replicaSet=response['setName'], ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one() @gen_test def test_cert_ssl_validation_none(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("can't test with auth") client = motor.MotorClient( test.env.fake_hostname_uri, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_NONE, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.admin.command('ismaster') @gen_test def test_cert_ssl_validation_hostname_fail(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("can't test with auth") client = motor.MotorClient( env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) response = yield client.admin.command('ismaster') with self.assertRaises(ConnectionFailure): # Create client with hostname 'server', not 'localhost', # which is what the server cert presents. client = motor.MotorClient( test.env.fake_hostname_uri, serverSelectionTimeoutMS=100, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one() if 'setName' in response: with self.assertRaises(ConnectionFailure): client = motor.MotorClient( test.env.fake_hostname_uri, serverSelectionTimeoutMS=100, replicaSet=response['setName'], ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one() @gen_test def test_mongodb_x509_auth(self): if 'EVERGREEN' in os.environ: raise SkipTest("TODO: fix on Evergreen") # Expects the server to be running with SSL config described above, # and with "--auth". if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") # self.env.uri includes username and password. authenticated_client = motor.MotorClient( test.env.uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield authenticated_client['$external'].add_user( MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) # Not authenticated. client = motor.MotorClient( "server", test.env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) with self.assertRaises(OperationFailure): yield client.motor_test.test.count() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), "server", test.env.port)) # SSL options aren't supported in the URI.... auth_uri_client = motor.MotorClient( uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield auth_uri_client.db.collection.find_one() # Cleanup. yield remove_all_users(authenticated_client['$external']) yield authenticated_client['$external'].logout() motor-1.2.1/test/tornado_tests/test_motor_web.py000066400000000000000000000214511323007662700221410ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals """Test utilities for using Motor with Tornado web applications.""" import datetime import email import hashlib import time import re import unittest import gridfs from tornado.testing import AsyncHTTPTestCase from tornado.web import Application import motor import motor.web import test from test.test_environment import env, CA_PEM, CLIENT_PEM # We're using Tornado's AsyncHTTPTestCase instead of our own MotorTestCase for # the convenience of self.fetch(). class GridFSHandlerTestBase(AsyncHTTPTestCase): def setUp(self): super(GridFSHandlerTestBase, self).setUp() self.fs = gridfs.GridFS(test.env.sync_cx.motor_test) # Make a 500k file in GridFS with filename 'foo' self.contents = b'Jesse' * 100 * 1024 self.contents_hash = hashlib.md5(self.contents).hexdigest() # Record when we created the file, to check the Last-Modified header self.put_start = datetime.datetime.utcnow().replace(microsecond=0) self.file_id = 'id' self.fs.delete(self.file_id) self.fs.put( self.contents, _id='id', filename='foo', content_type='my type') self.put_end = datetime.datetime.utcnow().replace(microsecond=0) self.assertTrue(self.fs.get_last_version('foo')) def motor_db(self, **kwargs): if env.mongod_started_with_ssl: kwargs.setdefault('ssl_certfile', CLIENT_PEM) kwargs.setdefault('ssl_ca_certs', CA_PEM) kwargs.setdefault('ssl', env.mongod_started_with_ssl) client = motor.MotorClient( test.env.uri, io_loop=self.io_loop, **kwargs) return client.motor_test def tearDown(self): self.fs.delete(self.file_id) super(GridFSHandlerTestBase, self).tearDown() def get_app(self): return Application([ ('/(.+)', motor.web.GridFSHandler, {'database': self.motor_db()})]) def stop(self, *args, **kwargs): # A stop() method more permissive about the number of its positional # arguments than AsyncHTTPTestCase.stop if len(args) == 1: AsyncHTTPTestCase.stop(self, args[0], **kwargs) else: AsyncHTTPTestCase.stop(self, args, **kwargs) def parse_date(self, d): date_tuple = email.utils.parsedate(d) return datetime.datetime.fromtimestamp(time.mktime(date_tuple)) def last_mod(self, response): """Parse the 'Last-Modified' header from an HTTP response into a datetime. """ return self.parse_date(response.headers['Last-Modified']) def expires(self, response): return self.parse_date(response.headers['Expires']) class GridFSHandlerTest(GridFSHandlerTestBase): def test_basic(self): # First request response = self.fetch('/foo') self.assertEqual(200, response.code) self.assertEqual(self.contents, response.body) self.assertEqual( len(self.contents), int(response.headers['Content-Length'])) self.assertEqual('my type', response.headers['Content-Type']) self.assertEqual('public', response.headers['Cache-Control']) self.assertTrue('Expires' not in response.headers) etag = response.headers['Etag'] last_mod_dt = self.last_mod(response) self.assertEqual(self.contents_hash, etag.strip('"')) self.assertTrue(self.put_start <= last_mod_dt <= self.put_end) # Now check we get 304 NOT MODIFIED responses as appropriate for ims_value in ( last_mod_dt, last_mod_dt + datetime.timedelta(seconds=1) ): response = self.fetch('/foo', if_modified_since=ims_value) self.assertEqual(304, response.code) self.assertEqual(b'', response.body) # If-Modified-Since in the past, get whole response back response = self.fetch( '/foo', if_modified_since=last_mod_dt - datetime.timedelta(seconds=1)) self.assertEqual(200, response.code) self.assertEqual(self.contents, response.body) # Matching Etag response = self.fetch('/foo', headers={'If-None-Match': etag}) self.assertEqual(304, response.code) self.assertEqual(b'', response.body) # Mismatched Etag response = self.fetch('/foo', headers={'If-None-Match': etag + 'a'}) self.assertEqual(200, response.code) self.assertEqual(self.contents, response.body) def test_404(self): response = self.fetch('/bar') self.assertEqual(404, response.code) def test_head(self): response = self.fetch('/foo', method='HEAD') # Get Etag and parse Last-Modified into a datetime etag = response.headers['Etag'] last_mod_dt = self.last_mod(response) # Test the result self.assertEqual(200, response.code) self.assertEqual(b'', response.body) # Empty body for HEAD request self.assertEqual( len(self.contents), int(response.headers['Content-Length'])) self.assertEqual('my type', response.headers['Content-Type']) self.assertEqual(self.contents_hash, etag.strip('"')) self.assertTrue(self.put_start <= last_mod_dt <= self.put_end) self.assertEqual('public', response.headers['Cache-Control']) def test_content_type(self): # Check that GridFSHandler uses file extension to guess Content-Type # if not provided for filename, expected_type in [ ('foo.jpg', 'jpeg'), ('foo.png', 'png'), ('ht.html', 'html'), ('jscr.js', 'javascript'), ]: # 'fs' is PyMongo's blocking GridFS self.fs.put(b'', filename=filename) for method in 'GET', 'HEAD': response = self.fetch('/' + filename, method=method) self.assertEqual(200, response.code) # mimetypes are platform-defined, be fuzzy self.assertTrue( response.headers['Content-Type'].lower().endswith( expected_type)) class TZAwareGridFSHandlerTest(GridFSHandlerTestBase): def motor_db(self): return super(TZAwareGridFSHandlerTest, self).motor_db(tz_aware=True) def test_tz_aware(self): now = datetime.datetime.utcnow() ago = now - datetime.timedelta(minutes=10) hence = now + datetime.timedelta(minutes=10) response = self.fetch('/foo', if_modified_since=ago) self.assertEqual(200, response.code) response = self.fetch('/foo', if_modified_since=hence) self.assertEqual(304, response.code) class CustomGridFSHandlerTest(GridFSHandlerTestBase): def get_app(self): class CustomGridFSHandler(motor.web.GridFSHandler): def get_gridfs_file(self, bucket, filename, request): # Test overriding the get_gridfs_file() method, path is # interpreted as file_id instead of filename. return bucket.open_download_stream(file_id=filename) def get_cache_time(self, path, modified, mime_type): return 10 def set_extra_headers(self, path, gridout): self.set_header('quux', 'fizzledy') return Application([ ('/(.+)', CustomGridFSHandler, {'database': self.motor_db()})]) def test_get_gridfs_file(self): # We overrode get_gridfs_file so we expect getting by filename *not* to # work now; we'll get a 404. We have to get by file_id now. response = self.fetch('/foo') self.assertEqual(404, response.code) response = self.fetch('/' + str(self.file_id)) self.assertEqual(200, response.code) self.assertEqual(self.contents, response.body) cache_control = response.headers['Cache-Control'] self.assertTrue(re.match(r'max-age=\d+', cache_control)) self.assertEqual(10, int(cache_control.split('=')[1])) expires = self.expires(response) # It should expire about 10 seconds from now self.assertTrue( datetime.timedelta(seconds=8) < expires - datetime.datetime.utcnow() < datetime.timedelta(seconds=12)) self.assertEqual('fizzledy', response.headers['quux']) if __name__ == '__main__': unittest.main() motor-1.2.1/test/utils.py000066400000000000000000000052111323007662700153510ustar00rootroot00000000000000# Copyright 2012-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals from collections import defaultdict from pymongo import monitoring """Utilities for testing Motor with any framework.""" import contextlib import functools import warnings def one(s): """Get one element of a set""" return next(iter(s)) def safe_get(dct, dotted_key, default=None): for key in dotted_key.split('.'): if key not in dct: return default dct = dct[key] return dct # Use as a decorator or in a "with" statement. def ignore_deprecations(fn=None): if fn: @functools.wraps(fn) def wrapper(*args, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) return fn(*args, **kwargs) return wrapper else: @contextlib.contextmanager def ignore_deprecations_context(): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) yield return ignore_deprecations_context() def get_primary_pool(client): for s in client.delegate._topology._servers.values(): if s.description.is_writable: return s.pool # Ignore auth commands like saslStart, so we can assert lsid is in all commands. class SessionTestListener(monitoring.CommandListener): def __init__(self): self.results = defaultdict(list) def started(self, event): if not event.command_name.startswith('sasl'): self.results['started'].append(event) def succeeded(self, event): if not event.command_name.startswith('sasl'): self.results['succeeded'].append(event) def failed(self, event): if not event.command_name.startswith('sasl'): self.results['failed'].append(event) def first_command_started(self): assert len(self.results['started']) >= 1, ( "No command-started events") return self.results['started'][0] def session_ids(client): return [s.session_id for s in client.delegate._topology._session_pool] motor-1.2.1/test/version.py000066400000000000000000000056621323007662700157100ustar00rootroot00000000000000# Copyright 2009-2015 MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Some tools for running tests based on MongoDB server version.""" class Version(tuple): def __new__(cls, *version): padded_version = cls._padded(version, 4) return super(Version, cls).__new__(cls, tuple(padded_version)) @classmethod def _padded(cls, iter, length, padding=0): l = list(iter) if len(l) < length: for _ in range(length - len(l)): l.append(padding) return l @classmethod def from_string(cls, version_string): mod = 0 bump_patch_level = False if version_string.endswith("+"): version_string = version_string[0:-1] mod = 1 elif version_string.endswith("-pre-"): version_string = version_string[0:-5] mod = -1 elif version_string.endswith("-"): version_string = version_string[0:-1] mod = -1 # Deal with '-rcX' substrings if '-rc' in version_string: version_string = version_string[0:version_string.find('-rc')] mod = -1 # Deal with git describe generated substrings elif '-' in version_string: version_string = version_string[0:version_string.find('-')] mod = -1 bump_patch_level = True version = [int(part) for part in version_string.split(".")] version = cls._padded(version, 3) # Make from_string and from_version_array agree. For example: # MongoDB Enterprise > db.runCommand('buildInfo').versionArray # [ 3, 2, 1, -100 ] # MongoDB Enterprise > db.runCommand('buildInfo').version # 3.2.0-97-g1ef94fe if bump_patch_level: version[-1] += 1 version.append(mod) return Version(*version) @classmethod def from_version_array(cls, version_array): version = list(version_array) if version[-1] < 0: version[-1] = -1 version = cls._padded(version, 3) return Version(*version) @classmethod def from_client(cls, client): info = client.server_info() if 'versionArray' in info: return cls.from_version_array(info['versionArray']) return cls.from_string(info['version']) def at_least(self, *other_version): return self >= Version(*other_version) def __str__(self): return ".".join(map(str, self)) motor-1.2.1/tox.ini000066400000000000000000000054101323007662700141740ustar00rootroot00000000000000# Tox (https://tox.readthedocs.io) is a tool for running tests in multiple # virtualenvs. "pip install tox" and run "tox" from this directory. # Adapted from Tornado's tox.ini. [tox] envlist = tornado4-{py27,pypy,pypy3,py34,py35,py36}, # Test Tornado on master in a few configurations. tornado_git-{py27,py34,py35,py36}, # Ensure the sphinx build has no errors or warnings. py3-sphinx-docs, # Run the doctests, include examples and tutorial, via Sphinx. py3-sphinx-doctest, # asyncio without Tornado. asyncio-{py34,py35,py36}, # Test PyMongo HEAD, not the latest release. py3-pymongo-master, # Apply PyMongo's test suite to Motor via Synchro. synchro [testenv] passenv = DB_IP DB_PORT DB_USER DB_PASSWORD CERT_DIR ASYNC_TEST_TIMEOUT basepython = py27,synchro: {env:PYTHON_BINARY:python2.7} py34: {env:PYTHON_BINARY:python3.4} py35: {env:PYTHON_BINARY:python3.5} py36: {env:PYTHON_BINARY:python3.6} pypy: {env:PYTHON_BINARY:pypy} pypy3: {env:PYTHON_BINARY:pypy3} # Default Python 3 when we don't care about minor version. py3: {env:PYTHON_BINARY:python3.6} deps = tornado4: tornado>=4,<5 tornado_git: git+https://github.com/tornadoweb/tornado.git {py27,pypy}: futures # aiohttp requires Python 3.4.2+. {py34,py35,py36}: aiohttp sphinx: sphinx sphinx: aiohttp sphinx: git+https://github.com/tornadoweb/tornado.git py3-pymongo-master: git+https://github.com/mongodb/mongo-python-driver.git py3-pymongo-master: tornado>=4,<5 synchro: pymongo==3.6.0 synchro: tornado>=4,<5 synchro: nose synchro: futures commands = python setup.py test --xunit-output=xunit-results {posargs} [testenv:tornado_git-py27] install_command = pip install --force-reinstall {opts} {packages} [testenv:tornado_git-py34] install_command = pip install --force-reinstall {opts} {packages} [testenv:tornado_git-py35] install_command = pip install --force-reinstall {opts} {packages} [testenv:tornado_git-py36] install_command = pip install --force-reinstall {opts} {packages} [testenv:py3-sphinx-docs] install_command = pip install --force-reinstall {opts} {packages} changedir = doc commands = sphinx-build -q -E -W -b html . {envtmpdir}/html {posargs} [testenv:py3-sphinx-doctest] setenv = PYTHONHASHSEED=0 changedir = doc commands = sphinx-build -q -E -b doctest . {envtmpdir}/doctest {posargs} [testenv:synchro] whitelist_externals = git cd commands = git clone https://github.com/mongodb/mongo-python-driver.git {envtmpdir}/mongo-python-driver git --work-tree {envtmpdir}/mongo-python-driver --git-dir {envtmpdir}/mongo-python-driver/.git checkout 3.6.0 python -m synchro.synchrotest -v -w {envtmpdir}/mongo-python-driver {posargs}