pax_global_header00006660000000000000000000000064145261257320014521gustar00rootroot0000000000000052 comment=edc0aaa6c597ab84e93424fd1bbeaee6d3c3544e pytest-order-1.2.0/000077500000000000000000000000001452612573200141625ustar00rootroot00000000000000pytest-order-1.2.0/.coveragerc000066400000000000000000000000241452612573200162770ustar00rootroot00000000000000[run] branch = True pytest-order-1.2.0/.github/000077500000000000000000000000001452612573200155225ustar00rootroot00000000000000pytest-order-1.2.0/.github/workflows/000077500000000000000000000000001452612573200175575ustar00rootroot00000000000000pytest-order-1.2.0/.github/workflows/performance.yml000066400000000000000000000021021452612573200225760ustar00rootroot00000000000000name: PerformanceTests on: [push, pull_request] defaults: run: shell: bash jobs: test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest] python-version: [3.7, 3.11] env: OS: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Get pip cache dir id: pip-cache run: | python -m pip install -U pip echo "::set-output name=dir::$(pip cache dir)" - name: pip cache uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: py${{ matrix.python-version }}-${{ matrix.os }}-pip - name: Test environment setup run: | python -m pip install wheel python -m pip install pytest python -m pip install pytest-dependency python -m pip install -e . - name: Run tests run: | python -m pytest -s perf_tests pytest-order-1.2.0/.github/workflows/pythontests.yml000066400000000000000000000035211452612573200227070ustar00rootroot00000000000000name: Testsuite on: [push, pull_request] defaults: run: shell: bash jobs: test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] include: - python-version: "pypy-3.7" os: ubuntu-latest - python-version: "pypy-3.9" os: ubuntu-latest - python-version: "pypy-3.10" os: ubuntu-latest env: OS: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Get pip cache dir id: pip-cache run: | python -m pip install -U pip # to ensure version > 20 to have cache dir echo "::set-output name=dir::$(pip cache dir)" - name: pip cache uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: py${{ matrix.python-version }}-${{ matrix.os }}-pip - name: Test environment setup run: | python -m pip install wheel python -m pip install pytest pytest-cov tox - name: Run tests run: | TOX_PYTHON_VERSION=$(if [[ ${{ matrix.python-version }} == pypy* ]]; then echo ${{ matrix.python-version }} | tr -d .-; else echo py${{ matrix.python-version }} | tr -d .-; fi) COV_CMD=$(if [ ${{ matrix.python-version }} == 3.10 ]; then echo "--cov=./pytest_order/ --cov-report=xml"; else echo ; fi) tox -e $(tox -l | grep "$TOX_PYTHON_VERSION" | paste -sd "," -) - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 if: ${{ success() && matrix.python-version == 3.10 }} with: env_vars: OS name: codecov-pytest-order pytest-order-1.2.0/.github/workflows/release-deploy.yml000066400000000000000000000013011452612573200232070ustar00rootroot00000000000000name: release-deploy on: release: types: [ published ] jobs: deploy: runs-on: ubuntu-latest strategy: fail-fast: true matrix: python-version: [ '3.10' ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Build package run: | python -m pip install --upgrade pip python -m pip install build python -m build - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} pytest-order-1.2.0/.gitignore000066400000000000000000000006531452612573200161560ustar00rootroot00000000000000*.py[cod] # C extensions *.so # Packages *.egg *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg lib lib64 __pycache__ # Virtualenvs env # Sphinx documentation docs/build # Installer logs pip-log.txt # Unit test / coverage reports .coverage .tox nosetests.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject # Misc .cache # PyCharm .idea/ # pyenv .python-version pytest-order-1.2.0/.pre-commit-config.yaml000066400000000000000000000026061452612573200204470ustar00rootroot00000000000000default_language_version: python: "3.10" repos: - repo: https://github.com/codespell-project/codespell rev: v2.2.6 hooks: - id: codespell args: - --ignore-words-list=wronly,afile - repo: https://github.com/psf/black rev: 23.11.0 hooks: - id: black args: [ --safe, --quiet ] - repo: https://github.com/asottile/blacken-docs rev: 1.16.0 hooks: - id: blacken-docs additional_dependencies: [ black==22.12.0 ] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: fix-encoding-pragma args: [ --remove ] - id: check-yaml - id: debug-statements language_version: python3 - repo: https://github.com/PyCQA/autoflake rev: v2.2.1 hooks: - id: autoflake name: autoflake args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"] language: python files: \.py$ - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: - id: flake8 language_version: python3 additional_dependencies: - flake8-bugbear args: ["--extend-ignore=E203", "--max-line-length=88"] - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.7.0 hooks: - id: mypy exclude: (docs|pyfakefs/tests) pytest-order-1.2.0/.readthedocs.yaml000066400000000000000000000004141452612573200174100ustar00rootroot00000000000000# Read the Docs configuration file for Sphinx projects # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details version: 2 build: os: ubuntu-22.04 tools: python: "3.11" sphinx: configuration: docs/source/conf.py fail_on_warning: true pytest-order-1.2.0/.vscode/000077500000000000000000000000001452612573200155235ustar00rootroot00000000000000pytest-order-1.2.0/.vscode/launch.json000066400000000000000000000006641452612573200176760ustar00rootroot00000000000000{ // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "name": "Test plugin", "type": "python", "request": "launch", "module": "pytest", "args": ["-v"] } ] } pytest-order-1.2.0/.vscode/settings.json000066400000000000000000000004131452612573200202540ustar00rootroot00000000000000{ "restructuredtext.confPath": "${workspaceFolder}\\docs\\source", "python.formatting.blackArgs": [ "--line-length", "88", "--skip-string-normalization" ], "python.formatting.provider": "black", "editor.rulers": [79, 87] } pytest-order-1.2.0/AUTHORS000066400000000000000000000005331452612573200152330ustar00rootroot00000000000000Frank Tobia Sergei Chipiga Ben Greene Adam Talsma Brian Maissy Jonas Zinn Andrew Gilbert Bartosz Peszek mrbean-bremen pytest-order-1.2.0/CHANGELOG.md000066400000000000000000000206351452612573200160010ustar00rootroot00000000000000# pytest-order Release Notes ## [Version 1.2.0](https://pypi.org/project/pytest-order/1.2.0/) (2023-11-18) Allows using custom markers for ordering. ### New features * added option `--order-marker-prefix` to allow using custom markers for ordering ### Infrastructure - added pre-commit hook for linters - added Python 3.12 to supported versions ## [Version 1.1.0](https://pypi.org/project/pytest-order/1.1.0/) (2023-03-10) Adds support for executing tests more than once using order marks. ### New features - added support for multiple test order markers ### Changes - removed official support for Python 3.6 ### Infrastructure - added pytest 7.x to CI tests - added Python 3.11 to and removed Python 3.6 from CI tests - moved documentation to [Read the Docs](https://pytest-order.readthedocs.io) ## [Version 1.0.1](https://pypi.org/project/pytest-order/1.0.1/) (2022-01-09) Bugfix release. ### Fixes - correctly handle combined class and test order markers, see [#45](https://github.com/pytest-dev/pytest-order/issues/45) - adapt code to changed type of `automark_dependency` in development branch of `pytest-dependency`, see [#58](https://github.com/pytest-dev/pytest-order/issues/58) ## [Version 1.0.0](https://pypi.org/project/pytest-order/1.0.0/) (2021-05-30) First Python 3 only version. ### Breaking changes - removed support for Python 2.7 and 3.5 - removed official support for pytest < 5.0.0 (older versions may still work, but they are not tested) - changed notation of relative markers in other modules - instead of using the dot notation, the standard pytest nodeid is used, see [#24](https://github.com/pytest-dev/pytest-order/issues/24) - using parametrized test names that include the parameter is no longer supported, just use the name without the parameter instead ### New features - added support for the ``pytest-dependency`` option ``automark_dependency`` - added support for relative ordering of parametrized tests using the test name without the parameter value, see [#38](https://github.com/pytest-dev/pytest-order/issues/38) ### Infrastructure - re-added Python 3.10 to CI tests (for pytest >= 6.2.4) - added type hints ## [Version 0.11.0](https://pypi.org/project/pytest-order/0.11.0/) (2021-04-11) Adds support for multiple relative markers for the same test. ### New features - added support for more than one relative marker for the same test ### Infrastructure - added Python 3.10 to CI tests ## [Version 0.10.0](https://pypi.org/project/pytest-order/0.10.0/) (2021-03-18) Adds support for class-level relative markers and directory level scope. ### New features - added support for class level relative markers, see [#7](https://github.com/pytest-dev/pytest-order/issues/7) - added option `--order-scope-level` which allows grouping tests on the same directory level, see [#8](https://github.com/pytest-dev/pytest-order/issues/8) ### Fixes - fixed sorting of dependency markers that depend on an item with the same name in different modules ### Infrastructure - added performance tests to prevent performance degradation - added automated deploy workflow step triggered on creating a tag ## [Version 0.9.5](https://pypi.org/project/pytest-order/0.9.5/) (2021-02-16) Introduces hierarchical ordering option and fixes ordering of session-scoped dependency markers. ### Changes - tests with unresolved relative markers are now handled like tests without order markers instead of being enqueued after all other tests ### New features - added `group-order-scope` option to allow hierarchical ordering on module and class scope, see [#6](https://github.com/pytest-dev/pytest-order/issues/6) ### Fixes - the dependency marker scope is now considered for resolving marker names (module scope had been assumed before) - dependency markers in session scope referenced by the nodeid are now correctly sorted if using the `order-dependencies` option ## [Version 0.9.4](https://pypi.org/project/pytest-order/0.9.4/) (2021-01-27) Patch release to make packaging easier. ### Infrastructure - use codecov instead of coveralls, that is failing - added pytest 6.2 to CI tests - added tests, examples and documentation to source package, see [#5](https://github.com/pytest-dev/pytest-order/issues/5) ## [Version 0.9.3](https://pypi.org/project/pytest-order/0.9.3/) (2021-01-14) Bugfix release. ### Fixes - fixed handling of more than one attribute in an order marker ## [Version 0.9.2](https://pypi.org/project/pytest-order/0.9.2/) (2020-11-13) Friday the 13th release. ### Changes - changed definition of classes in before/after markers (now uses `::` as delimiter) ### Fixes - fixed handling of before/after markers in different classes and modules - fixed handling of names in dependencies - did not match the actual behavior of `pytest-dependency` ## [Version 0.9.1](https://pypi.org/project/pytest-order/0.9.1/) (2020-11-11) This is a bugfix only release. ### Fixes - fixed handling of relative markers in classes - fixed handling of dependencies (could have been added twice) ## [Version 0.9.0](https://pypi.org/project/pytest-order/0.9.0/) (2020-11-08) This is the last major version that will support Python 2 - Python 2 support will be dropped in version 1.0. There is no timeline for that release, as there are currently no new features planned - further development will be demand-driven. ### Changes - removed support for pytest 3.6 (it still may work, just isn't tested anymore) ### New features - added configuration option for sparse sorting, e.g. the possibility to fill gaps between ordinals with unordered tests (see also [this issue](https://github.com/ftobia/pytest-ordering/issues/14) in `pytest-ordering`) - ignore ordering if it would break a dependency defined by the `pytest-dependency` plugin - experimental: added configuration option for ordering all dependencies defined by the `pytest-dependency` plugin - added ``index`` keyword for ordering as alternative to raw number ### Fixes - correctly handle combined index and dependency attributes ### Infrastructure - added list of open issues in `pytest-ordering` with respective state in `pytest-order` ## [Version 0.8.1](https://pypi.org/project/pytest-order/0.8.1/) (2020-11-02) ### New features - added configuration option for sorting scope, see [#2](https://github.com/pytest-dev/pytest-order/issues/2) ## [Version 0.8.0](https://pypi.org/project/pytest-order/0.8.0/) (2020-10-30) This release is mostly related to the consolidation of infrastructure (documentation build and tests in CI build) and documentation. ### Fixes - fixed the handling of unknown marker attributes (test had been skipped) ### Infrastructure - added automatic documentation build on change - added Python 3.9, pypy3 and pytest 6.0 and 6.1 to CI builds - use GitHub Actions for CI builds to speed them up, added Windows CI builds - added regression test for ``pytest-xdist`` (imported from [PR #52](https://github.com/ftobia/pytest-ordering/pull/52)) ## [Version 0.7.1](https://pypi.org/project/pytest-order/0.7.1/) (2020-10-24) Update after renaming the repository and the package. ### Changes - renamed repository and package from ``pytest-ordering2`` to ``pytest-order`` - changed the used marker from ``run`` to ``order``, removed all additional markers (see [#38](https://github.com/ftobia/pytest-ordering/issues/38)) ### Documentation - use separate documentation pages for release and development versions ## Version 0.7.0 (2020-10-22) Imported version from [pytest-ordering](https://github.com/ftobia/pytest-ordering), including some PRs (manually merged). Note: this version has been removed from PyPi to avoid confusion with the changed name in the next release. ### New features - added support for markers like run(before=...), run(after=), run("first") etc. (imported from [PR #37](https://github.com/ftobia/pytest-ordering/pull/37)) - added ``--indulgent-ordering`` to request that the sort from pytest-ordering be run before other plugins. This allows the built-in ``--failed-first`` implementation to override the ordering. (imported from [PR #50](https://github.com/ftobia/pytest-ordering/pull/50)) - include LICENSE file in distribution (imported from [PR #68](https://github.com/ftobia/pytest-ordering/pull/68)) ### Infrastructure - added more pytest versions, fix pytest-cov compatibility issue, remove Python 3.4, add Python 3.8 (imported from [PR #74](https://github.com/ftobia/pytest-ordering/pull/74)) - moved documentation to [GitHub Pages](https://pytest-dev.github.io/pytest-order/) pytest-order-1.2.0/LICENSE000066400000000000000000000021741452612573200151730ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2014 Frank Tobia. Modifications copyright 2020- by contributors, as listed in AUTHORS. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. pytest-order-1.2.0/MANIFEST.in000066400000000000000000000002541452612573200157210ustar00rootroot00000000000000include LICENSE include AUTHORS include CHANGELOG.md include README.md include tox.ini graft tests graft docs graft example prune docs/build global-exclude __pycache__/* pytest-order-1.2.0/README.md000066400000000000000000000140441452612573200154440ustar00rootroot00000000000000_pytest-order_ - a pytest plugin to order test execution ======================================================== [![PyPI version](https://badge.fury.io/py/pytest-order.svg)](https://pypi.org/project/pytest-order) [![Testsuite](https://github.com/pytest-dev/pytest-order/workflows/Testsuite/badge.svg)](https://github.com/pytest-dev/pytest-order/actions?query=workflow%3ATestsuite) [![DocBuild](https://readthedocs.org/projects/pytest-order/badge/?version=latest)](https://pytest-order.readthedocs.io/en/latest/?badge=latest) [![codecov](https://codecov.io/gh/pytest-dev/pytest-order/branch/main/graph/badge.svg?token=M9PHWZSHUU)](https://codecov.io/gh/pytest-dev/pytest-order) [![Python version](https://img.shields.io/pypi/pyversions/pytest-order.svg)](https://pypi.org/project/pytest-order) `pytest-order` is a pytest plugin that allows you to customize the order in which your tests are run. It uses the marker `order` that defines when a specific test shall run, either by using an ordinal number, or by specifying the relationship to other tests. `pytest-order` is a fork of [pytest-ordering](https://github.com/ftobia/pytest-ordering) that provides additional features like ordering relative to other tests. `pytest-order` works with Python 3.7 - 3.12, with pytest versions >= 5.0.0 for all versions up to Python 3.9, and for pytest >= 6.2.4 for Python >= 3.10. `pytest-order` runs on Linux, macOS and Windows. Documentation ------------- Apart from this overview, the following information is available: - usage documentation for the [latest release](https://pytest-order.readthedocs.io/en/stable/) - usage documentation for the [current main branch](https://pytest-order.readthedocs.io/en/latest/) - most examples shown in the documentation can also be found in the [repository](https://github.com/pytest-dev/pytest-order/tree/main/example) - the [Release Notes](https://github.com/pytest-dev/pytest-order/blob/main/CHANGELOG.md) with a list of changes in the latest versions - a [list of open issues](https://github.com/pytest-dev/pytest-order/blob/main/old_issues.md) in the original project and their handling in `pytest-order` Features -------- `pytest-order` provides the following features: - ordering of tests [by index](https://pytest-order.readthedocs.io/en/stable/usage.html#ordering-by-numbers) - ordering of tests both from the start and from the end (via negative index) - ordering of tests [relative to each other](https://pytest-order.readthedocs.io/en/stable/usage.html#order-relative-to-other-tests) (via the `before` and `after` marker attributes) - session-, module- and class-scope ordering via the [order-scope](https://pytest-order.readthedocs.io/en/stable/configuration.html#order-scope) option - directory scope ordering via the [order-scope-level](https://pytest-order.readthedocs.io/en/stable/configuration.html#order-scope-level) option - hierarchical module and class-level ordering via the [order-group-scope](https://pytest-order.readthedocs.io/en/stable/configuration.html#order-group-scope) option - ordering tests with `pytest-dependency` markers if using the [order-dependencies](https://pytest-order.readthedocs.io/en/stable/configuration.html#order-dependencies) option, more information about `pytest-dependency` compatibility [here](https://pytest-order.readthedocs.io/en/stable/other_plugins.html#relationship-with-pytest-dependency) - sparse ordering of tests via the [sparse-ordering](https://pytest-order.readthedocs.io/en/stable/configuration.html#sparse-ordering) option - usage of custom markers for ordering using the [sparse-ordering](https://pytest-order.readthedocs.io/en/stable/configuration.html#order-marker-prefix) option Overview -------- _(adapted from the original project)_ Have you ever wanted to easily run one of your tests before any others run? Or run some tests last? Or run this one test before that other test? Or make sure that this group of tests runs after this other group of tests? Now you can. Install with: pip install pytest-order This defines the ``order`` marker that you can use in your code with different attributes. For example, this code: import pytest @pytest.mark.order(2) def test_foo(): assert True @pytest.mark.order(1) def test_bar(): assert True yields the output: $ pytest test_foo.py -vv ============================= test session starts ============================== platform darwin -- Python 3.7.1, pytest-5.4.3, py-1.8.1, pluggy-0.13.1 -- env/bin/python plugins: order collected 2 items test_foo.py:7: test_bar PASSED test_foo.py:3: test_foo PASSED =========================== 2 passed in 0.01 seconds =========================== Contributing ------------ Contributions are very welcome. Tests can be run with [tox](https://tox.readthedocs.io/en/latest/), please ensure the coverage at least stays the same before you submit a pull request. License ------- Distributed under the terms of the [MIT](http://opensource.org/licenses/MIT) license, `pytest-order` is free and open source software. History ------- This is a fork of [pytest-ordering](https://github.com/ftobia/pytest-ordering). That project is not maintained anymore, and there are several helpful PRs that are now integrated into `pytest-order`. The idea and most of the initial code has been created by Frank Tobia, the author of that plugin, and [contributors](https://github.com/pytest-dev/pytest-order/blob/main/AUTHORS). While derived from `pytest_ordering`, `pytest-order` is **not** compatible with `pytest-ordering` due to the changed marker name (`order` instead of `run`). Additional markers defined `pytest_ordering` are all integrated into the `order` marker (for a rationale see also [this issue](https://github.com/ftobia/pytest-ordering/issues/38)). Ordering relative to other tests and all the configuration options are not available in the released version of `pytest-ordering`. However, most of these features are derived from or inspired by [issues](https://github.com/pytest-dev/pytest-order/blob/main/old_issues.md) and pull requests already existing in `pytest-ordering`. pytest-order-1.2.0/docs/000077500000000000000000000000001452612573200151125ustar00rootroot00000000000000pytest-order-1.2.0/docs/Makefile000066400000000000000000000152131452612573200165540ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pytest-order.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pytest-order.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/pytest-order" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pytest-order" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." pytest-order-1.2.0/docs/make.bat000066400000000000000000000151021452612573200165160ustar00rootroot00000000000000@ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source set I18NSPHINXOPTS=%SPHINXOPTS% source if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. xml to make Docutils-native XML files echo. pseudoxml to make pseudoxml-XML files for display purposes echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) %SPHINXBUILD% 2> nul if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pytest-order.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pytest-order.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdf" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf cd %BUILDDIR%/.. echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdfja" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf-ja cd %BUILDDIR%/.. echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "texinfo" ( %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo if errorlevel 1 exit /b 1 echo. echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. goto end ) if "%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 echo. echo.Build finished. The message catalogs are in %BUILDDIR%/locale. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) if "%1" == "xml" ( %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml if errorlevel 1 exit /b 1 echo. echo.Build finished. The XML files are in %BUILDDIR%/xml. goto end ) if "%1" == "pseudoxml" ( %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml if errorlevel 1 exit /b 1 echo. echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. goto end ) :end pytest-order-1.2.0/docs/requirements.txt000066400000000000000000000000161452612573200203730ustar00rootroot00000000000000sphinx>=2.3.1 pytest-order-1.2.0/docs/source/000077500000000000000000000000001452612573200164125ustar00rootroot00000000000000pytest-order-1.2.0/docs/source/conf.py000066400000000000000000000206541452612573200177200ustar00rootroot00000000000000from typing import List, Dict # pytest-order documentation build configuration file, created by # sphinx-quickstart on Mon Mar 17 18:20:44 2014. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. import os import sys sys.path.insert(0, os.path.abspath("../..")) from pytest_order import __version__ # noqa: E402 # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # needs_sphinx = "1.0" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named "sphinx.ext.*") or your custom # ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.doctest", "sphinx.ext.coverage", "sphinx.ext.viewcode", "sphinx.ext.githubpages", # puts .nojekyll file into source ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix of source filenames. source_suffix = ".rst" # The encoding of source files. # source_encoding = "utf-8-sig" # The master toctree document. master_doc = "index" # General information about the project. project = "pytest-order" # The version info for the project you"re documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = __version__ # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = "" # Else, today_fmt is used as the format for a strftime call. # today_fmt = "%B %d, %Y" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns: List[str] = [] # The reST default role (used for this markup: `text`) to use for all # documents. # default_role = None # If true, "()" will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "nature" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path: List[str] = [] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. # html_extra_path = [] # If not "", a "Last updated on:" timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = "%b %d, %Y" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_domain_indices = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = False # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. html_show_copyright = False # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = "" # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = "pytest-orderdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements: Dict[str, str] = { # The paper size ("letterpaper" or "a4paper"). # "papersize": "letterpaper", # The font size ("10pt", "11pt" or "12pt"). # "pointsize": "10pt", # Additional stuff for the LaTeX preamble. # "preamble": "", } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ( "index", "pytest-order.tex", "pytest-order Documentation", "Frank Tobia", "manual", ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # If true, show page references after internal links. # latex_show_pagerefs = False # If true, show URL addresses after external links. # latex_show_urls = False # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ( "index", "pytest-order", "pytest-order Documentation", ["Frank Tobia"], 1, ) ] # If true, show URL addresses after external links. # man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( "index", "pytest-order", "pytest-order Documentation", "Frank Tobia", "pytest-order", "One line description of project.", "Miscellaneous", ), ] # Documents to append as an appendix to all manuals. # texinfo_appendices = [] # If false, no module index is generated. # texinfo_domain_indices = True # How to display URL addresses: "footnote", "no", or "inline". # texinfo_show_urls = "footnote" # If true, do not generate a @detailmenu in the "Top" node"s menu. # texinfo_no_detailmenu = False pytest-order-1.2.0/docs/source/configuration.rst000066400000000000000000000433161452612573200220220ustar00rootroot00000000000000Configuration ============= There are a few command line options that change the behavior of the plugin. As with any pytest option, you can add the options to your ``pytest.ini`` if you want to have them applied to all tests automatically: .. code:: [pytest] ; always order tests with dependency markers addopts = --order-dependencies .. _order-scope: ``--order-scope`` ----------------- By default, tests are ordered per session, e.g. across all modules in the test run. Sometimes, you want to order tests per module or per test class instead. Consider that you have a growing number of test modules that you want to run simultaneously, with tests ordered per module. Per default you would need to make sure that the order numbers increases globally, if you want to run the test modules consecutively and order the test per module. If you use the option ``--order-scope=module``, there is no need for this. You can enumerate your tests starting with 0 or 1 in each module, and the tests will only be ordered inside each module. Using ``--order-scope=class`` additionally considers test classes--each test class is considered separately for ordering the tests. If a module has both test classes and separate test functions, these test functions are handled separately from the test classes. If a module has no test classes, the effect is the same as if using ``--order-scope=module``. For example consider two test modules: **tests/test_module1.py**: .. code:: import pytest @pytest.mark.order(2) def test2(): pass @pytest.mark.order(1) def test1(): pass **tests/test_module2.py**: .. code:: import pytest @pytest.mark.order(2) def test2(): pass @pytest.mark.order(1) def test1(): pass Here is what you get using session and module-scoped sorting:: $ pytest tests -vv ============================= test session starts ============================== ... tests/test_module1.py:9: test1 PASSED tests/test_module2.py:9: test1 PASSED tests/test_module1.py:5: test2 PASSED tests/test_module2.py:5: test2 PASSED :: $ pytest tests -vv --order-scope=module ============================= test session starts ============================== ... tests/test_module1.py:9: test1 PASSED tests/test_module1.py:5: test2 PASSED tests/test_module2.py:9: test1 PASSED tests/test_module2.py:5: test2 PASSED ``--order-scope-level`` ----------------------- This is an alternative option to define the order scope. It defines the directory level which is used as the order scope, counting from the root directory. The resulting scope is between the session and module scopes defined via ``--order-scope``, where ``--order-scope-level=0`` is the same as session scope, while setting the level to the number of test directory levels would result in module scope. Consider the following directory structure:: order_scope_level feature1 __init__.py test_a.py test_b.py feature2 __init__.py test_a.py test_b.py with the test contents: **test_a.py**: .. code:: import pytest @pytest.mark.order(4) def test_four(): pass @pytest.mark.order(3) def test_three(): pass **test_b.py**: .. code:: import pytest @pytest.mark.order(2) def test_two(): pass @pytest.mark.order(1) def test_one(): pass The idea here is to test each feature separately, while ordering the tests across the test modules for each feature. If we use session scope, we get:: $ pytest -v order_scope_level ============================= test session starts ============================== ... order_scope_level/feature1/test_a.py::test_one PASSED order_scope_level/feature2/test_a.py::test_one PASSED order_scope_level/feature1/test_a.py::test_two PASSED order_scope_level/feature2/test_a.py::test_two PASSED order_scope_level/feature1/test_b.py::test_three PASSED order_scope_level/feature2/test_b.py::test_three PASSED order_scope_level/feature1/test_b.py::test_four PASSED order_scope_level/feature2/test_b.py::test_four PASSED which mixes the features. Using module scope instead separates the features, but does not order the modules as wanted:: $ pytest -v --order-scope=module order_scope_level ============================= test session starts ============================== ... order_scope_level/feature1/test_a.py::test_three PASSED order_scope_level/feature1/test_a.py::test_four PASSED order_scope_level/feature1/test_b.py::test_one PASSED order_scope_level/feature1/test_b.py::test_two PASSED order_scope_level/feature2/test_a.py::test_three PASSED order_scope_level/feature2/test_a.py::test_four PASSED order_scope_level/feature2/test_b.py::test_one PASSED order_scope_level/feature2/test_b.py::test_two PASSED To get the wanted behavior, we can use ``--order-scope-level=2``, which keeps the first two directory levels:: $ pytest tests -v --order-scope-level=2 order_scope_level ============================= test session starts ============================== ... order_scope_level/feature1/test_b.py::test_one PASSED order_scope_level/feature1/test_b.py::test_two PASSED order_scope_level/feature1/test_a.py::test_three PASSED order_scope_level/feature1/test_a.py::test_four PASSED order_scope_level/feature2/test_b.py::test_one PASSED order_scope_level/feature2/test_b.py::test_two PASSED order_scope_level/feature2/test_a.py::test_three PASSED order_scope_level/feature2/test_a.py::test_four PASSED Note that using a level of 0 or 1 would cause the same result as session scope in this example, and any level greater than 2 would emulate module scope. ``--order-group-scope`` ----------------------- This option is also related to the order scope. It defines the scope inside which tests may be reordered. Consider you have several test modules which you want to order, but you don't want to mix the tests of several modules because the module setup is costly. In this case you can set the group order scope to "module", meaning that first the tests are ordered inside each module (the same as with the module order scope), but afterwards the modules themselves are sorted without changing the order inside each module. Consider these two test modules: **tests/test_module1.py**: .. code:: import pytest @pytest.mark.order(2) def test1(): pass def test2(): pass **tests/test_module2.py**: .. code:: import pytest @pytest.mark.order(1) def test1(): pass def test2(): pass Here is what you get using different scopes:: $ pytest tests -vv ============================= test session starts ============================== ... tests/test_module2.py:9: test1 PASSED tests/test_module1.py:9: test1 PASSED tests/test_module1.py:5: test2 PASSED tests/test_module2.py:5: test2 PASSED :: $ pytest tests -vv --order-scope=module ============================= test session starts ============================== ... tests/test_module1.py:9: test1 PASSED tests/test_module1.py:5: test2 PASSED tests/test_module2.py:9: test1 PASSED tests/test_module2.py:5: test2 PASSED :: $ pytest tests -vv --order-group-scope=module ============================= test session starts ============================== ... tests/test_module2.py:9: test1 PASSED tests/test_module2.py:5: test2 PASSED tests/test_module1.py:9: test1 PASSED tests/test_module1.py:5: test2 PASSED The ordering of the module groups is done based on the lowest non-negative order number present in the module (e.g. the order number of the first test). If only negative numbers are present, the highest negative number (e.g. the number of the last test) is used, and these modules will be ordered at the end. Modules without order numbers will be sorted between modules with a non-negative order number and modules with a negative order number, the same way tests are sorted inside a module. The group order scope defaults to the order scope. In this case the tests are ordered the same way as without the group order scope. The setting takes effect only if the scope is less than the order scope, e.g. there are three possibilities: - order scope "session", order group scope "module" - this is shown in the example above: first tests in each module are ordered, afterwards the modules - order scope "module", order group scope "class" - first orders tests inside each class, then the classes inside each module - order scope "session", order group scope "class" - first orders tests inside each class, then the classes inside each module, and finally the modules relatively to each other This option will also work with relative markers, and with dependency markers if using the :ref:`order-dependencies` option. Here is a similar example using relative markers: **tests/test_module1.py**: .. code:: import pytest @pytest.mark.order(after="test_module2.py::test1") def test1(): pass def test2(): pass **tests/test_module2.py**: .. code:: import pytest def test1(): pass @pytest.mark.order(before="test1") def test2(): pass Here is what you get using different scopes:: $ pytest tests -vv ============================= test session starts ============================== ... tests/test_module1.py:5: test2 PASSED tests/test_module2.py:9: test1 PASSED tests/test_module2.py:5: test2 PASSED tests/test_module1.py:9: test1 PASSED :: $ pytest tests -vv --order-group-scope=module ============================= test session starts ============================== ... tests/test_module2.py:9: test1 PASSED tests/test_module2.py:5: test2 PASSED tests/test_module1.py:9: test1 PASSED tests/test_module1.py:5: test2 PASSED You can see that in the second run the second test module is run before the first because of the dependency, but the tests inside each module remain in the same order as before. Note that using module scope as in the example above doesn't make sense here due to the dependencies between modules. This will also work with dependency markers if using the :ref:`order-dependencies` option. .. note:: This option will not work together well with the sparse ordering option. .. _order-dependencies: ``--order-dependencies`` ------------------------ This defines the behavior if the `pytest-dependency`_ plugin is used. By default, ``dependency`` marks are only considered if they coexist with an ``order`` mark. In this case it is checked if the ordering would break the dependency, and is ignored if this is the case. Consider the following: .. code:: python import pytest def test_a(): assert True @pytest.mark.dependency(depends=["test_a"]) @pytest.mark.order("first") def test_b(): assert True In this case, the ordering would break the dependency and is therefore ignored. This behavior is independent of the option. Now consider the following tests: .. code:: python import pytest @pytest.mark.dependency(depends=["test_b"]) def test_a(): assert True @pytest.mark.dependency def test_b(): assert True By default, ``test_a`` is not run, because it depends on ``test_b``, which is only run after ``test_b``:: $ pytest tests -vv ============================= test session starts ============================== ... test_dep.py::test_a SKIPPED test_dep.py::test_b PASSED If you use ``--order-dependencies``, this will change--the tests will now be reordered according to the dependency and both run:: $ pytest tests -vv --order-dependencies ============================= test session starts ============================== ... test_dep.py::test_b PASSED test_dep.py::test_a PASSED Note that a similar feature may be added to ``pytest-dependency`` - if this is done, this option will not be needed, but for the time being you can use both plugins together to get this behavior. Note that ``pytest-order`` does not replace ``pytest-dependency``--it just adds ordering to the existing functionality if needed. .. note:: ``pytest-dependency`` also has the possibility to `add dependencies at runtime`_ using ``pytest_dependency.depends``. These dependencies cannot be detected at collection time and therefore are not included in ordering. The same is true for the `dynamic compilation of marked parameters`_. .. _order-marker-prefix: ``--order-marker-prefix`` ------------------------- Consider the following: You have several groups of tests where you want to decide which test groups to execute in a certain test run. This is usually done using custom markers, so that you can filter the tests by the markers using the "-m" option: .. code:: python import pytest @pytest.mark.m3 def test_a(): assert True @pytest.mark.m1 def test_b(): assert True @pytest.mark.m2 def test_c(): assert True Running these you get:: $ pytest tests -vv -m "m2 or m3" ============================= test session starts ============================== ... test_module.py:5: test_a PASSED test_module.py:15: test_c PASSED Now consider that the test groups shall always be executed in a certain order, e.g. the group with the marker "m1" shall always be executed before the tests with "m2" etc. This can be achieved by adding an additional order marker to each test: .. code:: python import pytest @pytest.mark.order(3) @pytest.mark.m3 def test_a(): assert True @pytest.mark.order(1) @pytest.mark.m1 def test_b(): assert True etc. Running these you get the desired order:: $ pytest tests -vv -m "m2 or m3" ============================= test session starts ============================== ... test_module.py:18: test_c PASSED test_module.py:6: test_a PASSED This looks redundant and is also error-prone. If you want to order them instead just using your own marker (which has the order index already in the name), you can use the option ``--order-marker-prefix``. Running the original tests without any order marker gives you now:: $ pytest tests -vv -m "m2 or m3" --order-merker-prefix=m ============================= test session starts ============================== ... test_module.py:18: test_c PASSED test_module.py:6: test_a PASSED .. note:: As usually, you are responsible for registering your own markers, either in the code or in the ``pytest.ini`` file. If you forget this, pytest will give you warnings about unknown markers. .. _indulgent-ordering: ``--indulgent-ordering`` ------------------------ You may sometimes find that you want to suggest an ordering of tests, while allowing it to be overridden for good reason. For example, if you run your test suite in parallel and have a number of tests which are particularly slow, it might be desirable to start those tests running first, in order to optimize your completion time. You can use the ``pytest-order`` plugin to inform pytest of this. Now suppose you also want to prioritize tests which failed during the previous run, by using the ``--failed-first`` option. By default, pytest-order will override the ``--failed-first`` order, but by adding the ``--indulgent-ordering`` option, you can ask pytest to run the sort from pytest-order *before* the sort from ``--failed-first``, allowing the failed tests to be sorted to the front (note that in pytest versions from 6.0 on, this seems not to be needed anymore, at least in this specific case). .. _sparse-ordering: ``--sparse-ordering`` --------------------- Ordering tests by ordinals where some numbers are missing by default behaves the same as if the the ordinals are consecutive. For example, these tests: .. code:: python import pytest @pytest.mark.order(3) def test_two(): assert True def test_three(): assert True def test_four(): assert True @pytest.mark.order(1) def test_one(): assert True are executed in the same order as: .. code:: python import pytest @pytest.mark.order(1) def test_two(): assert True def test_three(): assert True def test_four(): assert True @pytest.mark.order(0) def test_one(): assert True e.g. you get:: $ pytest tests -vv ============================= test session starts ============================== ... test_module.py:13: test_one PASSED test_module.py:3: test_two PASSED test_module.py:6: test_three PASSED test_module.py:9: test_four PASSED The gaps between numbers, and the fact that the starting number is not 0, are ignored. This is consistent with the current behavior of ``pytest-ordering``. If you use the ``--sparse-ordering`` option, the behavior will change:: $ pytest tests -vv --sparse-ordering ============================= test session starts ============================== ... test_module.py:6: test_three PASSED test_module.py:13: test_one PASSED test_module.py:9: test_four PASSED test_module.py:3: test_two PASSED Now all missing numbers (starting with 0) are filled with unordered tests, as long as unordered tests are left. In the shown example, ``test_three`` is filled in for the missing number 0, and ``test_four`` is filled in for the missing number 2. This will also work for tests with negative order numbers (or the respective names). The missing ordinals are filled with unordered tests first from the start, then from the end if there are negative numbers, and the rest will be in between (e.g. between positive and negative numbers), as it is without this option. .. _`pytest-dependency`: https://pypi.org/project/pytest-dependency/ .. _`dynamic compilation of marked parameters`: https://pytest-dependency.readthedocs.io/en/stable/advanced.html#dynamic-compilation-of-marked-parameters .. _`add dependencies at runtime`: https://pytest-dependency.readthedocs.io/en/stable/usage.html#marking-dependencies-at-runtime pytest-order-1.2.0/docs/source/index.rst000066400000000000000000000003561452612573200202570ustar00rootroot00000000000000pytest-order - a plugin to order test execution =============================================== Content of the documentation ---------------------------- .. toctree:: :maxdepth: 2 intro usage configuration other_plugins pytest-order-1.2.0/docs/source/intro.rst000066400000000000000000000077251452612573200203120ustar00rootroot00000000000000Introduction ============ ``pytest-order`` is a pytest plugin which allows you to customize the order in which your tests are run. It provides the marker ``order``, that has attributes that define when your tests should run in relation to each other. These attributes can be absolute (i.e. first, or second-to-last) or relative (i.e. run this test before this other test). .. note:: It is generally considered bad practice to write tests that depend on each other. However, in reality this may still be needed due to performance issues, legacy code or other constraints. Still, before using this plugin, you should check if you can refactor your tests to remove the dependencies between tests. Relationship with pytest-ordering --------------------------------- ``pytest-order`` is a fork of `pytest-ordering `__, which is not maintained anymore. The idea and most of the code has been created by Frank Tobia, the author of that plugin, and contributors to the project. However, ``pytest-order`` is not compatible with ``pytest-ordering`` due to the changed marker name (``order`` instead of ``run``) and the removal of all other special markers for consistence (as has been discussed in `this issue `__). This also avoids clashes between the plugins if they are both installed. Here are examples for which markers correspond to markers in ``pytest-ordering``: - ``pytest.mark.order1``, ``pytest.mark.run(order=1)`` => ``pytest.mark.order(1)`` - ``pytest.mark.first`` => ``pytest.mark.order("first")`` Additionally, ``pytest-order`` provides a number of features (relative ordering, all configuration options) that are not available in ``pytest-ordering``. Supported Python and pytest versions ------------------------------------ ``pytest-order`` supports python 3.7 - 3.12 and pypy3, and is compatible with pytest 5.0.0 or newer (older versions may also work, but are not tested) for Python versions up to 3.9, and with pytest >= 6.2.4 for Python versions >= 3.10. All supported combinations of Python and pytest versions are tested in the CI builds. The plugin shall work under Linux, MacOs and Windows. Installation ------------ The latest released version can be installed from `PyPi `__: .. code:: bash pip install pytest-order The latest main branch can be installed from the GitHub sources: .. code:: bash pip install git+https://github.com/pytest-dev/pytest-order Examples -------- Most examples shown in this documentation can be also found in the repository under `example `__ as working test files. Quickstart ---------- Ordinarily pytest will run tests in the order that they appear in a module. For example, for the following tests: .. code:: python def test_foo(): assert True def test_bar(): assert True the output is something like:: $ pytest test_foo.py -vv ============================= test session starts ============================== platform darwin -- Python 3.7.1, pytest-5.4.3, py-1.8.1, pluggy-0.13.1 -- env/bin/python collected 2 items test_foo.py:2: test_foo PASSED test_foo.py:6: test_bar PASSED =========================== 2 passed in 0.01 seconds =========================== With ``pytest-order``, you can change the default ordering as follows: .. code:: python import pytest @pytest.mark.order(2) def test_foo(): assert True @pytest.mark.order(1) def test_bar(): assert True This will generate the output:: $ pytest test_foo.py -vv ============================= test session starts ============================== platform darwin -- Python 3.7.1, pytest-5.4.3, py-1.8.1, pluggy-0.13.1 -- env/bin/python plugins: order collected 2 items test_foo.py:7: test_bar PASSED test_foo.py:3: test_foo PASSED =========================== 2 passed in 0.01 seconds =========================== pytest-order-1.2.0/docs/source/other_plugins.rst000066400000000000000000000135311452612573200220310ustar00rootroot00000000000000Using pytest-order with other pytest plugins ============================================ Relationship with pytest-dependency ----------------------------------- The `pytest-dependency`_ plugin also manages dependencies between tests (skips tests that depend on skipped or failed tests), but doesn't do any ordering. If you want to execute the tests in a specific order to each other, you can use ``pytest-order``. If you want to skip or xfail tests dependent on other tests you can use ``pytest-dependency``. If you want to have both behaviors combined, you can use both plugins together with the option :ref:`order-dependencies`. As mentioned before, only static ``dependency`` markers are considered for ordering. Usage with other ordering plugins --------------------------------- There is a number of other pytest plugins that change the order in which tests are executed. Most of these plugins only reorder tests if given some command line options or in the presence of specific markers (as does ``pytest-order``). These plugins will not have any effect on ``pytest-order`` if not actively used. A few plugins always do reordering, most notably ``pytest-randomly``. `pytest-randomly`_ ~~~~~~~~~~~~~~~~~~ This plugin executes tests in a random order to avoid unknown test dependencies. The plugin is effective if installed and not actively disabled. ``pytest-order`` should still work correctly, because it is executed after ``pytest-randomly`` (except if you use the option :ref:`indulgent-ordering`). The marked tests will be ordered correctly, but the order of the unordered tests will change unpredictably. For example, if you run the following tests: .. code:: python import pytest @pytest.mark.order(1) def test_second(): assert True def test_third(): assert True def test_fourth(): assert True @pytest.mark.order(0) def test_first(): assert True the output could either be:: test_randomly.py::test_first PASSED test_randomly.py::test_second PASSED test_randomly.py::test_third PASSED test_randomly.py::test_fourth PASSED or: :: test_randomly.py::test_first PASSED test_randomly.py::test_second PASSED test_randomly.py::test_fourth PASSED test_randomly.py::test_third PASSED The same is true for relative ordering. The tests will be correctly ordered before and after the tests as configured, but all other tests will be in an arbitrary order. If you want the tests to execute in a random order except the ones that you know depend on each other, this can be a good strategy to prevent test dependencies. If you don't want the installed ``pytest-randomly`` be effective in a test run, you can exclude it explicitly on the command line:: python -m pytest -p no:randomly `pytest-reverse`_ ~~~~~~~~~~~~~~~~~ `pytest-reverse`_ is another plugin by the same author to find test dependencies by running tests in the reverse order. Other than ``pytest-randomly``, it is only effective if the tests are called with the command line option ``--reverse``. As with ``pytest-randomly``, you can use this plugin combined with ``pytest-ordering`` to make sure that all known test dependencies are handled, and possible unknown dependencies are found. `pytest-random-order`_ ~~~~~~~~~~~~~~~~~~~~~~ `pytest-random-order`_ is very similar to ``pytest-randomly``, except that the tests are only reordered if the option ``--random-order`` is given. Except from that, what was mentioned for ``pytest-randomly`` is also true for this plugin. `pytest-ordering`_ ~~~~~~~~~~~~~~~~~~ As mentioned, `pytest-ordering`_ can coexist with ``pytest-order`` due to the different marker names, but using markers of both plugins in the same test run is not recommended. One plugin may partially revert the effects of the other plugin in unpredictable ways, because the order in which they are executed is not deterministic. The same is true for other plugins that define the test order and are run last. `pytest-depends`_ ~~~~~~~~~~~~~~~~~ `pytest-depends`_ has a goal somewhat similar to `pytest-dependency`_ with additional ordering, but due to a known issue it always reorders the tests. If you have installed this plugin, the order of the unordered tests will change even without using it actively. `pytest-find-dependencies`_ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a small plugin by the same author as ``pytest-order`` that tries to find specific dependencies between tests by running a subset of them repeatedly in reverse order until the dependencies are found. This plugin would run the tests as ordered by any ordering plugin in the first run, but reverse the test order in the second run, so that already ordered tests are not run in the correct order. You have the possibility to exclude ordered tests completely by using the ``--markers-to-ignore`` option:: python -m pytest --find-dependencies --markers-to-ignore=order Usage with pytest-xdist ----------------------- The `pytest-xdist`_ plugin schedules tests unordered, and the order configured by ``pytest-order`` will normally not be preserved. But if we use the ``--dist=loadfile`` option, provided by ``xdist``, all tests from one file will be run in the same thread. So, to make the two plugins work together, we have to put each group of dependent tests in one file, and call pytest with ``--dist=loadfile`` (this is taken from `this issue `__). .. _`pytest-xdist`: https://pypi.org/project/pytest-xdist/ .. _`pytest-randomly`: https://pypi.org/project/pytest-randomly/ .. _`pytest-dependency`: https://pypi.org/project/pytest-dependency/ .. _`pytest-reverse`: https://pypi.org/project/pytest-reverse/ .. _`pytest-depends`: https://pypi.org/project/pytest-depends/ .. _`pytest-random-order`: https://pypi.org/project/pytest-random-order/ .. _`pytest-find-dependencies`: https://pypi.org/project/pytest-find-dependencies/ .. _`pytest-ordering`: https://pypi.org/project/pytest-ordering/ pytest-order-1.2.0/docs/source/usage.rst000066400000000000000000000312101452612573200202450ustar00rootroot00000000000000Usage ===== The above is a trivial example, but ordering is respected across test files. .. note:: The scope of the ordering is global per default, e.g. tests with lower ordinal numbers are always executed before tests with higher numbers in the same test session, regardless of the module and class they reside in. This can be changed by using the :ref:`order-scope` option. Ordering is done either absolutely, by using ordinal numbers that define the order, or relative to other tests, using the ``before`` and ``after`` attributes of the marker. Ordering by numbers ------------------- The order can be defined by ordinal numbers, or by ordinal strings. Order by index ~~~~~~~~~~~~~~ As already shown above, the order can be defined using ordinal numbers. There is a long form that uses the keyword ``index``, and a short form, that uses just the ordinal number--both are shown in the example below. The long form may be better readable if you want to combine it with a dependency marker (``before`` or ``after``, see below). Negative numbers are also allowed--they are used the same way as indices are used in Python lists, e.g. to count from the end: .. code:: python import pytest @pytest.mark.order(-2) def test_three(): assert True @pytest.mark.order(index=-1) def test_four(): assert True @pytest.mark.order(index=2) def test_two(): assert True @pytest.mark.order(1) def test_one(): assert True :: $ pytest test_foo.py -vv ============================= test session starts ============================== platform darwin -- Python 3.7.1, pytest-5.4.3, py-1.8.1, pluggy-0.13.1 -- env/bin/python plugins: order collected 4 items test_foo.py:17: test_one PASSED test_foo.py:12: test_two PASSED test_foo.py:3: test_three PASSED test_foo.py:7: test_four PASSED =========================== 4 passed in 0.02 seconds =========================== There is no limit for the numbers that can be used in this way. Order using ordinals ~~~~~~~~~~~~~~~~~~~~ Instead of the numbers, you can use ordinal names such as "first", "second", "last", and "second_to_last". These are convenience notations, and have the same effect as the numbers 0, 1, -1 and -2, respectively, that have been shown above: .. code:: python import pytest @pytest.mark.order("second_to_last") def test_three(): assert True @pytest.mark.order("last") def test_four(): assert True @pytest.mark.order("second") def test_two(): assert True @pytest.mark.order("first") def test_one(): assert True :: $ pytest test_foo.py -vv ============================= test session starts ============================== platform darwin -- Python 3.7.1, pytest-5.4.3, py-1.8.1, pluggy-0.13.1 -- env/bin/python plugins: order collected 4 items test_foo.py:17: test_one PASSED test_foo.py:12: test_two PASSED test_foo.py:3: test_three PASSED test_foo.py:7: test_four PASSED =========================== 4 passed in 0.02 seconds =========================== Convenience names are only defined for the first and the last 8 numbers. Here is the complete list with the corresponding numbers: - "first": 0 - "second": 1 - "third": 2 - "fourth": 3 - "fifth": 4 - "sixth": 5 - "seventh": 6 - "eighth": 7 - "last": -1 - "second_to_last": -2 - "third_to_last": -3 - "fourth_to_last": -4 - "fifth_to_last": -5 - "sixth_to_last": -6 - "seventh_to_last": -7 - "eighth_to_last": -8 Markers on class level ~~~~~~~~~~~~~~~~~~~~~~ If setting an ``order`` mark on class level, all tests in this class will be handled as having the same ordinal marker, e.g. the class as a whole will be reordered without changing the test order inside the test class: .. code:: python import pytest @pytest.mark.order(1) class Test1: def test_1(self): assert True def test_2(self): assert True @pytest.mark.order(0) class Test2: def test_1(self): assert True def test_2(self): assert True :: $ pytest -vv test_ordinal_class_mark.py ============================= test session starts ============================== ... collected 4 items test_ordinal_class_mark.py::Test2::test_1 PASSED test_ordinal_class_mark.py::Test2::test_2 PASSED test_ordinal_class_mark.py::Test1::test_1 PASSED test_ordinal_class_mark.py::Test1::test_2 PASSED Handling of unordered tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, tests with no ``order`` mark are executed after all tests with positive ordinal numbers (or the respective names), and before tests with negative ordinal numbers. The order of these tests in relationship to each other is not changed. This behavior will slightly change if the option :ref:`sparse-ordering` is used and the ordinals are not contiguous. Order relative to other tests ----------------------------- The test order can be defined relative to other tests, which are referenced by their name. The marker attributes ``before`` and ``after`` can be used to define the order relative to these tests: .. code:: python import pytest @pytest.mark.order(after="test_second") def test_third(): assert True def test_second(): assert True @pytest.mark.order(before="test_second") def test_first(): assert True :: $ pytest test_foo.py -vv ============================= test session starts ============================== platform darwin -- Python 3.7.1, pytest-5.4.3, py-1.8.1, pluggy-0.13.1 -- env/bin/python plugins: order collected 3 items test_foo.py:11: test_first PASSED test_foo.py:7: test_second PASSED test_foo.py:4: test_third PASSED =========================== 4 passed in 0.02 seconds =========================== Referencing of tests in other classes or modules ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If a test is referenced using the unqualified test name as shown in the example above, the test is assumed to be in the current module and the current class, if any. For tests in other classes in the same module the class name with a ``::`` suffix has to be prepended to the test name: .. code:: python import pytest class TestA: @pytest.mark.order(after="TestB::test_c") def test_a(self): assert True def test_b(self): assert True class TestB: def test_c(self): assert True If the referenced test lives in another module, you have to use the nodeid of the test, or a part of the nodeid that is sufficient to make it uniquely identifiable (the nodeid is the test ID that pytest prints if you run it with the ``-v`` option). Let's say we have the following module and test layout:: test_module_a.py TestA test_a test_b test_module_b.py test_a test_b test_module_c test_submodule.py test_1 test_2 Suppose the tests in ``test_module_b`` shall depend on tests in the other modules, this could be expressed like: **test_module_b.py** .. code:: python import pytest @pytest.mark.order(after="test_module_a.py::TestA::test_a") def test_a(): assert True @pytest.mark.order(before="test_module_c/test_submodule.py::test_2") def test_b(): assert True If an unknown test is referenced, a warning is issued and the execution order of the test in is not changed. Markers on class level ~~~~~~~~~~~~~~~~~~~~~~ As for ordinal markers, markers on class level are handled as if they are set to each individual test in the class. Additionally to referencing single tests, you can also reference test classes if using the ``before`` or ``after`` marker attributes: .. code:: python import pytest @pytest.mark.order(after="Test2") class Test1: def test_1(self): assert True def test_2(self): assert True class Test2: def test_1(self): assert True def test_2(self): assert True In this case, the tests in the marked class will be ordered behind all tests in the referenced class:: $ pytest -vv test_relative_class_mark.py ============================= test session starts ============================== ... collected 4 items test_relative_class_marker.py::Test2::test_1 PASSED test_relative_class_marker.py::Test2::test_2 PASSED test_relative_class_marker.py::Test1::test_1 PASSED test_relative_class_marker.py::Test1::test_2 PASSED Combination of absolute and relative ordering ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you combine absolute and relative order markers, the ordering is first done for the absolute markers (e.g. the ordinals), and afterwards for the relative ones. This means that relative ordering always takes preference: .. code:: python import pytest @pytest.mark.order(index=0, after="test_second") def test_first(): assert True @pytest.mark.order(1) def test_second(): assert True In this case, ``test_second`` will be executed before ``test_first``, regardless of the ordinal markers. Several relationships for the same marker ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you need to order a certain test relative to more than one other test, you can add more than one test name to the ``before`` or ``after`` marker attributes by using a list or tuple of test names: .. code:: python import pytest @pytest.mark.order(after=["test_second", "other_module.py::test_other"]) def test_first(): assert True def test_second(): assert True This will ensure that ``test_first`` is executed both after ``test_second`` and after ``test_other`` which resides in the module ``other_module.py``. Relationships with parameterized tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to reference parametrized tests, you can just use the test name without the parameter part, for example: .. code:: python import pytest @pytest.mark.order(after=["test_second"]) def test_first(): assert True @pytest.parametrize(param, [1, 2, 3]) def test_second(param): assert True Note that using the fully qualified test name, which would include the parameter (in this case ``test_second[1]``, ``test_second[2]`` etc) is not supported. Multiple test order markers --------------------------- More than one order marker can be set for the test. In this scenario test will be executed several times in the defined order. Combination of absolute and relative ordering ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: python import pytest @pytest.mark.order(1) @pytest.mark.order(-1) def test_one_and_seven(): pass @pytest.mark.order(2) @pytest.mark.order(-2) def test_two_and_six(): pass def test_four(): pass @pytest.mark.order(before="test_four") @pytest.mark.order(after="test_four") def test_three_and_five(): pass When a test has multiple order markers, each marker turns into a pytest ``ParameterSet``, so it will be run multiple times. :: ============================= test session starts ============================= collecting ... collected 7 items test_multiple_markers.py::test_one_and_seven[index=1] test_multiple_markers.py::test_two_and_six[index=2] test_multiple_markers.py::test_three_and_five[before=test_four] test_multiple_markers.py::test_four test_multiple_markers.py::test_three_and_five[after=test_four] test_multiple_markers.py::test_two_and_six[index=-2] test_multiple_markers.py::test_one_and_seven[index=-1] ============================== 7 passed in 0.02s ============================== Parametrized tests ~~~~~~~~~~~~~~~~~~ Although multiple test order markers create their own parametrization, it can be used with parametrized tests. .. code:: python import pytest @pytest.mark.order(1) @pytest.mark.order(3) @pytest.mark.parametrize("foo", ["aaa", "bbb"]) def test_one_and_three(foo): pass @pytest.mark.order(4) @pytest.mark.parametrize("bar", ["bbb", "ccc"]) @pytest.mark.order(2) def test_two_and_four(bar): pass :: collecting ... collected 8 items test_multiple_markers.py::test_one_and_three[index=1-aaa] PASSED [ 12%] test_multiple_markers.py::test_one_and_three[index=1-bbb] PASSED [ 25%] test_multiple_markers.py::test_two_and_four[index=2-bbb] PASSED [ 37%] test_multiple_markers.py::test_two_and_four[index=2-ccc] PASSED [ 50%] test_multiple_markers.py::test_one_and_three[index=3-aaa] PASSED [ 62%] test_multiple_markers.py::test_one_and_three[index=3-bbb] PASSED [ 75%] test_multiple_markers.py::test_two_and_four[index=4-bbb] PASSED [ 87%] test_multiple_markers.py::test_two_and_four[index=4-ccc] PASSED [100%] ============================== 8 passed in 0.02s ============================== pytest-order-1.2.0/example/000077500000000000000000000000001452612573200156155ustar00rootroot00000000000000pytest-order-1.2.0/example/__init__.py000066400000000000000000000000001452612573200177140ustar00rootroot00000000000000pytest-order-1.2.0/example/order_scope_level/000077500000000000000000000000001452612573200213105ustar00rootroot00000000000000pytest-order-1.2.0/example/order_scope_level/feature1/000077500000000000000000000000001452612573200230245ustar00rootroot00000000000000pytest-order-1.2.0/example/order_scope_level/feature1/__init__.py000066400000000000000000000000001452612573200251230ustar00rootroot00000000000000pytest-order-1.2.0/example/order_scope_level/feature1/test_a.py000066400000000000000000000001601452612573200246520ustar00rootroot00000000000000import pytest @pytest.mark.order(2) def test_two(): pass @pytest.mark.order(1) def test_one(): pass pytest-order-1.2.0/example/order_scope_level/feature1/test_b.py000066400000000000000000000001631452612573200246560ustar00rootroot00000000000000import pytest @pytest.mark.order(4) def test_four(): pass @pytest.mark.order(3) def test_three(): pass pytest-order-1.2.0/example/order_scope_level/feature2/000077500000000000000000000000001452612573200230255ustar00rootroot00000000000000pytest-order-1.2.0/example/order_scope_level/feature2/__init__.py000066400000000000000000000000001452612573200251240ustar00rootroot00000000000000pytest-order-1.2.0/example/order_scope_level/feature2/test_a.py000066400000000000000000000001601452612573200246530ustar00rootroot00000000000000import pytest @pytest.mark.order(2) def test_two(): pass @pytest.mark.order(1) def test_one(): pass pytest-order-1.2.0/example/order_scope_level/feature2/test_b.py000066400000000000000000000001631452612573200246570ustar00rootroot00000000000000import pytest @pytest.mark.order(4) def test_four(): pass @pytest.mark.order(3) def test_three(): pass pytest-order-1.2.0/example/test.py000066400000000000000000000003071452612573200171460ustar00rootroot00000000000000"""Reference test without ordering from the quick start chapter. See https://pytest-order.readthedocs.io/en/stable/#quickstart """ def test_foo(): assert True def test_bar(): assert True pytest-order-1.2.0/example/test_classes.py000066400000000000000000000006111452612573200206610ustar00rootroot00000000000000"""Shows how to use relative sorting in test classes, see See https://pytest-order.readthedocs.io/en/stable/#referencing-of-tests-in-other-classes-or-modules # noqa: E501 """ import pytest class TestA: @pytest.mark.order(after="TestB::test_c") def test_a(self): assert True def test_b(self): assert True class TestB: def test_c(self): assert True pytest-order-1.2.0/example/test_combined.py000066400000000000000000000005201452612573200210030ustar00rootroot00000000000000"""Shows how combined ordinal and relative sorting is handled. See https://pytest-order.readthedocs.io/en/stable/#combination-of-absolute-and-relative-ordering # noqa: E501 """ import pytest @pytest.mark.order(index=0, after="test_second") def test_first(): assert True @pytest.mark.order(1) def test_second(): assert True pytest-order-1.2.0/example/test_module1.py000066400000000000000000000003631452612573200205760ustar00rootroot00000000000000"""One of the test files showing the use of the --order-scope option. See https://pytest-order.readthedocs.io/en/stable/#order-scope """ import pytest @pytest.mark.order(2) def test2(): pass @pytest.mark.order(1) def test1(): pass pytest-order-1.2.0/example/test_module2.py000066400000000000000000000003631452612573200205770ustar00rootroot00000000000000"""One of the test files showing the use of the --order-scope option. See https://pytest-order.readthedocs.io/en/stable/#order-scope """ import pytest @pytest.mark.order(2) def test2(): pass @pytest.mark.order(1) def test1(): pass pytest-order-1.2.0/example/test_module_a.py000066400000000000000000000004411452612573200210120ustar00rootroot00000000000000"""One of the example files showing relative ordering between modules. See https://pytest-order.readthedocs.io/en/stable/#referencing-of-tests-in-other-classes-or-modules # noqa: E501 """ class TestA: def test_a(self): assert True def test_b(self): assert True pytest-order-1.2.0/example/test_module_c/000077500000000000000000000000001452612573200204435ustar00rootroot00000000000000pytest-order-1.2.0/example/test_module_c/__init__.py000066400000000000000000000000001452612573200225420ustar00rootroot00000000000000pytest-order-1.2.0/example/test_module_c/test_submodule.py000066400000000000000000000003571452612573200240600ustar00rootroot00000000000000"""One of the example files showing relative ordering between modules. See https://pytest-order.readthedocs.io/en/stable/#referencing-of-tests-in-other-classes-or-modules # noqa: E501 """ def test_1(): pass def test_2(): pass pytest-order-1.2.0/example/test_modules_b.py000066400000000000000000000006051452612573200212000ustar00rootroot00000000000000"""Example showing ordering relative to tests in other modules. See https://pytest-order.readthedocs.io/en/stable/#referencing-of-tests-in-other-classes-or-modules # noqa: E501 """ import pytest @pytest.mark.order(after="test_module_a.py::TestA::test_a") def test_a(): assert True @pytest.mark.order(before="test_module_c.py::test_submodule.test_2") def test_b(): assert True pytest-order-1.2.0/example/test_number.py000066400000000000000000000007261452612573200205230ustar00rootroot00000000000000"""Shows how to use both positive and negative numbers for sorting, and the use of the long form (index attribute) and short form (number only). See https://pytest-order.readthedocs.io/en/stable/#order-by-index """ import pytest @pytest.mark.order(-2) def test_three(): assert True @pytest.mark.order(index=-1) def test_four(): assert True @pytest.mark.order(index=2) def test_two(): assert True @pytest.mark.order(1) def test_one(): assert True pytest-order-1.2.0/example/test_ordinal_class_mark.py000066400000000000000000000004021452612573200230510ustar00rootroot00000000000000import pytest @pytest.mark.order(1) class Test1: def test_1(self): assert True def test_2(self): assert True @pytest.mark.order(0) class Test2: def test_1(self): assert True def test_2(self): assert True pytest-order-1.2.0/example/test_ordinals.py000066400000000000000000000006151452612573200210430ustar00rootroot00000000000000"""Shows how to use ordinal names for sorting. See https://pytest-order.readthedocs.io/en/stable/#order-using-ordinals """ import pytest @pytest.mark.order("second_to_last") def test_three(): assert True @pytest.mark.order("last") def test_four(): assert True @pytest.mark.order("second") def test_two(): assert True @pytest.mark.order("first") def test_one(): assert True pytest-order-1.2.0/example/test_quickstart.py000066400000000000000000000004141452612573200214170ustar00rootroot00000000000000"""Test with ordering (as opposed to test.py) from the quick start chapter. See https://pytest-order.readthedocs.io/en/stable/#quickstart """ import pytest @pytest.mark.order(2) def test_foo(): assert True @pytest.mark.order(1) def test_bar(): assert True pytest-order-1.2.0/example/test_relative.py000066400000000000000000000005261452612573200210440ustar00rootroot00000000000000"""Shows how to sort tests relatively to each other, see https://pytest-order.readthedocs.io/en/stable/#order-relative-to-other-tests """ import pytest @pytest.mark.order(after="test_second") def test_third(): assert True def test_second(): assert True @pytest.mark.order(before="test_second") def test_first(): assert True pytest-order-1.2.0/example/test_relative_class_marker.py000066400000000000000000000003701452612573200235670ustar00rootroot00000000000000import pytest @pytest.mark.order(after="Test2") class Test1: def test_1(self): assert True def test_2(self): assert True class Test2: def test_1(self): assert True def test_2(self): assert True pytest-order-1.2.0/example/test_sort_breaks_dependency.py000066400000000000000000000004631452612573200237450ustar00rootroot00000000000000"""Test showing that sorting is not done if it would break an existing dependency. See https://pytest-order.readthedocs.io/en/stable/#order-dependencies """ import pytest def test_a(): assert True @pytest.mark.dependency(depends=["test_a"]) @pytest.mark.order("first") def test_b(): assert True pytest-order-1.2.0/example/test_sort_dependency.py000066400000000000000000000004351452612573200224150ustar00rootroot00000000000000"""Test showing the behavior of the --order-dependencies option. See https://pytest-order.readthedocs.io/en/stable/#order-dependencies """ import pytest @pytest.mark.dependency(depends=["test_b"]) def test_a(): assert True @pytest.mark.dependency def test_b(): assert True pytest-order-1.2.0/example/test_sparse_ordering.py000066400000000000000000000005171452612573200224170ustar00rootroot00000000000000"""Test file showing the behavior of the --sparse-ordering option. See https://pytest-order.readthedocs.io/en/stable/#sparse-ordering """ import pytest @pytest.mark.order(3) def test_two(): assert True def test_three(): assert True def test_four(): assert True @pytest.mark.order(1) def test_one(): assert True pytest-order-1.2.0/mypy.ini000066400000000000000000000010611452612573200156570ustar00rootroot00000000000000[mypy] ; Ensures correct typing information inside non-typed functions check_untyped_defs = True ; Strict checking for None no_implicit_optional = True ; Disallow none and partial generics disallow_any_generics = True ; Point out pointless things warn_redundant_casts = True warn_unused_ignores = True ; Misc show_error_codes = True ; These are needed for VSCode ; Silences errors about missing imports ; (type-checks correctly when they're not missing tho) ignore_missing_imports = True ; Needed to position the underline correctly show_column_numbers = True pytest-order-1.2.0/old_issues.md000066400000000000000000000106151452612573200166600ustar00rootroot00000000000000List of issues in pytest-ordering --------------------------------- Tracks the state of all open issues in pytest-ordering for reference. - [Implement "before" and "after" markers](https://github.com/ftobia/pytest-ordering/issues/6) manually merged respective [PR](https://github.com/ftobia/pytest-ordering/pull/37) by Jonas Zinn :heavy_check_mark: - [Custom markers](https://github.com/ftobia/pytest-ordering/issues/10) will not be implemented (see [this issue](https://github.com/ftobia/pytest-ordering/issues/38)) :-1: - [Support for ordering testcases](https://github.com/ftobia/pytest-ordering/issues/12) unclear question, will ignore :-1: - [Test sparse ordinal behavior](https://github.com/ftobia/pytest-ordering/issues/14) - added tests from [this PR](https://github.com/ftobia/pytest-ordering/pull/29), - added `sparse-ordering` option, implemented behavior :heavy_check_mark: - [Doesn't work when using inside a class](https://github.com/ftobia/pytest-ordering/issues/18) seems to be fixed :heavy_check_mark: - [Allow ordering on per-class or per-module basis, instead of just per-session](https://github.com/ftobia/pytest-ordering/issues/20) added `--order-scope` option :heavy_check_mark: - [Ordering with multiple files](https://github.com/ftobia/pytest-ordering/issues/25) behavior can be configured with `--order-scope=module` :heavy_check_mark: - [Move to pytest-dev organization](https://github.com/ftobia/pytest-ordering/issues/32) handled in mirrored [issue](https://github.com/pytest-dev/pytest-order/issues/4) :heavy_check_mark: - [Ordering of test suite](https://github.com/ftobia/pytest-ordering/issues/33) don't understand the issue, ignoring :-1: - [Ensure compatibility with xdist](https://github.com/ftobia/pytest-ordering/issues/36) added/adapted [respective test](https://github.com/ftobia/pytest-ordering/pull/52) by Andrew Gilbert :heavy_check_mark: - [Standardize on a single marker name: "order"](https://github.com/ftobia/pytest-ordering/issues/38) implemented :heavy_check_mark: - [Remove not existing "relative ordering" feature from docs](https://github.com/ftobia/pytest-ordering/issues/39) obsolete, docs are up-to-date :heavy_check_mark: - [Ordering ignored on specification of multiple testcases](https://github.com/ftobia/pytest-ordering/issues/42) added respective test, seems to work correctly :heavy_check_mark: - [Order will work between different testClass](https://github.com/ftobia/pytest-ordering/issues/53) behavior can be configured with `--order-scope=class` :heavy_check_mark: - [Unknown mark warning](https://github.com/ftobia/pytest-ordering/issues/57) obsolete with registered marker :heavy_check_mark: - [pytest-ordering doesn't honor test dependencies](https://github.com/ftobia/pytest-ordering/issues/58) - ignore ordering if it would break a dependency - added configuration option for ordering all dependencies :heavy_check_mark: - [should pytest-ordering be deprecated in favor of pytest-dependency?](https://github.com/ftobia/pytest-ordering/issues/59) has been answered (`pytest-dependency` does not support ordering) :heavy_check_mark: - [py.test ordering doesn't works when methods with order greater than 9 are present](https://github.com/ftobia/pytest-ordering/issues/61) not reproducible, probably obsolete :-1: - [All ordering tests are failing (git develop)](https://github.com/ftobia/pytest-ordering/issues/62) not reproducible, will ignore :-1: - [Packaging the license file](https://github.com/ftobia/pytest-ordering/issues/63) done in [this PR](https://github.com/ftobia/pytest-ordering/pull/68) by Álvaro Mondéjar :heavy_check_mark: - [pytest ordering execute in reverse order](https://github.com/ftobia/pytest-ordering/issues/64) not reproducible, will ignore :-1: - [The module relative order don't working???](https://github.com/ftobia/pytest-ordering/issues/65) see [Implement "before" and "after" markers](https://github.com/ftobia/pytest-ordering/issues/6) :heavy_check_mark: - [Travis.CI results not showing up in GitHub](https://github.com/ftobia/pytest-ordering/issues/70) not an issue here :heavy_check_mark: - [Test ordering is completely broken](https://github.com/ftobia/pytest-ordering/issues/73) obsolete, shall work with respective `order` markers :heavy_check_mark: - [license is showing as UNKNOWN in pip show command.](https://github.com/ftobia/pytest-ordering/issues/75) fixed by adding the license to `setup.py` :heavy_check_mark: pytest-order-1.2.0/perf_tests/000077500000000000000000000000001452612573200163405ustar00rootroot00000000000000pytest-order-1.2.0/perf_tests/__init__.py000066400000000000000000000000001452612573200204370ustar00rootroot00000000000000pytest-order-1.2.0/perf_tests/test_dependencies.py000066400000000000000000000024171452612573200224030ustar00rootroot00000000000000from unittest import mock from textwrap import dedent import pytest from perf_tests.util import TimedSorter pytest_plugins = ["pytester"] @pytest.fixture def fixture_path_relative(testdir): for i_mod in range(10): test_name = testdir.tmpdir.join("test_dep_perf{}.py".format(i_mod)) test_contents = "import pytest\n" for i in range(40): test_contents += dedent( """ @pytest.mark.dependency(depends=["test_{}"]) def test_{}(): assert True """ ).format(i + 50, i) for i in range(60): test_contents += dedent( """ @pytest.mark.dependency def test_{}(): assert True """ ).format(i + 40) test_name.write(test_contents) yield testdir @mock.patch("pytest_order.plugin.Sorter", TimedSorter) def test_performance_dependency(fixture_path_relative): """Test performance of dependency markers that point to tests without an order mark (same as for test_relative does for after markers).""" TimedSorter.nr_marks = 400 fixture_path_relative.runpytest("--quiet", "--order-dependencies") assert TimedSorter.elapsed < 0.15 pytest-order-1.2.0/perf_tests/test_ordinal.py000066400000000000000000000015401452612573200214010ustar00rootroot00000000000000from unittest import mock from textwrap import dedent import pytest from perf_tests.util import TimedSorter pytest_plugins = ["pytester"] @pytest.fixture def fixture_path_ordinal(testdir): for i_mod in range(10): test_name = testdir.tmpdir.join("test_performance{}.py".format(i_mod)) test_contents = "import pytest\n" for i in range(100): test_contents += dedent( """ @pytest.mark.order({}) def test_{}(): assert True """ ).format(50 - i, i) test_name.write(test_contents) yield testdir @mock.patch("pytest_order.plugin.Sorter", TimedSorter) def test_performance_ordinal(fixture_path_ordinal): TimedSorter.nr_marks = 1000 fixture_path_ordinal.runpytest("--quiet") assert TimedSorter.elapsed < 0.02 pytest-order-1.2.0/perf_tests/test_relative.py000066400000000000000000000022421452612573200215640ustar00rootroot00000000000000from unittest import mock from textwrap import dedent import pytest from perf_tests.util import TimedSorter pytest_plugins = ["pytester"] @pytest.fixture def fixture_path_relative(testdir): for i_mod in range(10): test_name = testdir.tmpdir.join("test_relative_perf{}.py".format(i_mod)) test_contents = "import pytest\n" for i in range(40): test_contents += dedent( """ @pytest.mark.order(after="test_{}") def test_{}(): assert True """ ).format(i + 50, i) for i in range(60): test_contents += dedent( """ def test_{}(): assert True """ ).format(i + 40) test_name.write(test_contents) yield testdir @mock.patch("pytest_order.plugin.Sorter", TimedSorter) def test_performance_relative(fixture_path_relative): """Test performance of after markers that point to tests without an order mark (the usual case).""" TimedSorter.nr_marks = 400 fixture_path_relative.runpytest("--quiet") assert TimedSorter.elapsed < 0.15 pytest-order-1.2.0/perf_tests/test_relative_dense.py000066400000000000000000000023311452612573200227410ustar00rootroot00000000000000from unittest import mock from textwrap import dedent import pytest from perf_tests.util import TimedSorter pytest_plugins = ["pytester"] @pytest.fixture def fixture_path_relative_dense(testdir): for i_mod in range(10): test_name = testdir.tmpdir.join("test_relative_dense_perf{}.py".format(i_mod)) test_contents = "import pytest\n" for i in range(90): test_contents += dedent( """ @pytest.mark.order(after="test_{}") def test_{}(): assert True """ ).format(i + 10, i) for i in range(10): test_contents += dedent( """ def test_{}(): assert True """ ).format(i + 90) test_name.write(test_contents) yield testdir @mock.patch("pytest_order.plugin.Sorter", TimedSorter) def test_performance_relative(fixture_path_relative_dense): """Test performance of after markers that mostly point to tests with another order mark, so items are evaluated multiple times.""" TimedSorter.nr_marks = 900 fixture_path_relative_dense.runpytest("--quiet") assert TimedSorter.elapsed < 0.15 pytest-order-1.2.0/perf_tests/util.py000066400000000000000000000006601452612573200176710ustar00rootroot00000000000000import time from pytest_order.sorter import Sorter class TimedSorter(Sorter): elapsed = 0.0 nr_marks = 1000 def sort_items(self): self.__class__.elapsed = 0.0 start_time = time.time() items = super().sort_items() self.__class__.elapsed = (time.time() - start_time) / self.nr_marks * 1000 print("\nTime per test: {:.3f} ms".format(self.__class__.elapsed)) return items pytest-order-1.2.0/pytest_order/000077500000000000000000000000001452612573200167055ustar00rootroot00000000000000pytest-order-1.2.0/pytest_order/__init__.py000066400000000000000000000000261452612573200210140ustar00rootroot00000000000000__version__ = "1.2.0" pytest-order-1.2.0/pytest_order/item.py000066400000000000000000000165711452612573200202270ustar00rootroot00000000000000import sys from typing import Optional, List, Dict, Tuple, Generic, TypeVar from _pytest.python import Function from .settings import Scope, Settings _ItemType = TypeVar("_ItemType", "Item", "ItemGroup") class Item: """Represents a single test item.""" def __init__(self, item: Function) -> None: self.item: Function = item self.nr_rel_items: int = 0 self.order: Optional[int] = None self._node_id: Optional[str] = None def inc_rel_marks(self) -> None: if self.order is None: self.nr_rel_items += 1 def dec_rel_marks(self) -> None: if self.order is None: self.nr_rel_items -= 1 @property def module_path(self) -> str: return self.item.nodeid[: self.node_id.index("::")] def parent_path(self, level) -> str: return "/".join(self.module_path.split("/")[:level]) @property def node_id(self) -> str: if self._node_id is None: # in pytest < 4 the nodeid has an unwanted ::() part self._node_id = self.item.nodeid.replace("::()", "") return self._node_id class ItemList: """Handles a group of items with the same scope.""" def __init__( self, items: List[Item], settings: Settings, scope: Scope, rel_marks: List["RelativeMark[Item]"], dep_marks: List["RelativeMark[Item]"], ) -> None: self.items = items self.settings = settings self.scope = scope self.start_items: List[Tuple[int, List[Item]]] = [] self.end_items: List[Tuple[int, List[Item]]] = [] self.unordered_items: List[Item] = [] self._start_items: Dict[int, List[Item]] = {} self._end_items: Dict[int, List[Item]] = {} self.all_rel_marks = rel_marks self.all_dep_marks = dep_marks self.rel_marks = filter_marks(rel_marks, items) self.dep_marks = filter_marks(dep_marks, items) def collect_markers(self, item: Item) -> None: self.handle_order_mark(item) if item.nr_rel_items or item.order is None: self.unordered_items.append(item) def handle_order_mark(self, item: Item) -> None: if item.order is not None: if item.order < 0: self._end_items.setdefault(item.order, []).append(item) else: self._start_items.setdefault(item.order, []).append(item) def sort_numbered_items(self) -> List[Item]: self.start_items = sorted(self._start_items.items()) self.end_items = sorted(self._end_items.items()) sorted_list = [] index = 0 for order, items in self.start_items: if self.settings.sparse_ordering: while order > index and self.unordered_items: sorted_list.append(self.unordered_items.pop(0)) index += 1 sorted_list += items index += len(items) mid_index = len(sorted_list) index = -1 for order, items in reversed(self.end_items): if self.settings.sparse_ordering: while order < index and self.unordered_items: sorted_list.insert(mid_index, self.unordered_items.pop()) index -= 1 sorted_list[mid_index:mid_index] = items index -= len(items) sorted_list[mid_index:mid_index] = self.unordered_items return sorted_list def print_unhandled_items(self) -> None: msg = " ".join( [mark.item.node_id for mark in self.rel_marks] + [mark.item.node_id for mark in self.dep_marks] ) if msg: sys.stdout.write("\nWARNING: cannot execute test relative to others: ") sys.stdout.write(msg) sys.stdout.write(" - ignoring the marker.\n") sys.stdout.flush() def number_of_rel_groups(self) -> int: return len(self.rel_marks) + len(self.dep_marks) def handle_rel_marks(self, sorted_list: List[Item]) -> None: self.handle_relative_marks(self.rel_marks, sorted_list, self.all_rel_marks) def handle_dep_marks(self, sorted_list: List[Item]) -> None: self.handle_relative_marks(self.dep_marks, sorted_list, self.all_dep_marks) @staticmethod def handle_relative_marks( marks: List["RelativeMark[Item]"], sorted_list: List[Item], all_marks: List["RelativeMark[Item]"], ) -> None: for mark in reversed(marks): if move_item(mark, sorted_list): marks.remove(mark) all_marks.remove(mark) def group_order(self) -> Optional[int]: if self.start_items: return self.start_items[0][0] elif self.end_items: return self.end_items[-1][0] return None class ItemGroup: """ Holds a group of sorted items with the same group order scope. Used for sorting groups similar to Item for sorting items. """ def __init__( self, items: Optional[List[Item]] = None, order: Optional[int] = None ) -> None: self.items: List[Item] = items or [] self.order = order self.nr_rel_items = 0 def inc_rel_marks(self) -> None: if self.order is None: self.nr_rel_items += 1 def dec_rel_marks(self) -> None: if self.order is None: self.nr_rel_items -= 1 def extend(self, groups: List["ItemGroup"], order: Optional[int]) -> None: for group in groups: self.items.extend(group.items) self.order = order class RelativeMark(Generic[_ItemType]): """ Represents a marker for an item or an item group. Holds two related items or groups and their relationship. """ def __init__( self, item: _ItemType, item_to_move: _ItemType, move_after: bool, ) -> None: self.item: _ItemType = item self.item_to_move: _ItemType = item_to_move self.move_after: bool = move_after def filter_marks( marks: List[RelativeMark[_ItemType]], all_items: List[Item] ) -> List[RelativeMark[_ItemType]]: result = [] for mark in marks: if mark.item in all_items and mark.item_to_move in all_items: result.append(mark) else: mark.item_to_move.dec_rel_marks() return result def move_item(mark: RelativeMark[_ItemType], sorted_items: List[_ItemType]) -> bool: if ( mark.item not in sorted_items or mark.item_to_move not in sorted_items or mark.item.nr_rel_items ): return False pos_item = sorted_items.index(mark.item) pos_item_to_move = sorted_items.index(mark.item_to_move) if mark.item_to_move.order is not None and mark.item.order is None: # if the item to be moved has already been ordered numerically, # and the other item is not ordered, we move that one instead mark.move_after = not mark.move_after mark.item, mark.item_to_move = mark.item_to_move, mark.item pos_item, pos_item_to_move = pos_item_to_move, pos_item mark.item_to_move.dec_rel_marks() if mark.move_after: if pos_item_to_move < pos_item + 1: del sorted_items[pos_item_to_move] sorted_items.insert(pos_item, mark.item_to_move) else: if pos_item_to_move > pos_item: del sorted_items[pos_item_to_move] pos_item -= 1 sorted_items.insert(pos_item + 1, mark.item_to_move) return True pytest-order-1.2.0/pytest_order/plugin.py000066400000000000000000000121711452612573200205570ustar00rootroot00000000000000from typing import List import pytest from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.main import Session from _pytest.mark import Mark from _pytest.python import Function from .sorter import Sorter def pytest_configure(config: Config) -> None: """ Register the "order" marker and configure the plugin, depending on the CLI options. """ provided_by_pytest_order = ( "Provided by pytest-order. " "See also: https://pytest-order.readthedocs.io/" ) config_line = ( "order: specify ordering information for when tests should run " "in relation to one another. " + provided_by_pytest_order ) config.addinivalue_line("markers", config_line) # We need to dynamically add this `tryfirst` decorator to the plugin: # only when the CLI option is present should the decorator be added. # Thus, we manually run the decorator on the class function and # manually replace it. if config.getoption("indulgent_ordering"): wrapper = pytest.hookimpl(tryfirst=True) else: wrapper = pytest.hookimpl(trylast=True) OrderingPlugin.pytest_collection_modifyitems = wrapper( # type:ignore[attr-defined] modify_items ) config.pluginmanager.register(OrderingPlugin(), "orderingplugin") def pytest_addoption(parser: Parser) -> None: """Set up CLI option for pytest""" group = parser.getgroup("order") group.addoption( "--indulgent-ordering", action="store_true", dest="indulgent_ordering", help=( "Request that the sort order provided by pytest-order be applied " "before other sorting, allowing the other sorting to have priority" ), ) group.addoption( "--order-scope", action="store", dest="order_scope", help=( "Defines the scope used for ordering. Possible values are: " "'session' (default), 'module', and 'class'. " "Ordering is only done inside a scope." ), ) group.addoption( "--order-scope-level", action="store", type=int, dest="order_scope_level", help=( "Defines that the given directory level is used as order scope. " "Cannot be used with --order-scope. The value is a number " "that defines the hierarchical index of the directories used as " "order scope, starting with 0 at session scope." ), ) group.addoption( "--order-group-scope", action="store", dest="order_group_scope", help=( "Defines the scope used for order groups. Possible values are: " " 'session' (default), 'module', and 'class'. " "Ordering is first done inside a group, then between groups." ), ) group.addoption( "--sparse-ordering", action="store_true", dest="sparse_ordering", help=( "If there are gaps between ordinals, they are filled " "with unordered tests." ), ) group.addoption( "--order-dependencies", action="store_true", dest="order_dependencies", help=( "If set, dependencies added by pytest-dependency will be ordered " "if needed." ), ) group.addoption( "--order-marker-prefix", action="store", dest="order_marker_prefix", help=( "If set, markers starting with the given prefix followed by a number " "are handled like order markers with an index." ), ) def _get_mark_description(mark: Mark): if mark.kwargs: return ", ".join([f"{k}={v}" for k, v in mark.kwargs.items()]) elif mark.args: return f"index={mark.args[0]}" return mark def pytest_generate_tests(metafunc): """ Handle multiple pytest.mark.order decorators. Make parametrized tests with corresponding order marks. """ if getattr(metafunc, "function", False): if getattr(metafunc.function, "pytestmark", False): # Get list of order marks marks = metafunc.function.pytestmark order_marks = [mark for mark in marks if mark.name == "order"] if len(order_marks) > 1: # Remove all order marks metafunc.function.pytestmark = [ mark for mark in marks if mark.name != "order" ] # Prepare arguments for parametrization with order marks args = [ pytest.param(_get_mark_description(mark), marks=[mark]) for mark in order_marks ] if "order" not in metafunc.fixturenames: metafunc.fixturenames.append("order") metafunc.parametrize("order", args) class OrderingPlugin: """ Plugin implementation. By putting this in a class, we are able to dynamically register it after the CLI is parsed. """ def modify_items(session: Session, config: Config, items: List[Function]) -> None: sorter = Sorter(config, items) items[:] = sorter.sort_items() pytest-order-1.2.0/pytest_order/settings.py000066400000000000000000000047501452612573200211250ustar00rootroot00000000000000from enum import Enum from warnings import warn from _pytest.config import Config class Scope(Enum): CLASS = 1 MODULE = 2 SESSION = 3 class Settings: """Holds all configuration settings.""" valid_scopes = { "class": Scope.CLASS, "module": Scope.MODULE, "session": Scope.SESSION, } def __init__(self, config: Config) -> None: self.sparse_ordering: bool = config.getoption("sparse_ordering") self.order_dependencies: bool = config.getoption("order_dependencies") self.marker_prefix: str = config.getoption("order_marker_prefix") scope: str = config.getoption("order_scope") if scope in self.valid_scopes: self.scope: Scope = self.valid_scopes[scope] else: if scope is not None: warn( "Unknown order scope '{}', ignoring it. " "Valid scopes are 'session', 'module' and 'class'.".format(scope) ) self.scope = Scope.SESSION scope_level: int = config.getoption("order_scope_level") or 0 if scope_level != 0 and self.scope != Scope.SESSION: warn( "order-scope-level cannot be used together with " "--order-scope={}".format(scope) ) scope_level = 0 self.scope_level: int = scope_level group_scope: str = config.getoption("order_group_scope") if group_scope in self.valid_scopes: self.group_scope: Scope = self.valid_scopes[group_scope] else: if group_scope is not None: warn( "Unknown order group scope '{}', ignoring it. " "Valid scopes are 'session', 'module' and 'class'.".format( group_scope ) ) self.group_scope = self.scope if self.group_scope.value > self.scope.value: warn("Group scope is larger than order scope, ignoring it.") self.group_scope = self.scope try: auto_mark_dep = config.getini("automark_dependency") if isinstance(auto_mark_dep, str): auto_mark_dep = auto_mark_dep.lower() in ( "1", "yes", "y", "true", "t", "on", ) except ValueError: auto_mark_dep = False self.auto_mark_dep = auto_mark_dep pytest-order-1.2.0/pytest_order/sorter.py000066400000000000000000000502701452612573200206010ustar00rootroot00000000000000import sys from warnings import warn from contextlib import suppress from typing import Optional, List, Dict, Tuple, cast from _pytest.config import Config from _pytest.mark import Mark from _pytest.python import Function from .item import Item, ItemList, ItemGroup, filter_marks, move_item, RelativeMark from .settings import Settings, Scope try: from typing import OrderedDict except ImportError: # In Python <3.7.2, we need to stub it from collections import OrderedDict as OrderedDict_cls from typing import MutableMapping, TypeVar KT = TypeVar("KT") VT = TypeVar("VT") class OrderedDict(OrderedDict_cls, MutableMapping[KT, VT]): # type: ignore pass orders_map = { "first": 0, "second": 1, "third": 2, "fourth": 3, "fifth": 4, "sixth": 5, "seventh": 6, "eighth": 7, "last": -1, "second_to_last": -2, "third_to_last": -3, "fourth_to_last": -4, "fifth_to_last": -5, "sixth_to_last": -6, "seventh_to_last": -7, "eighth_to_last": -8, } class Sorter: """ Sort all items according to the given configuration. """ def __init__(self, config: Config, items: List[Function]) -> None: self.settings: Settings = Settings(config) self.items: List[Item] = [Item(item) for item in items] self.node_ids: Dict[str, Item] = OrderedDict() self.node_id_last: Dict[str, List[str]] = {} for item in self.items: self.node_ids[item.node_id] = item last_part = item.node_id.rpartition("::")[2] if "[" in last_part: last_part = last_part.rpartition("[")[0] # save last nodeid component to avoid to iterate over all # items for each label self.node_id_last.setdefault(last_part, []).append(item.node_id) self.rel_marks: List[RelativeMark[Item]] = [] self.dep_marks: List[RelativeMark[Item]] = [] def sort_items(self) -> List[Function]: """ Do the actual sorting and return the sorted items. """ self.collect_markers() if self.settings.scope == Scope.SESSION: if self.settings.scope_level > 0: dir_groups = directory_item_groups( self.items, self.settings.scope_level ) sorted_list = [] for items in dir_groups.values(): sorter = ScopeSorter( self.settings, items, self.rel_marks, self.dep_marks ) sorted_list.extend(sorter.sort_items()) else: sorter = ScopeSorter( self.settings, self.items, self.rel_marks, self.dep_marks, session_scope=True, ) sorted_list = sorter.sort_items() elif self.settings.scope == Scope.MODULE: module_groups = module_item_groups(self.items) sorted_list = [] for module_items in module_groups.values(): sorter = ScopeSorter( self.settings, module_items, self.rel_marks, self.dep_marks ) sorted_list.extend(sorter.sort_items()) else: # class scope class_groups = class_item_groups(self.items) sorted_list = [] for class_items in class_groups.values(): sorter = ScopeSorter( self.settings, class_items, self.rel_marks, self.dep_marks ) sorted_list.extend(sorter.sort_items()) return [item.item for item in sorted_list] def mark_binning( self, item: Item, dep_marks: Dict[Tuple[str, Scope, str], List[Item]], aliases: Dict[str, Item], ) -> None: """ Collect relevant markers for the given item. """ keys = item.item.keywords.keys() has_dependency = "dependency" in keys has_order = "order" in keys if not has_order and self.settings.marker_prefix: for key in keys: if key.startswith(self.settings.marker_prefix): try: index = int(key[len(self.settings.marker_prefix)]) item.order = index except ValueError: pass if has_dependency or self.settings.auto_mark_dep: self.handle_dependency_mark(item, has_order, dep_marks, aliases) if has_order: self.handle_order_marks(item) def handle_dependency_mark( self, item: Item, has_order: bool, dep_marks: Dict[Tuple[str, Scope, str], List[Item]], aliases: Dict[str, Item], ) -> None: # always order dependencies if an order mark is present # otherwise only if order-dependencies is set mark = item.item.get_closest_marker("dependency") name_mark = None if mark and (self.settings.order_dependencies or has_order): dependent_mark = mark.kwargs.get("depends") if dependent_mark: scope = scope_from_name(mark.kwargs.get("scope", "module")) prefix = scoped_node_id(item.node_id, scope) for name in dependent_mark: dep_marks.setdefault((name, scope, prefix), []).append(item) item.inc_rel_marks() # we always collect the names of the dependent items, because # we need them in both cases name_mark = mark.kwargs.get("name") # the default name in pytest-dependency is the nodeid or a part # of the nodeid, depending on the scope if not name_mark: name_mark = item.node_id aliases[name_mark] = item def handle_order_marks(self, item: Item) -> None: marks = item.item.iter_markers("order") for mark in marks: self.handle_order_mark(item, mark) def handle_order_mark(self, item: Item, mark: Mark) -> None: order = mark.args[0] if mark.args else mark.kwargs.get("index") if order is not None: if isinstance(order, int): order = int(order) elif order in orders_map: order = orders_map[order] else: warn("Unknown order attribute:'{}'".format(order)) order = None if item.order is None: item.order = order self.handle_relative_marks(item, mark) if order is not None: item.nr_rel_items = 0 def items_from_label(self, label: str, item: Item, is_cls_mark: bool) -> List[Item]: """ Return the list of matching items from the given label. The list contains one item for a single matching test, several items in the case of a matching parametrized test, or no item in case of an invalid label. """ item_id = item.node_id label_len = len(label) last_comp = label.split("/")[-1].split("::")[-1] items = [] with suppress(KeyError): node_ids = self.node_id_last[last_comp] for node_id in node_ids: if node_id.endswith(label): id_start = node_id[:-label_len] elif node_id.endswith("]") and node_id.rpartition("[")[0].endswith( label ): id_start = node_id.rpartition("[")[0][:-label_len] else: continue if is_cls_mark and id_start.count("::") == 2: continue if item_id.startswith(id_start): items.append(self.node_ids[node_id]) return items def items_from_class_label(self, label: str, item: Item) -> List[Item]: items = [] item_id = item.node_id label_len = len(label) for node_id in self.node_ids: if node_id.count("::") == 2: cls_index = node_id.rindex("::") if node_id[:cls_index].endswith(label): id_start = node_id[: cls_index - label_len] if item_id.startswith(id_start): items.append(self.node_ids[node_id]) return items def handle_before_or_after_mark( self, item: Item, mark: Mark, marker_name: str, is_after: bool ) -> bool: def is_class_mark() -> bool: if item.item.cls and item.item.parent: return item.item.parent.get_closest_marker("order") == mark return False def is_mark_for_class() -> bool: return "::" not in marker_name and is_class_mark() is_cls_mark = is_class_mark() items_for_label = self.items_from_label(marker_name, item, is_cls_mark) if items_for_label: for item_for_label in items_for_label: rel_mark = RelativeMark(item_for_label, item, move_after=is_after) if is_after or not is_cls_mark: self.rel_marks.append(rel_mark) else: self.rel_marks.insert(0, rel_mark) item.inc_rel_marks() return True else: if is_mark_for_class(): items = self.items_from_class_label(marker_name, item) for item_for_label in items: rel_mark = RelativeMark(item_for_label, item, move_after=is_after) if is_after: self.rel_marks.append(rel_mark) else: self.rel_marks.insert(0, rel_mark) item.inc_rel_marks() return len(items) > 0 return False def handle_relative_marks(self, item: Item, mark: Mark) -> bool: has_relative_marks = False before_marks = mark.kwargs.get("before", ()) if before_marks and not isinstance(before_marks, (list, tuple)): before_marks = (before_marks,) for before_mark in before_marks: if self.handle_before_or_after_mark( item, mark, before_mark, is_after=False ): has_relative_marks = True else: self.warn_about_unknown_test(item, before_mark) after_marks = mark.kwargs.get("after", ()) if after_marks and not isinstance(after_marks, (list, tuple)): after_marks = (after_marks,) for after_mark in after_marks: if self.handle_before_or_after_mark(item, mark, after_mark, is_after=True): has_relative_marks = True else: self.warn_about_unknown_test(item, after_mark) return has_relative_marks @staticmethod def warn_about_unknown_test(item: Item, rel_mark: str) -> None: sys.stdout.write( "\nWARNING: cannot execute '{}' relative to others: " "'{}' - ignoring the marker.".format(item.item.name, rel_mark) ) def collect_markers(self) -> None: aliases: Dict[str, Item] = {} dep_marks: Dict[Tuple[str, Scope, str], List[Item]] = {} for item in self.items: self.mark_binning(item, dep_marks, aliases) self.resolve_dependency_markers(dep_marks, aliases) def resolve_dependency_markers( self, dep_marks: Dict[Tuple[str, Scope, str], List[Item]], aliases: Dict[str, Item], ) -> None: for (name, _, prefix), items in dep_marks.items(): if name in aliases: for item in items: self.dep_marks.append( RelativeMark(aliases[name], item, move_after=True) ) else: label = "::".join((prefix, name)) if label in aliases: for item in items: self.dep_marks.append( RelativeMark(aliases[label], item, move_after=True) ) else: sys.stdout.write( "\nWARNING: Cannot resolve the dependency marker '{}' " "- ignoring it.".format(name) ) def module_item_groups(items: List[Item]) -> Dict[str, List[Item]]: """ Split items into groups per module. """ module_items: OrderedDict[str, List[Item]] = OrderedDict() for item in items: module_items.setdefault(item.module_path, []).append(item) return module_items def directory_item_groups(items: List[Item], level: int) -> Dict[str, List[Item]]: """ Split items into groups per directory at the given level. The level is relative to the root directory, which is at level 0. """ module_items: OrderedDict[str, List[Item]] = OrderedDict() for item in items: module_items.setdefault(item.parent_path(level), []).append(item) return module_items def class_item_groups(items: List[Item]) -> Dict[str, List[Item]]: """ Split items into groups per class. Items outside a class are sorted into a group per module. """ class_items: OrderedDict[str, List[Item]] = OrderedDict() for item in items: delimiter_index = item.node_id.index("::") if "::" in item.node_id[delimiter_index + 2 :]: delimiter_index = item.node_id.index("::", delimiter_index + 2) class_path = item.node_id[:delimiter_index] class_items.setdefault(class_path, []).append(item) return class_items class ScopeSorter: """ Sorts the items for the defined scope. """ def __init__( self, settings: Settings, items: List[Item], rel_marks: List[RelativeMark[Item]], dep_marks: List[RelativeMark[Item]], session_scope: bool = False, ) -> None: self.settings = settings self.items = items # no need to filter items in session scope if session_scope: self.rel_marks = rel_marks self.dep_marks = dep_marks else: self.rel_marks = filter_marks(rel_marks, self.items) self.dep_marks = filter_marks(dep_marks, self.items) def sort_items(self) -> List[Item]: if self.settings.group_scope.value < self.settings.scope.value: if self.settings.scope == Scope.SESSION: sorted_list = self.sort_in_session_scope() else: # module scope / class group scope sorted_list = self.sort_in_module_scope() else: sorted_list = self.sort_items_in_scope(self.items, Scope.SESSION).items return sorted_list def sort_in_session_scope(self) -> List[Item]: sorted_list = [] module_items = module_item_groups(self.items) if self.settings.group_scope == Scope.CLASS: module_groups = self.sort_class_groups(module_items) else: module_groups = [ self.sort_items_in_scope(item, Scope.MODULE) for item in module_items.values() ] sorter = GroupSorter( Scope.MODULE, module_groups, self.rel_marks, self.dep_marks ) for group in sorter.sorted_groups()[1]: sorted_list.extend(group.items) return sorted_list def sort_in_module_scope(self) -> List[Item]: sorted_list = [] class_items = class_item_groups(self.items) class_groups = [ self.sort_items_in_scope(item, Scope.CLASS) for item in class_items.values() ] sorter = GroupSorter(Scope.CLASS, class_groups, self.rel_marks, self.dep_marks) for group in sorter.sorted_groups()[1]: sorted_list.extend(group.items) return sorted_list def sort_class_groups(self, module_items: Dict[str, List[Item]]) -> List[ItemGroup]: module_groups = [] for module_item in module_items.values(): class_items = class_item_groups(module_item) class_groups = [ self.sort_items_in_scope(item, Scope.CLASS) for item in class_items.values() ] module_group = ItemGroup() sorter = GroupSorter( Scope.CLASS, class_groups, self.rel_marks, self.dep_marks ) group_order, class_groups = sorter.sorted_groups() module_group.extend(class_groups, group_order) module_groups.append(module_group) return module_groups def sort_items_in_scope(self, items: List[Item], scope: Scope) -> ItemGroup: item_list = ItemList( items, self.settings, scope, self.rel_marks, self.dep_marks ) for item in items: item_list.collect_markers(item) sorted_list = item_list.sort_numbered_items() still_left = 0 length = item_list.number_of_rel_groups() while length and still_left != length: still_left = length item_list.handle_rel_marks(sorted_list) item_list.handle_dep_marks(sorted_list) length = item_list.number_of_rel_groups() if length: item_list.print_unhandled_items() return ItemGroup(sorted_list, item_list.group_order()) def scope_from_name(name: str) -> Scope: if name == "module": return Scope.MODULE if name == "class": return Scope.CLASS return Scope.SESSION def scoped_node_id(node_id: str, scope: Scope) -> str: if scope == Scope.MODULE: return node_id[: node_id.index("::")] if scope == Scope.CLASS: return node_id[: node_id.rindex("::")] return "" class GroupSorter: """ Sorts groups of items. """ def __init__( self, scope: Scope, groups: List[ItemGroup], rel_marks: List[RelativeMark[Item]], dep_marks: List[RelativeMark[Item]], ) -> None: self.scope: Scope = scope self.groups: List[ItemGroup] = groups self.rel_marks: List[RelativeMark[ItemGroup]] = self.collect_group_marks( rel_marks ) self.dep_marks: List[RelativeMark[ItemGroup]] = self.collect_group_marks( dep_marks ) def collect_group_marks( self, marks: List[RelativeMark[Item]] ) -> List[RelativeMark[ItemGroup]]: group_marks: List[RelativeMark[ItemGroup]] = [] for mark in marks: group = self.group_for_item(mark.item) group_to_move = self.group_for_item(mark.item_to_move) if group is not None and group_to_move is not None: group_marks.append(RelativeMark(group, group_to_move, mark.move_after)) group_to_move.inc_rel_marks() return group_marks def group_for_item(self, item: Item) -> Optional[ItemGroup]: for group in self.groups: if item in group.items: return group return None def sorted_groups(self) -> Tuple[Optional[int], List[ItemGroup]]: group_order = self.sort_by_ordinal_markers() length = len(self.rel_marks) + len(self.dep_marks) if length == 0: return group_order, self.groups # handle relative markers the same way single items are handled still_left = 0 while length and still_left != length: still_left = length self.handle_rel_marks(self.rel_marks) self.handle_rel_marks(self.dep_marks) return group_order, self.groups def sort_by_ordinal_markers(self) -> Optional[int]: start_groups = [] middle_groups = [] end_groups = [] for group in self.groups: if group.order is None: middle_groups.append(group) elif group.order >= 0: start_groups.append(group) else: end_groups.append(group) start_groups = sorted(start_groups, key=lambda g: cast(int, g.order)) end_groups = sorted(end_groups, key=lambda g: cast(int, g.order)) self.groups = start_groups + middle_groups + end_groups if start_groups: group_order = start_groups[0].order elif end_groups: group_order = end_groups[-1].order else: group_order = None return group_order def handle_rel_marks(self, marks: List[RelativeMark[ItemGroup]]) -> None: for mark in reversed(marks): if move_item(mark, self.groups): marks.remove(mark) pytest-order-1.2.0/requirements.txt000066400000000000000000000000141452612573200174410ustar00rootroot00000000000000pytest>=5.0 pytest-order-1.2.0/requirements_dev.txt000066400000000000000000000001331452612573200203010ustar00rootroot00000000000000-r requirements.txt -e . pytest-mock>=1.11.0 pytest-xdist>=1.29.0 pytest-dependency>=0.5.1 pytest-order-1.2.0/setup.cfg000066400000000000000000000001721452612573200160030ustar00rootroot00000000000000[metadata] description-file = README.md license_files = LICENSE [bdist_wheel] universal=0 [tool:pytest] testpaths = tests pytest-order-1.2.0/setup.py000066400000000000000000000036111452612573200156750ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup import os __here__ = os.path.abspath(os.path.dirname(__file__)) from pytest_order import __version__ with open(os.path.join(__here__, "README.md")) as f: LONG_DESCRIPTION = f.read() setup( name="pytest-order", description="pytest plugin to run your tests in a specific order", long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", version=__version__, author="mrbean-bremen", author_email="hansemrbean@googlemail.com", url="https://github.com/pytest-dev/pytest-order", packages=["pytest_order"], license="MIT", entry_points={ "pytest11": [ "pytest_order = pytest_order.plugin", ] }, python_requires=">=3.6", install_requires=[ "pytest>=5.0; python_version < '3.10'", "pytest>=6.2.4; python_version >= '3.10'", ], classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Topic :: Software Development :: Testing", "Topic :: Software Development :: Quality Assurance", "Topic :: Utilities", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ], ) pytest-order-1.2.0/tests/000077500000000000000000000000001452612573200153245ustar00rootroot00000000000000pytest-order-1.2.0/tests/__init__.py000066400000000000000000000000001452612573200174230ustar00rootroot00000000000000pytest-order-1.2.0/tests/conftest.py000066400000000000000000000024161452612573200175260ustar00rootroot00000000000000import pytest from pytest_order.settings import Scope pytest_plugins = ["pytester"] @pytest.fixture def item_names_for(testdir): def _item_names_for(tests_content): def name(item): if item.cls: return item.cls.__name__ + "::" + item.name return item.name items = testdir.getitems(tests_content) hook = items[0].config.hook hook.pytest_collection_modifyitems( session=items[0].session, config=items[0].config, items=items ) return [name(item) for item in items] return _item_names_for @pytest.fixture def test_path(testdir): testdir.tmpdir.join("pytest.ini").write("[pytest]\nconsole_output_style = classic") yield testdir @pytest.fixture def ignore_settings(mocker): settings = mocker.patch("pytest_order.sorter.Settings") settings.return_value.sparse_ordering = False settings.return_value.order_dependencies = False settings.return_value.scope = Scope.SESSION settings.return_value.group_scope = Scope.SESSION settings.return_value.scope_level = 0 settings.return_value.marker_prefix = None yield settings @pytest.fixture def order_dependencies(ignore_settings): ignore_settings.return_value.order_dependencies = True yield pytest-order-1.2.0/tests/test_class_marks.py000066400000000000000000000113131452612573200212360ustar00rootroot00000000000000def test_ordinal_class_marks(item_names_for): tests_content = """ import pytest @pytest.mark.order(1) class Test1: def test_1(self): pass def test_2(self): pass @pytest.mark.order(0) class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test2::test_1", "Test2::test_2", "Test1::test_1", "Test1::test_2", ] def test_after_class_mark(item_names_for): tests_content = """ import pytest @pytest.mark.order(after="Test2") class Test1: def test_1(self): pass def test_2(self): pass class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test2::test_1", "Test2::test_2", "Test1::test_1", "Test1::test_2", ] def test_invalid_class_mark(item_names_for, capsys): tests_content = """ import pytest @pytest.mark.order(after="Test3") class Test1: def test_1(self): pass def test_2(self): pass class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test1::test_1", "Test1::test_2", "Test2::test_1", "Test2::test_2", ] out, err = capsys.readouterr() assert ( "WARNING: cannot execute 'test_2' relative to others: " "'Test3' - ignoring the marker" in out ) def test_before_class_mark(item_names_for): tests_content = """ import pytest class Test1: def test_1(self): pass def test_2(self): pass @pytest.mark.order(before="Test1") class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test2::test_1", "Test2::test_2", "Test1::test_1", "Test1::test_2", ] def test_after_class_marks_for_single_test_in_class(item_names_for): tests_content = """ import pytest @pytest.mark.order(after="Test2::test_1") class Test1: def test_1(self): pass def test_2(self): pass class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test2::test_1", "Test1::test_1", "Test1::test_2", "Test2::test_2", ] def test_before_class_marks_for_single_test_in_class(item_names_for): tests_content = """ import pytest class Test1: def test_1(self): pass def test_2(self): pass @pytest.mark.order(before="Test1::test_2") class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test1::test_1", "Test2::test_1", "Test2::test_2", "Test1::test_2", ] def test_after_class_marks_for_single_test(item_names_for): tests_content = """ import pytest @pytest.mark.order(after="test_1") class Test1: def test_1(self): pass def test_2(self): pass def test_1(): pass class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "test_1", "Test1::test_1", "Test1::test_2", "Test2::test_1", "Test2::test_2", ] def test_before_class_marks_for_single_test(item_names_for): tests_content = """ import pytest def test_1(): pass class Test1: def test_1(self): pass def test_2(self): pass @pytest.mark.order(before="test_1") class Test2: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test2::test_1", "Test2::test_2", "test_1", "Test1::test_1", "Test1::test_2", ] def test_rel_class_mark_with_order_mark(item_names_for): tests_content = """ import pytest class Test1: def test_1(self): pass def test_2(self): pass @pytest.mark.order(before="Test1") class Test2: @pytest.mark.order(2) def test_1(self): pass @pytest.mark.order(1) def test_2(self): pass """ assert item_names_for(tests_content) == [ "Test2::test_2", "Test2::test_1", "Test1::test_1", "Test1::test_2", ] pytest-order-1.2.0/tests/test_dependency.py000066400000000000000000000317101452612573200210550ustar00rootroot00000000000000import pytest def test_ignore_order_with_dependency(item_names_for): tests_content = """ import pytest @pytest.mark.dependency() def test_a(): pass @pytest.mark.dependency(depends=["test_a"]) @pytest.mark.order("first") def test_b(): pass """ assert item_names_for(tests_content) == ["test_a", "test_b"] def test_order_with_dependency(item_names_for): tests_content = """ import pytest @pytest.mark.dependency(depends=["test_b"]) @pytest.mark.order("second") def test_a(): pass @pytest.mark.dependency() def test_b(): pass """ assert item_names_for(tests_content) == ["test_b", "test_a"] @pytest.fixture(scope="module") def ordered_test(): yield ( """ import pytest @pytest.mark.dependency() def test_a(): pass @pytest.mark.dependency(depends=["test_a"]) def test_b(): pass """ ) def test_dependency_already_ordered_default(ordered_test, item_names_for): assert item_names_for(ordered_test) == ["test_a", "test_b"] def test_dependency_already_ordered_with_ordering( ordered_test, item_names_for, order_dependencies ): assert item_names_for(ordered_test) == ["test_a", "test_b"] @pytest.fixture(scope="module") def order_dependency_test(): yield ( """ import pytest @pytest.mark.dependency(depends=["test_b"]) def test_a(): pass @pytest.mark.dependency() def test_b(): pass """ ) def test_order_dependency_default(order_dependency_test, item_names_for): assert item_names_for(order_dependency_test) == ["test_a", "test_b"] def test_order_dependency_ordered( order_dependency_test, item_names_for, order_dependencies ): assert item_names_for(order_dependency_test) == ["test_b", "test_a"] @pytest.fixture(scope="module") def multiple_dependencies_test(): yield ( """ import pytest @pytest.mark.dependency(depends=["test_b", "test_c"]) def test_a(): pass @pytest.mark.dependency() def test_b(): pass @pytest.mark.dependency() def test_c(): pass """ ) def test_order_multiple_dependencies_default( multiple_dependencies_test, item_names_for ): assert item_names_for(multiple_dependencies_test) == ["test_a", "test_b", "test_c"] def test_order_multiple_dependencies_ordered( multiple_dependencies_test, item_names_for, order_dependencies ): assert item_names_for(multiple_dependencies_test) == ["test_b", "test_c", "test_a"] @pytest.fixture def no_dep_marks(test_path): test_path.makepyfile( test_auto=( """ import pytest @pytest.mark.dependency(depends=["test_b", "test_c"]) def test_a(): pass def test_b(): pass def test_c(): pass """ ) ) yield test_path def test_order_dependencies_no_auto_mark(no_dep_marks): no_dep_marks.makefile( ".ini", pytest=( """ [pytest] automark_dependency = 0 console_output_style = classic """ ), ) result = no_dep_marks.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=2, skipped=1) result.stdout.fnmatch_lines( [ "test_auto.py::test_a SKIPPED*", "test_auto.py::test_b PASSED", "test_auto.py::test_c PASSED", ] ) def test_order_dependencies_auto_mark(no_dep_marks): no_dep_marks.makefile( ".ini", pytest=( """ [pytest] automark_dependency = 1 console_output_style = classic """ ), ) result = no_dep_marks.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=3, failed=0) result.stdout.fnmatch_lines( [ "test_auto.py::test_b PASSED", "test_auto.py::test_c PASSED", "test_auto.py::test_a PASSED", ] ) @pytest.fixture(scope="module") def named_dependency_test(): yield ( """ import pytest @pytest.mark.dependency(depends=["my_test"]) def test_a(): pass @pytest.mark.dependency(name="my_test") def test_b(): pass def test_c(): pass """ ) def test_order_named_dependency_default(named_dependency_test, item_names_for): assert item_names_for(named_dependency_test) == ["test_a", "test_b", "test_c"] def test_order_named_dependency_ordered( named_dependency_test, item_names_for, order_dependencies ): assert item_names_for(named_dependency_test) == ["test_b", "test_a", "test_c"] def test_dependency_in_class(item_names_for, order_dependencies): tests_content = """ import pytest class Test: @pytest.mark.dependency(depends=["Test::test_c"]) def test_a(self): assert True @pytest.mark.dependency(depends=["Test::test_c"]) def test_b(self): assert True @pytest.mark.dependency() def test_c(self): assert True """ assert item_names_for(tests_content) == [ "Test::test_c", "Test::test_a", "Test::test_b", ] def test_unresolved_dependency_in_class(item_names_for, order_dependencies, capsys): tests_content = """ import pytest class Test: @pytest.mark.dependency(depends=["test_c"]) def test_a(self): assert True @pytest.mark.dependency(depends=["test_c"]) def test_b(self): assert True @pytest.mark.dependency() def test_c(self): assert True """ assert item_names_for(tests_content) == [ "Test::test_a", "Test::test_b", "Test::test_c", ] out, err = capsys.readouterr() warning = "Cannot resolve the dependency marker 'test_c' - ignoring it" assert warning in out def test_named_dependency_in_class(item_names_for, order_dependencies): tests_content = """ import pytest class Test: @pytest.mark.dependency(name="test_1", depends=["test_3"]) def test_a(self): assert True @pytest.mark.dependency(name="test_2", depends=["test_3"]) def test_b(self): assert True @pytest.mark.dependency(name="test_3") def test_c(self): assert True """ assert item_names_for(tests_content) == [ "Test::test_c", "Test::test_a", "Test::test_b", ] def test_dependencies_in_classes(item_names_for, order_dependencies): tests_content = """ import pytest class TestA: @pytest.mark.dependency(depends=["test_2"]) def test_a(self): assert True @pytest.mark.dependency(depends=["TestB::test_b"]) def test_b(self): assert True def test_c(self): assert True class TestB: @pytest.mark.dependency(name="test_2") def test_a(self): assert True @pytest.mark.dependency() def test_b(self): assert True def test_c(self): assert True """ assert item_names_for(tests_content) == [ "TestA::test_c", "TestB::test_a", "TestA::test_a", "TestB::test_b", "TestA::test_b", "TestB::test_c", ] def test_class_scope_dependencies(item_names_for, order_dependencies): tests_content = """ import pytest class TestA: @pytest.mark.dependency(depends=["test_c"], scope='class') def test_a(self): assert True def test_b(self): assert True @pytest.mark.dependency def test_c(self): assert True """ assert item_names_for(tests_content) == [ "TestA::test_b", "TestA::test_c", "TestA::test_a", ] @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_named_dependency_in_modules(test_path): test_path.makepyfile( test_ndep1=( """ import pytest class Test1: def test_one(self): assert True @pytest.mark.dependency( depends=['dep2_test_one'], scope='session' ) def test_two(self): assert True """ ), test_ndep2=( """ import pytest @pytest.mark.dependency(name='dep2_test_one') def test_one(): assert True def test_two(): assert True """ ), ) result = test_path.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=4, failed=0) result.stdout.fnmatch_lines( [ "test_ndep1.py::Test1::test_one PASSED", "test_ndep2.py::test_one PASSED", "test_ndep1.py::Test1::test_two PASSED", "test_ndep2.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_dependency_in_modules(test_path): test_path.makepyfile( test_unnamed_dep1=( """ import pytest class Test1: def test_one(self): assert True @pytest.mark.dependency( depends=['test_unnamed_dep2.py::test_one'], scope='session', ) def test_two(self): assert True """ ), test_unnamed_dep2=( """ import pytest @pytest.mark.dependency def test_one(): assert True def test_two(): assert True """ ), ) result = test_path.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=4, failed=0) result.stdout.fnmatch_lines( [ "test_unnamed_dep1.py::Test1::test_one PASSED", "test_unnamed_dep2.py::test_one PASSED", "test_unnamed_dep1.py::Test1::test_two PASSED", "test_unnamed_dep2.py::test_two PASSED", ] ) def test_same_dependency_in_modules(test_path): # regression test - make sure that the same dependency in different # modules works correctly test_path.makepyfile( test_module_dep1=( """ import pytest @pytest.mark.dependency(depends=['test_two']) def test_one(): assert True @pytest.mark.dependency def test_two(): assert True """ ), test_module_dep2=( """ import pytest @pytest.mark.dependency(depends=['test_two']) def test_one(): assert True @pytest.mark.dependency def test_two(): assert True """ ), ) result = test_path.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=4, failed=0) result.stdout.fnmatch_lines( [ "test_module_dep1.py::test_two PASSED", "test_module_dep1.py::test_one PASSED", "test_module_dep2.py::test_two PASSED", "test_module_dep2.py::test_one PASSED", ] ) def test_unknown_dependency(item_names_for, order_dependencies, capsys): tests_content = """ import pytest class Test: def test_a(self): assert True @pytest.mark.dependency(depends=["test_3"]) def test_b(self): assert True def test_c(self): assert True """ assert item_names_for(tests_content) == [ "Test::test_a", "Test::test_b", "Test::test_c", ] out, err = capsys.readouterr() warning = "Cannot resolve the dependency marker 'test_3' - ignoring it." assert warning in out def test_unsupported_order_with_dependency(item_names_for): test_content = """ import pytest @pytest.mark.dependency(depends=["test_2"]) @pytest.mark.order("unknown") def test_1(): pass def test_2(): pass """ with pytest.warns(UserWarning, match="Unknown order attribute:'unknown'"): assert item_names_for(test_content) == ["test_1", "test_2"] pytest-order-1.2.0/tests/test_indulgent_ordering.py000066400000000000000000000020741452612573200226220ustar00rootroot00000000000000def test_run_marker_registered(test_path): test_path.makepyfile( test_failing=( """ import pytest @pytest.mark.order("second") def test_me_second(): assert True def test_that_fails(): assert False @pytest.mark.order("first") def test_me_first(): assert True """ ) ) result = test_path.runpytest("-v") result.assert_outcomes(passed=2, failed=1) result.stdout.fnmatch_lines( [ "test_failing.py::test_me_first PASSED", "test_failing.py::test_me_second PASSED", "test_failing.py::test_that_fails FAILED", ] ) result = test_path.runpytest("-v", "--ff", "--indulgent-ordering") result.assert_outcomes(passed=2, failed=1) result.stdout.fnmatch_lines( [ "test_failing.py::test_that_fails FAILED", "test_failing.py::test_me_first PASSED", "test_failing.py::test_me_second PASSED", ] ) pytest-order-1.2.0/tests/test_marker_prefix.py000066400000000000000000000067211452612573200216010ustar00rootroot00000000000000import pytest @pytest.fixture(scope="module") def marker_test_file(): yield ( """ import pytest @pytest.mark.my3 def test_a(): pass @pytest.mark.my1 def test_b(): pass @pytest.mark.my2 def test_c(): pass """ ) @pytest.fixture def marker_test(test_path, marker_test_file): test_path.makepyfile(test_marker=marker_test_file) yield test_path def test_no_ordering(marker_test_file, item_names_for): assert item_names_for(marker_test_file) == ["test_a", "test_b", "test_c"] def test_order_with_marker_prefix(marker_test): result = marker_test.runpytest("-v", "--order-marker-prefix=my") result.assert_outcomes(passed=3, skipped=0) result.stdout.fnmatch_lines( [ "test_marker.py::test_b PASSED", "test_marker.py::test_c PASSED", "test_marker.py::test_a PASSED", ] ) def test_order_with_marker_prefix_filtered(marker_test): result = marker_test.runpytest("-v", "--order-marker-prefix=my", "-m", "my2 or my3") result.assert_outcomes(passed=2, skipped=0) result.stdout.fnmatch_lines( [ "test_marker.py::test_c PASSED", "test_marker.py::test_a PASSED", ] ) def test_no_ordering_with_incorrect_marker_prefix(marker_test): result = marker_test.runpytest("-v", "--order-marker-prefix=mi") result.assert_outcomes(passed=3, skipped=0) result.stdout.fnmatch_lines( [ "test_marker.py::test_a PASSED", "test_marker.py::test_b PASSED", "test_marker.py::test_c PASSED", ] ) def test_no_ordering_with_shorter_marker_prefix(marker_test): result = marker_test.runpytest("-v", "--order-marker-prefix=m") result.assert_outcomes(passed=3, skipped=0) result.stdout.fnmatch_lines( [ "test_marker.py::test_a PASSED", "test_marker.py::test_b PASSED", "test_marker.py::test_c PASSED", ] ) def test_marker_prefix_does_not_interfere_with_order_marks(test_path): test_path.makepyfile( test_marker=( """ import pytest @pytest.mark.order(3) def test_a(): pass @pytest.mark.order(1) def test_b(): pass @pytest.mark.order(2) def test_c(): pass """ ) ) result = test_path.runpytest("-v", "--order-marker-prefix=m") result.assert_outcomes(passed=3, skipped=0) result.stdout.fnmatch_lines( [ "test_marker.py::test_b PASSED", "test_marker.py::test_c PASSED", "test_marker.py::test_a PASSED", ] ) def test_mix_marker_prefix_with_order_marks(test_path): test_path.makepyfile( test_marker=( """ import pytest @pytest.mark.order(3) def test_a(): pass @pytest.mark.my1 def test_b(): pass @pytest.mark.my2 def test_c(): pass """ ) ) result = test_path.runpytest("-v", "--order-marker-prefix=my") result.assert_outcomes(passed=3, skipped=0) result.stdout.fnmatch_lines( [ "test_marker.py::test_b PASSED", "test_marker.py::test_c PASSED", "test_marker.py::test_a PASSED", ] ) pytest-order-1.2.0/tests/test_misc.py000066400000000000000000000021501452612573200176660ustar00rootroot00000000000000import re import pytest import pytest_order def test_version_exists(): assert hasattr(pytest_order, "__version__") def test_version_valid(): # check for PEP 440 conform version assert re.match( r"\d+(\.\d+)*((a|b|rc)\d+)?(\.post\d)?(\.dev\d)?$", pytest_order.__version__, ) def test_markers_registered(capsys): pytest.main(["--markers"]) out, err = capsys.readouterr() assert "@pytest.mark.order" in out # only order is supported as marker assert out.count("Provided by pytest-order.") == 1 def tests_working_without_dependency(test_path): """Make sure no other plugins are needed in settings.""" test_path.makepyfile( test_a=( """ import pytest def test_a(): pass @pytest.mark.order(0) def test_b(): pass """ ) ) result = test_path.runpytest( "-v", "-p", "no:xdist", "-p", "no:dependency", "-p", "no:mock" ) result.stdout.fnmatch_lines( [ "test_a.py::test_b PASSED", "test_a.py::test_a PASSED", ] ) pytest-order-1.2.0/tests/test_multiple_ordering.py000066400000000000000000000051351452612573200224650ustar00rootroot00000000000000def test_multiple_markers(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_1(): pass @pytest.mark.order(1) @pytest.mark.order(3) def test_2(): pass """ assert item_names_for(test_content) == [ "test_2[index=1]", "test_1", "test_2[index=3]", ] def test_with_relative_markers(item_names_for): test_content = """ import pytest def test_1(): pass @pytest.mark.order(before="test_1") @pytest.mark.order(2) def test_2(): pass @pytest.mark.order(1) @pytest.mark.order(before="test_1") def test_3(): pass @pytest.mark.order(-1) @pytest.mark.order(-3) def test_4(): pass @pytest.mark.order(-2) @pytest.mark.order(-4) def test_5(): pass """ assert item_names_for(test_content) == [ "test_3[index=1]", "test_2[index=2]", "test_3[before=test_1]", "test_2[before=test_1]", "test_1", "test_5[index=-4]", "test_4[index=-3]", "test_5[index=-2]", "test_4[index=-1]", ] def test_multiple_markers_with_parametrization(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_1(): pass @pytest.mark.parametrize("arg", ["aaaaa", "bbbbb", "ccccc", "ddddd"]) @pytest.mark.order(1) @pytest.mark.order(-1) def test_2(arg): pass @pytest.mark.order(3) def test_3(): pass """ assert item_names_for(test_content) == [ "test_2[index=1-aaaaa]", "test_2[index=1-bbbbb]", "test_2[index=1-ccccc]", "test_2[index=1-ddddd]", "test_1", "test_3", "test_2[index=-1-aaaaa]", "test_2[index=-1-bbbbb]", "test_2[index=-1-ccccc]", "test_2[index=-1-ddddd]", ] def test_multiple_markers_in_class(item_names_for): test_content = """ import pytest class TestA: @pytest.mark.order(1) @pytest.mark.order(3) def test_1_and_3(): pass @pytest.mark.order(-1) def test_4(): pass @pytest.mark.order(2) def test_2(): pass """ assert item_names_for(test_content) == [ "TestA::test_1_and_3[index=1]", "test_2", "TestA::test_1_and_3[index=3]", "TestA::test_4", ] pytest-order-1.2.0/tests/test_order_group_scope.py000066400000000000000000000137741452612573200224710ustar00rootroot00000000000000import pytest @pytest.fixture def fixture_path(test_path): test_path.makepyfile( test_clss=( """ import pytest class Test1: @pytest.mark.order(2) def test_two(self): assert True def test_one(self): assert True class Test2: def test_two(self): assert True @pytest.mark.order(1) def test_one(self): assert True @pytest.mark.order(-1) def test_two(): assert True def test_one(): assert True """ ), test_fcts1=( """ import pytest @pytest.mark.order(5) def test1_two(): assert True def test1_one(): assert True """ ), test_fcts2=( """ import pytest @pytest.mark.order(0) def test2_two(): assert True def test2_one(): assert True """ ), test_fcts3=( """ import pytest @pytest.mark.order(-2) def test3_two(): assert True def test3_one(): assert True """ ), test_fcts4=( """ import pytest def test4_one(): assert True def test4_two(): assert True """ ), ) yield test_path def test_session_scope(fixture_path): result = fixture_path.runpytest("-v") result.assert_outcomes(passed=14, failed=0) result.stdout.fnmatch_lines( [ "test_fcts2.py::test2_two PASSED", "test_clss.py::Test2::test_one PASSED", "test_clss.py::Test1::test_two PASSED", "test_fcts1.py::test1_two PASSED", "test_clss.py::Test1::test_one PASSED", "test_clss.py::Test2::test_two PASSED", "test_clss.py::test_one PASSED", "test_fcts1.py::test1_one PASSED", "test_fcts2.py::test2_one PASSED", "test_fcts3.py::test3_one PASSED", "test_fcts4.py::test4_one PASSED", "test_fcts4.py::test4_two PASSED", "test_fcts3.py::test3_two PASSED", "test_clss.py::test_two PASSED", ] ) def test_module_group_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-group-scope=module") result.assert_outcomes(passed=14, failed=0) result.stdout.fnmatch_lines( [ "test_fcts2.py::test2_two PASSED", "test_fcts2.py::test2_one PASSED", "test_clss.py::Test2::test_one PASSED", "test_clss.py::Test1::test_two PASSED", "test_clss.py::Test1::test_one PASSED", "test_clss.py::Test2::test_two PASSED", "test_clss.py::test_one PASSED", "test_clss.py::test_two PASSED", "test_fcts1.py::test1_two PASSED", "test_fcts1.py::test1_one PASSED", "test_fcts4.py::test4_one PASSED", "test_fcts4.py::test4_two PASSED", "test_fcts3.py::test3_one PASSED", "test_fcts3.py::test3_two PASSED", ] ) def test_class_group_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-group-scope=class") result.assert_outcomes(passed=14, failed=0) result.stdout.fnmatch_lines( [ "test_fcts2.py::test2_two PASSED", "test_fcts2.py::test2_one PASSED", "test_clss.py::Test2::test_one PASSED", "test_clss.py::Test2::test_two PASSED", "test_clss.py::Test1::test_two PASSED", "test_clss.py::Test1::test_one PASSED", "test_clss.py::test_one PASSED", "test_clss.py::test_two PASSED", "test_fcts1.py::test1_two PASSED", "test_fcts1.py::test1_one PASSED", "test_fcts4.py::test4_one PASSED", "test_fcts4.py::test4_two PASSED", "test_fcts3.py::test3_one PASSED", "test_fcts3.py::test3_two PASSED", ] ) def test_class_group_scope_module_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-group-scope=class", "--order-scope=module" ) result.assert_outcomes(passed=14, failed=0) result.stdout.fnmatch_lines( [ "test_clss.py::Test2::test_one PASSED", "test_clss.py::Test2::test_two PASSED", "test_clss.py::Test1::test_two PASSED", "test_clss.py::Test1::test_one PASSED", "test_clss.py::test_one PASSED", "test_clss.py::test_two PASSED", "test_fcts1.py::test1_two PASSED", "test_fcts1.py::test1_one PASSED", "test_fcts2.py::test2_two PASSED", "test_fcts2.py::test2_one PASSED", "test_fcts3.py::test3_one PASSED", "test_fcts3.py::test3_two PASSED", "test_fcts4.py::test4_one PASSED", "test_fcts4.py::test4_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="Warning does not appear in output in pytest < 3.8", ) def test_invalid_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-group-scope=function") result.assert_outcomes(passed=14, failed=0) result.stdout.fnmatch_lines( ["*UserWarning: Unknown order group scope 'function', ignoring it.*"] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="Warning does not appear in output in pytest < 3.8", ) def test_ignored_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-group-scope=session", "--order-scope=module" ) result.assert_outcomes(passed=14, failed=0) result.stdout.fnmatch_lines( ["*UserWarning: Group scope is larger than order scope, ignoring it."] ) pytest-order-1.2.0/tests/test_order_group_scope_dep.py000066400000000000000000000124451452612573200233130ustar00rootroot00000000000000import pytest @pytest.fixture def fixture_path(test_path): test_path.makepyfile( test_dep1=( """ import pytest class Test1: @pytest.mark.dependency(depends=['Test2::test_two']) def test_one(self): assert True @pytest.mark.dependency(depends=['test_three'], scope="class") def test_two(self): assert True @pytest.mark.dependency def test_three(self): assert True class Test2: def test_one(self): assert True @pytest.mark.dependency( depends=['test_dep3.py::test_two'], scope='session' ) def test_two(self): assert True """ ), test_dep2=( """ import pytest @pytest.mark.dependency( depends=['test_dep3.py::test_two'], scope='session' ) def test_one(): assert True def test_two(): assert True """ ), test_dep3=( """ import pytest def test_one(): assert True @pytest.mark.dependency def test_two(): assert True """ ), test_dep4=( """ import pytest def test_one(): assert True def test_two(): assert True """ ), ) yield test_path @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_session_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=11, failed=0) result.stdout.fnmatch_lines( [ "test_dep1.py::Test1::test_three PASSED", "test_dep1.py::Test1::test_two PASSED", "test_dep1.py::Test2::test_one PASSED", "test_dep2.py::test_two PASSED", "test_dep3.py::test_one PASSED", "test_dep3.py::test_two PASSED", "test_dep1.py::Test2::test_two PASSED", "test_dep1.py::Test1::test_one PASSED", "test_dep2.py::test_one PASSED", "test_dep4.py::test_one PASSED", "test_dep4.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_module_group_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-dependencies", "--order-group-scope=module" ) result.assert_outcomes(passed=11, failed=0) result.stdout.fnmatch_lines( [ "test_dep3.py::test_one PASSED", "test_dep3.py::test_two PASSED", "test_dep1.py::Test1::test_three PASSED", "test_dep1.py::Test1::test_two PASSED", "test_dep1.py::Test2::test_one PASSED", "test_dep1.py::Test2::test_two PASSED", "test_dep1.py::Test1::test_one PASSED", "test_dep2.py::test_one PASSED", "test_dep2.py::test_two PASSED", "test_dep4.py::test_one PASSED", "test_dep4.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_class_group_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-dependencies", "--order-group-scope=class" ) result.assert_outcomes(passed=11, failed=0) result.stdout.fnmatch_lines( [ "test_dep3.py::test_one PASSED", "test_dep3.py::test_two PASSED", "test_dep1.py::Test2::test_one PASSED", "test_dep1.py::Test2::test_two PASSED", "test_dep1.py::Test1::test_one PASSED", "test_dep1.py::Test1::test_three PASSED", "test_dep1.py::Test1::test_two PASSED", "test_dep2.py::test_one PASSED", "test_dep2.py::test_two PASSED", "test_dep4.py::test_one PASSED", "test_dep4.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_class_group_scope_module_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-dependencies", "--order-group-scope=class", "--order-scope=module", ) result.assert_outcomes(passed=8, skipped=3) result.stdout.fnmatch_lines( [ "test_dep1.py::Test2::test_one PASSED", "test_dep1.py::Test2::test_two SKIPPED*", "test_dep1.py::Test1::test_one SKIPPED*", "test_dep1.py::Test1::test_three PASSED", "test_dep1.py::Test1::test_two PASSED", "test_dep2.py::test_one SKIPPED*", "test_dep2.py::test_two PASSED", "test_dep3.py::test_one PASSED", "test_dep3.py::test_two PASSED", "test_dep4.py::test_one PASSED", "test_dep4.py::test_two PASSED", ] ) pytest-order-1.2.0/tests/test_order_group_scope_named_dep.py000066400000000000000000000123241452612573200244530ustar00rootroot00000000000000import pytest @pytest.fixture def fixture_path(test_path): test_path.makepyfile( test_named_dep1=( """ import pytest class Test1: @pytest.mark.dependency(depends=['Test2_test2']) def test_one(self): assert True def test_two(self): assert True class Test2: def test_one(self): assert True @pytest.mark.dependency( name='Test2_test2', depends=['dep3_test_two'], scope='session', ) def test_two(self): assert True """ ), test_named_dep2=( """ import pytest @pytest.mark.dependency( depends=['dep3_test_two'], scope='session' ) def test_one(): assert True def test_two(): assert True """ ), test_named_dep3=( """ import pytest def test_one(): assert True @pytest.mark.dependency(name='dep3_test_two') def test_two(): assert True """ ), test_named_dep4=( """ import pytest def test_one(): assert True def test_two(): assert True """ ), ) yield test_path @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_session_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-dependencies") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_named_dep1.py::Test1::test_two PASSED", "test_named_dep1.py::Test2::test_one PASSED", "test_named_dep2.py::test_two PASSED", "test_named_dep3.py::test_one PASSED", "test_named_dep3.py::test_two PASSED", "test_named_dep1.py::Test2::test_two PASSED", "test_named_dep1.py::Test1::test_one PASSED", "test_named_dep2.py::test_one PASSED", "test_named_dep4.py::test_one PASSED", "test_named_dep4.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_module_group_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-dependencies", "--order-group-scope=module" ) result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_named_dep3.py::test_one PASSED", "test_named_dep3.py::test_two PASSED", "test_named_dep1.py::Test1::test_two PASSED", "test_named_dep1.py::Test2::test_one PASSED", "test_named_dep1.py::Test2::test_two PASSED", "test_named_dep1.py::Test1::test_one PASSED", "test_named_dep2.py::test_one PASSED", "test_named_dep2.py::test_two PASSED", "test_named_dep4.py::test_one PASSED", "test_named_dep4.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_class_group_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-dependencies", "--order-group-scope=class" ) result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_named_dep3.py::test_one PASSED", "test_named_dep3.py::test_two PASSED", "test_named_dep1.py::Test2::test_one PASSED", "test_named_dep1.py::Test2::test_two PASSED", "test_named_dep1.py::Test1::test_one PASSED", "test_named_dep1.py::Test1::test_two PASSED", "test_named_dep2.py::test_one PASSED", "test_named_dep2.py::test_two PASSED", "test_named_dep4.py::test_one PASSED", "test_named_dep4.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="pytest-dependency < 0.5 does not support session scope", ) def test_class_group_scope_module_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-dependencies", "--order-group-scope=class", "--order-scope=module", ) result.assert_outcomes(passed=7, skipped=3) result.stdout.fnmatch_lines( [ "test_named_dep1.py::Test2::test_one PASSED", "test_named_dep1.py::Test2::test_two SKIPPED*", "test_named_dep1.py::Test1::test_one SKIPPED*", "test_named_dep1.py::Test1::test_two PASSED", "test_named_dep2.py::test_one SKIPPED*", "test_named_dep2.py::test_two PASSED", "test_named_dep3.py::test_one PASSED", "test_named_dep3.py::test_two PASSED", "test_named_dep4.py::test_one PASSED", "test_named_dep4.py::test_two PASSED", ] ) pytest-order-1.2.0/tests/test_order_group_scope_relative.py000066400000000000000000000117051452612573200243540ustar00rootroot00000000000000import pytest @pytest.fixture def fixture_path(test_path): test_path.makepyfile( test_rel1=( """ import pytest class Test1: @pytest.mark.order(after='Test2::test_two') def test_one(self): assert True def test_two(self): assert True class Test2: @pytest.mark.order(after='invalid') def test_one(self): assert True @pytest.mark.order(after='test_rel3.py::test_two') def test_two(self): assert True """ ), test_rel2=( """ def test_one(): assert True def test_two(): assert True """ ), test_rel3=( """ import pytest def test_one(): assert True @pytest.mark.order(before='test_rel2.py::test_one') def test_two(): assert True """ ), test_rel4=( """ import pytest def test_one(): assert True def test_two(): assert True """ ), ) yield test_path def test_session_scope(fixture_path): result = fixture_path.runpytest("-v") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "*'invalid' - ignoring the marker.*", "test_rel1.py::Test1::test_two PASSED", "test_rel1.py::Test2::test_one PASSED", "test_rel3.py::test_two PASSED", "test_rel1.py::Test2::test_two PASSED", "test_rel1.py::Test1::test_one PASSED", "test_rel2.py::test_one PASSED", "test_rel2.py::test_two PASSED", "test_rel3.py::test_one PASSED", "test_rel4.py::test_one PASSED", "test_rel4.py::test_two PASSED", ] ) def test_module_group_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-group-scope=module") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_rel3.py::test_one PASSED", "test_rel3.py::test_two PASSED", "test_rel1.py::Test1::test_two PASSED", "test_rel1.py::Test2::test_one PASSED", "test_rel1.py::Test2::test_two PASSED", "test_rel1.py::Test1::test_one PASSED", "test_rel2.py::test_one PASSED", "test_rel2.py::test_two PASSED", "test_rel4.py::test_one PASSED", "test_rel4.py::test_two PASSED", ] ) def test_class_group_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-group-scope=class") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_rel3.py::test_one PASSED", "test_rel3.py::test_two PASSED", "test_rel1.py::Test2::test_one PASSED", "test_rel1.py::Test2::test_two PASSED", "test_rel1.py::Test1::test_one PASSED", "test_rel1.py::Test1::test_two PASSED", "test_rel2.py::test_one PASSED", "test_rel2.py::test_two PASSED", "test_rel4.py::test_one PASSED", "test_rel4.py::test_two PASSED", ] ) def test_class_group_scope_module_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-group-scope=class", "--order-scope=module" ) result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_rel1.py::Test2::test_one PASSED", "test_rel1.py::Test2::test_two PASSED", "test_rel1.py::Test1::test_one PASSED", "test_rel1.py::Test1::test_two PASSED", "test_rel2.py::test_one PASSED", "test_rel2.py::test_two PASSED", "test_rel3.py::test_one PASSED", "test_rel3.py::test_two PASSED", "test_rel4.py::test_one PASSED", "test_rel4.py::test_two PASSED", ] ) def test_rel_class_mark_with_order_mark(test_path): test_path.makepyfile( test_class_rel=""" import pytest class Test1: def test_1(self): pass @pytest.mark.order(1) def test_2(self): pass @pytest.mark.order(before="Test1") class Test2: def test_1(self): pass @pytest.mark.order(1) def test_2(self): pass """ ) result = test_path.runpytest("-v", "--order-group-scope=class") result.assert_outcomes(passed=4, failed=0) result.stdout.fnmatch_lines( [ "test_class_rel.py::Test2::test_2 PASSED", "test_class_rel.py::Test2::test_1 PASSED", "test_class_rel.py::Test1::test_2 PASSED", "test_class_rel.py::Test1::test_1 PASSED", ] ) pytest-order-1.2.0/tests/test_order_scope.py000066400000000000000000000077321452612573200212520ustar00rootroot00000000000000import pytest @pytest.fixture def fixture_path(test_path): test_path.makepyfile( test_classes=( """ import pytest class Test1: @pytest.mark.order("last") def test_two(self): assert True @pytest.mark.order("first") def test_one(self): assert True class Test2: @pytest.mark.order("last") def test_two(self): assert True @pytest.mark.order("first") def test_one(self): assert True @pytest.mark.order("last") def test_two(): assert True @pytest.mark.order("first") def test_one(): assert True """ ), test_functions1=( """ import pytest @pytest.mark.order("last") def test1_two(): assert True @pytest.mark.order("first") def test1_one(): assert True """ ), test_functions2=( """ import pytest @pytest.mark.order("last") def test2_two(): assert True @pytest.mark.order("first") def test2_one(): assert True """ ), ) yield test_path def test_session_scope(fixture_path): result = fixture_path.runpytest("-v") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_classes.py::Test1::test_one PASSED", "test_classes.py::Test2::test_one PASSED", "test_classes.py::test_one PASSED", "test_functions1.py::test1_one PASSED", "test_functions2.py::test2_one PASSED", "test_classes.py::Test1::test_two PASSED", "test_classes.py::Test2::test_two PASSED", "test_classes.py::test_two PASSED", "test_functions1.py::test1_two PASSED", "test_functions2.py::test2_two PASSED", ] ) def test_module_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-scope=module") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_classes.py::Test1::test_one PASSED", "test_classes.py::Test2::test_one PASSED", "test_classes.py::test_one PASSED", "test_classes.py::Test1::test_two PASSED", "test_classes.py::Test2::test_two PASSED", "test_classes.py::test_two PASSED", "test_functions1.py::test1_one PASSED", "test_functions1.py::test1_two PASSED", "test_functions2.py::test2_one PASSED", "test_functions2.py::test2_two PASSED", ] ) def test_class_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-scope=class") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( [ "test_classes.py::Test1::test_one PASSED", "test_classes.py::Test1::test_two PASSED", "test_classes.py::Test2::test_one PASSED", "test_classes.py::Test2::test_two PASSED", "test_classes.py::test_one PASSED", "test_classes.py::test_two PASSED", "test_functions1.py::test1_one PASSED", "test_functions1.py::test1_two PASSED", "test_functions2.py::test2_one PASSED", "test_functions2.py::test2_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="Warning does not appear in output in pytest < 3.8", ) def test_invalid_scope(fixture_path): result = fixture_path.runpytest("-v", "--order-scope=function") result.assert_outcomes(passed=10, failed=0) result.stdout.fnmatch_lines( ["*UserWarning: Unknown order scope 'function', ignoring it.*"] ) pytest-order-1.2.0/tests/test_order_scope_level.py000066400000000000000000000111751452612573200224350ustar00rootroot00000000000000from textwrap import dedent import pytest @pytest.fixture def fixture_path(test_path): test_a_contents = dedent( """ import pytest @pytest.mark.order(4) def test_four(): pass @pytest.mark.order(3) def test_three(): pass """ ) test_b_contents = dedent( """ import pytest @pytest.mark.order(2) def test_two(): pass @pytest.mark.order(1) def test_one(): pass """ ) for i in range(3): sub_path = "feature{}".format(i) test_path.mkpydir(sub_path) path = test_path.tmpdir.join(sub_path, "test_a.py") path.write(test_a_contents) path = test_path.tmpdir.join(sub_path, "test_b.py") path.write(test_b_contents) yield test_path def test_session_scope(fixture_path): result = fixture_path.runpytest("-v") result.assert_outcomes(passed=12, failed=0) result.stdout.fnmatch_lines( [ "feature0/test_b.py::test_one PASSED", "feature1/test_b.py::test_one PASSED", "feature2/test_b.py::test_one PASSED", "feature0/test_b.py::test_two PASSED", "feature1/test_b.py::test_two PASSED", "feature2/test_b.py::test_two PASSED", "feature0/test_a.py::test_three PASSED", "feature1/test_a.py::test_three PASSED", "feature2/test_a.py::test_three PASSED", "feature0/test_a.py::test_four PASSED", "feature1/test_a.py::test_four PASSED", "feature2/test_a.py::test_four PASSED", ] ) def test_dir_level0(fixture_path, capsys): """Same as session scope.""" result = fixture_path.runpytest("-v", "--order-scope-level=0") result.assert_outcomes(passed=12, failed=0) result.stdout.fnmatch_lines( [ "feature0/test_b.py::test_one PASSED", "feature1/test_b.py::test_one PASSED", "feature2/test_b.py::test_one PASSED", "feature0/test_b.py::test_two PASSED", "feature1/test_b.py::test_two PASSED", "feature2/test_b.py::test_two PASSED", "feature0/test_a.py::test_three PASSED", "feature1/test_a.py::test_three PASSED", "feature2/test_a.py::test_three PASSED", "feature0/test_a.py::test_four PASSED", "feature1/test_a.py::test_four PASSED", "feature2/test_a.py::test_four PASSED", ] ) def test_dir_level1(fixture_path, capsys): """Orders per feature path.""" result = fixture_path.runpytest("-v", "--order-scope-level=1") result.assert_outcomes(passed=12, failed=0) result.stdout.fnmatch_lines( [ "feature0/test_b.py::test_one PASSED", "feature0/test_b.py::test_two PASSED", "feature0/test_a.py::test_three PASSED", "feature0/test_a.py::test_four PASSED", "feature1/test_b.py::test_one PASSED", "feature1/test_b.py::test_two PASSED", "feature1/test_a.py::test_three PASSED", "feature1/test_a.py::test_four PASSED", "feature2/test_b.py::test_one PASSED", "feature2/test_b.py::test_two PASSED", "feature2/test_a.py::test_three PASSED", "feature2/test_a.py::test_four PASSED", ] ) def test_dir_level2(fixture_path, capsys): """Same as module scope.""" result = fixture_path.runpytest("-v", "--order-scope-level=2") result.assert_outcomes(passed=12, failed=0) result.stdout.fnmatch_lines( [ "feature0/test_a.py::test_three PASSED", "feature0/test_a.py::test_four PASSED", "feature0/test_b.py::test_one PASSED", "feature0/test_b.py::test_two PASSED", "feature1/test_a.py::test_three PASSED", "feature1/test_a.py::test_four PASSED", "feature1/test_b.py::test_one PASSED", "feature1/test_b.py::test_two PASSED", "feature2/test_a.py::test_three PASSED", "feature2/test_a.py::test_four PASSED", "feature2/test_b.py::test_one PASSED", "feature2/test_b.py::test_two PASSED", ] ) @pytest.mark.skipif( pytest.__version__.startswith("3.7."), reason="Warning does not appear in output in pytest < 3.8", ) def test_invalid_scope(fixture_path): result = fixture_path.runpytest( "-v", "--order-scope=module", "--order-scope-level=1" ) result.assert_outcomes(passed=12, failed=0) result.stdout.fnmatch_lines( [ "*UserWarning: order-scope-level cannot be used " "together with --order-scope=module*" ] ) pytest-order-1.2.0/tests/test_ordering.py000066400000000000000000000167211452612573200205550ustar00rootroot00000000000000import pytest def test_no_marks(item_names_for): tests_content = """ def test_1(): pass def test_2(): pass """ assert item_names_for(tests_content) == ["test_1", "test_2"] def test_first_mark(item_names_for): tests_content = """ import pytest def test_1(): pass @pytest.mark.order("first") def test_2(): pass """ assert item_names_for(tests_content) == ["test_2", "test_1"] def test_last_mark(item_names_for): tests_content = """ import pytest @pytest.mark.order("last") def test_1(): pass def test_2(): pass """ assert item_names_for(tests_content) == ["test_2", "test_1"] def test_first_last_marks(item_names_for): tests_content = """ import pytest @pytest.mark.order("last") def test_1(): pass @pytest.mark.order("first") def test_2(): pass def test_3(): pass """ assert item_names_for(tests_content) == ["test_2", "test_3", "test_1"] def test_order_marks(item_names_for): tests_content = """ import pytest @pytest.mark.order(-1) def test_1(): pass @pytest.mark.order(-2) def test_2(): pass @pytest.mark.order(1) def test_3(): pass """ assert item_names_for(tests_content) == ["test_3", "test_2", "test_1"] def test_order_marks_by_index(item_names_for): tests_content = """ import pytest @pytest.mark.order(index=-1) def test_1(): pass @pytest.mark.order(index=-2) def test_2(): pass @pytest.mark.order(index=1) def test_3(): pass """ assert item_names_for(tests_content) == ["test_3", "test_2", "test_1"] def test_non_contiguous_positive(item_names_for): tests_content = """ import pytest @pytest.mark.order(10) def test_1(): pass @pytest.mark.order(20) def test_2(): pass @pytest.mark.order(5) def test_3(): pass """ assert item_names_for(tests_content) == ["test_3", "test_1", "test_2"] def test_non_contiguous_negative(item_names_for): tests_content = """ import pytest @pytest.mark.order(-10) def test_1(): pass @pytest.mark.order(-20) def test_2(): pass @pytest.mark.order(-5) def test_3(): pass """ assert item_names_for(tests_content) == ["test_2", "test_1", "test_3"] def test_non_contiguous_inc_zero(item_names_for): tests_content = """ import pytest @pytest.mark.order(10) def test_1(): pass @pytest.mark.order(20) def test_2(): pass @pytest.mark.order(5) def test_3(): pass @pytest.mark.order(-10) def test_4(): pass @pytest.mark.order(-20) def test_5(): pass @pytest.mark.order(-5) def test_6(): pass @pytest.mark.order(0) def test_7(): pass """ assert item_names_for(tests_content) == [ "test_7", "test_3", "test_1", "test_2", "test_5", "test_4", "test_6", ] def test_non_contiguous_inc_none(item_names_for): tests_content = """ import pytest @pytest.mark.order(5) def test_1(): pass @pytest.mark.order(0) def test_2(): pass @pytest.mark.order(1) def test_3(): pass @pytest.mark.order(-1) def test_4(): pass @pytest.mark.order(-5) def test_5(): pass def test_6(): pass """ assert item_names_for(tests_content) == [ "test_2", "test_3", "test_1", "test_6", "test_5", "test_4", ] def test_first_mark_class(item_names_for): tests_content = """ import pytest def test_1(): pass @pytest.mark.order("first") class TestSuite: def test_3(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "TestSuite::test_3", "TestSuite::test_2", "test_1", ] def test_last_mark_class(item_names_for): tests_content = """ import pytest @pytest.mark.order("last") class TestSuite: def test_1(self): pass def test_2(self): pass def test_3(): pass """ assert item_names_for(tests_content) == [ "test_3", "TestSuite::test_1", "TestSuite::test_2", ] def test_first_last_mark_class(item_names_for): tests_content = """ import pytest @pytest.mark.order("last") class TestLast: def test_1(self): pass def test_2(self): pass def test_0(): pass @pytest.mark.order("first") class TestFirst: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "TestFirst::test_1", "TestFirst::test_2", "test_0", "TestLast::test_1", "TestLast::test_2", ] def test_order_mark_class(item_names_for): tests_content = """ import pytest @pytest.mark.order(-1) class TestLast: def test_1(self): pass def test_2(self): pass @pytest.mark.order(0) def test_0(): pass @pytest.mark.order(-2) class TestFirst: def test_1(self): pass def test_2(self): pass """ assert item_names_for(tests_content) == [ "test_0", "TestFirst::test_1", "TestFirst::test_2", "TestLast::test_1", "TestLast::test_2", ] def test_run_ordinals(item_names_for): test_content = """ import pytest @pytest.mark.order("second_to_last") def test_three(): pass @pytest.mark.order("last") def test_four(): pass @pytest.mark.order("second") def test_two(): pass @pytest.mark.order("first") def test_one(): pass """ assert item_names_for(test_content) == [ "test_one", "test_two", "test_three", "test_four", ] def test_sparse_numbers(item_names_for): test_content = """ import pytest @pytest.mark.order(4) def test_two(): assert True def test_three(): assert True @pytest.mark.order(2) def test_one(): assert True """ assert item_names_for(test_content) == ["test_one", "test_two", "test_three"] def test_quickstart(item_names_for): test_content = """ import pytest def test_foo(): pass def test_bar(): pass """ assert item_names_for(test_content) == ["test_foo", "test_bar"] def test_quickstart2(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_foo(): pass @pytest.mark.order(1) def test_bar(): pass """ assert item_names_for(test_content) == ["test_bar", "test_foo"] def test_unsupported_order(item_names_for): test_content = """ import pytest @pytest.mark.order("unknown") def test_1(): pass def test_2(): pass """ with pytest.warns(UserWarning, match="Unknown order attribute:'unknown'"): assert item_names_for(test_content) == ["test_1", "test_2"] pytest-order-1.2.0/tests/test_relative_ordering.py000066400000000000000000000255171452612573200224530ustar00rootroot00000000000000from textwrap import dedent import pytest def test_relative(item_names_for): test_content = """ import pytest @pytest.mark.order(after="test_second") def test_third(): pass def test_second(): pass @pytest.mark.order(before="test_second") def test_first(): pass """ assert item_names_for(test_content) == ["test_first", "test_second", "test_third"] def test_relative2(item_names_for): test_content = """ import pytest @pytest.mark.order(after="test_second") def test_third(): pass def test_second(): pass @pytest.mark.order(before="test_second") def test_first(): pass def test_five(): pass @pytest.mark.order(before="test_five") def test_four(): pass """ assert item_names_for(test_content) == [ "test_first", "test_second", "test_third", "test_four", "test_five", ] def test_relative3(item_names_for): test_content = """ import pytest @pytest.mark.order(after="test_second") def test_third(): pass def test_second(): pass @pytest.mark.order(before="test_second") def test_first(): pass def test_five(): pass @pytest.mark.order(before="test_five") def test_four(): pass """ assert item_names_for(test_content) == [ "test_first", "test_second", "test_third", "test_four", "test_five", ] def test_relative_in_class(item_names_for): tests_content = """ import pytest class Test: @pytest.mark.order(after="test_b") def test_a(self): pass def test_b(self): pass def test_c(self): pass """ assert item_names_for(tests_content) == [ "Test::test_b", "Test::test_a", "Test::test_c", ] def test_relative_in_classes(item_names_for): tests_content = """ import pytest class TestA: @pytest.mark.order(after="TestB::test_b") def test_a(self): pass @pytest.mark.order(after="test_c") def test_b(self): pass def test_c(self): pass class TestB: @pytest.mark.order(before="TestA::test_c") def test_a(self): pass def test_b(self): pass def test_c(self): pass """ assert item_names_for(tests_content) == [ "TestB::test_a", "TestA::test_c", "TestA::test_b", "TestB::test_b", "TestA::test_a", "TestB::test_c", ] @pytest.fixture def fixture_path(test_path): test_path.makepyfile( mod1_test=( """ import pytest class TestA: @pytest.mark.order(after="mod2_test.py::TestB::test_b") def test_a(self): pass @pytest.mark.order(after="sub/mod3_test.py::test_b") def test_b(self): pass def test_c(self): pass """ ), mod2_test=( """ import pytest class TestB: @pytest.mark.order(before="mod1_test.py::TestA::test_c") def test_a(self): pass def test_b(self): pass def test_c(self): pass """ ), ) test_path.mkpydir("sub") path = test_path.tmpdir.join("sub", "mod3_test.py") path.write( dedent( """ import pytest @pytest.mark.order(before="mod2_test.py::TestB::test_c") def test_a(): pass def test_b(): pass def test_c(): pass """ ) ) yield test_path def test_relative_in_modules(fixture_path): result = fixture_path.runpytest("-v") result.assert_outcomes(passed=9, failed=0) result.stdout.fnmatch_lines( [ "mod2_test.py::TestB::test_a PASSED", "mod1_test.py::TestA::test_c PASSED", "mod2_test.py::TestB::test_b PASSED", "mod1_test.py::TestA::test_a PASSED", "sub/mod3_test.py::test_a PASSED", "mod2_test.py::TestB::test_c PASSED", "sub/mod3_test.py::test_b PASSED", "mod1_test.py::TestA::test_b PASSED", "sub/mod3_test.py::test_c PASSED", ] ) def test_false_insert(item_names_for): test_content = """ import pytest @pytest.mark.order(after="test_a") def test_third(): pass def test_second(): pass @pytest.mark.order(before="test_b") def test_first(): pass """ assert item_names_for(test_content) == ["test_third", "test_second", "test_first"] def test_mixed_markers1(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_1(): pass @pytest.mark.order(after="test_1") def test_2(): pass @pytest.mark.order(1) def test_3(): pass """ assert item_names_for(test_content) == ["test_3", "test_1", "test_2"] def test_mixed_markers2(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_1(): pass @pytest.mark.order(1) def test_2(): pass @pytest.mark.order(before="test_2") def test_3(): pass """ assert item_names_for(test_content) == ["test_3", "test_2", "test_1"] def test_combined_markers1(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_1(): pass def test_2(): pass @pytest.mark.order(index=1, before="test_2") def test_3(): pass """ assert item_names_for(test_content) == ["test_3", "test_1", "test_2"] def test_combined_markers2(item_names_for): test_content = """ import pytest def test_1(): pass @pytest.mark.order(index=2, before="test_1") def test_2(): pass @pytest.mark.order(index=1, before="test_1") def test_3(): pass """ assert item_names_for(test_content) == ["test_3", "test_2", "test_1"] def test_combined_markers3(item_names_for): test_content = """ import pytest def test_1(): pass @pytest.mark.order(index=2, before="test_3") def test_2(): pass @pytest.mark.order(index=1, before="test_1") def test_3(): pass """ assert item_names_for(test_content) == ["test_2", "test_3", "test_1"] def test_mixed_markers4(item_names_for): test_content = """ import pytest @pytest.mark.order(2) def test_1(): pass @pytest.mark.order(index=1, after="test_3") def test_2(): pass def test_3(): pass """ assert item_names_for(test_content) == ["test_3", "test_2", "test_1"] def test_multiple_markers_in_same_test(item_names_for): test_content = """ import pytest @pytest.mark.order(after=["test_3", "test_4", "test_5"]) def test_1(): pass def test_2(): pass def test_3(): pass @pytest.mark.order(before=["test_3", "test_2"]) def test_4(): pass def test_5(): pass """ assert item_names_for(test_content) == [ "test_4", "test_2", "test_3", "test_5", "test_1", ] def test_dependency_after_unknown_test(item_names_for, capsys): test_content = """ import pytest @pytest.mark.order(after="some_module.py::test_2") def test_1(): pass def test_2(): pass """ assert item_names_for(test_content) == ["test_1", "test_2"] out, err = capsys.readouterr() warning = ( "cannot execute 'test_1' relative to others: 'some_module.py::test_2' " "- ignoring the marker" ) assert warning in out def test_dependency_before_unknown_test(item_names_for, capsys): test_content = """ import pytest def test_1(): pass @pytest.mark.order(before="test_4") def test_2(): pass def test_3(): pass """ assert item_names_for(test_content) == ["test_1", "test_2", "test_3"] out, err = capsys.readouterr() warning = ( "cannot execute 'test_2' relative to others: 'test_4' " "- ignoring the marker" ) assert warning in out def test_dependency_in_class_before_unknown_test(item_names_for, capsys): test_content = """ import pytest class Test: def test_1(self): pass @pytest.mark.order(before="test_4") def test_2(self): pass def test_3(self): pass """ assert item_names_for(test_content) == [ "Test::test_1", "Test::test_2", "Test::test_3", ] out, err = capsys.readouterr() warning = ( "cannot execute 'test_2' relative to others: 'test_4' " "- ignoring the marker" ) assert warning in out def test_dependency_loop(item_names_for, capsys): test_content = """ import pytest @pytest.mark.order(after="test_3") def test_1(): pass @pytest.mark.order(1) def test_2(): pass @pytest.mark.order(before="test_1") def test_3(): pass """ assert item_names_for(test_content) == ["test_2", "test_1", "test_3"] out, err = capsys.readouterr() warning = ( "cannot execute test relative to others: " "test_dependency_loop.py::test_3" ) assert warning in out def test_dependency_on_parametrized_test(item_names_for): test_content = """ import pytest @pytest.mark.order(after="test_2") def test_1(): pass @pytest.mark.parametrize("arg", ["aaaaa", "bbbbb", "ccccc", "ddddd"]) def test_2(arg): pass @pytest.mark.order(before="test_2") def test_3(): pass """ assert item_names_for(test_content) == [ "test_3", "test_2[aaaaa]", "test_2[bbbbb]", "test_2[ccccc]", "test_2[ddddd]", "test_1", ] pytest-order-1.2.0/tests/test_sparse_ordinals.py000066400000000000000000000123531452612573200221310ustar00rootroot00000000000000import pytest @pytest.fixture def sparse_ordering(ignore_settings): ignore_settings.return_value.sparse_ordering = True yield @pytest.fixture(scope="module") def first_test(): yield ( """ import pytest def test_1(): pass @pytest.mark.order("first") def test_2(): pass """ ) def test_first_default(first_test, item_names_for): assert item_names_for(first_test) == ["test_2", "test_1"] def test_first_sparse(first_test, item_names_for, sparse_ordering): assert item_names_for(first_test) == ["test_2", "test_1"] @pytest.fixture(scope="module") def second_test(): yield ( """ import pytest def test_1(): pass def test_2(): pass def test_3(): pass def test_4(): pass @pytest.mark.order("second") def test_5(): pass """ ) def test_second_default(second_test, item_names_for): assert item_names_for(second_test) == [ "test_5", "test_1", "test_2", "test_3", "test_4", ] def test_second_sparse(second_test, item_names_for, sparse_ordering): assert item_names_for(second_test) == [ "test_1", "test_5", "test_2", "test_3", "test_4", ] @pytest.fixture(scope="module") def third_test(): yield ( """ import pytest def test_1(): pass def test_2(): pass def test_3(): pass @pytest.mark.order("third") def test_4(): pass def test_5(): pass """ ) def test_third_default(third_test, item_names_for): assert item_names_for(third_test) == [ "test_4", "test_1", "test_2", "test_3", "test_5", ] def test_third_sparse(third_test, item_names_for, sparse_ordering): assert item_names_for(third_test) == [ "test_1", "test_2", "test_4", "test_3", "test_5", ] @pytest.fixture(scope="module") def second_to_last_test(): yield ( """ import pytest def test_1(): pass @pytest.mark.order("second_to_last") def test_2(): pass def test_3(): pass def test_4(): pass def test_5(): pass """ ) def test_second_to_last_default(second_to_last_test, item_names_for): assert item_names_for(second_to_last_test) == [ "test_1", "test_3", "test_4", "test_5", "test_2", ] def test_second_to_last_sparse(second_to_last_test, item_names_for, sparse_ordering): assert item_names_for(second_to_last_test) == [ "test_1", "test_3", "test_4", "test_2", "test_5", ] @pytest.fixture(scope="module") def last_test(): yield ( """ import pytest @pytest.mark.order("last") def test_1(): pass def test_2(): pass """ ) def test_last_default(last_test, item_names_for): assert item_names_for(last_test) == ["test_2", "test_1"] def test_last_sparse(last_test, item_names_for, sparse_ordering): assert item_names_for(last_test) == ["test_2", "test_1"] @pytest.fixture(scope="module") def first_last_test(): yield ( """ import pytest @pytest.mark.order("last") def test_1(): pass @pytest.mark.order("first") def test_2(): pass def test_3(): pass """ ) def test_first_last_default(first_last_test, item_names_for): assert item_names_for(first_last_test) == ["test_2", "test_3", "test_1"] def test_first_last_sparse(first_last_test, item_names_for, sparse_ordering): assert item_names_for(first_last_test) == ["test_2", "test_3", "test_1"] @pytest.fixture(scope="module") def duplicate_numbers_test(): yield ( """ import pytest @pytest.mark.order(1) def test_1(): pass @pytest.mark.order(1) def test_2(): pass def test_3(): pass def test_4(): pass @pytest.mark.order(4) def test_5(): pass """ ) def test_duplicate_numbers_default(duplicate_numbers_test, item_names_for): assert item_names_for(duplicate_numbers_test) == [ "test_1", "test_2", "test_5", "test_3", "test_4", ] def test_duplicate_numbers_sparse( duplicate_numbers_test, item_names_for, sparse_ordering ): assert item_names_for(duplicate_numbers_test) == [ "test_3", "test_1", "test_2", "test_4", "test_5", ] @pytest.fixture(scope="module") def end_items_test(): yield ( """ import pytest @pytest.mark.order(-2) def test_1(): pass @pytest.mark.order(-4) def test_2(): pass def test_3(): pass def test_4(): pass def test_5(): pass """ ) def test_end_items_default(end_items_test, item_names_for): assert item_names_for(end_items_test) == [ "test_3", "test_4", "test_5", "test_2", "test_1", ] def test_end_items_sparse(end_items_test, item_names_for, sparse_ordering): assert item_names_for(end_items_test) == [ "test_3", "test_2", "test_4", "test_1", "test_5", ] pytest-order-1.2.0/tests/test_xdist_handling.py000066400000000000000000000032261452612573200217370ustar00rootroot00000000000000from textwrap import dedent import pytest import pytest_order def test_xdist_ordering(tmpdir): testname = str(tmpdir.join("first_test.py")) with open(testname, "w") as fi: fi.write( dedent( """ import pytest val = 1 @pytest.mark.order("second") def test_second_integer(): global val assert val == 2 val += 1 def test_last_integer(): assert val == 3 @pytest.mark.order("first") def test_first_integer(): global val assert val == 1 val += 1 """ ) ) testname = str(tmpdir.join("second_test.py")) with open(testname, "w") as fi: fi.write( dedent( """ import pytest val = "frog" @pytest.mark.order("second") def test_second_string(): global val assert val == "goat" val = "fish" def test_last_string(): assert val == "fish" @pytest.mark.order("first") def test_first_string(): global val assert val == "frog" val = "goat" """ ) ) # With `loadfile`, the tests should pass args = ["-n3", "--dist=loadfile", str(tmpdir)] ret = pytest.main(args, [pytest_order]) assert ret == 0 # Without `loadfile`, the tests should fail args = ["-n3", str(tmpdir)] ret = pytest.main(args, [pytest_order]) assert ret == 1 pytest-order-1.2.0/tox.ini000066400000000000000000000013171452612573200154770ustar00rootroot00000000000000[tox] envlist = {py37,py38,py39,pypy37,pypy39}-pytest{50,51,52,53,54,60,61,62,70,73} {py310,py311,py312,pypy310}-pytest{624,70,73} [testenv] deps = pytest50: pytest>=5.0,<5.1 pytest51: pytest>=5.1,<5.2 pytest52: pytest>=5.2,<5.3 pytest53: pytest>=5.3,<5.4 pytest54: pytest>=5.4,<6.0 pytest60: pytest>=6.0,<6.1 pytest61: pytest>=6.1,<6.2 pytest62: pytest>=6.2,<6.3 pytest624: pytest>=6.2.4,<6.3 pytest70: pytest>=7.0,<7.1 pytest73: pytest>=7.3,<7.4 pytest-cov<2.10 pytest{50,51,52,53,54}: pytest-xdist<2.0.0 pytest{60,61,62,624,70,73}: pytest-xdist pytest-dependency>=0.5.1 pytest-mock passenv = * commands = python -m pytest {env:COV_CMD} tests