pax_global_header00006660000000000000000000000064144334244260014520gustar00rootroot0000000000000052 comment=e8a554ee0846c61c6f0403a94f9f2f959a2bdc74 django-cte-1.3.0/000077500000000000000000000000001443342442600135345ustar00rootroot00000000000000django-cte-1.3.0/.github/000077500000000000000000000000001443342442600150745ustar00rootroot00000000000000django-cte-1.3.0/.github/workflows/000077500000000000000000000000001443342442600171315ustar00rootroot00000000000000django-cte-1.3.0/.github/workflows/tests.yml000066400000000000000000000043131443342442600210170ustar00rootroot00000000000000name: django-cte tests on: push: branches: [master] pull_request: branches: [master] jobs: tests: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python: ['3.7', '3.8', '3.9'] # Time to switch to pytest or nose2? # nosetests is broken on 3.10 # AttributeError: module 'collections' has no attribute 'Callable' # https://github.com/nose-devs/nose/issues/1099 django: - 'Django>=2.2,<3.0' - 'Django>=3.0,<3.1' - 'Django>=3.1,<3.2' - 'Django>=3.2,<3.3' - 'Django>=4.0,<4.1' experimental: [false] include: - python: '3.10' django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' experimental: true # NOTE this job will appear to pass even when it fails because of # `continue-on-error: true`. Github Actions apparently does not # have this feature, similar to Travis' allow-failure, yet. # https://github.com/actions/toolkit/issues/399 exclude: - python: '3.7' django: 'Django>=4.0,<4.1' services: postgres: image: postgres:latest env: POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_USER: postgres ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - name: Setup run: | python --version pip install --upgrade pip wheel pip install "${{ matrix.django }}" psycopg2-binary nose flake8 - name: Run tests env: DB_SETTINGS: >- { "ENGINE":"django.db.backends.postgresql_psycopg2", "NAME":"django_cte", "USER":"postgres", "PASSWORD":"postgres", "HOST":"localhost", "PORT":"5432" } run: nosetests continue-on-error: ${{ matrix.experimental }} - name: Check style run: flake8 django_cte/ tests/ django-cte-1.3.0/.gitignore000066400000000000000000000022051443342442600155230ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ django-cte-1.3.0/CHANGELOG.md000066400000000000000000000033471443342442600153540ustar00rootroot00000000000000# Django CTE change log ## 1.3.0 - 2023-05-24 - Add support for Materialized CTEs. - Fix: add EXPLAIN clause in correct position when using `.explain()` method. ## v1.2.1 - 2022-07-07 - Fix compatibility with non-CTE models. ## v1.2.0 - 2022-03-30 - Add support for Django 3.1, 3.2 and 4.0. - Quote the CTE table name if needed. - Resolve `OuterRef` in CTE `Subquery`. - Fix default `CTEManager` so it can use `from_queryset` corectly. - Fix for Django 3.0.5+. ## v1.1.5 - 2020-02-07 - Django 3 compatibility. Thank you @tim-schilling and @ryanhiebert! ## v1.1.4 - 2018-07-30 - Python 3 compatibility. ## v1.1.3 - 2018-06-19 - Fix CTE alias bug. ## v1.1.2 - 2018-05-22 - Use `_default_manager` instead of `objects`. ## v1.1.1 - 2018-04-13 - Fix recursive CTE pickling. Note: this is currently [broken on Django master](https://github.com/django/django/pull/9134#pullrequestreview-112057277). ## v1.1.0 - 2018-04-09 - `With.queryset()` now uses the CTE model's manager to create a new `QuerySet`, which makes it easier to work with custom `QuerySet` classes. ## v1.0.0 - 2018-04-04 - BACKWARD INCOMPATIBLE CHANGE: `With.queryset()` no longer accepts a `model` argument. - Improve `With.queryset()` to select directly from the CTE rather than joining to anoter QuerySet. - Refactor `With.join()` to use real JOIN clause. ## v0.1.4 - 2018-03-21 - Fix related field attname masking CTE column. ## v0.1.3 - 2018-03-15 - Add `django_cte.raw.raw_cte_sql` for constructing CTEs with raw SQL. ## v0.1.2 - 2018-02-21 - Improve error on bad recursive reference. - Add more tests. - Add change log. - Improve README. - PEP-8 style fixes. ## v0.1.1 - 2018-02-21 - Fix readme formatting on PyPI. ## v0.1 - 2018-02-21 - Initial implementation. django-cte-1.3.0/LICENSE000066400000000000000000000027441443342442600145500ustar00rootroot00000000000000Copyright (c) 2018, Dimagi Inc., and individual contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name Dimagi, nor the names of its contributors, may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIMAGI INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. django-cte-1.3.0/README.md000066400000000000000000000015221443342442600150130ustar00rootroot00000000000000# Common Table Expressions with Django [![Build Status](https://travis-ci.com/dimagi/django-cte.png)](https://travis-ci.com/dimagi/django-cte) [![PyPI version](https://badge.fury.io/py/django-cte.svg)](https://badge.fury.io/py/django-cte) ## Installation ``` pip install django-cte ``` ## Documentation The [django-cte documentation](https://dimagi.github.io/django-cte/) shows how to use Common Table Expressions with the Django ORM. ## Running tests ``` cd django-cte mkvirtualenv cte # or however you choose to setup your environment pip install django nose flake8 nosetests flake8 --config=setup.cfg ``` All feature and bug contributions are expected to be covered by tests. ## Uploading to PyPI Package and upload the generated files. ``` pip install -r pkg-requires.txt python setup.py sdist bdist_wheel twine upload dist/* ``` django-cte-1.3.0/django_cte/000077500000000000000000000000001443342442600156315ustar00rootroot00000000000000django-cte-1.3.0/django_cte/__init__.py000066400000000000000000000002361443342442600177430ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals from .cte import CTEManager, CTEQuerySet, With # noqa __version__ = "1.3.0" django-cte-1.3.0/django_cte/cte.py000066400000000000000000000146561443342442600167720ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals from django.db.models import Manager from django.db.models.query import Q, QuerySet, ValuesIterable from django.db.models.sql.datastructures import BaseTable from .join import QJoin, INNER from .meta import CTEColumnRef, CTEColumns from .query import CTEQuery __all__ = ["With", "CTEManager", "CTEQuerySet"] class With(object): """Common Table Expression query object: `WITH ...` :param queryset: A queryset to use as the body of the CTE. :param name: Optional name parameter for the CTE (default: "cte"). This must be a unique name that does not conflict with other entities (tables, views, functions, other CTE(s), etc.) referenced in the given query as well any query to which this CTE will eventually be added. :param materialized: Optional parameter (default: False) which enforce using of MATERIALIZED statement for supporting databases. """ def __init__(self, queryset, name="cte", materialized=False): self.query = None if queryset is None else queryset.query self.name = name self.col = CTEColumns(self) self.materialized = materialized def __getstate__(self): return (self.query, self.name, self.materialized) def __setstate__(self, state): self.query, self.name, self.materialized = state self.col = CTEColumns(self) def __repr__(self): return "".format(self.name) @classmethod def recursive(cls, make_cte_queryset, name="cte", materialized=False): """Recursive Common Table Expression: `WITH RECURSIVE ...` :param make_cte_queryset: Function taking a single argument (a not-yet-fully-constructed cte object) and returning a `QuerySet` object. The returned `QuerySet` normally consists of an initial statement unioned with a recursive statement. :param name: See `name` parameter of `__init__`. :param materialized: See `materialized` parameter of `__init__`. :returns: The fully constructed recursive cte object. """ cte = cls(None, name, materialized) cte.query = make_cte_queryset(cte).query return cte def join(self, model_or_queryset, *filter_q, **filter_kw): """Join this CTE to the given model or queryset This CTE will be refernced by the returned queryset, but the corresponding `WITH ...` statement will not be prepended to the queryset's SQL output; use `.with_cte(cte)` to achieve that outcome. :param model_or_queryset: Model class or queryset to which the CTE should be joined. :param *filter_q: Join condition Q expressions (optional). :param **filter_kw: Join conditions. All LHS fields (kwarg keys) are assumed to reference `model_or_queryset` fields. Use `cte.col.name` on the RHS to recursively reference CTE query columns. For example: `cte.join(Book, id=cte.col.id)` :returns: A queryset with the given model or queryset joined to this CTE. """ if isinstance(model_or_queryset, QuerySet): queryset = model_or_queryset.all() else: queryset = model_or_queryset._default_manager.all() join_type = filter_kw.pop("_join_type", INNER) query = queryset.query # based on Query.add_q: add necessary joins to query, but no filter q_object = Q(*filter_q, **filter_kw) map = query.alias_map existing_inner = set(a for a in map if map[a].join_type == INNER) on_clause, _ = query._add_q(q_object, query.used_aliases) query.demote_joins(existing_inner) parent = query.get_initial_alias() query.join(QJoin(parent, self.name, self.name, on_clause, join_type)) return queryset def queryset(self): """Get a queryset selecting from this CTE This CTE will be referenced by the returned queryset, but the corresponding `WITH ...` statement will not be prepended to the queryset's SQL output; use `.with_cte(cte)` to achieve that outcome. :returns: A queryset. """ cte_query = self.query qs = cte_query.model._default_manager.get_queryset() query = CTEQuery(cte_query.model) query.join(BaseTable(self.name, None)) query.default_cols = cte_query.default_cols if cte_query.annotations: for alias, value in cte_query.annotations.items(): col = CTEColumnRef(alias, self.name, value.output_field) query.add_annotation(col, alias) if cte_query.values_select: query.set_values(cte_query.values_select) qs._iterable_class = ValuesIterable query.annotation_select_mask = cte_query.annotation_select_mask qs.query = query return qs def _resolve_ref(self, name): return self.query.resolve_ref(name) class CTEQuerySet(QuerySet): """QuerySet with support for Common Table Expressions""" def __init__(self, model=None, query=None, using=None, hints=None): # Only create an instance of a Query if this is the first invocation in # a query chain. if query is None: query = CTEQuery(model) super(CTEQuerySet, self).__init__(model, query, using, hints) def with_cte(self, cte): """Add a Common Table Expression to this queryset The CTE `WITH ...` clause will be added to the queryset's SQL output (after other CTEs that have already been added) so it can be referenced in annotations, filters, etc. """ qs = self._clone() qs.query._with_ctes.append(cte) return qs def as_manager(cls): # Address the circular dependency between # `CTEQuerySet` and `CTEManager`. manager = CTEManager.from_queryset(cls)() manager._built_with_as_manager = True return manager as_manager.queryset_only = True as_manager = classmethod(as_manager) class CTEManager(Manager.from_queryset(CTEQuerySet)): """Manager for models that perform CTE queries""" @classmethod def from_queryset(cls, queryset_class, class_name=None): if not issubclass(queryset_class, CTEQuerySet): raise TypeError( "models with CTE support need to use a CTEQuerySet") return super(CTEManager, cls).from_queryset( queryset_class, class_name=class_name) django-cte-1.3.0/django_cte/expressions.py000066400000000000000000000036121443342442600205670ustar00rootroot00000000000000import django from django.db.models import Subquery class CTESubqueryResolver(object): def __init__(self, annotation): self.annotation = annotation def resolve_expression(self, *args, **kw): # source: django.db.models.expressions.Subquery.resolve_expression # --- begin copied code (lightly adapted) --- # # Need to recursively resolve these. def resolve_all(child): if hasattr(child, 'children'): [resolve_all(_child) for _child in child.children] if hasattr(child, 'rhs'): child.rhs = resolve(child.rhs) def resolve(child): if hasattr(child, 'resolve_expression'): resolved = child.resolve_expression(*args, **kw) # Add table alias to the parent query's aliases to prevent # quoting. if hasattr(resolved, 'alias') and \ resolved.alias != resolved.target.model._meta.db_table: get_query(clone).external_aliases.add(resolved.alias) return resolved return child # --- end copied code --- # if django.VERSION < (3, 0): def get_query(clone): return clone.queryset.query else: def get_query(clone): return clone.query # NOTE this uses the old (pre-Django 3) way of resolving. # Should a different technique should be used on Django 3+? clone = self.annotation.resolve_expression(*args, **kw) if isinstance(self.annotation, Subquery): for cte in getattr(get_query(clone), '_with_ctes', []): resolve_all(cte.query.where) for key, value in cte.query.annotations.items(): if isinstance(value, Subquery): cte.query.annotations[key] = resolve(value) return clone django-cte-1.3.0/django_cte/join.py000066400000000000000000000062531443342442600171500ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals from django.db.models.sql.constants import INNER class QJoin(object): """Join clause with join condition from Q object clause :param parent_alias: Alias of parent table. :param table_name: Name of joined table. :param table_alias: Alias of joined table. :param on_clause: Query `where_class` instance represenging the ON clause. :param join_type: Join type (INNER or LOUTER). """ filtered_relation = None def __init__(self, parent_alias, table_name, table_alias, on_clause, join_type=INNER, nullable=None): self.parent_alias = parent_alias self.table_name = table_name self.table_alias = table_alias self.on_clause = on_clause self.join_type = join_type # LOUTER or INNER self.nullable = join_type != INNER if nullable is None else nullable @property def identity(self): return ( self.__class__, self.table_name, self.parent_alias, self.join_type, self.on_clause, ) def __hash__(self): return hash(self.identity) def __eq__(self, other): if not isinstance(other, QJoin): return NotImplemented return self.identity == other.identity def equals(self, other): return self.identity == other.identity def as_sql(self, compiler, connection): """Generate join clause SQL""" on_clause_sql, params = self.on_clause.as_sql(compiler, connection) if self.table_alias == self.table_name: alias = '' else: alias = ' %s' % self.table_alias qn = compiler.quote_name_unless_alias sql = '%s %s%s ON %s' % ( self.join_type, qn(self.table_name), alias, on_clause_sql ) return sql, params def relabeled_clone(self, change_map): return self.__class__( parent_alias=change_map.get(self.parent_alias, self.parent_alias), table_name=self.table_name, table_alias=change_map.get(self.table_alias, self.table_alias), on_clause=self.on_clause.relabeled_clone(change_map), join_type=self.join_type, nullable=self.nullable, ) class join_field: # `Join.join_field` is used internally by `Join` as well as in # `QuerySet.resolve_expression()`: # # isinstance(table, Join) # and table.join_field.related_model._meta.db_table != alias # # Currently that does not apply here since `QJoin` is not an # instance of `Join`, although maybe it should? Maybe this # should have `related_model._meta.db_table` return # `.table_name` or `.table_alias`? # # `PathInfo.join_field` is another similarly named attribute in # Django that has a much more complicated interface, but luckily # seems unrelated to `Join.join_field`. class related_model: class _meta: # for QuerySet.set_group_by(allow_aliases=True) local_concrete_fields = () django-cte-1.3.0/django_cte/meta.py000066400000000000000000000070371443342442600171400ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals import weakref from django.db.models.expressions import Col, Expression class CTEColumns(object): def __init__(self, cte): self._cte = weakref.ref(cte) def __getattr__(self, name): return CTEColumn(self._cte(), name) class CTEColumn(Expression): def __init__(self, cte, name, output_field=None): self._cte = cte self.table_alias = cte.name self.name = self.alias = name self._output_field = output_field def __repr__(self): return "<{} {}.{}>".format( self.__class__.__name__, self._cte.name, self.name, ) @property def _ref(self): if self._cte.query is None: raise ValueError( "cannot resolve '{cte}.{name}' in recursive CTE setup. " "Hint: use ExpressionWrapper({cte}.col.{name}, " "output_field=...)".format(cte=self._cte.name, name=self.name) ) ref = self._cte._resolve_ref(self.name) if ref is self or self in ref.get_source_expressions(): raise ValueError("Circular reference: {} = {}".format(self, ref)) return ref @property def target(self): return self._ref.target @property def output_field(self): # required to fix error caused by django commit # 9d519d3dc4e5bd1d9ff3806b44624c3e487d61c1 if self._cte.query is None: raise AttributeError if self._output_field is not None: return self._output_field return self._ref.output_field def as_sql(self, compiler, connection): qn = compiler.quote_name_unless_alias ref = self._ref if isinstance(ref, Col) and self.name == "pk": column = ref.target.column else: column = self.name return "%s.%s" % (qn(self.table_alias), qn(column)), [] def relabeled_clone(self, relabels): if self.table_alias is not None and self.table_alias in relabels: clone = self.copy() clone.table_alias = relabels[self.table_alias] return clone return self class CTEColumnRef(Expression): def __init__(self, name, cte_name, output_field): self.name = name self.cte_name = cte_name self.output_field = output_field self._alias = None def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False): if query: clone = self.copy() clone._alias = self._alias or query.table_map.get( self.cte_name, [self.cte_name])[0] return clone return super(CTEColumnRef, self).resolve_expression( query, allow_joins, reuse, summarize, for_save) def relabeled_clone(self, change_map): if ( self.cte_name not in change_map and self._alias not in change_map ): return super(CTEColumnRef, self).relabeled_clone(change_map) clone = self.copy() if self.cte_name in change_map: clone._alias = change_map[self.cte_name] if self._alias in change_map: clone._alias = change_map[self._alias] return clone def as_sql(self, compiler, connection): qn = compiler.quote_name_unless_alias table = self._alias or compiler.query.table_map.get( self.cte_name, [self.cte_name])[0] return "%s.%s" % (qn(table), qn(self.name)), [] django-cte-1.3.0/django_cte/query.py000066400000000000000000000130121443342442600173450ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals import django from django.db import connections from django.db.models.sql import DeleteQuery, Query, UpdateQuery from django.db.models.sql.compiler import ( SQLCompiler, SQLDeleteCompiler, SQLUpdateCompiler, ) from .expressions import CTESubqueryResolver class CTEQuery(Query): """A Query which processes SQL compilation through the CTE compiler""" def __init__(self, *args, **kwargs): super(CTEQuery, self).__init__(*args, **kwargs) self._with_ctes = [] def combine(self, other, connector): if other._with_ctes: if self._with_ctes: raise TypeError("cannot merge queries with CTEs on both sides") self._with_ctes = other._with_ctes[:] return super(CTEQuery, self).combine(other, connector) def get_compiler(self, using=None, connection=None, *args, **kwargs): """ Overrides the Query method get_compiler in order to return a CTECompiler. """ # Copy the body of this method from Django except the final # return statement. We will ignore code coverage for this. if using is None and connection is None: # pragma: no cover raise ValueError("Need either using or connection") if using: connection = connections[using] # Check that the compiler will be able to execute the query for alias, aggregate in self.annotation_select.items(): connection.ops.check_expression_support(aggregate) # Instantiate the custom compiler. klass = COMPILER_TYPES.get(self.__class__, CTEQueryCompiler) return klass(self, connection, using, *args, **kwargs) def add_annotation(self, annotation, *args, **kw): annotation = CTESubqueryResolver(annotation) super(CTEQuery, self).add_annotation(annotation, *args, **kw) def __chain(self, _name, klass=None, *args, **kwargs): klass = QUERY_TYPES.get(klass, self.__class__) clone = getattr(super(CTEQuery, self), _name)(klass, *args, **kwargs) clone._with_ctes = self._with_ctes[:] return clone if django.VERSION < (2, 0): def clone(self, klass=None, *args, **kwargs): return self.__chain("clone", klass, *args, **kwargs) else: def chain(self, klass=None): return self.__chain("chain", klass) class CTECompiler(object): @classmethod def generate_sql(cls, connection, query, as_sql): if query.combinator: return as_sql() ctes = [] params = [] for cte in query._with_ctes: compiler = cte.query.get_compiler(connection=connection) qn = compiler.quote_name_unless_alias cte_sql, cte_params = compiler.as_sql() template = cls.get_cte_query_template(cte) ctes.append(template.format(name=qn(cte.name), query=cte_sql)) params.extend(cte_params) explain_query = getattr(query, "explain_query", None) sql = [] if explain_query: explain_format = getattr(query, "explain_format", None) explain_options = getattr(query, "explain_options", {}) sql.append( connection.ops.explain_query_prefix( explain_format, **explain_options ) ) # this needs to get set to False so that the base as_sql() doesn't # insert the EXPLAIN statement where it would end up between the # WITH ... clause and the final SELECT query.explain_query = False if ctes: # Always use WITH RECURSIVE # https://www.postgresql.org/message-id/13122.1339829536%40sss.pgh.pa.us sql.extend(["WITH RECURSIVE", ", ".join(ctes)]) base_sql, base_params = as_sql() if explain_query: query.explain_query = explain_query sql.append(base_sql) params.extend(base_params) return " ".join(sql), tuple(params) @classmethod def get_cte_query_template(cls, cte): if cte.materialized: return "{name} AS MATERIALIZED ({query})" return "{name} AS ({query})" class CTEUpdateQuery(UpdateQuery, CTEQuery): pass class CTEDeleteQuery(DeleteQuery, CTEQuery): pass QUERY_TYPES = { UpdateQuery: CTEUpdateQuery, DeleteQuery: CTEDeleteQuery, } class CTEQueryCompiler(SQLCompiler): def as_sql(self, *args, **kwargs): def _as_sql(): return super(CTEQueryCompiler, self).as_sql(*args, **kwargs) return CTECompiler.generate_sql(self.connection, self.query, _as_sql) class CTEUpdateQueryCompiler(SQLUpdateCompiler): def as_sql(self, *args, **kwargs): def _as_sql(): return super(CTEUpdateQueryCompiler, self).as_sql(*args, **kwargs) return CTECompiler.generate_sql(self.connection, self.query, _as_sql) class CTEDeleteQueryCompiler(SQLDeleteCompiler): # NOTE: it is currently not possible to execute delete queries that # reference CTEs without patching `QuerySet.delete` (Django method) # to call `self.query.chain(sql.DeleteQuery)` instead of # `sql.DeleteQuery(self.model)` def as_sql(self, *args, **kwargs): def _as_sql(): return super(CTEDeleteQueryCompiler, self).as_sql(*args, **kwargs) return CTECompiler.generate_sql(self.connection, self.query, _as_sql) COMPILER_TYPES = { CTEUpdateQuery: CTEUpdateQueryCompiler, CTEDeleteQuery: CTEDeleteQueryCompiler, } django-cte-1.3.0/django_cte/raw.py000066400000000000000000000021611443342442600167740ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals def raw_cte_sql(sql, params, refs): """Raw CTE SQL :param sql: SQL query (string). :param params: List of bind parameters. :param refs: Dict of output fields: `{"name": }`. :returns: Object that can be passed to `With`. """ class raw_cte_ref(object): def __init__(self, output_field): self.output_field = output_field def get_source_expressions(self): return [] class raw_cte_compiler(object): def __init__(self, connection): self.connection = connection def as_sql(self): return sql, params def quote_name_unless_alias(self, name): return self.connection.ops.quote_name(name) class raw_cte_queryset(object): class query(object): @staticmethod def get_compiler(connection): return raw_cte_compiler(connection) @staticmethod def resolve_ref(name): return raw_cte_ref(refs[name]) return raw_cte_queryset django-cte-1.3.0/docs/000077500000000000000000000000001443342442600144645ustar00rootroot00000000000000django-cte-1.3.0/docs/_config.yml000066400000000000000000000001201443342442600166040ustar00rootroot00000000000000title: django-cte author: Dimagi markdown: kramdown kramdown: toc_levels: 2..3django-cte-1.3.0/docs/index.md000066400000000000000000000312041443342442600161150ustar00rootroot00000000000000# Common Table Expressions with Django * Table of contents (this line will not be displayed). {:toc} A Common Table Expression acts like a temporary table or view that exists only for the duration of the query it is attached to. django-cte allows common table expressions to be attached to normal Django ORM queries. ## Prerequisite: A Model with a "CTEManager" The custom manager class, `CTEManager`, constructs `CTEQuerySet`s, which have all of the same features as normal `QuerySet`s and also support CTE queries. ```py from django_cte import CTEManager class Order(Model): objects = CTEManager() id = AutoField(primary_key=True) region = ForeignKey("Region", on_delete=CASCADE) amount = IntegerField(default=0) class Meta: db_table = "orders" ``` ## Simple Common Table Expressions Simple CTEs are constructed using `With(...)`. A CTE can be joined to a model or other `CTEQuerySet` using its `join(...)` method, which creates a new queryset with a `JOIN` and `ON` condition. Finally, the CTE is added to the resulting queryset using `with_cte(cte)`, which adds the `WITH` expression before the main `SELECT` query. ```py from django_cte import With cte = With( Order.objects .values("region_id") .annotate(total=Sum("amount")) ) orders = ( # FROM orders INNER JOIN cte ON orders.region_id = cte.region_id cte.join(Order, region=cte.col.region_id) # Add `WITH ...` before `SELECT ... FROM orders ...` .with_cte(cte) # Annotate each Order with a "region_total" .annotate(region_total=cte.col.total) ) print(orders.query) # print SQL ``` The `orders` SQL, after formatting for readability, would look something like this: ```sql WITH RECURSIVE "cte" AS ( SELECT "orders"."region_id", SUM("orders"."amount") AS "total" FROM "orders" GROUP BY "orders"."region_id" ) SELECT "orders"."id", "orders"."region_id", "orders"."amount", "cte"."total" AS "region_total" FROM "orders" INNER JOIN "cte" ON "orders"."region_id" = "cte"."region_id" ``` The `orders` query is a query set containing annotated `Order` objects, just as you would get from a query like `Order.objects.annotate(region_total=...)`. Each `Order` object will be annotated with a `region_total` attribute, which is populated with the value of the corresponding total from the joined CTE query. You may have noticed the CTE in this query uses `WITH RECURSIVE` even though this is not a [Recursive Common Table Expression](#recursive-common-table-expressions). The `RECURSIVE` keyword is always used, even for non-recursive CTEs. On databases such as PostgreSQL and SQLite this has no effect other than allowing recursive CTEs to be included in the WITH block. ## Recursive Common Table Expressions Recursive CTE queries allow fundamentally new types of queries that are not otherwise possible. First, a model for the example. ```py class Region(Model): objects = CTEManager() name = TextField(primary_key=True) parent = ForeignKey("self", null=True, on_delete=CASCADE) class Meta: db_table = "region" ``` Recursive CTEs are constructed using `With.recursive()`, which takes as its first argument a function that constructs and returns a recursive query. Recursive queries have two elements: first a non-recursive query element, and second a recursive query element. The second is typically attached to the first using `QuerySet.union()`. ```py def make_regions_cte(cte): # non-recursive: get root nodes return Region.objects.filter( parent__isnull=True ).values( "name", path=F("name"), depth=Value(0, output_field=IntegerField()), ).union( # recursive union: get descendants cte.join(Region, parent=cte.col.name).values( "name", path=Concat( cte.col.path, Value(" / "), F("name"), output_field=TextField(), ), depth=cte.col.depth + Value(1, output_field=IntegerField()), ), all=True, ) cte = With.recursive(make_regions_cte) regions = ( cte.join(Region, name=cte.col.name) .with_cte(cte) .annotate( path=cte.col.path, depth=cte.col.depth, ) .filter(depth=2) .order_by("path") ) ``` `Region` objects returned by this query will have `path` and `depth` attributes. The results will be ordered by `path` (hierarchically by region name). The SQL produced by this query looks something like this: ```sql WITH RECURSIVE "cte" AS ( SELECT "region"."name", "region"."name" AS "path", 0 AS "depth" FROM "region" WHERE "region"."parent_id" IS NULL UNION ALL SELECT "region"."name", "cte"."path" || ' / ' || "region"."name" AS "path", "cte"."depth" + 1 AS "depth" FROM "region" INNER JOIN "cte" ON "region"."parent_id" = "cte"."name" ) SELECT "region"."name", "region"."parent_id", "cte"."path" AS "path", "cte"."depth" AS "depth" FROM "region" INNER JOIN "cte" ON "region"."name" = "cte"."name" WHERE "cte"."depth" = 2 ORDER BY "path" ASC ``` ## Named Common Table Expressions It is possible to add more than one CTE to a query. To do this, each CTE must have a unique name. `With(queryset)` returns a CTE with the name `'cte'` by default, but that can be overridden: `With(queryset, name='custom')` or `With.recursive(make_queryset, name='custom')`. This allows each CTE to be referenced uniquely within a single query. Also note that a CTE may reference other CTEs in the same query. Example query with two CTEs, and the second (`totals`) CTE references the first (`rootmap`): ```py def make_root_mapping(rootmap): return Region.objects.filter( parent__isnull=True ).values( "name", root=F("name"), ).union( rootmap.join(Region, parent=rootmap.col.name).values( "name", root=rootmap.col.root, ), all=True, ) rootmap = With.recursive(make_root_mapping, name="rootmap") totals = With( rootmap.join(Order, region_id=rootmap.col.name) .values( root=rootmap.col.root, ).annotate( orders_count=Count("id"), region_total=Sum("amount"), ), name="totals", ) root_regions = ( totals.join(Region, name=totals.col.root) # Important: add both CTEs to the final query .with_cte(rootmap) .with_cte(totals) .annotate( # count of orders in this region and all subregions orders_count=totals.col.orders_count, # sum of order amounts in this region and all subregions region_total=totals.col.region_total, ) ) ``` And the resulting SQL. ```sql WITH RECURSIVE "rootmap" AS ( SELECT "region"."name", "region"."name" AS "root" FROM "region" WHERE "region"."parent_id" IS NULL UNION ALL SELECT "region"."name", "rootmap"."root" AS "root" FROM "region" INNER JOIN "rootmap" ON "region"."parent_id" = "rootmap"."name" ), "totals" AS ( SELECT "rootmap"."root" AS "root", COUNT("orders"."id") AS "orders_count", SUM("orders"."amount") AS "region_total" FROM "orders" INNER JOIN "rootmap" ON "orders"."region_id" = "rootmap"."name" GROUP BY "rootmap"."root" ) SELECT "region"."name", "region"."parent_id", "totals"."orders_count" AS "orders_count", "totals"."region_total" AS "region_total" FROM "region" INNER JOIN "totals" ON "region"."name" = "totals"."root" ``` ## Selecting FROM a Common Table Expression Sometimes it is useful to construct queries where the final `FROM` clause contains only common table expression(s). This is possible with `With(...).queryset()`. Each returned row may be a model object: ```py cte = With( Order.objects .annotate(region_parent=F("region__parent_id")), ) orders = cte.queryset().with_cte(cte) ``` And the resulting SQL: ```sql WITH RECURSIVE "cte" AS ( SELECT "orders"."id", "orders"."region_id", "orders"."amount", "region"."parent_id" AS "region_parent" FROM "orders" INNER JOIN "region" ON "orders"."region_id" = "region"."name" ) SELECT "cte"."id", "cte"."region_id", "cte"."amount", "cte"."region_parent" AS "region_parent" FROM "cte" ``` It is also possible to do the same with `values(...)` queries: ```py cte = With( Order.objects .values( "region_id", region_parent=F("region__parent_id"), ) .distinct() ) values = cte.queryset().with_cte(cte).filter(region_parent__isnull=False) ``` Which produces this SQL: ```sql WITH RECURSIVE "cte" AS ( SELECT DISTINCT "orders"."region_id", "region"."parent_id" AS "region_parent" FROM "orders" INNER JOIN "region" ON "orders"."region_id" = "region"."name" ) SELECT "cte"."region_id", "cte"."region_parent" AS "region_parent" FROM "cte" WHERE "cte"."region_parent" IS NOT NULL ``` ## Custom QuerySets and Managers Custom `QuerySet`s that will be used in CTE queries should be derived from `CTEQuerySet`. ```py class LargeOrdersQueySet(CTEQuerySet): return self.filter(amount__gt=100) class Order(Model): large = LargeOrdersQueySet.as_manager() ``` Custom `CTEQuerySet`s can also be used with custom `CTEManager`s. ```py class CustomManager(CTEManager): ... class Order(Model): large = CustomManager.from_queryset(LargeOrdersQueySet)() objects = CustomManager() ``` ## Experimental: Left Outer Join Django does not provide precise control over joins, but there is an experimental way to perform a `LEFT OUTER JOIN` with a CTE query using the `_join_type` keyword argument of `With.join(...)`. ```py from django.db.models.sql.constants import LOUTER totals = With( Order.objects .values("region_id") .annotate(total=Sum("amount")) .filter(total__gt=100) ) orders = ( totals .join(Order, region=totals.col.region_id, _join_type=LOUTER) .with_cte(totals) .annotate(region_total=totals.col.total) ) ``` Which produces the following SQL ```sql WITH RECURSIVE "cte" AS ( SELECT "orders"."region_id", SUM("orders"."amount") AS "total" FROM "orders" GROUP BY "orders"."region_id" HAVING SUM("orders"."amount") > 100 ) SELECT "orders"."id", "orders"."region_id", "orders"."amount", "cte"."total" AS "region_total" FROM "orders" LEFT OUTER JOIN "cte" ON "orders"."region_id" = "cte"."region_id" ``` WARNING: as noted, this feature is experimental. There may be scenarios where Django automatically converts a `LEFT OUTER JOIN` to an `INNER JOIN` in the process of building the query. Be sure to test your queries to ensure they produce the desired SQL. ## Materialized CTE Both PostgreSQL 12+ and sqlite 3.35+ supports `MATERIALIZED` keyword for CTE queries. To enforce using of this keyword add `materialized` as a parameter of `With(..., materialized=True)`. ```py cte = With( Order.objects.values('id'), materialized=True ) ``` Which produces this SQL: ```sql WITH RECURSIVE "cte" AS MATERIALIZED ( SELECT "orders"."id" FROM "orders" ) ... ``` ## Raw CTE SQL Some queries are easier to construct with raw SQL than with the Django ORM. `raw_cte_sql()` is one solution for situations like that. The down-side is that each result field in the raw query must be explicitly mapped to a field type. The up-side is that there is no need to compromise result-set expressiveness with the likes of `Manager.raw()`. A short example: ```py from django.db.models import IntegerField, TextField from django_cte.raw import raw_cte_sql cte = With(raw_cte_sql( """ SELECT region_id, AVG(amount) AS avg_order FROM orders WHERE region_id = %s GROUP BY region_id """, ["moon"], { "region_id": TextField(), "avg_order": IntegerField(), }, )) moon_avg = ( cte .join(Region, name=cte.col.region_id) .annotate(avg_order=cte.col.avg_order) .with_cte(cte) ) ``` Which produces this SQL: ```sql WITH RECURSIVE "cte" AS ( SELECT region_id, AVG(amount) AS avg_order FROM orders WHERE region_id = 'moon' GROUP BY region_id ) SELECT "region"."name", "region"."parent_id", "cte"."avg_order" AS "avg_order" FROM "region" INNER JOIN "cte" ON "region"."name" = "cte"."region_id" ``` **WARNING**: Be very careful when writing raw SQL. Use bind parameters to prevent SQL injection attacks. ## More Advanced Use Cases A few more advanced techniques as well as example query results can be found in the tests: - [`test_cte.py`](https://github.com/dimagi/django-cte/blob/master/tests/test_cte.py) - [`test_recursive.py`](https://github.com/dimagi/django-cte/blob/master/tests/test_recursive.py) - [`test_raw.py`](https://github.com/dimagi/django-cte/blob/master/tests/test_raw.py) django-cte-1.3.0/pkg-requires.txt000066400000000000000000000000571443342442600167150ustar00rootroot00000000000000setuptools>=38.6.0 twine>=1.11.0 wheel>=0.31.0 django-cte-1.3.0/setup.cfg000066400000000000000000000001311443342442600153500ustar00rootroot00000000000000[bdist_wheel] universal=1 [metadata] license_file = LICENSE [flake8] exclude = ./build django-cte-1.3.0/setup.py000066400000000000000000000037221443342442600152520ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals import os import re from io import open from setuptools import find_packages, setup def get_version(filename): path = os.path.join(os.path.dirname(__file__), filename) with open(path, encoding="utf-8") as handle: content = handle.read() return re.search(r'__version__ = "([^"]+)"', content).group(1) def read_md(filename): path = os.path.join(os.path.dirname(__file__), filename) with open(path, encoding='utf-8') as handle: return handle.read() setup( name='django-cte', version=get_version('django_cte/__init__.py'), description='Common Table Expressions (CTE) for Django', long_description=read_md('README.md'), long_description_content_type='text/markdown', maintainer='Daniel Miller', maintainer_email='millerdev@gmail.com', url='https://github.com/dimagi/django-cte', license='BSD License', packages=find_packages(exclude=['tests']), include_package_data=True, classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', 'Framework :: Django', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Framework :: Django', 'Framework :: Django :: 2', 'Framework :: Django :: 2.2', 'Framework :: Django :: 3', 'Framework :: Django :: 3.0', 'Framework :: Django :: 3.1', 'Framework :: Django :: 3.2', 'Framework :: Django :: 4', 'Framework :: Django :: 4.0', 'Topic :: Software Development :: Libraries :: Python Modules', ], ) django-cte-1.3.0/tests/000077500000000000000000000000001443342442600146765ustar00rootroot00000000000000django-cte-1.3.0/tests/__init__.py000066400000000000000000000006201443342442600170050ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals import os import django # django setup must occur before importing models os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") django.setup() from .django_setup import init_db, destroy_db # noqa def setup(): """Initialize database for nosetests""" init_db() def teardown(): destroy_db() django-cte-1.3.0/tests/django_setup.py000066400000000000000000000041211443342442600177300ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals import sys from django.db import connection from .models import KeyPair, Region, Order is_initialized = False def init_db(): global is_initialized if is_initialized: return is_initialized = True # replace sys.stdout for prompt to delete database old_stdout = sys.stdout sys.stdout = sys.__stdout__ try: connection.creation.create_test_db(verbosity=0) finally: sys.stdout = old_stdout setup_data() def destroy_db(): connection.creation.destroy_test_db(verbosity=0) def setup_data(): regions = {None: None} for name, parent in [ ("sun", None), ("mercury", "sun"), ("venus", "sun"), ("earth", "sun"), ("moon", "earth"), ("mars", "sun"), ("deimos", "mars"), ("phobos", "mars"), ("proxima centauri", None), ("proxima centauri b", "proxima centauri"), ("bernard's star", None), ]: region = Region(name=name, parent=regions[parent]) region.save() regions[name] = region for region, amount in [ ("sun", 1000), ("mercury", 10), ("mercury", 11), ("mercury", 12), ("venus", 20), ("venus", 21), ("venus", 22), ("venus", 23), ("earth", 30), ("earth", 31), ("earth", 32), ("earth", 33), ("moon", 1), ("moon", 2), ("moon", 3), ("mars", 40), ("mars", 41), ("mars", 42), ("proxima centauri", 2000), ("proxima centauri b", 10), ("proxima centauri b", 11), ("proxima centauri b", 12), ]: order = Order(amount=amount, region=regions[region]) order.save() for key, value, parent in [ ("level 1", 1, None), ("level 2", 1, "level 1"), ("level 2", 2, "level 1"), ("level 3", 1, "level 2"), ]: parent = parent and KeyPair.objects.filter(key=parent).first() KeyPair.objects.create(key=key, value=value, parent=parent) django-cte-1.3.0/tests/models.py000066400000000000000000000035411443342442600165360ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals from django.db.models import ( CASCADE, Model, AutoField, CharField, ForeignKey, IntegerField, TextField, ) from django_cte import CTEManager, CTEQuerySet class LT40QuerySet(CTEQuerySet): def lt40(self): return self.filter(amount__lt=40) class LT30QuerySet(CTEQuerySet): def lt30(self): return self.filter(amount__lt=30) class LT25QuerySet(CTEQuerySet): def lt25(self): return self.filter(amount__lt=25) class LTManager(CTEManager): pass class Region(Model): objects = CTEManager() name = TextField(primary_key=True) parent = ForeignKey("self", null=True, on_delete=CASCADE) class Meta: db_table = "region" class User(Model): id = AutoField(primary_key=True) name = TextField() class Meta: db_table = "user" class Order(Model): objects = CTEManager() id = AutoField(primary_key=True) region = ForeignKey(Region, on_delete=CASCADE) amount = IntegerField(default=0) user = ForeignKey(User, null=True, on_delete=CASCADE) class Meta: db_table = "orders" class OrderFromLT40(Order): class Meta: proxy = True objects = CTEManager.from_queryset(LT40QuerySet)() class OrderLT40AsManager(Order): class Meta: proxy = True objects = LT40QuerySet.as_manager() class OrderCustomManagerNQuery(Order): class Meta: proxy = True objects = LTManager.from_queryset(LT25QuerySet)() class OrderCustomManager(Order): class Meta: proxy = True objects = LTManager() class KeyPair(Model): objects = CTEManager() key = CharField(max_length=32) value = IntegerField(default=0) parent = ForeignKey("self", null=True, on_delete=CASCADE) class Meta: db_table = "keypair" django-cte-1.3.0/tests/settings.py000066400000000000000000000010231443342442600171040ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals import os import json if "DB_SETTINGS" in os.environ: _db_settings = json.loads(os.environ["DB_SETTINGS"]) else: # sqlite3 by default # must be sqlite3 >= 3.8.3 supporting WITH clause # must be sqlite3 >= 3.35.0 supporting MATERIALIZED option _db_settings = { "ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:", } DATABASES = {'default': _db_settings} INSTALLED_APPS = ["tests"] SECRET_KEY = "test" django-cte-1.3.0/tests/test_cte.py000066400000000000000000000421341443342442600170660ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals from __future__ import print_function from unittest import SkipTest from django.db.models import IntegerField, TextField from django.db.models.aggregates import Count, Max, Min, Sum from django.db.models.expressions import ( Exists, ExpressionWrapper, F, OuterRef, Subquery, ) from django.db.models.sql.constants import LOUTER from django.test import TestCase from django_cte import With from django_cte import CTEManager from .models import Order, Region, User int_field = IntegerField() text_field = TextField() class TestCTE(TestCase): def test_simple_cte_query(self): cte = With( Order.objects .values("region_id") .annotate(total=Sum("amount")) ) orders = ( # FROM orders INNER JOIN cte ON orders.region_id = cte.region_id cte.join(Order, region=cte.col.region_id) # Add `WITH ...` before `SELECT ... FROM orders ...` .with_cte(cte) # Annotate each Order with a "region_total" .annotate(region_total=cte.col.total) ) print(orders.query) data = sorted((o.amount, o.region_id, o.region_total) for o in orders) self.assertEqual(data, [ (1, 'moon', 6), (2, 'moon', 6), (3, 'moon', 6), (10, 'mercury', 33), (10, 'proxima centauri b', 33), (11, 'mercury', 33), (11, 'proxima centauri b', 33), (12, 'mercury', 33), (12, 'proxima centauri b', 33), (20, 'venus', 86), (21, 'venus', 86), (22, 'venus', 86), (23, 'venus', 86), (30, 'earth', 126), (31, 'earth', 126), (32, 'earth', 126), (33, 'earth', 126), (40, 'mars', 123), (41, 'mars', 123), (42, 'mars', 123), (1000, 'sun', 1000), (2000, 'proxima centauri', 2000), ]) def test_cte_name_escape(self): totals = With( Order.objects .filter(region__parent="sun") .values("region_id") .annotate(total=Sum("amount")), name="mixedCaseCTEName" ) orders = ( totals .join(Order, region=totals.col.region_id) .with_cte(totals) .annotate(region_total=totals.col.total) .order_by("amount") ) self.assertTrue( str(orders.query).startswith('WITH RECURSIVE "mixedCaseCTEName"')) def test_cte_queryset(self): sub_totals = With( Order.objects .values(region_parent=F("region__parent_id")) .annotate(total=Sum("amount")), ) regions = ( Region.objects.all() .with_cte(sub_totals) .annotate( child_regions_total=Subquery( sub_totals.queryset() .filter(region_parent=OuterRef("name")) .values("total"), ), ) .order_by("name") ) print(regions.query) data = [(r.name, r.child_regions_total) for r in regions] self.assertEqual(data, [ ("bernard's star", None), ('deimos', None), ('earth', 6), ('mars', None), ('mercury', None), ('moon', None), ('phobos', None), ('proxima centauri', 33), ('proxima centauri b', None), ('sun', 368), ('venus', None) ]) def test_cte_queryset_with_model_result(self): cte = With( Order.objects .annotate(region_parent=F("region__parent_id")), ) orders = cte.queryset().with_cte(cte) print(orders.query) data = sorted( (x.region_id, x.amount, x.region_parent) for x in orders)[:5] self.assertEqual(data, [ ("earth", 30, "sun"), ("earth", 31, "sun"), ("earth", 32, "sun"), ("earth", 33, "sun"), ("mars", 40, "sun"), ]) self.assertTrue( all(isinstance(x, Order) for x in orders), repr([x for x in orders]), ) def test_cte_queryset_with_join(self): cte = With( Order.objects .annotate(region_parent=F("region__parent_id")), ) orders = ( cte.queryset() .with_cte(cte) .annotate(parent=F("region__parent_id")) .order_by("region_id", "amount") ) print(orders.query) data = [(x.region_id, x.region_parent, x.parent) for x in orders][:5] self.assertEqual(data, [ ("earth", "sun", "sun"), ("earth", "sun", "sun"), ("earth", "sun", "sun"), ("earth", "sun", "sun"), ("mars", "sun", "sun"), ]) def test_cte_queryset_with_values_result(self): cte = With( Order.objects .values( "region_id", region_parent=F("region__parent_id"), ) .distinct() ) values = ( cte.queryset() .with_cte(cte) .filter(region_parent__isnull=False) ) print(values.query) def key(item): return item["region_parent"], item["region_id"] data = sorted(values, key=key)[:5] self.assertEqual(data, [ {'region_id': 'moon', 'region_parent': 'earth'}, { 'region_id': 'proxima centauri b', 'region_parent': 'proxima centauri', }, {'region_id': 'earth', 'region_parent': 'sun'}, {'region_id': 'mars', 'region_parent': 'sun'}, {'region_id': 'mercury', 'region_parent': 'sun'}, ]) def test_named_simple_ctes(self): totals = With( Order.objects .filter(region__parent="sun") .values("region_id") .annotate(total=Sum("amount")), name="totals", ) region_count = With( Region.objects .filter(parent="sun") .values("parent") .annotate(num=Count("name")), name="region_count", ) orders = ( region_count.join( totals.join(Order, region=totals.col.region_id), region__parent=region_count.col.parent_id ) .with_cte(totals) .with_cte(region_count) .annotate(region_total=totals.col.total) .annotate(region_count=region_count.col.num) .order_by("amount") ) print(orders.query) data = [( o.amount, o.region_id, o.region_count, o.region_total, ) for o in orders] self.assertEqual(data, [ (10, 'mercury', 4, 33), (11, 'mercury', 4, 33), (12, 'mercury', 4, 33), (20, 'venus', 4, 86), (21, 'venus', 4, 86), (22, 'venus', 4, 86), (23, 'venus', 4, 86), (30, 'earth', 4, 126), (31, 'earth', 4, 126), (32, 'earth', 4, 126), (33, 'earth', 4, 126), (40, 'mars', 4, 123), (41, 'mars', 4, 123), (42, 'mars', 4, 123), ]) def test_named_ctes(self): def make_root_mapping(rootmap): return Region.objects.filter( parent__isnull=True ).values( "name", root=F("name"), ).union( rootmap.join(Region, parent=rootmap.col.name).values( "name", root=rootmap.col.root, ), all=True, ) rootmap = With.recursive(make_root_mapping, name="rootmap") totals = With( rootmap.join(Order, region_id=rootmap.col.name) .values( root=rootmap.col.root, ).annotate( orders_count=Count("id"), region_total=Sum("amount"), ), name="totals", ) root_regions = ( totals.join(Region, name=totals.col.root) .with_cte(rootmap) .with_cte(totals) .annotate( # count of orders in this region and all subregions orders_count=totals.col.orders_count, # sum of order amounts in this region and all subregions region_total=totals.col.region_total, ) ) print(root_regions.query) data = sorted( (r.name, r.orders_count, r.region_total) for r in root_regions ) self.assertEqual(data, [ ('proxima centauri', 4, 2033), ('sun', 18, 1374), ]) def test_materialized_option(self): totals = With( Order.objects .filter(region__parent="sun") .values("region_id") .annotate(total=Sum("amount")), materialized=True ) orders = ( totals .join(Order, region=totals.col.region_id) .with_cte(totals) .annotate(region_total=totals.col.total) .order_by("amount") ) self.assertTrue( str(orders.query).startswith( 'WITH RECURSIVE "cte" AS MATERIALIZED' ) ) def test_update_cte_query(self): cte = With( Order.objects .values(region_parent=F("region__parent_id")) .annotate(total=Sum("amount")) .filter(total__isnull=False) ) # not the most efficient query, but it exercises CTEUpdateQuery Order.objects.all().with_cte(cte).filter(region_id__in=Subquery( cte.queryset() .filter(region_parent=OuterRef("region_id")) .values("region_parent") )).update(amount=Subquery( cte.queryset() .filter(region_parent=OuterRef("region_id")) .values("total") )) data = set((o.region_id, o.amount) for o in Order.objects.filter( region_id__in=["earth", "sun", "proxima centauri", "mars"] )) self.assertEqual(data, { ('earth', 6), ('mars', 40), ('mars', 41), ('mars', 42), ('proxima centauri', 33), ('sun', 368), }) def test_delete_cte_query(self): raise SkipTest( "this test will not work until `QuerySet.delete` (Django method) " "calls `self.query.chain(sql.DeleteQuery)` instead of " "`sql.DeleteQuery(self.model)`" ) cte = With( Order.objects .values(region_parent=F("region__parent_id")) .annotate(total=Sum("amount")) .filter(total__isnull=False) ) Order.objects.all().with_cte(cte).annotate( cte_has_order=Exists( cte.queryset() .values("total") .filter(region_parent=OuterRef("region_id")) ) ).filter(cte_has_order=False).delete() data = [(o.region_id, o.amount) for o in Order.objects.all()] self.assertEqual(data, [ ('sun', 1000), ('earth', 30), ('earth', 31), ('earth', 32), ('earth', 33), ('proxima centauri', 2000), ]) def test_outerref_in_cte_query(self): # This query is meant to return the difference between min and max # order of each region, through a subquery min_and_max = With( Order.objects .filter(region=OuterRef("pk")) .values('region') # This is to force group by region_id .annotate( amount_min=Min("amount"), amount_max=Max("amount"), ) .values('amount_min', 'amount_max') ) regions = ( Region.objects .annotate( difference=Subquery( min_and_max.queryset().with_cte(min_and_max).annotate( difference=ExpressionWrapper( F('amount_max') - F('amount_min'), output_field=int_field, ), ).values('difference')[:1], output_field=IntegerField() ) ) .order_by("name") ) print(regions.query) data = [(r.name, r.difference) for r in regions] self.assertEqual(data, [ ("bernard's star", None), ('deimos', None), ('earth', 3), ('mars', 2), ('mercury', 2), ('moon', 2), ('phobos', None), ('proxima centauri', 0), ('proxima centauri b', 2), ('sun', 0), ('venus', 3) ]) def test_experimental_left_outer_join(self): totals = With( Order.objects .values("region_id") .annotate(total=Sum("amount")) .filter(total__gt=100) ) orders = ( totals .join(Order, region=totals.col.region_id, _join_type=LOUTER) .with_cte(totals) .annotate(region_total=totals.col.total) ) print(orders.query) self.assertIn("LEFT OUTER JOIN", str(orders.query)) self.assertNotIn("INNER JOIN", str(orders.query)) data = sorted((o.region_id, o.amount, o.region_total) for o in orders) self.assertEqual(data, [ ('earth', 30, 126), ('earth', 31, 126), ('earth', 32, 126), ('earth', 33, 126), ('mars', 40, 123), ('mars', 41, 123), ('mars', 42, 123), ('mercury', 10, None), ('mercury', 11, None), ('mercury', 12, None), ('moon', 1, None), ('moon', 2, None), ('moon', 3, None), ('proxima centauri', 2000, 2000), ('proxima centauri b', 10, None), ('proxima centauri b', 11, None), ('proxima centauri b', 12, None), ('sun', 1000, 1000), ('venus', 20, None), ('venus', 21, None), ('venus', 22, None), ('venus', 23, None), ]) def test_non_cte_subquery(self): """ Verifies that subquery annotations are handled correctly when the subquery model doesn't use the CTE manager, and the query results match expected behavior """ self.assertNotIsInstance(User.objects, CTEManager) sub_totals = With( Order.objects .values(region_parent=F("region__parent_id")) .annotate( total=Sum("amount"), # trivial subquery example testing existence of # a user for the order non_cte_subquery=Exists( User.objects.filter(pk=OuterRef("user_id")) ), ), ) regions = ( Region.objects.all() .with_cte(sub_totals) .annotate( child_regions_total=Subquery( sub_totals.queryset() .filter(region_parent=OuterRef("name")) .values("total"), ), ) .order_by("name") ) print(regions.query) data = [(r.name, r.child_regions_total) for r in regions] self.assertEqual(data, [ ("bernard's star", None), ('deimos', None), ('earth', 6), ('mars', None), ('mercury', None), ('moon', None), ('phobos', None), ('proxima centauri', 33), ('proxima centauri b', None), ('sun', 368), ('venus', None) ]) def test_explain(self): """ Verifies that using .explain() prepends the EXPLAIN clause in the correct position """ totals = With( Order.objects .filter(region__parent="sun") .values("region_id") .annotate(total=Sum("amount")), name="totals", ) region_count = With( Region.objects .filter(parent="sun") .values("parent") .annotate(num=Count("name")), name="region_count", ) orders = ( region_count.join( totals.join(Order, region=totals.col.region_id), region__parent=region_count.col.parent_id ) .with_cte(totals) .with_cte(region_count) .annotate(region_total=totals.col.total) .annotate(region_count=region_count.col.num) .order_by("amount") ) # the test db (sqlite3) doesn't support EXPLAIN, so let's just check # to make sure EXPLAIN is at the top orders.query.explain_query = True self.assertTrue(str(orders.query).startswith("EXPLAIN ")) django-cte-1.3.0/tests/test_manager.py000066400000000000000000000067511443342442600177320ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import unicode_literals from __future__ import print_function from django.db.models.expressions import F from django.db.models.query import QuerySet from django.test import TestCase from django_cte import With, CTEQuerySet, CTEManager from .models import ( Order, OrderFromLT40, OrderLT40AsManager, OrderCustomManagerNQuery, OrderCustomManager, LT40QuerySet, LTManager, LT25QuerySet, ) class TestCTE(TestCase): def test_cte_queryset_correct_defaultmanager(self): self.assertEqual(type(Order._default_manager), CTEManager) self.assertEqual(type(Order.objects.all()), CTEQuerySet) def test_cte_queryset_correct_from_queryset(self): self.assertEqual(type(OrderFromLT40.objects.all()), LT40QuerySet) def test_cte_queryset_correct_queryset_as_manager(self): self.assertEqual(type(OrderLT40AsManager.objects.all()), LT40QuerySet) def test_cte_queryset_correct_manager_n_from_queryset(self): self.assertIsInstance( OrderCustomManagerNQuery._default_manager, LTManager) self.assertEqual(type( OrderCustomManagerNQuery.objects.all()), LT25QuerySet) def test_cte_create_manager_from_non_cteQuery(self): class BrokenQuerySet(QuerySet): "This should be a CTEQuerySet if we want this to work" with self.assertRaises(TypeError): CTEManager.from_queryset(BrokenQuerySet)() def test_cte_queryset_correct_limitedmanager(self): self.assertEqual(type(OrderCustomManager._default_manager), LTManager) # Check the expected even if not ideal behavior occurs self.assertIsInstance(OrderCustomManager.objects.all(), CTEQuerySet) def test_cte_queryset_with_from_queryset(self): self.assertEqual(type(OrderFromLT40.objects.all()), LT40QuerySet) cte = With( OrderFromLT40.objects .annotate(region_parent=F("region__parent_id")) .filter(region__parent_id="sun") ) orders = ( cte.queryset() .with_cte(cte) .lt40() # custom queryset method .order_by("region_id", "amount") ) print(orders.query) data = [(x.region_id, x.amount, x.region_parent) for x in orders] self.assertEqual(data, [ ("earth", 30, "sun"), ("earth", 31, "sun"), ("earth", 32, "sun"), ("earth", 33, "sun"), ('mercury', 10, 'sun'), ('mercury', 11, 'sun'), ('mercury', 12, 'sun'), ('venus', 20, 'sun'), ('venus', 21, 'sun'), ('venus', 22, 'sun'), ('venus', 23, 'sun'), ]) def test_cte_queryset_with_custom_queryset(self): cte = With( OrderCustomManagerNQuery.objects .annotate(region_parent=F("region__parent_id")) .filter(region__parent_id="sun") ) orders = ( cte.queryset() .with_cte(cte) .lt25() # custom queryset method .order_by("region_id", "amount") ) print(orders.query) data = [(x.region_id, x.amount, x.region_parent) for x in orders] self.assertEqual(data, [ ('mercury', 10, 'sun'), ('mercury', 11, 'sun'), ('mercury', 12, 'sun'), ('venus', 20, 'sun'), ('venus', 21, 'sun'), ('venus', 22, 'sun'), ('venus', 23, 'sun'), ]) django-cte-1.3.0/tests/test_raw.py000066400000000000000000000034141443342442600171020ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals from django.db.models import IntegerField, TextField from django.test import TestCase from django_cte import With from django_cte.raw import raw_cte_sql from .models import Region int_field = IntegerField() text_field = TextField() class TestRawCTE(TestCase): def test_raw_cte_sql(self): cte = With(raw_cte_sql( """ SELECT region_id, AVG(amount) AS avg_order FROM orders WHERE region_id = %s GROUP BY region_id """, ["moon"], {"region_id": text_field, "avg_order": int_field}, )) moon_avg = ( cte .join(Region, name=cte.col.region_id) .annotate(avg_order=cte.col.avg_order) .with_cte(cte) ) print(moon_avg.query) data = [(r.name, r.parent.name, r.avg_order) for r in moon_avg] self.assertEqual(data, [('moon', 'earth', 2)]) def test_raw_cte_sql_name_escape(self): cte = With( raw_cte_sql( """ SELECT region_id, AVG(amount) AS avg_order FROM orders WHERE region_id = %s GROUP BY region_id """, ["moon"], {"region_id": text_field, "avg_order": int_field}, ), name="mixedCaseCTEName" ) moon_avg = ( cte .join(Region, name=cte.col.region_id) .annotate(avg_order=cte.col.avg_order) .with_cte(cte) ) self.assertTrue( str(moon_avg.query).startswith( 'WITH RECURSIVE "mixedCaseCTEName"') ) django-cte-1.3.0/tests/test_recursive.py000066400000000000000000000252431443342442600203240ustar00rootroot00000000000000from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals import pickle from unittest import SkipTest from django.db.models import IntegerField, TextField from django.db.models.expressions import ( Case, Exists, ExpressionWrapper, F, OuterRef, Q, Value, When, ) from django.db.models.functions import Concat from django.db.utils import DatabaseError from django.test import TestCase from django_cte import With from .models import KeyPair, Region int_field = IntegerField() text_field = TextField() class TestRecursiveCTE(TestCase): def test_recursive_cte_query(self): def make_regions_cte(cte): return Region.objects.filter( # non-recursive: get root nodes parent__isnull=True ).values( "name", path=F("name"), depth=Value(0, output_field=int_field), ).union( # recursive union: get descendants cte.join(Region, parent=cte.col.name).values( "name", path=Concat( cte.col.path, Value(" / "), F("name"), output_field=text_field, ), depth=cte.col.depth + Value(1, output_field=int_field), ), all=True, ) cte = With.recursive(make_regions_cte) regions = ( cte.join(Region, name=cte.col.name) .with_cte(cte) .annotate( path=cte.col.path, depth=cte.col.depth, ) .filter(depth=2) .order_by("path") ) print(regions.query) data = [(r.name, r.path, r.depth) for r in regions] self.assertEqual(data, [ ('moon', 'sun / earth / moon', 2), ('deimos', 'sun / mars / deimos', 2), ('phobos', 'sun / mars / phobos', 2), ]) def test_recursive_cte_reference_in_condition(self): def make_regions_cte(cte): return Region.objects.filter( parent__isnull=True ).values( "name", path=F("name"), depth=Value(0, output_field=int_field), is_planet=Value(0, output_field=int_field), ).union( cte.join( Region, parent=cte.col.name ).annotate( # annotations for filter and CASE/WHEN conditions parent_name=ExpressionWrapper( cte.col.name, output_field=text_field, ), parent_depth=ExpressionWrapper( cte.col.depth, output_field=int_field, ), ).filter( ~Q(parent_name="mars"), ).values( "name", path=Concat( cte.col.path, Value("\x01"), F("name"), output_field=text_field, ), depth=cte.col.depth + Value(1, output_field=int_field), is_planet=Case( When(parent_depth=0, then=Value(1)), default=Value(0), output_field=int_field, ), ), all=True, ) cte = With.recursive(make_regions_cte) regions = cte.join(Region, name=cte.col.name).with_cte(cte).annotate( path=cte.col.path, depth=cte.col.depth, is_planet=cte.col.is_planet, ).order_by("path") data = [(r.path.split("\x01"), r.is_planet) for r in regions] print(data) self.assertEqual(data, [ (["bernard's star"], 0), (['proxima centauri'], 0), (['proxima centauri', 'proxima centauri b'], 1), (['sun'], 0), (['sun', 'earth'], 1), (['sun', 'earth', 'moon'], 0), (['sun', 'mars'], 1), # mars moons excluded: parent_name != 'mars' (['sun', 'mercury'], 1), (['sun', 'venus'], 1), ]) def test_recursive_cte_with_empty_union_part(self): def make_regions_cte(cte): return Region.objects.none().union( cte.join(Region, parent=cte.col.name), all=True, ) cte = With.recursive(make_regions_cte) regions = cte.join(Region, name=cte.col.name).with_cte(cte) print(regions.query) try: self.assertEqual(regions.count(), 0) except DatabaseError: raise SkipTest( "Expected failure: QuerySet omits `EmptyQuerySet` from " "UNION queries resulting in invalid CTE SQL" ) # -- recursive query "cte" does not have the form # -- non-recursive-term UNION [ALL] recursive-term # WITH RECURSIVE cte AS ( # SELECT "tests_region"."name", "tests_region"."parent_id" # FROM "tests_region", "cte" # WHERE "tests_region"."parent_id" = ("cte"."name") # ) # SELECT COUNT(*) # FROM "tests_region", "cte" # WHERE "tests_region"."name" = ("cte"."name") def test_circular_ref_error(self): def make_bad_cte(cte): # NOTE: not a valid recursive CTE query return cte.join(Region, parent=cte.col.name).values( depth=cte.col.depth + 1, ) cte = With.recursive(make_bad_cte) regions = cte.join(Region, name=cte.col.name).with_cte(cte) with self.assertRaises(ValueError) as context: print(regions.query) self.assertIn("Circular reference:", str(context.exception)) def test_attname_should_not_mask_col_name(self): def make_regions_cte(cte): return Region.objects.filter( name="moon" ).values( "name", "parent_id", ).union( cte.join(Region, name=cte.col.parent_id).values( "name", "parent_id", ), all=True, ) cte = With.recursive(make_regions_cte) regions = ( Region.objects.all() .with_cte(cte) .annotate(_ex=Exists( cte.queryset() .values(value=Value("1", output_field=int_field)) .filter(name=OuterRef("name")) )) .filter(_ex=True) .order_by("name") ) print(regions.query) data = [r.name for r in regions] self.assertEqual(data, ['earth', 'moon', 'sun']) def test_pickle_recursive_cte_queryset(self): def make_regions_cte(cte): return Region.objects.filter( parent__isnull=True ).annotate( depth=Value(0, output_field=int_field), ).union( cte.join(Region, parent=cte.col.name).annotate( depth=cte.col.depth + Value(1, output_field=int_field), ), all=True, ) cte = With.recursive(make_regions_cte) regions = cte.queryset().with_cte(cte).filter(depth=2).order_by("name") pickled_qs = pickle.loads(pickle.dumps(regions)) data = [(r.name, r.depth) for r in pickled_qs] self.assertEqual(data, [(r.name, r.depth) for r in regions]) self.assertEqual(data, [('deimos', 2), ('moon', 2), ('phobos', 2)]) def test_alias_change_in_annotation(self): def make_regions_cte(cte): return Region.objects.filter( parent__name="sun", ).annotate( value=F('name'), ).union( cte.join( Region.objects.all().annotate( value=F('name'), ), parent_id=cte.col.name, ), all=True, ) cte = With.recursive(make_regions_cte) query = cte.queryset().with_cte(cte) exclude_leaves = With(cte.queryset().filter( parent__name='sun', ).annotate( value=Concat(F('name'), F('name')) ), name='value_cte') query = query.annotate( _exclude_leaves=Exists( exclude_leaves.queryset().filter( name=OuterRef("name"), value=OuterRef("value"), ) ) ).filter(_exclude_leaves=True).with_cte(exclude_leaves) print(query.query) # Nothing should be returned. self.assertFalse(query) def test_alias_as_subquery(self): # This test covers CTEColumnRef.relabeled_clone def make_regions_cte(cte): return KeyPair.objects.filter( parent__key="level 1", ).annotate( rank=F('value'), ).union( cte.join( KeyPair.objects.all().order_by(), parent_id=cte.col.id, ).annotate( rank=F('value'), ), all=True, ) cte = With.recursive(make_regions_cte) children = cte.queryset().with_cte(cte) xdups = With(cte.queryset().filter( parent__key="level 1", ).annotate( rank=F('value') ).values('id', 'rank'), name='xdups') children = children.annotate( _exclude=Exists( ( xdups.queryset().filter( id=OuterRef("id"), rank=OuterRef("rank"), ) ) ) ).filter(_exclude=True).with_cte(xdups) print(children.query) query = KeyPair.objects.filter(parent__in=children) print(query.query) print(children.query) self.assertEqual(query.get().key, 'level 3') # Tests the case in which children's query was modified since it was # used in a subquery to define `query` above. self.assertEqual( list(c.key for c in children), ['level 2', 'level 2'] ) def test_materialized(self): # This test covers MATERIALIZED option in SQL query def make_regions_cte(cte): return KeyPair.objects.all() cte = With.recursive(make_regions_cte, materialized=True) query = KeyPair.objects.with_cte(cte) print(query.query) self.assertTrue( str(query.query).startswith('WITH RECURSIVE "cte" AS MATERIALIZED') )