pax_global_header00006660000000000000000000000064132044742360014517gustar00rootroot0000000000000052 comment=857b249ead8223dd6190ef39832ed03f8731015f Multicorn-1.3.4/000077500000000000000000000000001320447423600135005ustar00rootroot00000000000000Multicorn-1.3.4/.coveragerc000077500000000000000000000002331320447423600156220ustar00rootroot00000000000000[run] branch = True [report] exclude_lines = pragma: no cover raise NotImplementedError assert False __repr__ __str__ __unicode__ Multicorn-1.3.4/.git_hooks/000077500000000000000000000000001320447423600155445ustar00rootroot00000000000000Multicorn-1.3.4/.gitignore000077500000000000000000000001431320447423600154710ustar00rootroot00000000000000*.sw? tags *.so *.o *egg-info sql/multicorn--*.sql *.pyc /build /dist /results doc/_build doc/dist Multicorn-1.3.4/.gitmodules000077500000000000000000000001311320447423600156530ustar00rootroot00000000000000[submodule ".git_hooks"] path = .git_hooks url = git://github.com/Kozea/.git_hooks.git Multicorn-1.3.4/.pylintrc000066400000000000000000000001021320447423600153360ustar00rootroot00000000000000[MESSAGES CONTROL] disable= # Too few public methods R0903, Multicorn-1.3.4/CHANGELOG000066400000000000000000000074221320447423600147170ustar00rootroot000000000000001.3.4: - Add compatibility with PostgreSQL 10 1.3.3: - Add compatibility with PostgreSQL 9.6 - Fix bug with typecasting of params 1.3.2: - Fixes invalid sizes in makeConst calls. (thanks to Dickson S. Guedes for the report, github user rastkok for the fix) 1.3.1: - Fixes issues with nullsfirst / nullslast in sqlalchemyfdw (thanks to KyleLilly) 1.3.0: - Add support for SORT pushdown (Thanks to Julien Rouhaud !) - Add support for EXPLAIN - Follow API changes from 9.5 rc1 - Improve python 2.6 compatibility 1.2.4: - Restore compatibility with python 2.6 (Thanks to Asya Kamsky !) - Follow API change from 9.5 beta2 - Fix structured fs tests - Fix missing reference counting on PyNone (Thanks to github user "tchornyi") 1.2.3: - Fix issue with NULL handling with the sequence protocol - Fix API change in PostgreSQL 9.5 1.2.2: - Ensure that tests can success on pgxn-testers infrastructure 1.2.1: - fix bug in error reporting code wrt to encodings (thanks to Thomas Vondra and the pgxn-testers infrastructure for uncovering this one) 1.2.0: - Add support for IMPORT FOREIGN SCHEMA - Fix issue with pymssql behavior (#100, thanks to homelink for reporting it) 1.1.1: - Fixes to the build system on MacOSX (Ray Ruvinskiy) - Fix bug with dropped attributes during DML operations - Add sphinx documentation 1.1.0: - ldap_fdw: - use the ldap3 module for python3 compatibility (Guillaume Ayoub) - allow types other than varchar (Guillaume Ayoub) - sqlalchemy_fdw: explictly pass "stream_results" options - fix some bugs wrt array handling - Build: - add support for VPATH builds (Markus Wanner, Christoph Berg) - rssfdw: - more flexibility for rss feeds 1.0.4: - Add support for upcoming AFTER TRIGGERs on foreign tables. - Fix reference counting bug in messages - Add indivual options for sqlalchemy fdw - Fix bugs with refcounting - Fix bug with unary operator 1.0.3: - Fix bugs with types reflection in sqlalchemyfdw (Thanks to naoshika for reporting it) - Fix bug regarding ARRAY detection - Small cleanups in Py_None reference counting - Better guard against errors during fdw initialization 1.0.2: - Fix bug when an exception occurs during iteration (Thanks to johnmudd for reporting it) - Add missing PYTHON_LDFLAGS when compiling with PYTHON_OVERRIDE (Thanks to frensley for reporting it) 1.0.1: - Use rollback/commit hooks on 9.2 too. 1.0.0: - 9.3 writable API support - Python3 experimental support - Local evaluation of the right side of any "VAR OPERATOR EXPR" expressions. 1.0.0beta1: - Fix many many memory related bugs - Add gcfdw to help debugging python reference counting bugs. - Let the structuredfs work with file-like objects. 0.9.2.1: - Fix bug in log_to_postgres utility function, leading to SIGABRT from python. 0.9.2: - Move to PG 9.2 fdw API. - Add support to declared parameterized paths 0.9.1 - Improvements to the imap fdw - last supported version with pg 9.1. 0.0.9 - Break backwards compatibility: the column definition to the foreign data wrapper constructor is now a dictionary of column names to their types. - Add python2.6 compatibility - Add sqlalchemy foreign data wrapper - Remove sqlite foreign data wrapper 0.0.8 - Added minimal documentation in the doc folder 0.0.7 - Fix bug with 64bits platform on text and bytea types 0.0.6 - Fix operand switching, replacing the operator by its commutator - Removed ANY, CONTAINS, IN support for a more general approach - Fix many bugs in imapfdw 0.0.5 - Fix bug on Postgresql exceptions management - Replaced imaplib by imapclient for better server compatibility Multicorn-1.3.4/LICENSE000066400000000000000000000015751320447423600145150ustar00rootroot00000000000000Copyright (c) 2013, Kozea Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL KOZEA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF KOZEA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KOZEA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND KOZEA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. Multicorn-1.3.4/META.json000077500000000000000000000030331320447423600151230ustar00rootroot00000000000000{ "name": "multicorn", "abstract": "Multicorn Python bindings for Postgres 9.2+ Foreign Data Wrapper", "description": "The Multicorn Foreign Data Wrapper allows you to write foreign data wrappers in python.", "version": "__VERSION__", "maintainer": "Ronan Dunklau ", "license": { "PostgreSQL": "http://www.postgresql.org/about/licence" }, "resources": { "repository": { "url": "git://github.com/Kozea/multicorn.git", "type": "git" }, "homepage": "http://multicorn.readthedocs.org", "bugtracker": { "web": "https://github.com/Kozea/Multicorn/issues", "mailto": "multicorn@librelist.org" } }, "provides": { "multicorn": { "abstract": "Multicorn Python bindings for Postgres 9.2 Foreign Data Wrapper", "file": "sql/multicorn.sql", "docfile": "doc/multicorn.md", "version": "__VERSION__" } }, "prereqs": { "runtime": { "requires": { "PostgreSQL": "9.2.0", "Python": "2.6.0" } } }, "release_status": "stable", "generated_by": "Kozea", "meta-spec": { "version": "1.0.0", "url": "http://pgxn.org/meta/spec.txt" }, "tags": [ "multicorn", "python", "csv", "ldap", "imap", "filesystem", "SQL MED", "foreign data wrapper", "fdw", "external data" ] } Multicorn-1.3.4/Makefile000077500000000000000000000114761320447423600151540ustar00rootroot00000000000000srcdir = . MODULE_big = multicorn OBJS = src/errors.o src/python.o src/query.o src/multicorn.o DATA = $(filter-out $(wildcard sql/*--*.sql),$(wildcard sql/*.sql)) DOCS = $(wildcard $(srcdir)/doc/*.md) EXTENSION = multicorn EXTVERSION = $(shell grep default_version $(srcdir)/$(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") all: preflight-check sql/$(EXTENSION)--$(EXTVERSION).sql directories.stamp: [ -d sql ] || mkdir sql [ -d src ] || mkdir src touch $@ $(OBJS): directories.stamp install: python_code sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql directories.stamp cp $< $@ preflight-check: $(srcdir)/preflight-check.sh python_code: setup.py cp $(srcdir)/setup.py ./setup--$(EXTVERSION).py sed -i -e "s/__VERSION__/$(EXTVERSION)-dev/g" ./setup--$(EXTVERSION).py $(PYTHON) ./setup--$(EXTVERSION).py install rm ./setup--$(EXTVERSION).py release-zip: all git archive --format zip --prefix=multicorn-$(EXTVERSION)/ --output ./multicorn-$(EXTVERSION).zip HEAD unzip ./multicorn-$(EXTVERSION).zip rm ./multicorn-$(EXTVERSION).zip sed -i -e "s/__VERSION__/$(EXTVERSION)/g" ./multicorn-$(EXTVERSION)/META.json ./multicorn-$(EXTVERSION)/setup.py ./multicorn-$(EXTVERSION)/python/multicorn/__init__.py zip -r ./multicorn-$(EXTVERSION).zip ./multicorn-$(EXTVERSION)/ rm ./multicorn-$(EXTVERSION) -rf coverage: lcov -d . -c -o lcov.info --no-external genhtml --show-details --legend --output-directory=coverage --title="Multicorn Code Coverage" --no-branch-coverage --num-spaces=4 --prefix=./src/ `find . -name lcov.info -print` DATA = sql/$(EXTENSION)--$(EXTVERSION).sql EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql ./multicorn-$(EXTVERSION).zip directories.stamp PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) REGRESS = virtual_tests include $(PGXS) with_python_no_override = no ifeq ($(with_python),yes) with_python_no_override = yes endif ifdef PYTHON_OVERRIDE with_python_no_override = no endif ifeq ($(with_python_no_override),yes) SHLIB_LINK = $(python_libspec) $(python_additional_libs) $(filter -lintl,$(LIBS)) override CPPFLAGS := -I. -I$(srcdir) $(python_includespec) $(CPPFLAGS) override PYTHON = python${python_version} else ifdef PYTHON_OVERRIDE override PYTHON = ${PYTHON_OVERRIDE} endif ifeq (${PYTHON}, ) override PYTHON = python endif python_version = $(shell ${PYTHON} --version 2>&1 | cut -d ' ' -f 2 | cut -d '.' -f 1-2) PYTHON_CONFIG ?= python${python_version}-config PY_LIBSPEC = $(shell ${PYTHON_CONFIG} --libs) PY_INCLUDESPEC = $(shell ${PYTHON_CONFIG} --includes) PY_CFLAGS = $(shell ${PYTHON_CONFIG} --cflags) PY_LDFLAGS = $(shell ${PYTHON_CONFIG} --ldflags) SHLIB_LINK += $(PY_LIBSPEC) $(PY_LDFLAGS) $(PY_ADDITIONAL_LIBS) $(filter -lintl,$(LIBS)) override PG_CPPFLAGS := $(PY_INCLUDESPEC) $(PG_CPPFLAGS) override CPPFLAGS := $(PG_CPPFLAGS) $(CPPFLAGS) endif ifeq ($(PORTNAME),darwin) override LDFLAGS += -undefined dynamic_lookup -bundle_loader $(shell $(PG_CONFIG) --bindir)/postgres endif PYTHON_TEST_VERSION ?= $(python_version) PG_TEST_VERSION ?= $(MAJORVERSION) SUPPORTS_WRITE=$(shell expr ${PG_TEST_VERSION} \>= 9.3) SUPPORTS_IMPORT=$(shell expr ${PG_TEST_VERSION} \>= 9.5) UNSUPPORTS_SQLALCHEMY=$(shell python -c "import sqlalchemy;import psycopg2" 1> /dev/null 2>&1; echo $$?) TESTS = test-$(PYTHON_TEST_VERSION)/sql/multicorn_cache_invalidation.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_column_options_test.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_error_test.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_logger_test.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_planner_test.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_regression_test.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_sequence_test.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_test_date.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_test_dict.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_test_list.sql \ test-$(PYTHON_TEST_VERSION)/sql/multicorn_test_sort.sql ifeq (${UNSUPPORTS_SQLALCHEMY}, 0) TESTS += test-$(PYTHON_TEST_VERSION)/sql/multicorn_alchemy_test.sql endif ifeq (${SUPPORTS_WRITE}, 1) TESTS += test-$(PYTHON_TEST_VERSION)/sql/write_filesystem.sql \ test-$(PYTHON_TEST_VERSION)/sql/write_savepoints.sql \ test-$(PYTHON_TEST_VERSION)/sql/write_test.sql ifeq (${UNSUPPORTS_SQLALCHEMY}, 0) TESTS += test-$(PYTHON_TEST_VERSION)/sql/write_sqlalchemy.sql endif endif ifeq (${SUPPORTS_IMPORT}, 1) TESTS += test-$(PYTHON_TEST_VERSION)/sql/import_test.sql ifeq (${UNSUPPORTS_SQLALCHEMY}, 0) TESTS += test-$(PYTHON_TEST_VERSION)/sql/import_sqlalchemy.sql endif endif REGRESS = $(patsubst test-$(PYTHON_TEST_VERSION)/sql/%.sql,%,$(TESTS)) REGRESS_OPTS = --inputdir=test-$(PYTHON_TEST_VERSION) --load-language=plpgsql $(info Python version is $(python_version)) Multicorn-1.3.4/README.md000077500000000000000000000010221320447423600147550ustar00rootroot00000000000000[![PGXN version](https://badge.fury.io/pg/multicorn.svg)](https://badge.fury.io/pg/multicorn) [![Build Status](https://jenkins.dalibo.info/buildStatus/public/Multicorn)]() Multicorn ========= Multicorn Python Wrapper for Postgresql 9.2+ Foreign Data Wrapper The Multicorn Foreign Data Wrapper allows you to fetch foreign data in Python in your PostgreSQL server. Documentation available at : http://multicorn.readthedocs.org/en/latest/ Multicorn is distributed under the PostgreSQL license. See the LICENSE file for details. Multicorn-1.3.4/doc/000077500000000000000000000000001320447423600142455ustar00rootroot00000000000000Multicorn-1.3.4/doc/Makefile000066400000000000000000000151661320447423600157160ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Multicorn.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Multicorn.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Multicorn" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Multicorn" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." Multicorn-1.3.4/doc/_static/000077500000000000000000000000001320447423600156735ustar00rootroot00000000000000Multicorn-1.3.4/doc/_static/css/000077500000000000000000000000001320447423600164635ustar00rootroot00000000000000Multicorn-1.3.4/doc/_static/css/custom.css000066400000000000000000000014301320447423600205050ustar00rootroot00000000000000.api_compatibility { line-height: 1.3em; } .api_compatibility > i { margin-left: 1em; text-opacity: 0; width: 40px; height: 32px; padding-left: 16px; padding-right: 16px; padding-top: 16px; padding-bottom: 16px; font-size: 2em; text-align: right; background-repeat: no-repeat; background-position: left, center; } .api_compatibility > i.checked { color: green; } .api_compatibility > i.unchecked { color: red; } .compat-read { background-image: url('../img/glyphicons-352-book-open.png'); } .compat-write { background-image: url('../img/glyphicons-151-edit.png'); } .compat-transaction { background-image: url('../img/glyphicons-142-database-plus.png'); } .compat-import_schema { background-image: url('../img/glyphicons-419-disk-import.png'); } Multicorn-1.3.4/doc/_static/img/000077500000000000000000000000001320447423600164475ustar00rootroot00000000000000Multicorn-1.3.4/doc/_static/img/glyphicons-142-database-plus.png000066400000000000000000000005121320447423600243610ustar00rootroot00000000000000PNG  IHDRUtEXtSoftwareAdobe ImageReadyqe<IDATxb?50 nE,Ai4 Y" .~h&#TLTH"CEHB,P @u}A )@iy% 7: KrR A4}y}IENDB`Multicorn-1.3.4/doc/api.rst000066400000000000000000000042141320447423600155510ustar00rootroot00000000000000*** API *** The API is split into two modules: the ``multicorn`` module and the `utils` module: - The ``multicorn`` module contains the whole API needed for implementing a Foreign Data Wrapper. - The ``utils`` module contains logging and error reporting functions, which are ultimately implemented as calls to the PostgreSQL API. Implementing an FDW =================== Implementing an FDW is as simple as implementing the :py:class:`~multicorn.ForeignDataWrapper` class. Required API ------------ .. py:currentmodule:: multicorn.ForeignDataWrapper This subset of the API allows your ForeignDataWrapper to be used for read-only queries. You have to implement the following methods: - :py:meth:`__init__` - :py:meth:`execute` .. note:: In the documentation, FDWs implementing this API will be marked with: .. api_compat:: :read: Write API --------- To implement full write capabilites, the following property must be implemented: .. autoattribute:: multicorn.ForeignDataWrapper.rowid_column In addition to that, you should implement each DML operation as you see fit: - :py:meth:`insert` - :py:meth:`update` - :py:meth:`delete` .. note:: In the documentation, FDWs implementing this API will be marked with: .. api_compat:: :write: Transactional API ----------------- Transactional Capabilities can be implemented with the following methods: .. automethod:: multicorn.ForeignDataWrapper.begin .. automethod:: multicorn.ForeignDataWrapper.pre_commit .. automethod:: multicorn.ForeignDataWrapper.rollback .. automethod:: multicorn.ForeignDataWrapper.sub_begin .. automethod:: multicorn.ForeignDataWrapper.sub_commit .. automethod:: multicorn.ForeignDataWrapper.sub_rollback .. note:: In the documentation, FDWs implementing this API will be marked with: .. api_compat:: :transaction: Full API ======== .. autoclass:: multicorn.ForeignDataWrapper :special-members: __init__ :members: .. autoclass:: multicorn.SortKey .. autoclass:: multicorn.Qual :members: .. autoclass:: multicorn.ColumnDefinition :members: .. autoclass:: multicorn.TableDefinition :members: Multicorn-1.3.4/doc/conf.py000066400000000000000000000233411320447423600155470ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Multicorn documentation build configuration file, created by # sphinx-quickstart on Thu Dec 4 08:56:10 2014. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os from sphinx.builders.html import StandaloneHTMLBuilder StandaloneHTMLBuilder.css_files = ["_static/css/custom.css"] + StandaloneHTMLBuilder.css_files # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) sys.path.insert(0, os.path.abspath('../python/')) sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinxcontrib.napoleon', 'sphinx.ext.autosummary', 'multicorn_directives' ] intersphinx_mapping = { 'python': ('http://docs.python.org/', None), 'psutil': ('http://pythonhosted.org/psutil', None), } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Multicorn' copyright = u"""2011-2014, Kozea""" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '1.1' # The full version, including alpha/beta/rc tags. release = '1.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # on_rtd is whether we are on readthedocs.org import os on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'Multicorndoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'Multicorn.tex', u'Multicorn Documentation', u'Ronan Dunklau, Florian Mounier', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'multicorn', u'Multicorn Documentation', [u'Ronan Dunklau, Florian Mounier'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Multicorn', u'Multicorn Documentation', u'Ronan Dunklau, Florian Mounier', 'Multicorn', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. rst_prolog = """ :mailaddress: multicorn@librelist.com :mailarchives: http://librelist.com/browser/multicorn :buglink: https://github.com/Kozea/Multicorn/issues :codelink: https://github.com/Kozea/Multicorn """ rst_epilog = """ .. _Foreign Data Wrapper: http://www.postgresql.org/docs/current/static/ddl-foreign-data.html .. |multicorn_pgxn_download| replace:: http://api.pgxn.org/dist/multicorn/{release}/multicorn-{release}.zip .. |multicorn_release| replace:: {release} """.format(release=release) import sys from mock import Mock as BaseMock class Mock(BaseMock): @classmethod def __getattr__(cls, name): return Mock() MOCK_MODULES = ['ldap3', 'lxml', 'imapclient'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) Multicorn-1.3.4/doc/contribute.rst000066400000000000000000000012541320447423600171570ustar00rootroot00000000000000************ Contribute ************ Send Us an Mail =============== Want to write kind words? You can send a mail on _`our Librelist mailing-list ` and even take a look at .. meta: mailarchives_`the archives`. If you use Multicorn in production, we would love to hear about your use-case ! Report Bugs =========== Found a bug? Want a new feature? Report a new issue on the `Multicorn bug-tracker on GitHub `. Hack ==== Interested in hacking? Feel free to clone the `git repository on GitHub ` if you want to add new features, fix bugs or update documentation. Multicorn-1.3.4/doc/foreign-data-wrappers.rst000066400000000000000000000007641320447423600212070ustar00rootroot00000000000000****************************** Included Foreign Data Wrappers ****************************** Multicorn is bundled with a small set of Foreign Data Wrappers, which you can use or customize for your needs. .. toctree:: :maxdepth: 2 /foreign-data-wrappers/sqlalchemyfdw.rst /foreign-data-wrappers/fsfdw.rst /foreign-data-wrappers/imapfdw.rst /foreign-data-wrappers/ldapfdw.rst /foreign-data-wrappers/csvfdw.rst /foreign-data-wrappers/rssfdw.rst /foreign-data-wrappers/processfdw.rst Multicorn-1.3.4/doc/foreign-data-wrappers/000077500000000000000000000000001320447423600204465ustar00rootroot00000000000000Multicorn-1.3.4/doc/foreign-data-wrappers/csvfdw.rst000066400000000000000000000001421320447423600224710ustar00rootroot00000000000000CSV Foreign Data Wrapper ************************ .. automodule:: multicorn.csvfdw :synopsis: Multicorn-1.3.4/doc/foreign-data-wrappers/fsfdw.rst000066400000000000000000000003051320447423600223070ustar00rootroot00000000000000FileSystem Foreign Data Wrapper ******************************* .. automodule:: multicorn.fsfdw :synopsis: ReStructuredText FDW ~~~~~~~~~~~~~~~~~~~~ .. automodule:: multicorn.fsfdw.restfsfdw Multicorn-1.3.4/doc/foreign-data-wrappers/imapfdw.rst000066400000000000000000000001451320447423600226270ustar00rootroot00000000000000Imap Foreign Data Wrapper ************************* .. automodule:: multicorn.imapfdw :synopsis: Multicorn-1.3.4/doc/foreign-data-wrappers/ldapfdw.rst000066400000000000000000000001441320447423600226200ustar00rootroot00000000000000LDAP Foreign Data Wrapper ************************* .. automodule:: multicorn.ldapfdw :synopsis: Multicorn-1.3.4/doc/foreign-data-wrappers/processfdw.rst000066400000000000000000000001561320447423600233610ustar00rootroot00000000000000Process Foreign Data Wrapper **************************** .. automodule:: multicorn.processfdw :synopsis: Multicorn-1.3.4/doc/foreign-data-wrappers/rssfdw.rst000066400000000000000000000001421320447423600225050ustar00rootroot00000000000000RSS foreign data wrapper ************************ .. automodule:: multicorn.rssfdw :synopsis: Multicorn-1.3.4/doc/foreign-data-wrappers/sqlalchemyfdw.rst000066400000000000000000000001671320447423600240470ustar00rootroot00000000000000SQLAlchemy Foreign Data Wrapper ******************************* .. automodule:: multicorn.sqlalchemyfdw :synopsis: Multicorn-1.3.4/doc/getting-started.rst000066400000000000000000000017511320447423600201100ustar00rootroot00000000000000***** Usage ***** The multicorn foreign data wrapper is not different from other foreign data wrappers. To use it, you have to: - Create the extension in the target database. As a PostgreSQL super user, run the following SQL: .. code-block:: sql CREATE EXTENSION multicorn; - Create a server. In the SQL ``OPTIONS`` clause, you must provide an options named wrapper, containing the fully-qualified class name of the concrete python foreign data wrapper you wish to use. want to use: .. code-block:: sql CREATE SERVER multicorn_imap FOREIGN DATA WRAPPER multicorn options ( wrapper 'multicorn.imapfdw.ImapFdw' ); You can then proceed on with the actual foreign tables creation, and pass them the needed options. Each foreign data wrapper supports its own set of options, and may interpret the columns definitions differently. You should look at the documentation for the specific :doc:`Foreign Data Wraper documentation ` Multicorn-1.3.4/doc/implementing-an-fdw.rst000066400000000000000000000003011320447423600206330ustar00rootroot00000000000000************** Writing an FDW ************** If you want to write an FDW, we recommend you start with the :ref:`tutorial`. .. toctree:: :maxdepth: 1 api.rst implementing-tutorial.rst Multicorn-1.3.4/doc/implementing-tutorial.rst000066400000000000000000000264671320447423600213470ustar00rootroot00000000000000.. _tutorial: ************************* Tutorial: Writing an FDW ************************* Multicorn provides a simple interface for writing foreign data wrappers: the ``multicorn.ForeignDataWrapper`` interface. Implementing a foreign data wrapper is as simple as inheriting from ``multicorn.ForeignDataWrapper`` and implemening the ``execute`` method. What are we trying to achieve ? =============================== Supposing we want to implement a foreign data wrapper which only returns a set of 20 rows, containing in each column the name of the column itself concatenated with the number of the line. The goal of this tutorial is to be able to execute this: .. code-block:: sql CREATE FOREIGN TABLE constanttable ( test character varying, test2 character varying ) server multicorn_srv options ( wrapper 'myfdw.ConstantForeignDataWrapper' ) SELECT * from constanttable; And obtain this as a result: .. code-block:: bash test | test2 ---------+---------- test 0 | test2 0 test 1 | test2 1 test 2 | test2 2 test 3 | test2 3 test 4 | test2 4 test 5 | test2 5 test 6 | test2 6 test 7 | test2 7 test 8 | test2 8 test 9 | test2 9 test 10 | test2 10 test 11 | test2 11 test 12 | test2 12 test 13 | test2 13 test 14 | test2 14 test 15 | test2 15 test 16 | test2 16 test 17 | test2 17 test 18 | test2 18 test 19 | test2 19 (20 lignes) How do we do that ? =================== The fdw described above is pretty simple, implementing it should be easy ! First things first, we have to create a new python module. This can be achieved with the most simple ``setup.py`` file: .. code-block:: python import subprocess from setuptools import setup, find_packages, Extension setup( name='myfdw', version='0.0.1', author='Ronan Dunklau', license='Postgresql', packages=['myfdw'] ) But let's see the whole code. To be usable with the above ``CREATE FOREIGN TABLE`` statement, this module should be named ``myfdw``. .. code-block:: python from multicorn import ForeignDataWrapper class ConstantForeignDataWrapper(ForeignDataWrapper): def __init__(self, options, columns): super(ConstantForeignDataWrapper, self).__init__(options, columns) self.columns = columns def execute(self, quals, columns): for index in range(20): line = {} for column_name in self.columns: line[column_name] = '%s %s' % (column_name, index) yield line You should have the following directory structure: .. code-block:: bash . |-- myfdw/ | `-- __init__.py `-- setup.py To install it, just run ``python setup.py install``, and the file will be copied to your global python installation, which should be the one your PostgreSQL instance is using. And that's it ! You just created your first foreign data wrapper. But let's look a bit more thoroughly to the class... The first thing to do (although optional, since you can implement the interface via duck-typing), is to import the base class and subclass it: .. code-block:: python from multicorn import ForeignDataWrapper class ConstantForeignDataWrapper(ForeignDataWrapper): The init method must accept two arguments ``options`` A dictionary of options given in the ``OPTIONS`` clause of the ``CREATE FOREIGN TABLE`` statement, minus the wrapper option. ``columns`` A mapping of the columns names given during the table creation, associated to their types. Ex: {'test': 'character varying'} Our access point do not need any options, thus we will only need to keep a reference to the columns: .. code-block:: python def __init__(self, options, columns): super(ConstantForeignDataWrapper, self).__init__(options, columns) self.columns = columns The execute method is the core of the API. It is called with a list of ``Qual`` objects, and a list column names, which we will ignore for now but more on that `later <#optimizations>`_. This method must return an iterable of the resulting lines. Each line can be either a list containing an item by column, or a dictonary mappning the column names to their value. For this example, we chose to build a dictionary. Each column contains the concatenation of the column name and the line index. .. code-block:: python def execute(self, quals): for index in range(20): line = {} for column_name in self.columns: line[column_name] = '%s %s' % (column_name, index) yield line And that's it ! Write API ========= Since PostgreSQL 9.3, foreign data wrappers can implement a write API. In multicorn, this involves defining which column will be used as a primary key (mandatory) and implementing the following methods at your discretion: .. code-block:: python def insert(self, new_values) def update(self, old_values, new_values) def delete(self, old_values) Each of these arguments will be dictionaries, containing at least the column you defined as a primary key, and the values to insert or those which have changed (for an update). In addition, other values may be present depending on the query involved. These methods should return a dictionary containing the new values (after insertion or update). This will be used in the case of RETURNING clauses of the form: .. code-block:: sql INSERT INTO my_ft VALUES (some_value) RETURNING *; You can return new values if the values that were given in sql are not the ones that are actually stored (think about default values, triggers...). The row_id_column attribute must be set to the name of a column acting as a primary key. For example: .. code-block:: python class MyFDW(ForeignDataWrapper): def __init__(self, fdw_options, fdw_columns): self.row_id_column = fdw_columns.keys()[0] If you want to handle transaction hooks, you can implement the following methods: .. code-block:: python def commit(self) def rollback(self) def pre_commit(self) The pre_commit method will be called just before the local transaction commits. You can raise an exception here to abort the current transaction were your remote commit to fail. The commit method will be called just at commit time, while the rollback method will be called whenever the local transaction is rollbacked. Optimizations ============= As was noted in the code commentaries, the execute methods accept a ``quals`` argument. This argument is a list of quals object, which are defined in `multicorn/__init__.py`_. A Qual object defines a simple condition wich can be used by the foreign data wrapper to restrict the number of the results. The Qual class defines three instance's attributes: - field_name: the name of the column concerned by the condition. - operator: the name of the operator. - value: the value expressed in the condition. Let's suppose we write the following query: .. code-block:: sql SELECT * from constanttable where test = 'test 2' and test2 like '%3%'; The method execute would be called with the following quals: .. code-block:: python [Qual('test', '=', 'test 2'), Qual('test', '~~', '3')] Now you can use this information to reduce the set of results to return to the postgresql server. .. note:: You don't HAVE to enforce those quals, Postgresql will check them anyway. It's nonetheless useful to reduce the amount of results you fetch over the network, for example. .. _multicorn/__init__.py: https://github.com/Kozea/Multicorn/blob/master/python/multicorn/__init__.py Similarly, the columns argument contains the list of needed columns. You can use this information to reduce the amount of data that has to be fetched. For example, the following query: .. code-block:: sql select test, test2 from constanttable; would result in the following columns argument: .. code-block:: python ['test', 'test2'] Once again, if you returns more than these columns everything should be fine. Parameterized paths ------------------- The python FDW implementor can affect the planner by implementing the get_path_keys and get_rel_size methods. .. code-block:: python def get_rel_size(self, quals, columns): This method must return a tuple of the form (expected_number_of_row, expected_mean_width_of_a_row (in bytes)). The quals and columns arguments can be used to compute those estimates. For example, the imapfdw computes a huge width whenever the payload column is requested. .. code-block:: python def get_path_keys(self): This method must return a list of tuple of the form (column_name, expected_number_of_row). The expected_number_of_row must be computed as if a "where column_name = some_value" filter were applied. This helps the planner to estimate parameterized paths cost, and change the plan accordingly. For example, informing the planner that a filter on a column may return exactly one row, instead of the full billion, may help it on deciding to use a nested-loop instead of a full sequential scan. Error reporting =============== In the `multicorn.utils`_ module lies a simple utility function, ``log_to_postgres``. .. _multicorn.utils: https://github.com/Kozea/Multicorn/blob/master/python/multicorn/utils.py This function is mapped to the Postgresql function erreport. It accepts three arguments: ``message`` (required) A python string containing the message to report. ``level`` (optional, defaults to ``logging.INFO``) The severity of the message. The following values are accepted: ``logging.DEBUG`` Maps to a postgresql DEBUG1 message. In most configurations, it won't show at all. ``logging.INFO`` Maps to a postgresql NOTICE message. A NOTICE message is passed to the client, as well as in the server logs. ``logging.WARNING`` Maps to a postgresql WARNING message. A WARNING message is passed to the client, as well as in the server logs. ``logging.ERROR`` Maps to a postgresql ERROR message. An ERROR message is passed to the client, as well as in the server logs. .. important:: An ERROR message results in the current transaction being aborted. Think about the consequences when you use it ! ``logging.CRITICAL`` Maps to a postgresql FATAL message. Causes the current server process to abort. .. important:: A CRITICAL message results in the current server process to be aborted Think about the consequences when you use it ! ``hint`` (optional) An hint given to the user to resolve the cause of the message (ex:``Try adding the missing option in the table creation statement``) Foreign Data Wrapper lifecycle ============================== The foreign data wrapper associated to a table is instantiated on a per-process basis, and it happens when the first query is run against it. Usually, postgresql server processes are spawned on a per-connection basis. During the life time of a server process, the instance is cached. That means that if you have to keep references to resources such as connections, you should establish them in the ``__init__`` method and cache them as instance attributes. Multicorn-1.3.4/doc/index.rst000066400000000000000000000014211320447423600161040ustar00rootroot00000000000000.. Multicorn documentation master file, created by sphinx-quickstart on Thu Dec 4 08:56:10 2014. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Multicorn ######### Multicorn is a PostgreSQL 9.1+ extension meant to make `Foreign Data Wrapper`_ development easy, by allowing the programmer to use the Python programming language. If you just wanto use it as soon as possible, jump straight to the `installation` section. Contents: .. toctree:: :maxdepth: 1 installation.rst getting-started.rst foreign-data-wrappers.rst implementing-an-fdw.rst third-party-fdw.rst internals.rst contribute.rst Indices and tables ################## * :ref:`genindex` * :ref:`modindex` * :ref:`search` Multicorn-1.3.4/doc/installation.rst000066400000000000000000000017001320447423600174760ustar00rootroot00000000000000************ Installation ************ Requirements ============ - Postgresql 9.1+ - Postgresql development packages - Python development packages - python 2.6 or >= python 3.3 as your default python If you are using *PostgreSQL 9.1*, you should use the 0.9.1 release. If you are using *PostgreSQL 9.2* or superior, you should use the 1.0.0 series. (Currently 1.0.1). If you are using Debian, a packaging effort is ongoing for PostgreSQL 9.4. You can install it from `here `_. With the `pgxn client`_:: pgxn install multicorn From pgxn: .. parsed-literal:: wget |multicorn_pgxn_download| unzip multicorn-|multicorn_release| cd multicorn-|multicorn_release| make && sudo make install From source:: git clone git://github.com/Kozea/Multicorn.git cd Multicorn make && make install .. _pgxn client: http://pgxnclient.projects.postgresql.org/ Multicorn-1.3.4/doc/internals.rst000066400000000000000000000003761320447423600170040ustar00rootroot00000000000000************************* Multicorn Internal Design ************************* This part is more geared toward those who may want to hack on Multicorn itself. PostgreSQL C API ================ The PostgreSQL C API follows a pretty simple workflow. Multicorn-1.3.4/doc/multicorn.md000077500000000000000000000032101320447423600166020ustar00rootroot00000000000000Multicorn ========= Synopsis -------- Multicorn is a PostgreSQL 9.2+ extension allowing to write Foreign Data Wrappers in python. It is bundled with some foreign data wrappers. *More comprehensive documentation can be found at http://multicorn.org* Usage ----- Create the extension (as a super user, on your target database): CREATE EXTENSION multicorn; Define a foreign server for the specific python foreign data wrappers you want to use: CREATE SERVER my_server_name FOREIGN DATA WRAPPER multicorn options ( wrapper 'python.class.Name' ) Where *python.class.Name* is a string defining which foreign data wrapper class to use. Ex, for the Imap foreign data wrapper: CREATE SERVER multicorn_imap FOREIGN DATA WRAPPER multicorn options ( wrapper 'multicorn.imapfdw.ImapFdw' ); Once you have a server set up, you can create foreign tables on your server. The foreign table must be supplied its required options. Ex: create foreign table gmail ( "Message-ID" character varying, "From" character varying, "Subject" character varying, "payload" character varying, "flags" character varying[], "To" character varying) server multicorn_imap options ( host 'imap.gmail.com', port '465', payload_column 'payload', flags_column 'flags', ssl 'True', login 'mylogin', password 'mypassword' ); For a documentation on the existing foreign data wrappers, see http://multicorn.org/foreign-data-wrappers/ Multicorn-1.3.4/doc/multicorn_directives/000077500000000000000000000000001320447423600205025ustar00rootroot00000000000000Multicorn-1.3.4/doc/multicorn_directives/__init__.py000066400000000000000000000036411320447423600226170ustar00rootroot00000000000000# -*- coding: utf-8 -*- from docutils.parsers.rst import Directive from docutils.parsers.rst import directives from docutils.nodes import Element from sphinx.builders.html import StandaloneHTMLBuilder class api_compat(Element): def __init__(self, api=None): self.api = api or {} super(api_compat, self).__init__() def visit_api_compat_node_html(self, node): self.body.append(u'Supports: %s' % "".join( [u'%s' % (key, "checked" if val else "unchecked", key, u"✓" if val else u"✗") for (key, val) in node.api.items()])) def depart_api_compat_node_html(self, node): self.body.append("") def visit_api_compat_node_text(self, node): self.add_text("Supported API: %s" % ",".join(node.api)) def depart_api_compat_node_text(self, node): pass def visit_api_compat_node_latex(self, node): # TODO: make it render in latex classes = node.get('classes', []) self.body.append('\n\\begin{notice}\n') self.body.append("Supported API: %s" % ",".join(node.api)) def depart_api_compat_node_latex(self, node): self.body.append('\\end{notice}\n') def setup(app): app.add_directive('api_compat', APICompatDirective) app.add_node(api_compat, html=(visit_api_compat_node_html, depart_api_compat_node_html), latex=(visit_api_compat_node_latex, depart_api_compat_node_latex), text=(visit_api_compat_node_text, depart_api_compat_node_text)) class APICompatDirective(Directive): has_content = True option_spec = { 'read': directives.flag, 'write': directives.flag, 'transaction': directives.flag, 'import_schema': directives.flag } def run(self): values = {key: key in self.options for key in self.option_spec} return [api_compat(api=values)] Multicorn-1.3.4/doc/requirements.txt000066400000000000000000000000671320447423600175340ustar00rootroot00000000000000sphinx sphinxcontrib-napoleon mock sqlalchemy funcsigs Multicorn-1.3.4/doc/third-party-fdw.rst000066400000000000000000000026361320447423600200330ustar00rootroot00000000000000**************** Third-Party-FDWs **************** In addition to the built-in fdws shipped with Multicorn, there are some third-party modules available on the net. rethinkdb-multicorn-postgresql-fdw ================================== A FDW for accessing RethinkDB databases .. api_compat:: :read: :write: repository: https://github.com/wilsonrmsorg/rethinkdb-multicorn-postgresql-fdw/ Hive FDW ======== Access data stored in Apache Hive tables. .. api_compat:: :read: repository: https://github.com/youngwookim/hive-fdw-for-postgresql dockerfdw ========= A FDW for interacting with docker containers .. api_compat:: :read: :write: repository https://github.com/paultag/dockerfdw fb-psql ======= Access data using Facebook FQL API .. api_compat:: :read: repository https://github.com/mrwilson/fb-psql telemetry-fdw ============= Reads data from OpenStack / Telemetry. .. api_compat:: :read: repository https://github.com/hhamalai/telemetry-fdw s3csv_fdw ========= Reads data from CSV files stored on Amazon S3. .. api_compat:: :read: repository https://github.com/eligoenergy/s3csv_fdw.git S3Fdw ===== Reads data from JSON files stored on Amazon S3 .. api_compat:: :read: repository https://github.com/blakedw/s3fdw.git Database.com FDW ================ .. api_compat:: :read: repository https://github.com/metadaddy-sfdc/Database.com-FDW-for-PostgreSQL.git Multicorn-1.3.4/multicorn.control000077500000000000000000000002411320447423600171160ustar00rootroot00000000000000comment = 'Multicorn Python bindings for Postgres 9.2.* Foreign Data Wrapper' default_version = '1.3.4' module_pathname = '$libdir/multicorn' relocatable = true Multicorn-1.3.4/preflight-check.sh000077500000000000000000000012651320447423600171020ustar00rootroot00000000000000#!/bin/bash PG_CONFIG=$(which pg_config) PY_VERSION=$(python --version 2>&1 | awk '{ print substr($2,1,3)}') PY27_VERSION=$(python2.7 --version 2>&1 | awk '{ print substr($2,1,3)}') if [ -z "${PG_CONFIG}" ]; then echo "No pg_config found in your path." echo "Please check if you installed the PostgreSQL development packages." exit 1 fi if [ ! -x "${PG_CONFIG}" ]; then echo "No pg_config found in your path." echo "Please check if you installed the PostgreSQL development packages." exit 1 fi if [ ${PY_VERSION} != "2.7" ] && [ ${PY_VERSION} != "2.6" ]; then if [ ${PY27_VERSION} != "2.7" ]; then echo "Found Python $PY_VERSION, but 2.6 is required." exit 2 fi fi Multicorn-1.3.4/python/000077500000000000000000000000001320447423600150215ustar00rootroot00000000000000Multicorn-1.3.4/python/multicorn/000077500000000000000000000000001320447423600170355ustar00rootroot00000000000000Multicorn-1.3.4/python/multicorn/__init__.py000077500000000000000000000540131320447423600211540ustar00rootroot00000000000000# -*- coding: utf-8 -*- """ Base multicorn module. This module contains all the python code needed by the multicorn C extension to postgresql. You should install it in the python path available to the user running postgresql (usually, the system wide python installation). """ import sys from collections import namedtuple try: from collections import OrderedDict except ImportError: import collections from ordereddict import OrderedDict collections.OrderedDict = OrderedDict __version__ = '__VERSION__' ANY = object() ALL = object() UNBOUND = object() SortKey = namedtuple("SortKey", ["attname", "attnum", "is_reversed", "nulls_first", "collate"]) """ A SortKey describes the sort of one column an SQL query requested. A query can request the sort of zero, one or multiple columns. Therefore, a list of SortKey is provided to the ForeignDataWrapper, containing zero, one or more SortKey. Attributes: attname(str): The name of the column to sort as defined in the postgresql table. attnum(int): The position of the column to sort as defined in the postgresql table. is_reversed(bool): True is the query requested a DESC order. nulls_first(bool): If True, NULL values must appears at the beginning. Otherwise, they must appear at the end. collate(str): The collation name to use to sort the data, as appearing in the postgresql cluster. """ class Qual(object): """A Qual describes a postgresql qualifier. A qualifier is here defined as an expression of the type:: col_name operator value For example:: mycolumn > 3 mycolumn = ANY(1,2,3) mycolumn ~~ ALL('A%','AB%', '%C') Attributes: field_name (str): The name of the column as defined in the postgresql table. operator (str or tuple): The name of the operator if a string. Example: =, <=, ~~ (for a like clause) If it is a tuple, then the tuple is of the form (operator name, ANY or ALL). The tuple represents a comparison of the form WHERE field = ANY(1, 2, 3), which is the internal representation of WHERE field IN (1, 2, 3) value (object): The constant value on the right side """ def __init__(self, field_name, operator, value): """Constructs a qual object. Instantiated from the C extension with the field name, operator and value extracted from the postgresql where clause. Accepts every field from the qual. """ self.field_name = field_name self.operator = operator self.value = value @property def is_list_operator(self): """ Returns: True if this qual represents an array expr, False otherwise """ return isinstance(self.operator, tuple) @property def list_any_or_all(self): """ Returns: ANY if and only if: - this qual is a list operator - the operator applies as an 'ANY' clause (eg, = ANY(1,2,3)) ALL if and only if: - this is a list operator - the operator applies as an 'ALL' clause (eg, > ALL(1, 2, 3)) None if this is not a list operator. """ if self.is_list_operator: return ANY if self.operator[1] else ALL return None def __repr__(self): if self.is_list_operator: value = '%s(%s)' % ( 'ANY' if self.list_any_or_all == ANY else 'ALL', self.value) operator = self.operator[0] else: value = self.value operator = self.operator return ("%s %s %s" % (self.field_name, operator, value)) def __eq__(self, other): if isinstance(other, Qual): return (self.field_name == other.field_name and self.operator == other.operator and self.value == other.value) return False def __hash__(self): return hash((self.field_name, self.operator, self.value)) class ForeignDataWrapper(object): """Base class for all foreign data wrapper instances. Though not required, ForeignDataWrapper implementation should inherit from this class. """ _startup_cost = 20 def __init__(self, fdw_options, fdw_columns): """The foreign data wrapper is initialized on the first query. Args: fdw_options (dict): The foreign data wrapper options. It is a dictionary mapping keys from the sql "CREATE FOREIGN TABLE" statement options. It is left to the implementor to decide what should be put in those options, and what to do with them. fdw_columns (dict): The foreign datawrapper columns. It is a dictionary mapping the column names to their ColumnDefinition. """ pass def get_rel_size(self, quals, columns): """ Method called from the planner to estimate the resulting relation size for a scan. It will help the planner in deciding between different types of plans, according to their costs. Args: quals (list): A list of Qual instances describing the filters applied to this scan. columns (list): The list of columns that must be returned. Returns: A tuple of the form (expected_number_of_rows, avg_row_width (in bytes)) """ return (100000000, len(columns) * 100) def can_sort(self, sortkeys): """ Method called from the planner to ask the FDW what are the sorts it can enforced, to avoid PostgreSQL to sort the data after retreiving all the rows. These sorts can come from explicit ORDER BY clauses, but also GROUP BY and DISTINCT clauses. The FDW has to inspect every sort, and respond which one are handled. The sorts are cumulatives. For example:: col1 ASC col2 DESC means that the FDW must render the tuples sorted by col1 ascending and col2 descending. Args: sortkeys (list): A list of :class:`SortKey` representing all the sorts the query must enforce. Return: The list of cumulative SortKey, for which the FDW can enforce the sort. """ return [] def get_path_keys(self): u""" Method called from the planner to add additional Path to the planner. By default, the planner generates an (unparameterized) path, which can be reasoned about like a SequentialScan, optionally filtered. This method allows the implementor to declare other Paths, corresponding to faster access methods for specific attributes. Such a parameterized path can be reasoned about like an IndexScan. For example, with the following query:: select * from foreign_table inner join local_table using(id); where foreign_table is a foreign table containing 100000 rows, and local_table is a regular table containing 100 rows. The previous query would probably be transformed to a plan similar to this one:: ┌────────────────────────────────────────────────────────────────────────────────────┐ │ QUERY PLAN │ ├────────────────────────────────────────────────────────────────────────────────────┤ │ Hash Join (cost=57.67..4021812.67 rows=615000 width=68) │ │ Hash Cond: (foreign_table.id = local_table.id) │ │ -> Foreign Scan on foreign_table (cost=20.00..4000000.00 rows=100000 width=40) │ │ -> Hash (cost=22.30..22.30 rows=1230 width=36) │ │ -> Seq Scan on local_table (cost=0.00..22.30 rows=1230 width=36) │ └────────────────────────────────────────────────────────────────────────────────────┘ But with a parameterized path declared on the id key, with the knowledge that this key is unique on the foreign side, the following plan might get chosen:: ┌───────────────────────────────────────────────────────────────────────┐ │ QUERY PLAN │ ├───────────────────────────────────────────────────────────────────────┤ │ Nested Loop (cost=20.00..49234.60 rows=615000 width=68) │ │ -> Seq Scan on local_table (cost=0.00..22.30 rows=1230 width=36) │ │ -> Foreign Scan on remote_table (cost=20.00..40.00 rows=1 width=40)│ │ Filter: (id = local_table.id) │ └───────────────────────────────────────────────────────────────────────┘ Returns: A list of tuples of the form: (key_columns, expected_rows), where key_columns is a tuple containing the columns on which the path can be used, and expected_rows is the number of rows this path might return for a simple lookup. For example, the return value corresponding to the previous scenario would be:: [(('id',), 1)] """ return [] def explain(self, quals, columns, sortkeys=None, verbose=False): """Hook called on explain. The arguments are the same as the :meth:`execute`, with the addition of a "verbose" keyword arg for when the EXPLAIN is called with the VERBOSE option. Returns: An iterable of strings to display in the EXPLAIN output. """ return [] def execute(self, quals, columns, sortkeys=None): """Execute a query in the foreign data wrapper. This method is called at the first iteration. This is where the actual remote query execution takes place. Multicorn makes no assumption about the particular behavior of a ForeignDataWrapper, and will NOT remove any qualifiers from the PostgreSQL quals list. That means the quals will be rechecked anyway. Typically, an implementation would: - initialize (or reuse) some sort of connection to the remote system - transform the quals and columns arguments to a representation suitable for the remote system - fetch the data according to this query - return it to the C-extension. Although any iterable can be returned, it is strongly advised to implement this method as a generator to prevent loading the whole dataset in memory. Args: quals (list): A list of :class:`Qual` instances, containing the basic where clauses in the query. columns (list): A list of columns that postgresql is going to need. You should return AT LEAST those columns when returning a dict. If returning a sequence, every column from the table should be in the sequence. sortkeys (list): A list of :class:`SortKey` that the FDW said it can enforce. Returns: An iterable of python objects which can be converted back to PostgreSQL. Currently, such objects are: - sequences containing exactly as much columns as the underlying tables - dictionaries mapping column names to their values. If the sortkeys wasn't empty, the FDW has to return the data in the expected order. """ pass @property def rowid_column(self): """ Returns: A column name which will act as a rowid column, for delete/update operations. One can think of it as a primary key. This can be either an existing column name, or a made-up one. This column name should be subsequently present in every returned resultset. """ raise NotImplementedError("This FDW does not support the writable API") def insert(self, values): """ Insert a tuple defined by ''values'' in the foreign table. Args: values (dict): a dictionary mapping column names to column values Returns: A dictionary containing the new values. These values can differ from the ``values`` argument if any one of them was changed or inserted by the foreign side. For example, if a key is auto generated. """ raise NotImplementedError("This FDW does not support the writable API") def update(self, oldvalues, newvalues): """ Update a tuple containing ''oldvalues'' to the ''newvalues''. Args: oldvalues (dict): a dictionary mapping from column names to previously known values for the tuple. newvalues (dict): a dictionary mapping from column names to new values for the tuple. Returns: A dictionary containing the new values. See :method:``insert`` for information about this return value. """ raise NotImplementedError("This FDW does not support the writable API") def delete(self, oldvalues): """ Delete a tuple identified by ``oldvalues`` Args: oldvalues (dict): a dictionary mapping from column names to previously known values for the tuple. Returns: None """ raise NotImplementedError("This FDW does not support the writable API") def pre_commit(self): """ Hook called just before a commit is issued, on PostgreSQL >=9.3. This is where the transaction should tentatively commited. """ pass def rollback(self): """ Hook called when the transaction is rollbacked. """ pass def commit(self): """ Hook called at commit time. On PostgreSQL >= 9.3, the pre_commit hook should be preferred. """ pass def end_scan(self): """ Hook called at the end of a foreign scan. """ pass def end_modify(self): """ Hook called at the end of a foreign modify (DML operations) """ pass def begin(self, serializable): """ Hook called at the beginning of a transaction. """ pass def sub_begin(self, level): """ Hook called at the beginning of a subtransaction. """ pass def sub_rollback(self, level): """ Hook called when a subtransaction is rollbacked. """ pass def sub_commit(self, level): """ Hook called when a subtransaction is committed. """ pass @classmethod def import_schema(self, schema, srv_options, options, restriction_type, restricts): """ Hook called on an IMPORT FOREIGN SCHEMA command. Args: schema (str): the foreign schema to import srv_options (dict): options defined at the server level options (dict): options defined at the IMPORT FOREIGN SCHEMA statement level restriction_type (str): One of 'limit', 'except' or None restricts (list): a list of tables as passed to the LIMIT TO or EXCEPT clause Returns: list: a list of :class:`multicorn.TableDefinition` """ raise NotImplementedError( "This FDW does not support IMPORT FOREIGN SCHEMA") class TransactionAwareForeignDataWrapper(ForeignDataWrapper): def __init__(self, fdw_options, fdw_columns): super(TransactionAwareForeignDataWrapper, self).__init__( fdw_options, fdw_columns) self._init_transaction_state() def _init_transaction_state(self): self.current_transaction_state = [] def insert(self, values): self.current_transaction_state.append(('insert', values)) def update(self, oldvalues, newvalues): self.current_transaction_state.append( ('update', (oldvalues, newvalues))) def delete(self, oldvalues): self.current_transaction_state.append(('delete', oldvalues)) def rollback(self): self._init_transaction_state() """Code from python2.7 importlib.import_module.""" """Backport of importlib.import_module from 3.x.""" # While not critical (and in no way guaranteed!), it would be nice to keep this # code compatible with Python 2.3. def _resolve_name(name, package, level): """Return the absolute name of the module to be imported.""" if not hasattr(package, 'rindex'): raise ValueError("'package' not set to a string") dot = len(package) for x in range(level, 1, -1): try: dot = package.rindex('.', 0, dot) except ValueError: raise ValueError("attempted relative import beyond top-level " "package") return "%s.%s" % (package[:dot], name) def import_module(name, package=None): """Import a module. The 'package' argument is required when performing a relative import. It specifies the package to use as the anchor point from which to resolve the relative import to an absolute import. """ if name.startswith('.'): if not package: raise TypeError("relative imports require the 'package' argument") level = 0 for character in name: if character != '.': break level += 1 name = _resolve_name(name[level:], package, level) __import__(name) return sys.modules[name] def get_class(module_path): """ Internal function called from c code to import a foreign data wrapper. Args: module_path (str): A fully qualified name for a class. to import Ex: multicorn.csvfdw.CsvFdw. Returns: the class designated by module_path. """ module_path.split(".") wrapper_class = module_path.split(".")[-1] module_name = ".".join(module_path.split(".")[:-1]) module = import_module(module_name) return getattr(module, wrapper_class) def quote_identifier(value): return '"' + value.replace('"', '""') + '"' def quote_option(value): return "'" + value.replace("'", "''") + "'" def dict_to_optionstring(options): return ",\n".join( "%s %s" % (key, quote_option(value)) for key, value in sorted(options.items())) class ColumnDefinition(object): """ Definition of Foreign Table Column. Attributes: column_name (str): the name of the column type_oid (int): the internal OID of the PostgreSQL type typmod (int): the type modifier (ex: VARCHAR(12)) type_name (str): the formatted type name, with the modifier (ex: VARCHAR(12)) base_type_name (str): the base type name, withou modifier (ex: VARCHAR) options (dict): a mapping of option names to option values, as strings. """ def __init__(self, column_name, type_oid=0, typmod=0, type_name="", base_type_name="", options=None): self.column_name = column_name self.type_oid = type_oid self.typmod = typmod self.type_name = type_name self.base_type_name = base_type_name self.options = options or {} def __repr__(self): return "%s(%s, %i, %s%s)" % ( self.__class__.__name__, self.column_name, self.type_oid, self.type_name, " options %s" % self.options if self.options else "") def to_statement(self): stmt = "%s %s" % ( quote_identifier(self.column_name), self.type_name) if self.options: stmt += " OPTIONS ( %s )" % dict_to_optionstring(self.options) return stmt class TableDefinition(object): """ Definition of a Foreign Table. Attributes: table_name (str): the name of the table columns (str): a list of :class:`ColumnDefinition` objects options (dict): a dictionary containing the table-level options. """ def __init__(self, table_name, schema=None, columns=None, options=None): self.table_name = table_name self.columns = columns or [] self.options = options or {} def to_statement(self, schema_name, server_name): """ Generates the CREATE FOREIGN TABLE statement associated with this definition. """ parts = [] parts.append("CREATE FOREIGN TABLE %s.%s (" % (quote_identifier(schema_name), quote_identifier(self.table_name))) parts.append(",\n".join(col.to_statement() for col in self.columns)) parts.append(" \n ) SERVER %s " % quote_identifier(server_name)) if self.options: parts.append(" OPTIONS (") parts.append(dict_to_optionstring(self.options)) parts.append(")") return '\n'.join(parts) Multicorn-1.3.4/python/multicorn/compat.py000066400000000000000000000004311320447423600206700ustar00rootroot00000000000000 try: unicode_ = unicode except NameError: # Python3 unicode_ = str try: basestring_ = basestring except NameError: # Python3 basestring_ = str try: bytes('') bytes_ = bytes except TypeError: # Python3 bytes_ = lambda x: bytes(x, 'utf8') Multicorn-1.3.4/python/multicorn/csvfdw.py000077500000000000000000000066241320447423600207160ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access data stored in `CSV files`_. Each column defined in the table will be mapped, in order, against columns in the CSV file. .. api_compat:: :read: .. _CSV files: http://en.wikipedia.org/wiki/Comma-separated_values Dependencies ------------ No dependency outside the standard python distribution. Options ---------------- ``filename`` (required) The full path to the CSV file containing the data. This file must be readable to the postgres user. ``delimiter`` The CSV delimiter (defaults to ``,``). ``quotechar`` The CSV quote character (defaults to ``"``). ``skip_header`` The number of lines to skip (defaults to ``0``). Usage example ------------- Supposing you want to parse the following CSV file, located in ``/tmp/test.csv``:: Year,Make,Model,Length 1997,Ford,E350,2.34 2000,Mercury,Cougar,2.38 You can declare the following table: .. code-block:: sql CREATE SERVER csv_srv foreign data wrapper multicorn options ( wrapper 'multicorn.csvfdw.CsvFdw' ); create foreign table csvtest ( year numeric, make character varying, model character varying, length numeric ) server csv_srv options ( filename '/tmp/test.csv', skip_header '1', delimiter ','); select * from csvtest; .. code-block:: bash year | make | model | length ------+---------+--------+-------- 1997 | Ford | E350 | 2.34 2000 | Mercury | Cougar | 2.38 (2 lines) """ from . import ForeignDataWrapper from .utils import log_to_postgres from logging import WARNING import csv class CsvFdw(ForeignDataWrapper): """A foreign data wrapper for accessing csv files. Valid options: - filename : full path to the csv file, which must be readable by the user running postgresql (usually postgres) - delimiter : the delimiter used between fields. Default: "," """ def __init__(self, fdw_options, fdw_columns): super(CsvFdw, self).__init__(fdw_options, fdw_columns) self.filename = fdw_options["filename"] self.delimiter = fdw_options.get("delimiter", ",") self.quotechar = fdw_options.get("quotechar", '"') self.skip_header = int(fdw_options.get('skip_header', 0)) self.columns = fdw_columns def execute(self, quals, columns): with open(self.filename) as stream: reader = csv.reader(stream, delimiter=self.delimiter) count = 0 checked = False for line in reader: if count >= self.skip_header: if not checked: # On first iteration, check if the lines are of the # appropriate length checked = True if len(line) > len(self.columns): log_to_postgres("There are more columns than " "defined in the table", WARNING) if len(line) < len(self.columns): log_to_postgres("There are less columns than " "defined in the table", WARNING) yield line[:len(self.columns)] count += 1 Multicorn-1.3.4/python/multicorn/fsfdw/000077500000000000000000000000001320447423600201465ustar00rootroot00000000000000Multicorn-1.3.4/python/multicorn/fsfdw/__init__.py000077500000000000000000000415701320447423600222710ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access data stored in various files, in a filesystem. The files are looked up based on a pattern, and parts of the file's path are mapped to various columns, as well as the file's content itself. .. api_compat:: :read: :write: :transaction: Dependencies ------------ No dependency outside the standard python distribution. Options ------- ``root_dir`` (required) The base directory from which the pattern is evaluated. The files in this directory should be readable by the PostgreSQL user. Ex: ``/var/www/``. ``pattern`` (required) A pattern defining which files to match, and wich parts of the file path are used as columns. A column name between braces defines a mapping from a path part to a column. Ex: ``{artist}/{album}/{trackno} - {trackname}.ogg``. ``content_column`` If set, defines which column will contain the actual file content. ``filename_column`` If set, defines which column will contain the full filename. ``file_mode`` (default: 700) The unix permission mask to be used when creating files. Usage Example ------------- Supposing you want to access files in a directory structured like this:: base_dir/ artist1/ album1/ 01 - title1.ogg 02 - title2.ogg album2/ 01 - title1.ogg 02 - title2.ogg artist2/ album1/ 01 - title1.ogg 02 - title2.ogg album2/ 01 - title1.ogg 02 - title2.ogg You can access those files using a foreign table like this: .. code-block:: sql CREATE SERVER filesystem_srv foreign data wrapper multicorn options ( wrapper 'multicorn.fsfdw.FilesystemFdw' ); CREATE FOREIGN TABLE musicfilesystem ( artist character varying, album character varying, track integer, title character varying, content bytea, filename character varying ) server filesystem_srv options( root_dir 'base_dir', pattern '{artist}/{album}/{track} - {title}.ogg', content_column 'content', filename_column 'filename') Example: .. code-block:: sql SELECT count(track), artist, album from musicfilesystem group by artist, album; :: count | artist | album -------+---------+-------- 2 | artist1 | album2 2 | artist1 | album1 2 | artist2 | album2 2 | artist2 | album1 (4 lines) A filesystem foreign data wrapper. This foreign data wrapper is based on StructuredDirectory, see https://github.com/Kozea/StructuredFS. """ from multicorn import TransactionAwareForeignDataWrapper from multicorn.fsfdw.structuredfs import StructuredDirectory from multicorn.utils import log_to_postgres from multicorn.compat import unicode_ from logging import ERROR, WARNING import os import errno class FilesystemFdw(TransactionAwareForeignDataWrapper): """ A filesystem foreign data wrapper. The foreign data wrapper accepts the following options: root_dir -- The base dir for searching the file pattern -- The pattern for looking for file, starting from the root_dir. See :class:`StructuredDirectory`. content_column -- The column's name which contains the file content. (defaults to None) filename_column -- The column's name wich contains the full filename. """ def __init__(self, options, columns): super(FilesystemFdw, self).__init__(options, columns) root_dir = options.get('root_dir') pattern = options.get('pattern') self.content_column = options.get('content_column', None) self.filename_column = options.get('filename_column', None) self.file_mode = int(options.get('file_mode', '700'), 8) self.structured_directory = StructuredDirectory( root_dir, pattern, file_mode=self.file_mode) self.folder_columns = [key[0] for key in self.structured_directory._path_parts_properties if key] # Keep a set of files that should not be seen inside the transaction, # because they have "logically" been deleted, but are not yet commited self.invisible_files = set() # Keep a dictionary of updated content. self.updated_content = dict() # Assume 100 files/folder per folder self.total_files = 100 ** len(pattern.split('/')) if self.filename_column: if self.filename_column not in columns: log_to_postgres("The filename column (%s) does not exist" "in the column list" % self.filename_column, ERROR, "You should try to create your table with an " "additional column: \n" "%s character varying" % self.filename_column) else: columns.pop(self.filename_column) if self.content_column: if self.content_column not in columns: log_to_postgres("The content column (%s) does not exist" "in the column list" % self.content_column, ERROR, "You should try to create your table with an " "additional column: \n" "%s bytea" % self.content_column) else: columns.pop(self.content_column) if len(self.structured_directory.properties) < len(columns): missing_columns = set(columns.keys()).difference( self.structured_directory.properties) log_to_postgres("Some columns are not mapped in the structured fs", level=WARNING, hint="Remove the following columns: %s " % missing_columns) def get_rel_size(self, quals, columns): """Helps the planner by returning costs For the width, we assume 30 for every returned column, + 1 million for the content column. For the number of rows, we assume 100 files per folder. So, if we filter down to the last folder, thats only 100 files. To one level up, that's already 100^2. """ # TODO: find a way to give useful stats. cond = self._equals_cond(quals) nb_total = len(self.folder_columns) nb_fixes = len([key for key in cond if key in self.folder_columns]) nb_rows = 100 ** (nb_total - nb_fixes) if self.filename_column in cond: nb_rows = 1 width = len(columns) * 30 if self.content_column in columns: width += 1000000 return (nb_rows, width) def _equals_cond(self, quals): return dict((qual.field_name, unicode_(qual.value)) for qual in quals if qual.operator == '=') def get_path_keys(self): """Return the path keys for parameterized path. The structured fs can manage equal filters on its directory patterns, so that can be used. """ values = [((self.filename_column,), 1)] folders = self.folder_columns for i in range(1, len(folders) + 1): values.append((folders[:i], 100 ** (len(folders) - i))) return values def execute(self, quals, columns): """Execute method. The FilesystemFdw performs some optimizations based on the filesystem structure. """ return self.items_to_dicts(self.get_items(quals, columns), columns) def get_items(self, quals, columns): filename_column = self.filename_column for qual in quals: if qual.field_name == filename_column and qual.operator == '=': item = self.structured_directory.from_filename( unicode_(qual.value)) if item is not None and os.path.exists(item.full_filename): return [item] else: return [] properties = self.structured_directory.properties return self.structured_directory.get_items(**dict( (qual.field_name, unicode_(qual.value)) for qual in quals if qual.operator == '=' and qual.field_name in properties)) def items_to_dicts(self, items, columns): content_column = self.content_column filename_column = self.filename_column has_content = content_column and content_column in columns has_filename = filename_column and filename_column in columns for item in items: if item.full_filename in self.invisible_files: continue new_item = dict(item) if has_content: content = self.updated_content.get(item.full_filename, None) if content is None: content = item.read() new_item[content_column] = content if has_filename: new_item[filename_column] = item.filename yield new_item def _item_from_dml(self, values): content = values.pop(self.content_column, None) filename = values.pop(self.filename_column, None) item_from_filename = None item_from_values = None if filename: item_from_filename = self.structured_directory.from_filename( filename) properties_columns = self.structured_directory.properties supplied_keys = set(key for (key, value) in values.items() if value is not None) if len(supplied_keys) != 0: if properties_columns != supplied_keys: log_to_postgres("The following columns are necessary: %s" % properties_columns.difference(supplied_keys), level=ERROR, hint="You can also insert an item by providing" " only the filename and content columns") values = dict([(key, str(value)) for key, value in values.items()]) item_from_values = self.structured_directory.create(**values) elif item_from_filename is None: log_to_postgres("The filename, or all pattern columns are needed.", level=ERROR) if item_from_filename and item_from_values: if item_from_filename != item_from_values: log_to_postgres("The columns inferred from the" " filename do not match the supplied columns.", level=ERROR, hint="Remove either the filename column" " or the properties column from your " " statement, or ensure they match") item = item_from_filename or item_from_values item.content = content return item def _report_pk_violation(self, item): keys = sorted(item.keys()) values = [item[key] for key in keys] log_to_postgres("Duplicate key value violates filesystem" " integrity.", detail="Key (%s)=(%s) already exists" % (', '.join(keys), ', '.join(values)), level=ERROR) def insert(self, values): item = self._item_from_dml(values) # Ensure that the file is created. try: item.open(shared_lock=False, fail_if='exists') except OSError as e: if e.errno == errno.EEXIST: self._report_pk_violation(item) else: raise # Keep track of it for pre_commit time. self.invisible_files.discard(item.full_filename) self.updated_content[item.full_filename] = item.content super(FilesystemFdw, self).insert(item) # Update the "generated" column values. return_value = dict(item) return_value[self.filename_column] = item.filename return_value[self.content_column] = item.content return return_value def update(self, oldfilename, newvalues): # The "oldfilename" file should exist. olditem = self.structured_directory.from_filename(oldfilename) olditem.content = self.updated_content.get(olditem.full_filename, olditem.content) if not olditem.content: # No content yet olditem.content = olditem.read() new_filename = newvalues.get(self.filename_column, oldfilename) filename_changed = new_filename != oldfilename values = dict([(key, (None if value is None else str(value))) for key, value in newvalues.items() if key not in (self.filename_column, self.content_column)]) values_changed = dict(olditem) != values # Check for null values in the "important" parts null_columns = [key for key in self.structured_directory.properties if values[key] is None] if null_columns: log_to_postgres("Null value in columns (%s) are not allowed" % ', '.join(null_columns), level=ERROR, detail="Failing row contains (%s)" % ', '.join(sorted(val if val is not None else 'NULL' for val in values.values()))) if filename_changed: if values_changed: # Keep everything to not bypass conflict detection values = dict(list(olditem.items()) + list(values.items())) else: # Keep only the filename values = {self.filename_column: new_filename} else: values = dict(list(olditem.items()) + list(values.items())) newitem = self._item_from_dml(values) newitem.content = newvalues.get(self.content_column, olditem.content) self.updated_content[newitem.full_filename] = newitem.content olditem.open(shared_lock=False, fail_if='missing') # Ensure that the new file doesnt exists if they are # not the same. if olditem.full_filename != newitem.full_filename: try: newitem.open(shared_lock=False, fail_if='exists') except OSError as e: if e.errno == errno.EEXIST: self._report_pk_violation(newitem) else: raise self.invisible_files.add(olditem.full_filename) self.invisible_files.discard(newitem.full_filename) super(FilesystemFdw, self).update(olditem, newitem) return_value = dict(newitem) return_value[self.filename_column] = newitem.filename return_value[self.content_column] = newitem.content return return_value def delete(self, rowid): item = self.structured_directory.from_filename(rowid) # Ensure that the file exists, and is locked. item.open(False, fail_if='missing') self.invisible_files.add(item.full_filename) super(FilesystemFdw, self).delete(item) def _post_xact_cleanup(self): self._init_transaction_state() self.invisible_files = set() self.structured_directory.clear_cache(only_shared=False) self.updated_content = {} def pre_commit(self): for operation, values in self.current_transaction_state: if operation == 'insert': values.write() elif operation == 'update': olditem, newitem = values if olditem.full_filename == newitem.full_filename: fd = olditem.open(shared_lock=False) else: fd = newitem.open(shared_lock=False) os.unlink(olditem.full_filename) self.structured_directory.clear_cache_entry( olditem.full_filename) newitem.write(fd) elif operation == 'delete': values.remove() self.structured_directory.clear_cache_entry( values.full_filename) self._post_xact_cleanup() def rollback(self): for operation, values in ( self.current_transaction_state[::-1]): if operation == 'insert': values.remove() elif operation == 'update': old_value, new_value = values if new_value.full_filename != old_value.full_filename: os.unlink(new_value.full_filename) old_value.write() self._post_xact_cleanup() @property def rowid_column(self): return self.filename_column def end_scan(self): self.structured_directory.clear_cache(only_shared=True) # For compatibility from multicorn.fsfdw.restfsfdw import ReStructuredTextFdw Multicorn-1.3.4/python/multicorn/fsfdw/docutils_meta.py000066400000000000000000000045531320447423600233630ustar00rootroot00000000000000""" Use low-level docutils API to extract metadata from ReStructuredText files. """ try: from collections import OrderedDict except ImportError: from ordereddict import OrderedDict from threading import Lock from functools import wraps from os.path import getmtime from docutils.core import publish_doctree def extract_meta(filename): """Read meta-data from a reStructuredText file and return a dict. The 'title' and 'subtitle' keys are special-cased, but other keys are read from the `docinfo` element. """ with open(filename) as file_obj: content = file_obj.read() meta = {} for element in publish_doctree(content): if element.tagname in ('title', 'subtitle'): meta[element.tagname] = element.astext() elif element.tagname == 'docinfo': for field in element: if field.tagname == 'field': name, body = field.children meta[name.astext().lower()] = body.astext() else: meta[field.tagname.lower()] = field.astext() return meta def mtime_lru_cache(function, max_size=100): """File mtime-based least-recently-used cache. :param function: A function that takes a filename as its single parameter. The file should exist, and the function's return value should only depend on the contents of the file. Return a decorated function that caches at most the ``max_size`` value. Least recently used value are dropped first. Cached values are invalidated when the files's modification time changes. Inspired from functools.lru_cache, which only exists in Python 3.2+. """ lock = Lock() # OrderedDict isn't threadsafe cache = OrderedDict() # ordered least recent to most recent @wraps(function) def wrapper(filename): mtime = getmtime(filename) with lock: if filename in cache: old_mtime, result = cache.pop(filename) if old_mtime == mtime: # Move to the end cache[filename] = old_mtime, result return result result = function(filename) with lock: cache[filename] = mtime, result # at the end if len(cache) > max_size: cache.popitem(last=False) return result return wrapper Multicorn-1.3.4/python/multicorn/fsfdw/restfsfdw.py000066400000000000000000000035321320447423600225320ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access metadata stored in ReStructured Text files, in a filesystem. The files are looked up based on a pattern, and parts of the file's path are mapped to various columns, as well as the file's content itself. The options are exactly the same as ``multicorn.fsfdw`` itself. If a column name is prefixed by ``rest_``, it will not be mapped to a part of the pattern but looked up in the metadata from the ReST document. """ from multicorn.fsfdw import FilesystemFdw class ReStructuredTextFdw(FilesystemFdw): """A filesystem with reStructuredText metadata foreign data wrapper. The foreign data wrapper accepts the same options as FilesystemFdw. Any column with a name in rest_* is set to the metadata value with the corresponding key. (Eg. rest_title is set to the title of the document.) """ def __init__(self, options, columns): from multicorn.fsfdw.docutils_meta import mtime_lru_cache, extract_meta # TODO: make max_size configurable? self.extract_meta = mtime_lru_cache(extract_meta, max_size=1000) columns = dict((name, column) for name, column in columns.items() if not name.startswith('rest_')) super(ReStructuredTextFdw, self).__init__(options, columns) def execute(self, quals, columns): items = self.get_items(quals, columns) keys = [(name, name[5:]) # len('rest_') == 5 for name in columns if name.startswith('rest_')] if keys: items = self.add_meta(items, keys) return self.items_to_dicts(items, columns) def add_meta(self, items, keys): extract_meta = self.extract_meta for item in items: meta = extract_meta(item.full_filename) for column, key in keys: item[column] = meta.get(key) yield item Multicorn-1.3.4/python/multicorn/fsfdw/structuredfs.py000066400000000000000000000373671320447423600232750ustar00rootroot00000000000000""" Handle nicely a set of files in a structured directory. """ import os import sys import io import re import errno import string import collections import fcntl from multicorn.compat import unicode_, basestring_ vformat = string.Formatter().vformat try: str.isidentifier except AttributeError: # Python 2 # http://docs.python.org/py3k/reference/lexical_analysis.html#identifiers # the uppercase and lowercase letters A through Z, the underscore _ # and, except for the first character, the digits 0 through 9. _IDENTIFIERS_RE = re.compile('^[a-zA-Z_][a-zA-Z_0-9]*$') def isidentifier(string): """ Return whether the given string is a valid Python identifier. """ return _IDENTIFIERS_RE.match(string) is not None else: # Python 3 def isidentifier(string): """ Return whether the given string is a valid Python identifier. """ return string.isidentifier() def _tokenize_pattern(pattern): """ Return an iterable of tokens from a string pattern. >>> list(_tokenize_pattern('{category}/{number}_{name}.txt')) [('property', 'category'), ('path separator', '/'), ('category', 'number'), ('literal', '_'), ('category', 'name'), ('path separator', '/')] """ # We could re-purpose the parser for str.format() and use string.Formatter, # but we do not want to parse conversions and format specs. in_field = False field_name = None char_list = list(pattern) for prev_char, char, next_char in zip( [None] + char_list[:-1], char_list, char_list[1:] + [None]): if in_field: if char == '}': yield 'property', field_name field_name = None in_field = False else: field_name += char else: if char == '/': yield 'path separator', char elif char in '{}' and next_char == char: # Two brakets are parsed as one. Ignore the first one. pass elif char == '}' and prev_char != char: raise ValueError("Single '}' encountered in format string") elif char == '{' and prev_char != char: in_field = True field_name = '' else: # Includes normal chars but also an escaped bracket. yield 'literal', char if in_field: raise ValueError("Unmatched '{' in format string") # Artificially add this token to simplify the parser below yield 'path separator', '/' def _parse_pattern(pattern): r""" Parse a string pattern and return (path_parts_re, path_parts_properties) >>> _parse_pattern('{category}/{number}_{name}.txt') ( ( .*)$'>, .*)\_(?P.*)\.txt$'> ), ( ('category',) ('number', 'name') ), ) """ # A list of list of names path_parts_properties = [] # The next list of names, being built properties = [] # A set of all names all_properties = set() # A list of compiled re objects path_parts_re = [] # The pattern being built for the next re next_re = '' for token_type, token in _tokenize_pattern(pattern): if token_type == 'path separator': if not next_re: raise ValueError('A slash-separated part is empty in %r' % pattern) path_parts_re.append(re.compile('^%s$' % next_re)) next_re = '' path_parts_properties.append(tuple(properties)) properties = [] elif token_type == 'property': if not isidentifier(token): raise ValueError('Invalid property name for Filesystem: %r. ' 'Must be a valid identifier' % token) if token in all_properties: raise ValueError('Property name %r appears more than once ' 'in the pattern %r.' % (token, pattern)) all_properties.add(token) properties.append(token) next_re += '(?P<%s>.*)' % token elif token_type == 'literal': next_re += re.escape(token) else: assert False, 'Unexpected token type: ' + token_type # Always end with an artificial '/' token so that the last regex is # in path_parts_re. assert token_type == 'path separator' return tuple(path_parts_re), tuple(path_parts_properties) def strict_unicode(value): """ Make sure that value is either unicode or (on Py 2.x) an ASCII string, and return it in unicode. Raise otherwise. """ if not isinstance(value, basestring_): raise TypeError('Filename property values must be of type ' 'unicode, got %r.' % value) return unicode_(value) class Item(collections.Mapping): """ Represents a single file in a :class:`StructuredDirectory`. Can be used as a mapping (dict-like) to access properties. Note that at a given point in time, the actual file for an Item may or may not exist in the filesystem. """ def __init__(self, directory, properties, content=b''): properties = dict(properties) keys = set(properties) missing = directory.properties - keys if missing: raise ValueError('Missing properties: %s', ', '.join(missing)) extra = keys - directory.properties if extra: raise ValueError('Unknown properties: %s', ', '.join(extra)) self.directory = directory self._properties = {} self.content = content # TODO: check for ambiguities. # eg. with pattern = '{a}_{b}', values {'a': '1_2', 'b': '3'} and # {'a': '1', 'b': '2_3'} both give the same filename. for name, value in properties.items(): value = strict_unicode(value) if '/' in value: raise ValueError('Property values can not contain a slash.') self._properties[name] = value @property def filename(self): """ Return the normalized (slash-separated) filename for the item, relative to the root. """ return vformat(self.directory.pattern, [], self) @property def full_filename(self): """ Return the absolute filename for the item, in OS-specific form. """ return self.directory._join(self.filename.split('/')) def open(self, shared_lock=True, fail_if=None): """Open the file underlying this item, if it is not in the cache. Shared_lock is a boolean indicating whether a shared or exclusive lock should be acquired. fail_if can be either None, "exists", or "missing". """ self._fd, is_shared = self.directory.cache.get(self.full_filename, (None, False)) if shared_lock: if self._fd is None: # Open it with a shared lock self._fd = os.open(self.full_filename, os.O_RDONLY | os.O_SYNC) fcntl.flock(self._fd, fcntl.LOCK_SH) self.directory.cache[self.full_filename] = (self._fd, shared_lock) # Do nothing if we already have a file descriptor else: if (self._fd is None or not (fcntl.fcntl(self._fd, fcntl.F_GETFL) & os.O_RDWR)): # Open it with an exclusive lock, sync mode, and fail if the # file already exists. dirname = os.path.dirname(self.full_filename) if not os.path.exists(dirname): umask = os.umask(0) os.makedirs(dirname, self.directory.file_mode) os.umask(umask) flags = os.O_SYNC | os.O_RDWR if fail_if == 'exists': flags = flags | os.O_CREAT | os.O_EXCL elif fail_if is None: flags = flags | os.O_CREAT if self._fd is not None: os.close(self._fd) umask = os.umask(0) self._fd = os.open(self.full_filename, flags, self.directory.file_mode) os.umask(umask) fcntl.flock(self._fd, fcntl.LOCK_EX) self.directory.cache[self.full_filename] = (self._fd, shared_lock) return self._fd def read(self): """ Return the content of the file as a bytestring. :raises IOError: if the file does not exist in the filesystem. """ fd = self.open(True, fail_if='missing') os.lseek(fd, 0, os.SEEK_SET) iof = io.open(fd, 'rb', closefd=False) content = iof.read() iof.close() return content def write(self, fd=None): if fd is None: fd = self.open(False) os.ftruncate(fd, 0) os.lseek(fd, 0, os.SEEK_SET) # Do not use a buffer, to ensure that the file is written in one # syscall. iof = io.open(fd, 'wb', closefd=False, buffering=0) if isinstance(self.content, unicode_): self.content = self.content.encode(sys.getfilesystemencoding()) if self.content is not None: iof.write(self.content) iof.close() def remove(self): os.unlink(self.full_filename) # collections.Mapping interface: def __len__(self): return len(self._properties) def __iter__(self): return iter(self._properties) def __getitem__(self, name): return self._properties[name] def __setitem__(self, name, value): self._properties[name] = value class StructuredDirectory(object): """ :param root_dir: Path to the root directory :param pattern: Pattern for files in this directory, eg. '{category}/{number}_{name}.txt' """ def __init__(self, root_dir, pattern, file_mode=0o700): self.root_dir = unicode_(root_dir) self.pattern = unicode_(pattern) # Cache for file descriptors. self.cache = {} parts_re, parts_properties = _parse_pattern(self.pattern) self.file_mode = file_mode self._path_parts_re = parts_re self._path_parts_properties = parts_properties self.properties = set(prop for part in parts_properties for prop in part) def create(self, **values): """ Return a new ``Item`` associated to this directory with the given ``values``. The file for this item may or may not exist. """ return Item(self, values) def from_filename(self, filename): """ Return an ``Item`` from a slash-separated ``filename`` relative to the root. Return ``None`` if ``filename`` does not match ``pattern``. Assuming a matching filename:: f = 'a/b/c' directory.from_filename(f).filename == f The file for this item may or may not exist. """ values = {} parts = filename.split('/') if len(parts) != len(self._path_parts_re): return None for part, part_re in zip(parts, self._path_parts_re): match = part_re.match(part) if match is None: return None values.update(match.groupdict()) return Item(self, values) def get_items(self, **fixed_values): """ Return an iterable of :class:`Item` objects for all files that match the given ``properties``. """ keys = set(fixed_values) extra = keys - self.properties if extra: raise ValueError('Unknown properties: %s', ', '.join(extra)) # Pre-compute everything we know about the request without looking # at the filesystem. # `fixed` is a list, one element for each "part" of the pattern. # Each element is a (fixed_part, fixed_part_values) tuple. # fixed_part: the whole part if it is completly fixed, or None # fixed_part_values: (name, value) pairs for$ fixed values # for this part. fixed = [] for pattern_part, part_properties in zip( self.pattern.split('/'), self._path_parts_properties): fixed_part_values = tuple( (name, fixed_values[name]) for name in part_properties if name in fixed_values ) if len(fixed_part_values) == len(part_properties): # All properties for this part are fixed fixed_part = vformat(pattern_part, [], dict(fixed_part_values)) else: fixed_part = None fixed.append((fixed_part, fixed_part_values)) return self._walk((), (), fixed) def clear_cache_entry(self, key): value, shared = self.cache.pop(key) os.close(value) def clear_cache(self, only_shared=False): for key, (value, shared) in list(self.cache.items()): if (not only_shared) or shared: self.clear_cache_entry(key) def _walk(self, previous_path_parts, previous_values, fixed): """ Called for each directory or sub-directory. """ # Empty previous_path_parts means look in root_dir, depth = 0 depth = len(previous_path_parts) # If the pattern has N path parts, "leaf" files are at depth = N-1 is_leaf = (depth == len(self._path_parts_re) - 1) for name, part_values in self._find_matching_names( previous_path_parts, fixed): path_parts = previous_path_parts + (name,) values = previous_values + tuple(part_values) filename = self._join(path_parts) if is_leaf: if os.path.isfile(filename): yield Item(self, values) # Do not check if filename is a directory or even exists, # let listdir() raise later. else: for item in self._walk(path_parts, values, fixed): yield item def _find_matching_names(self, previous_path_parts, fixed): """ Yield names and parsed values that match the request in a directory. """ depth = len(previous_path_parts) fixed_part, fixed_part_values = fixed[depth] if fixed_part is not None: yield fixed_part, fixed_part_values return try: names = self._listdir(previous_path_parts) except OSError as exc: if depth > 0 and exc.errno in [errno.ENOENT, errno.ENOTDIR]: # Does not exist or is not a directory, just return # without yielding any name. # If depth == 0, we're listing the root directory. Still raise # in that case. return else: # Re-raise other errors raise for name in names: match = self._path_parts_re[depth].match(name) if match is None: continue part_values = match.groupdict() if all(part_values[name] == value for name, value in fixed_part_values): yield name, list(part_values.items()) def _join(self, path_parts): """ Return a full filesystem path from parts relative to the root. """ # root_dir is unicode, so the join result should be unicode return os.path.join(self.root_dir, *path_parts) def _listdir(self, path_parts): """ Wrap os.listdir to make it monkey-patchable in tests. """ return os.listdir(self._join(path_parts)) Multicorn-1.3.4/python/multicorn/fsfdw/test.py000066400000000000000000000264011320447423600215020ustar00rootroot00000000000000# coding: utf8 """ Tests for StructuredFS. """ import os import sys import functools import tempfile import shutil from contextlib import contextmanager from multicorn.compat import unicode_, bytes_ import pytest from .structuredfs import StructuredDirectory, Item from .docutils_meta import mtime_lru_cache, extract_meta def with_tempdir(function): def wrapper(): directory = tempfile.mkdtemp() try: return function(directory) finally: shutil.rmtree(directory) wrapper.__doc__ = function.__doc__ wrapper.__name__ = function.__name__ return wrapper @contextmanager def assert_raises(exception_class, message_part): """ Check that an exception is raised and its message contains some string. """ try: yield except exception_class as exception: assert message_part.lower() in exception.args[0].lower() else: assert 0, 'Did not raise %s' % exception_class @with_tempdir def test_parser(tempdir): """ Test the pattern parser. """ make = functools.partial(StructuredDirectory, tempdir) with assert_raises(ValueError, 'slash-separated part is empty'): assert make('') with assert_raises(ValueError, 'slash-separated part is empty'): assert make('/a') with assert_raises(ValueError, 'slash-separated part is empty'): assert make('a/') with assert_raises(ValueError, 'slash-separated part is empty'): assert make('a//b') with assert_raises(ValueError, 'more than once'): assert make('{foo}/{foo}') with assert_raises(ValueError, 'Invalid property name'): assert make('{}') with assert_raises(ValueError, 'Invalid property name'): assert make('{0foo}') with assert_raises(ValueError, 'Invalid property name'): assert make('{foo/bar}') with assert_raises(ValueError, 'Invalid property name'): assert make('{foo!r}') with assert_raises(ValueError, 'Invalid property name'): assert make('{foo:s}') with assert_raises(ValueError, "unmatched '{'"): assert make('foo{bar') with assert_raises(ValueError, "single '}'"): assert make('foo}bar') bin = make('{category}/{num}_{name}.bin') assert bin.properties == set(['category', 'num', 'name']) assert bin._path_parts_properties == (('category',), ('num', 'name')) bin = make('{category}/{{num}}_{name}.bin') assert bin.properties == set(['category', 'name']) assert bin._path_parts_properties == (('category',), ('name',)) @with_tempdir def test_filenames(tempdir): binary = StructuredDirectory(tempdir, '{category}/{num}_{name}.bin') text = StructuredDirectory(tempdir, '{category}/{num}_{name}.txt') # No file created yet assert os.listdir(tempdir) == [] # Create some files for path_parts in [ # Matching the pattern ['lipsum', '4_foo.bin'], ['lipsum', '4_foo.txt'], # Not matching the pattern ['lipsum', '4_foo'], ['lipsum', '4-foo.txt'], ['lipsum', '4_bar.txt', 'baz'], ['lipsum', '4'], ['dolor']]: filename = os.path.join(tempdir, *path_parts) dirname = os.path.dirname(filename) # Create parent directories as needed if not os.path.exists(dirname): os.makedirs(dirname) # Create an empty file open(filename, 'wb').close() assert [i.filename for i in text.get_items()] == ['lipsum/4_foo.txt'] assert [i.filename for i in binary.get_items()] == ['lipsum/4_foo.bin'] @with_tempdir def test_items(tempdir): """ Test the :class:`Item` class. """ binary = StructuredDirectory(tempdir, '{category}/{num}_{name}.bin') text = StructuredDirectory(tempdir, '{category}/{num}_{name}.txt') with assert_raises(ValueError, 'Missing properties'): text.create(category='lipsum') with assert_raises(ValueError, 'Unknown properties'): text.create(category='lipsum', num='4', name='foo', bang='bar') with assert_raises(TypeError, 'must be of type unicode'): text.create(category='lipsum', num=4, name='foo') with assert_raises(ValueError, 'can not contain a slash'): text.create(category='lipsum', num='4', name='foo/bar') values = dict(category='lipsum', num='4', name='foo') assert Item(binary, values).filename == 'lipsum/4_foo.bin' assert Item(text, values).filename == 'lipsum/4_foo.txt' # No file created yet assert os.listdir(tempdir) == [] # Create a file directly os.mkdir(os.path.join(text.root_dir, 'lipsum')) open(os.path.join(text.root_dir, 'lipsum', '4_foo.txt'), 'wb').close() # Create a file from an Item i1 = text.create(category='lipsum', num='5', name='bar') i1.content = 'BAR' i1.write() item_foo, item_bar, = sorted(text.get_items(), key=lambda item: item['num']) assert len(item_foo) == 3 assert dict(item_foo) == dict(category='lipsum', num='4', name='foo') assert item_foo.read() == bytes_('') assert len(item_bar) == 3 assert dict(item_bar) == dict(category='lipsum', num='5', name='bar') assert item_bar.read() == bytes_('BAR') content = b'Hello,\xc2\xa0W\xc3\xb6rld!'.decode('utf-8') item_foo.content = content item_foo.write() assert item_foo.read().decode('utf8') == content item_foo.content = content.encode('utf8') item_foo.write() assert item_foo.read().decode('utf8') == content item_foo.remove() with pytest.raises(OSError): item_foo.remove() assert [i.filename for i in text.get_items()] == ['lipsum/5_bar.txt'] item_bar.remove() assert [i.filename for i in text.get_items()] == [] @with_tempdir def test_get_items(tempdir): """ Test the results of :meth:`StructuredDirectory.get_items` """ text = StructuredDirectory(tempdir, '{category}/{num}_{name}.txt') i1 = text.create(category='lipsum', num='4', name='foo') i1.content = 'FOO' i1.write() i2 = text.create(category='lipsum', num='5', name='bar') i2.content = 'BAR' i2.write() def filenames(**properties): return [i.filename for i in text.get_items(**properties)] assert filenames(num='9') == [] assert filenames(num='5', name='UUU') == [] assert filenames(num='5') == ['lipsum/5_bar.txt'] assert filenames(num='5', name='bar') == ['lipsum/5_bar.txt'] assert sorted(filenames()) == ['lipsum/4_foo.txt', 'lipsum/5_bar.txt'] with assert_raises(ValueError, 'Unknown properties'): filenames(fiz='5') @with_tempdir def test_from_filename(tempdir): """ Test the results of :meth:`StructuredDirectory.from_filename` """ text = StructuredDirectory(tempdir, '{category}/{num}_{name}.txt') assert text.from_filename('lipsum/4_foo.txt/bar') is None assert text.from_filename('lipsum') is None assert text.from_filename('lipsum/4') is None assert text.from_filename('lipsum/4_foo.bin') is None matching = text.from_filename('lipsum/4_foo.txt') assert dict(matching) == dict(category='lipsum', num='4', name='foo') assert matching.filename == 'lipsum/4_foo.txt' @with_tempdir def test_optimizations(tempdir): """ Test that :meth:`StructuredDirectory.get_items` doesn’t do more calls to :func:`os.listdir` than needed. """ text = StructuredDirectory(tempdir, '{cat}/{org}_{name}/{id}') listed = [] real_listdir = text._listdir def listdir_mock(parts): listed.append('/'.join(parts)) return real_listdir(parts) text._listdir = listdir_mock contents = {} def create(**values): item = Item(text, values) assert values['id'] not in contents # Make sure ids are unique content = item.filename.encode('ascii') item.content = content item.write() contents[values['id']] = content def assert_listed(properties, expected_ids, expected_listed): del listed[:] expected_contents = set(contents[num] for num in expected_ids) results = [item.read() for item in text.get_items(**properties)] assert set(results) == expected_contents assert set(listed) == set(expected_listed) create(cat='lipsum', org='a', name='foo', id='1') # No fixed values: all directories on the path are listed. assert_listed(dict(), ['1'], ['', 'lipsum', 'lipsum/a_foo']) # The category was fixed, no need to listdir() the root. assert_listed(dict(cat='lipsum'), ['1'], ['lipsum', 'lipsum/a_foo']) # The num and name were fixed, no need to listdir() the lipsum dir. assert_listed(dict(org='a', name='foo'), ['1'], ['', 'lipsum/a_foo']) # All filename properties were fixed, no need to listdir() anything assert_listed(dict(cat='lipsum', org='a', name='foo', id='1'), ['1'], []) create(cat='lipsum', org='b', name='foo', id='2') create(cat='dolor', org='c', name='bar', id='3') assert_listed(dict(), ['1', '2', '3'], ['', 'lipsum', 'dolor', 'lipsum/a_foo', 'lipsum/b_foo', 'dolor/c_bar']) # No need to listdir() the root assert_listed(dict(cat='lipsum'), ['1', '2'], ['lipsum', 'lipsum/a_foo', 'lipsum/b_foo']) # No need to listdir() the root assert_listed(dict(cat='dolor'), ['3'], ['dolor', 'dolor/c_bar']) # org='b' is not a whole part so we still need to listdir() lipsum, # but can filter out some deeper directories assert_listed(dict(org='b'), ['2'], ['', 'lipsum', 'dolor', 'lipsum/b_foo']) # Does not list the root and directry tries to list 'nonexistent' assert_listed(dict(cat='nonexistent'), [], ['nonexistent']) @with_tempdir def test_docutils_meta(tempdir): def counting(filename): counting.n_calls += 1 return extract_meta(filename) counting.n_calls = 0 wrapper = mtime_lru_cache(counting, max_size=2) def extract(filename): return wrapper(os.path.join(tempdir, filename)) rest_1 = ''' The main title ============== Second title ------------ :Author: Me Content ''' meta_1 = {'title': 'The main title', 'subtitle': 'Second title', 'author': 'Me'} rest_2 = ''' First title =========== :Author: Myself :Summary: Lorem ipsum dolor sit amet Not a subtitle -------------- Content ''' meta_2 = {'title': 'First title', 'author': 'Myself', 'summary': 'Lorem ipsum\ndolor sit amet'} def write(filename, content): with open(os.path.join(tempdir, filename), 'w') as file_obj: file_obj.write(content) write('first.rst', rest_1) write('second.rst', rest_2) assert counting.n_calls == 0 assert extract('first.rst') == meta_1 assert counting.n_calls == 1 assert extract('first.rst') == meta_1 # cached assert counting.n_calls == 1 assert extract('second.rst') == meta_2 assert counting.n_calls == 2 write('third.rst', rest_1) assert extract('third.rst') == meta_1 # Exceeds the cache size assert counting.n_calls == 3 write('third.rst', rest_2) assert extract('third.rst') == meta_2 assert counting.n_calls == 4 assert extract('first.rst') == meta_1 # Not cached anymore assert counting.n_calls == 5 if __name__ == '__main__': pytest.main([__file__] + sys.argv) Multicorn-1.3.4/python/multicorn/gcfdw.py000066400000000000000000000030741320447423600205050ustar00rootroot00000000000000from multicorn import ForeignDataWrapper import gc import sys from multicorn.compat import unicode_ class MyClass(object): def __init__(self, num, rand): self.num = num self.rand = rand class GCForeignDataWrapper(ForeignDataWrapper): def execute(self, quals, columns): gc.collect() result = [] for obj in gc.get_objects(): tobj = type(obj) if isinstance(obj, bytes): obj = obj.decode('utf8') elif isinstance(obj, unicode_): pass else: try: obj = bytes(obj).decode('utf8') except (UnicodeEncodeError, UnicodeDecodeError): try: obj = unicode_(obj) except (UnicodeEncodeError, UnicodeDecodeError): obj = unicode_("") result.append({'object': obj, 'type': unicode_(tobj), 'id': unicode_(id(obj)), 'refcount': unicode_(sys.getrefcount(obj))}) return result class MemStressFDW(ForeignDataWrapper): def __init__(self, options, columns): self.nb = int(options.get('nb', 100000)) self.options = options self.columns = columns super(MemStressFDW, self).__init__(options, columns) def execute(self, quals, columns): for i in range(self.nb): num = i / 100. yield {'value': str(MyClass(i, num)), 'i': i, 'num': num} Multicorn-1.3.4/python/multicorn/gitfdw.py000066400000000000000000000023131320447423600206720ustar00rootroot00000000000000""" A Git foreign data wrapper """ from . import ForeignDataWrapper import brigit class GitFdw(ForeignDataWrapper): """A Git foreign data wrapper. The git foreign data wrapper accepts the following options: path -- the absolute path to the git repo. It must be readable by the user running postgresql (usually, postgres). encoding -- the file encoding. Defaults to "utf-8". """ def __init__(self, fdw_options, fdw_columns): super(GitFdw, self).__init__(fdw_options, fdw_columns) self.path = fdw_options["path"] self.encoding = fdw_options.get("encoding", "utf-8") def execute(self, quals, columns): def enc(unicode_str): """Encode the string in the self given encoding.""" return unicode_str.encode(self.encoding) for log in brigit.Git(self.path).pretty_log(): yield { 'author_name': enc(log["author"]['name']), 'author_email': enc(log["author"]['email']), 'message': enc(log['message']), 'hash': enc(log['hash']), 'date': log['datetime'].isoformat() } Multicorn-1.3.4/python/multicorn/googlefdw.py000066400000000000000000000023261320447423600213670ustar00rootroot00000000000000""" A foreign data wrapper for performing google searches. """ from . import ForeignDataWrapper import json import urllib def google(search): """Retrieves results from google using the json api""" query = urllib.urlencode({'q': search}) url = ('http://ajax.googleapis.com/ajax/' 'services/search/web?v=1.0&%s' % query) response = urllib.urlopen(url) results = response.read() results = json.loads(results) data = results['responseData'] hits = data['results'] for hit in hits: yield {'url': hit['url'].encode("utf-8"), 'title': hit["titleNoFormatting"].encode("utf-8"), 'search': search.encode("utf-8")} class GoogleFdw(ForeignDataWrapper): """A Google search foreign data wrapper. Parses the quals to find anything ressembling a search criteria, and returns the google search result for it. Available columns are: url, title, search. """ def execute(self, quals, columns): if not quals: return ("No search specified",) for qual in quals: if qual.field_name == "search" or qual.operator == "=": return google(qual.value) Multicorn-1.3.4/python/multicorn/imapfdw.py000066400000000000000000000255121320447423600210430ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access mails from an IMAP mailbox. Column names are mapped to IMAP headers, and two special columns may contain the mail payload and its flags. .. api_compat:: :read: Dependencies ------------- imaplib Options -------- ``host`` (required) The IMAP host to connect to. ``port`` (required) The IMAP host port to connect to. ``login`` (required) The login to connect with. ``password`` (required) The password to connect with. The login and password options should be set as a user mapping options, so as not to be stored in plaintext. See `the create user mapping documentation`_ .. _the create user mapping documentation: http://www.postgresql.org/docs/9.1/static/sql-createusermapping.html ``payload_column`` The name of the column which will store the payload. ``flags_column`` The name of the column which will store the IMAP flags, as an array of strings. ``ssl`` Wether to use ssl or not ``imap_server_charset`` The name of the charset used for IMAP search commands. Defaults to UTF8. For the cyrus IMAP server, it should be set to "utf-8". ``internal_date_column`` The column to use as the INTERNALDATE imap header. Server side filtering --------------------- The imap fdw tries its best to convert postgresql quals into imap filters. The following quals are pushed to the server: - equal, not equal, like, not like comparison - = ANY, = NOT ANY ntThese conditions are matched against the headers, or the body itself. The imap FDW will fetch only what is needed by the query: you should thus avoid requesting the payload_column if you don't need it. """ from . import ForeignDataWrapper, ANY, ALL from .utils import log_to_postgres, ERROR, WARNING from imaplib import IMAP4 import re from multicorn.compat import basestring_ from email.header import decode_header from imapclient import IMAPClient from itertools import islice try: from itertools import zip_longest except ImportError: from itertools import izip_longest as zip_longest from functools import reduce STANDARD_FLAGS = { 'seen': 'Seen', 'flagged': 'Flagged', 'delete': 'Deleted', 'draft': 'Draft', 'recent': 'Recent' } SEARCH_HEADERS = ['BCC', 'CC', 'FROM', 'TO'] def compact_fetch(messages): """Compact result in ranges. For example, [1, 2, 3, 4, 10, 11, 12, 14, 17, 18, 19, 21, 92] can be compacted in ['1:4', '10:12', '14', '17:19', '21', '92'] """ first_i = messages[0] for (i, inext) in zip_longest(messages, islice(messages, 1, None)): if inext == i + 1: continue elif first_i != i: yield '%s:%s' % (first_i, i) first_i = inext else: yield "%s" % i first_i = inext class NoMatchPossible(Exception): """An exception raised when the conditions can NOT be met by any message, ever.""" def make_or(values): """Create an imap OR filter based on a list of conditions to be or'ed""" values = [x for x in values if x not in (None, '()')] if values: if len(values) > 1: return reduce(lambda x, y: '(OR %s %s)' % (x, y), values) else: return values[0] class ImapFdw(ForeignDataWrapper): """An imap foreign data wrapper """ def __init__(self, options, columns): super(ImapFdw, self).__init__(options, columns) self._imap_agent = None self.host = options.get('host', None) if self.host is None: log_to_postgres('You MUST set the imap host', ERROR) self.port = options.get('port', None) self.ssl = options.get('ssl', False) self.login = options.get('login', None) self.password = options.get('password', None) self.folder = options.get('folder', 'INBOX') self.imap_server_charset = options.get('imap_server_charset', 'UTF8') self.columns = columns self.payload_column = options.get('payload_column', None) self.flags_column = options.get('flags_column', None) self.internaldate_column = options.get('internaldate_column', None) def get_rel_size(self, quals, columns): """Inform the planner that it can be EXTREMELY costly to use the payload column, and that a query on Message-ID will return only one row.""" width = len(columns) * 100 nb_rows = 1000000 if self.payload_column in columns: width += 100000000000 nb_rows = nb_rows / (10 ** len(quals)) for qual in quals: if qual.field_name.lower() == 'in-reply-to' and\ qual.operator == '=': nb_rows = 10 if (qual.field_name.lower() == 'message-id' and qual.operator == '='): nb_rows = 1 break return (nb_rows, width) def _create_agent(self): self._imap_agent = IMAPClient(self.host, self.port, ssl=self.ssl) if self.login: self._imap_agent.login(self.login, self.password) self._imap_agent.select_folder(self.folder) @property def imap_agent(self): if self._imap_agent is None: self._create_agent() try: self._imap_agent.select_folder(self.folder) except IMAP4.abort: self._create_agent() return self._imap_agent def get_path_keys(self): """Helps the planner by supplying a list of list of access keys, as well as a row estimate for each one.""" return [(('Message-ID',), 1), (('From',), 100), (('To',), 100), (('In-Reply-To',), 10)] def _make_condition(self, key, operator, value): if operator not in ('~~', '!~~', '=', '<>', '@>', '&&', '~~*', '!~~*'): # Do not manage special operators return '' if operator in ('~~', '!~~', '~~*', '!~~*') and\ isinstance(value, basestring_): # 'Normalize' the sql like wildcards if value.startswith(('%', '_')): value = value[1:] if value.endswith(('%', '_')): value = value[:-1] if re.match(r'.*[^\\][_%]', value): return '' value = value.replace('\\%', '%').replace('\\_', '_') prefix = '' if operator in ('!~~', '<>', '!~~*'): if key == self.flags_column: prefix = 'UN' else: prefix = 'NOT ' if isinstance(value, basestring_): if value.lower() in STANDARD_FLAGS: prefix = '' value = value.upper() if key == self.flags_column: if operator == '@>': # Contains on flags return ' '.join(['%s%s' % (prefix, (STANDARD_FLAGS.get(atom.lower(), '%s %s' % ('KEYWORD', atom)))) for atom in value]) elif operator == '&&': # Overlaps on flags => Or values = ['(%s%s)' % (prefix, (STANDARD_FLAGS.get(atom.lower(), '%s %s' % ('KEYWORD', atom)))) for atom in value] return make_or(values) else: value = '\\\\%s' % value elif key == self.payload_column: value = 'TEXT "%s"' % value elif key in SEARCH_HEADERS: value = '%s "%s"' % (key, value) else: # Special case for Message-ID and In-Reply-To: # zero-length strings are forbidden so dont bother # searching them if not value: raise NoMatchPossible() prefix = 'HEADER ' value = '%s "%s"' % (key, value) return '%s%s' % (prefix, value) def extract_conditions(self, quals): """Build an imap search criteria string from a list of quals""" conditions = [] for qual in quals: # Its a list, so we must translate ANY to OR, and ALL to AND if qual.list_any_or_all == ANY: values = [ '(%s)' % self._make_condition(qual.field_name, qual.operator[0], value) for value in qual.value] conditions.append(make_or(values)) elif qual.list_any_or_all == ALL: conditions.extend([ self._make_condition(qual.field_name, qual.operator[0], value) for value in qual.value]) else: # its not a list, so everything is fine conditions.append(self._make_condition( qual.field_name, qual.operator, qual.value)) conditions = [x for x in conditions if x not in (None, '()')] return conditions def execute(self, quals, columns): # The header dictionary maps columns to their imap search string col_to_imap = {} headers = [] for column in list(columns): if column == self.payload_column: col_to_imap[column] = 'BODY[TEXT]' elif column == self.flags_column: col_to_imap[column] = 'FLAGS' elif column == self.internaldate_column: col_to_imap[column] = 'INTERNALDATE' else: col_to_imap[column] = ('BODY[HEADER.FIELDS (%s)]' % column.upper()) headers.append(column) try: conditions = self.extract_conditions(quals) or ['ALL'] except NoMatchPossible: matching_mails = [] else: matching_mails = self.imap_agent.search( charset=self.imap_server_charset, criteria=conditions) if matching_mails: data = self.imap_agent.fetch(list(compact_fetch(matching_mails)), list(col_to_imap.values())) item = {} for msg in data.values(): for column, key in col_to_imap.items(): item[column] = msg[key] if column in headers: item[column] = item[column].split(':', 1)[-1].strip() values = decode_header(item[column]) for decoded_header, charset in values: # Values are of the from "Header: value" if charset: try: item[column] = decoded_header.decode( charset) except LookupError: log_to_postgres('Unknown encoding: %s' % charset, WARNING) else: item[column] = decoded_header yield item Multicorn-1.3.4/python/multicorn/ldapfdw.py000077500000000000000000000124601320447423600210360ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access directory servers via the LDAP protocol. Tested with OpenLDAP. It supports: simple bind, multiple scopes (subtree, base, etc) .. api_compat: :read: Dependencies ------------ If using Multicorn >= 1.1.0, you will need the `ldap3`_ library: .. _ldap3: http://pythonhosted.org/python3-ldap/ For prior version, you will need the `ldap`_ library: .. _ldap: http://www.python-ldap.org/ Required options ---------------- ``uri`` (string) The URI for the server, for example "ldap://localhost". ``path`` (string) The base in which the search is performed, for example "dc=example,dc=com". ``objectclass`` (string) The objectClass for which is searched, for example "inetOrgPerson". ``scope`` (string) The scope: one, sub or base. Optional options ---------------- ``binddn`` (string) The binddn for example 'cn=admin,dc=example,dc=com'. ``bindpwd`` (string) The credentials for the binddn. Usage Example ------------- To search for a person definition: .. code-block:: sql CREATE SERVER ldap_srv foreign data wrapper multicorn options ( wrapper 'multicorn.ldapfdw.LdapFdw' ); CREATE FOREIGN TABLE ldapexample ( mail character varying, cn character varying, description character varying ) server ldap_srv options ( uri 'ldap://localhost', path 'dc=lab,dc=example,dc=com', scope 'sub', binddn 'cn=Admin,dc=example,dc=com', bindpwd 'admin', objectClass '*' ); select * from ldapexample; .. code-block:: bash mail | cn | description -----------------------+----------------+-------------------- test@example.com | test | admin@example.com | admin | LDAP administrator someuser@example.com | Some Test User | (3 rows) """ from . import ForeignDataWrapper import ldap3 from multicorn.utils import log_to_postgres, ERROR from multicorn.compat import unicode_ SPECIAL_CHARS = { ord('*'): '\\2a', ord('('): '\\28', ord(')'): '\29', ord('\\'): '\\5c', ord('\x00'): '\\00', ord('/'): '\\2f' } class LdapFdw(ForeignDataWrapper): """An Ldap Foreign Wrapper. The following options are required: uri -- the ldap URI to connect. (ex: 'ldap://localhost') address -- the ldap host to connect. (obsolete) path -- the ldap path (ex: ou=People,dc=example,dc=com) objectClass -- the ldap object class (ex: 'inetOrgPerson') scope -- the ldap scope (one, sub or base) binddn -- the ldap bind DN (ex: 'cn=Admin,dc=example,dc=com') bindpwd -- the ldap bind Password """ def __init__(self, fdw_options, fdw_columns): super(LdapFdw, self).__init__(fdw_options, fdw_columns) if "address" in fdw_options: self.ldapuri = "ldap://" + fdw_options["address"] else: self.ldapuri = fdw_options["uri"] self.ldap = ldap3.Connection( ldap3.Server(self.ldapuri), user=fdw_options.get("binddn", None), password=fdw_options.get("bindpwd", None), client_strategy=ldap3.STRATEGY_SYNC_RESTARTABLE) self.path = fdw_options["path"] self.scope = self.parse_scope(fdw_options.get("scope", None)) self.object_class = fdw_options["objectclass"] self.field_list = fdw_columns self.field_definitions = dict( (name.lower(), field) for name, field in self.field_list.items()) self.array_columns = [ col.column_name for name, col in self.field_definitions.items() if col.type_name.endswith('[]')] def execute(self, quals, columns): request = unicode_("(objectClass=%s)") % self.object_class for qual in quals: if isinstance(qual.operator, tuple): operator = qual.operator[0] else: operator = qual.operator if operator in ("=", "~~"): if hasattr(qual.value, "translate"): baseval = qual.value.translate(SPECIAL_CHARS) val = (baseval.replace("%", "*") if operator == "~~" else baseval) else: val = qual.value request = unicode_("(&%s(%s=%s))") % ( request, qual.field_name, val) self.ldap.search( self.path, request, self.scope, attributes=list(self.field_definitions)) for entry in self.ldap.response: # Case insensitive lookup for the attributes litem = dict() for key, value in entry["attributes"].items(): if key.lower() in self.field_definitions: pgcolname = self.field_definitions[key.lower()].column_name if pgcolname in self.array_columns: value = value else: value = value[0] litem[pgcolname] = value yield litem def parse_scope(self, scope=None): if scope in (None, "", "one"): return ldap3.SEARCH_SCOPE_SINGLE_LEVEL elif scope == "sub": return ldap3.SEARCH_SCOPE_WHOLE_SUBTREE elif scope == "base": return ldap3.SEARCH_SCOPE_BASE_OBJECT else: log_to_postgres("Invalid scope specified: %s" % scope, ERROR) Multicorn-1.3.4/python/multicorn/processfdw.py000066400000000000000000000051761320447423600215770ustar00rootroot00000000000000""" Purpose ------- This foreign data wrapper can be to list processes according to the psutil module. The column names are mapped to the :py:class:`psutil.Process` attributes. .. api_compat: :read: Usage Example ------------- .. code-block:: sql create foreign table processes ( pid int, ppid int, name text, exe text, cmdline text, create_time timestamptz, status text, cwd text, uids int[], gids int[], terminal text, nice int, ionice float[], rlimit int[], num_ctx_switches bigint[], num_fds int, num_threads int, cpu_times interval[], cpu_percent float, cpu_affinity bigint[], memory_info bigint[], memory_percent float, open_files text[], connections text[], is_running bool ) server process_server .. code-block:: bash ro= select name, cmdline, cpu_percent from processes where name = 'postgres'; name | cmdline | cpu_percent ----------+----------------------------------------------+------------- postgres | ['/home/ro/pgdev/bin/postgres'] | 0 postgres | ['postgres: checkpointer process '] | 0 postgres | ['postgres: writer process '] | 0 postgres | ['postgres: wal writer process '] | 0 postgres | ['postgres: autovacuum launcher process '] | 0 postgres | ['postgres: stats collector process '] | 0 postgres | ['postgres: ro ro [local] SELECT'] | 9 (7 rows) Options ------- No options. """ from . import ForeignDataWrapper from datetime import datetime import psutil DATE_COLUMNS = ['create_time'] class ProcessFdw(ForeignDataWrapper): """A foreign datawrapper for querying system stats. It accepts no options. You can define any column named after a statgrab column. See the statgrab documentation. """ def _convert(self, key, value): if key in DATE_COLUMNS: if isinstance(value, (list, tuple)): return [datetime.fromtimestamp(v) for v in value] else: return datetime.fromtimestamp(value) return value def execute(self, quals, columns): for process in psutil.process_iter(): yield dict([(key, self._convert(key, value)) for key, value in process.as_dict(columns).items()]) Multicorn-1.3.4/python/multicorn/rssfdw.py000066400000000000000000000155161320447423600207270ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access items from an rss feed. The column names are mapped to the elements inside an item. An rss item has the following strcture: .. code-block:: xml Title 2011-01-02 http://example.com/test http://example.com/test Small description You can access every element by defining a column with the same name. Be careful to match the case! Example: pubDate should be quoted like this: ``pubDate`` to preserve the uppercased ``D``. .. api_compat:: :read: Dependencies ------------ You will need the `lxml`_ library. .. _lxml: http://lxml.de/ Required options ----------------- ``url`` (string) The RSS feed URL. Usage Example ------------- .. _Radicale: http://radicale.org If you want to parse the `radicale`_ rss feed, you can use the following definition: .. code-block:: sql CREATE SERVER rss_srv foreign data wrapper multicorn options ( wrapper 'multicorn.rssfdw.RssFdw' ); CREATE FOREIGN TABLE radicalerss ( "pubDate" timestamp, description character varying, title character varying, link character varying ) server rss_srv options ( url 'http://radicale.org/rss/' ); select "pubDate", title, link from radicalerss limit 10; .. code-block:: bash pubDate | title | link ---------------------+----------------------------------+---------------------------------------------- 2011-09-27 06:07:42 | Radicale 0.6.2 | http://radicale.org/news#2011-09-27@06:07:42 2011-08-28 13:20:46 | Radicale 0.6.1, Changes, Future | http://radicale.org/news#2011-08-28@13:20:46 2011-08-01 08:54:43 | Radicale 0.6 Released | http://radicale.org/news#2011-08-01@08:54:43 2011-07-02 20:13:29 | Feature Freeze for 0.6 | http://radicale.org/news#2011-07-02@20:13:29 2011-05-01 17:24:33 | Ready for WSGI | http://radicale.org/news#2011-05-01@17:24:33 2011-04-30 10:21:12 | Apple iCal Support | http://radicale.org/news#2011-04-30@10:21:12 2011-04-25 22:10:59 | Two Features and One New Roadmap | http://radicale.org/news#2011-04-25@22:10:59 2011-04-10 20:04:33 | New Features | http://radicale.org/news#2011-04-10@20:04:33 2011-04-02 12:11:37 | Radicale 0.5 Released | http://radicale.org/news#2011-04-02@12:11:37 2011-02-03 23:35:55 | Jabber Room and iPhone Support | http://radicale.org/news#2011-02-03@23:35:55 (10 lignes) """ from . import ForeignDataWrapper from datetime import datetime, timedelta from lxml import etree try: from urllib.request import urlopen except ImportError: from urllib import urlopen from logging import ERROR, WARNING from multicorn.utils import log_to_postgres import json def element_to_dict(element): """ This method takes a lxml element and return a json string containing the element attributes and a text key and a child node. >>> test = lambda x: sorted([(k, sorted(v.items())) if isinstance(v, dict) else (k, [sorted(e.items()) for e in v]) if isinstance(v, list) else (k, v) for k, v in element_to_dict(etree.fromstring(x)).items()]) >>> test('') [('attributes', {'a1': 'v1'}), ('children', []), ('tag', 't'), ('text', '')] >>> test('Txt') [('attributes', {'a1': 'v1'}), ('children', []), ('tag', 't'), ('text', 'Txt')] >>> test('TxtSub1Txt2Sub2Txt3') [('attributes', {}), ('children', [[('attributes', {'a1': 'v1'}), ('children', []), ('tag', 's1'), ('text', 'Sub1')], [('attributes', {'a2': 'v2'}), ('children', []), ('tag', 's2'), ('text', 'Sub2')]]), ('tag', 't'), ('text', 'Txt')] """ return { 'tag': etree.QName(element.tag).localname, 'text': element.text or '', 'attributes': dict(element.attrib), 'children': [element_to_dict(e) for e in element] } class RssFdw(ForeignDataWrapper): """An rss foreign data wrapper. The following options are accepted: url -- The rss feed urls. The columns named are parsed, and are used as xpath expression on each item xml node. Exemple: a column named "pubDate" would return the pubDate element of an rss item. """ def __init__(self, options, columns): super(RssFdw, self).__init__(options, columns) self.url = options.get('url', None) self.cache = (None, None) self.cache_duration = options.get('cache_duration', None) if self.cache_duration is not None: self.cache_duration = timedelta(seconds=int(self.cache_duration)) if self.url is None: log_to_postgres("You MUST set an url when creating the table!", ERROR) self.columns = columns self.default_namespace_prefix = options.pop( 'default_namespace_prefix', None) self.item_root = options.pop('item_root', 'item') def get_namespaces(self, xml): ns = dict(xml.nsmap) if None in ns: ns[self.default_namespace_prefix] = ns.pop(None) return ns def make_item_from_xml(self, xml_elem): """Internal method used for parsing item xml element from the columns definition.""" item = {} for prop, column in self.columns.items(): value = xml_elem.xpath( prop, namespaces=self.get_namespaces(xml_elem)) if value: if column.type_name.startswith('json'): item[prop] = json.dumps([ element_to_dict(val) for val in value]) # There should be a better way # oid is 1009 ? elif column.type_name.endswith('[]'): item[prop] = [elem.text for elem in value] else: item[prop] = getattr(value[0], 'text', value[0]) return item def execute(self, quals, columns): """Quals are ignored.""" if self.cache_duration is not None: date, values = self.cache if values is not None: if (datetime.now() - date) < self.cache_duration: return values try: xml = etree.fromstring(urlopen(self.url).read()) items = [self.make_item_from_xml(elem) for elem in xml.xpath( '//%s' % self.item_root, namespaces=self.get_namespaces(xml))] self.cache = (datetime.now(), items) return items except etree.ParseError: log_to_postgres("Malformed xml, returning nothing") except IOError: log_to_postgres("Cannot retrieve '%s'" % self.url, WARNING) Multicorn-1.3.4/python/multicorn/sqlalchemyfdw.py000066400000000000000000000411351320447423600222560ustar00rootroot00000000000000""" Purpose ------- This fdw can be used to access data stored in a remote RDBMS. Through the use of sqlalchemy, many different rdbms engines are supported. .. api_compat:: :read: :write: :transaction: :import_schema: Dependencies ------------ You will need the `sqlalchemy`_ library, as well as a suitable dbapi driver for the remote database. You can find a list of supported RDBMs, and their associated dbapi drivers and connection strings in the `sqlalchemy dialects documentation`_. .. _sqlalchemy dialects documentation: http://docs.sqlalchemy.org/en/latest/dialects/ .. _sqlalchemy: http://www.sqlalchemy.org/ Connection options ~~~~~~~~~~~~~~~~~~ Connection options can be passed either with a db-url, or with a combination of individual connection parameters. If both a ``db_url`` and individual parameters are used, the parameters will override the value found in the ``db_url``. In both cases, at least the ``drivername`` should be passed, either as the url scheme in the ``db_url`` or using the ``drivername`` parameter. ``db_url`` An sqlalchemy connection string. Examples: - mysql: `mysql://:@/` - mssql: `mssql://:@` See the `sqlalchemy dialects documentation`_. for documentation. ``username`` The remote username. ``password`` The remote password ``host`` The remote host ``database`` The remote database ``port`` The remote port Other options --------------- ``tablename`` (required) The table name in the remote RDBMS. ``primary_key`` Identifies a column which is a primary key in the remote RDBMS. This options is required for INSERT, UPDATE and DELETE operations ``schema`` The schema in which this table resides on the remote side When defining the table, the local column names will be used to retrieve the remote column data. Moreover, the local column types will be used to interpret the results in the remote table. Sqlalchemy being aware of the differences between database implementations, it will convert each value from the remote database to python using the converter inferred from the column type, and convert it back to a postgresql suitable form. What does it do to reduce the amount of fetched data ? ------------------------------------------------------ - `quals` are pushed to the remote database whenever possible. This include simple operators : - equality, inequality (=, <>, >, <, <=, >=) - like, ilike and their negations - IN clauses with scalars, = ANY (array) - NOT IN clauses, != ALL (array) - the set of needed columns is pushed to the remote_side, and only those columns will be fetched. Sort push-down support ---------------------- Since the rules about NULL ordering are different for every database vendor, and many of them don't support the NULLS FIRST, NULLS LAST clause, this FDW tries to not generate any NULLS FIRST / LAST clause if the requested order matches what the remote system would do by default. Additionnaly, if it is found that a query can't be executed while keeping the same NULL ordering (because the remote system doesn't support the NULL ordering clause), the sort will not be pushed down. To check the SQL query that will be sent to the remote system, use EXPLAIN: .. code-block:: sql postgres=# explain select * from testalchemy order by id DESC NULLS FIRST; QUERY PLAN ------------------------------------------------------------------------------------------------------------------ Foreign Scan on testalchemy (cost=20.00..50000000000.00 rows=100000000 width=500) Multicorn: SELECT basetable.atimestamp, basetable.anumeric, basetable.adate, basetable.avarchar, basetable.id FROM basetable ORDER BY basetable.id DESC (3 lignes) Temps : 167,856 ms postgres=# explain select * from testalchemy order by id DESC NULLS LAST; QUERY PLAN ------------------------------------------------------------------------------------------------------------------ Foreign Scan on testalchemy (cost=20.00..50000000000.00 rows=100000000 width=500) Multicorn: SELECT basetable.atimestamp, basetable.anumeric, basetable.adate, basetable.avarchar, basetable.id FROM basetable ORDER BY basetable.id DESC NULLS LAST (3 lignes) Usage example ------------- For a connection to a remote mysql database (you'll need a mysql dbapi driver, such as pymysql): .. code-block:: sql CREATE SERVER alchemy_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw' ); create foreign table mysql_table ( column1 integer, column2 varchar ) server alchemy_srv options ( tablename 'table', db_url 'mysql://myuser:mypassword@myhost/mydb' ); """ from . import ForeignDataWrapper, TableDefinition, ColumnDefinition from .utils import log_to_postgres, ERROR, WARNING, DEBUG from sqlalchemy import create_engine from sqlalchemy.engine.url import make_url, URL from sqlalchemy.sql import select, operators as sqlops, and_ from sqlalchemy.sql.expression import nullsfirst, nullslast # Handle the sqlalchemy 0.8 / 0.9 changes try: from sqlalchemy.sql import sqltypes except ImportError: from sqlalchemy import types as sqltypes from sqlalchemy.schema import Table, Column, MetaData from sqlalchemy.dialects.oracle import base as oracle_dialect from sqlalchemy.dialects.postgresql.base import ( ARRAY, ischema_names, PGDialect, NUMERIC) import re import operator def compose(*funs): if len(funs) == 0: raise ValueError("At least one function is necessary for compose") if len(funs) == 1: return funs[0] else: result_fun = compose(*funs[1:]) return lambda *args, **kwargs: funs[0](result_fun(*args, **kwargs)) def not_(function): return compose(operator.inv, function) def _parse_url_from_options(fdw_options): if fdw_options.get('db_url'): url = make_url(fdw_options.get('db_url')) else: if 'drivername' not in fdw_options: log_to_postgres('Either a db_url, or drivername and other ' 'connection infos are needed', ERROR) url = URL(fdw_options['drivername']) for param in ('username', 'password', 'host', 'database', 'port'): if param in fdw_options: setattr(url, param, fdw_options[param]) return url OPERATORS = { '=': operator.eq, '<': operator.lt, '>': operator.gt, '<=': operator.le, '>=': operator.ge, '<>': operator.ne, '~~': sqlops.like_op, '~~*': sqlops.ilike_op, '!~~*': not_(sqlops.ilike_op), '!~~': not_(sqlops.like_op), ('=', True): sqlops.in_op, ('<>', False): not_(sqlops.in_op) } CONVERSION_MAP = { oracle_dialect.NUMBER: NUMERIC } SORT_SUPPORT = { 'mssql': {'default': 'lower', 'support': False}, 'postgresql': {'default': 'higher', 'support': True}, 'mysql': {'default': 'lower', 'support': False}, 'oracle': {'default': 'higher', 'support': True}, 'sqlite': {'default': 'lower', 'support': False} } class SqlAlchemyFdw(ForeignDataWrapper): """An SqlAlchemy foreign data wrapper. The sqlalchemy foreign data wrapper performs simple selects on a remote database using the sqlalchemy framework. Accepted options: db_url -- the sqlalchemy connection string. schema -- (optional) schema name to qualify table name with tablename -- the table name in the remote database. """ def __init__(self, fdw_options, fdw_columns): super(SqlAlchemyFdw, self).__init__(fdw_options, fdw_columns) if 'tablename' not in fdw_options: log_to_postgres('The tablename parameter is required', ERROR) self.metadata = MetaData() url = _parse_url_from_options(fdw_options) self.engine = create_engine(url) schema = fdw_options['schema'] if 'schema' in fdw_options else None tablename = fdw_options['tablename'] sqlacols = [] for col in fdw_columns.values(): col_type = self._get_column_type(col.type_name) sqlacols.append(Column(col.column_name, col_type)) self.table = Table(tablename, self.metadata, schema=schema, *sqlacols) self.transaction = None self._connection = None self._row_id_column = fdw_options.get('primary_key', None) def _need_explicit_null_ordering(self, key): support = SORT_SUPPORT[self.engine.dialect.name] default = support['default'] no = None if key.is_reversed: no = nullsfirst if default == 'higher' else nullslast else: no = nullslast if default == 'higher' else nullsfirst if key.nulls_first: if no != nullsfirst: return nullsfirst return None else: if no != nullslast: return nullslast return None def can_sort(self, sortkeys): if SORT_SUPPORT.get(self.engine.dialect.name) is None: # We have no idea about defaults return [] can_order_null = SORT_SUPPORT[self.engine.dialect.name]['support'] if (any((self._need_explicit_null_ordering(x) is not None for x in sortkeys)) and not can_order_null): return [] return sortkeys def explain(self, quals, columns, sortkeys=None, verbose=False): sortkeys = sortkeys or [] statement = self._build_statement(quals, columns, sortkeys) return [str(statement)] def _build_statement(self, quals, columns, sortkeys): statement = select([self.table]) clauses = [] for qual in quals: operator = OPERATORS.get(qual.operator, None) if operator: clauses.append(operator(self.table.c[qual.field_name], qual.value)) else: log_to_postgres('Qual not pushed to foreign db: %s' % qual, WARNING) if clauses: statement = statement.where(and_(*clauses)) if columns: columns = [self.table.c[col] for col in columns] else: columns = self.table.c statement = statement.with_only_columns(columns) orders = [] for sortkey in sortkeys: column = self.table.c[sortkey.attname] if sortkey.is_reversed: column = column.desc() if sortkey.collate: column = column.collate('"%s"' % sortkey.collate) null_ordering = self._need_explicit_null_ordering(sortkey) if null_ordering: column = null_ordering(column) statement = statement.order_by(column) return statement def execute(self, quals, columns, sortkeys=None): """ The quals are turned into an and'ed where clause. """ sortkeys = sortkeys or [] statement = self._build_statement(quals, columns, sortkeys) log_to_postgres(str(statement), DEBUG) rs = (self.connection .execution_options(stream_results=True) .execute(statement)) # Workaround pymssql "trash old results on new query" # behaviour (See issue #100) if self.engine.driver == 'pymssql' and self.transaction is not None: rs = list(rs) for item in rs: yield dict(item) @property def connection(self): if self._connection is None: self._connection = self.engine.connect() return self._connection def begin(self, serializable): self.transaction = self.connection.begin() def pre_commit(self): if self.transaction is not None: self.transaction.commit() self.transaction = None def commit(self): # Pre-commit hook does this on 9.3 if self.transaction is not None: self.transaction.commit() self.transaction = None def rollback(self): if self.transaction is not None: self.transaction.rollback() self.transaction = None @property def rowid_column(self): if self._row_id_column is None: log_to_postgres( 'You need to declare a primary key option in order ' 'to use the write features') return self._row_id_column def insert(self, values): self.connection.execute(self.table.insert(values=values)) def update(self, rowid, newvalues): self.connection.execute( self.table.update() .where(self.table.c[self._row_id_column] == rowid) .values(newvalues)) def delete(self, rowid): self.connection.execute( self.table.delete() .where(self.table.c[self._row_id_column] == rowid)) def _get_column_type(self, format_type): """Blatant ripoff from PG_Dialect.get_column_info""" # strip (*) from character varying(5), timestamp(5) # with time zone, geometry(POLYGON), etc. attype = re.sub(r'\(.*\)', '', format_type) # strip '[]' from integer[], etc. attype = re.sub(r'\[\]', '', attype) is_array = format_type.endswith('[]') charlen = re.search('\(([\d,]+)\)', format_type) if charlen: charlen = charlen.group(1) args = re.search('\((.*)\)', format_type) if args and args.group(1): args = tuple(re.split('\s*,\s*', args.group(1))) else: args = () kwargs = {} if attype == 'numeric': if charlen: prec, scale = charlen.split(',') args = (int(prec), int(scale)) else: args = () elif attype == 'double precision': args = (53, ) elif attype == 'integer': args = () elif attype in ('timestamp with time zone', 'time with time zone'): kwargs['timezone'] = True if charlen: kwargs['precision'] = int(charlen) args = () elif attype in ('timestamp without time zone', 'time without time zone', 'time'): kwargs['timezone'] = False if charlen: kwargs['precision'] = int(charlen) args = () elif attype == 'bit varying': kwargs['varying'] = True if charlen: args = (int(charlen),) else: args = () elif attype in ('interval', 'interval year to month', 'interval day to second'): if charlen: kwargs['precision'] = int(charlen) args = () elif charlen: args = (int(charlen),) coltype = ischema_names.get(attype, None) if coltype: coltype = coltype(*args, **kwargs) if is_array: coltype = ARRAY(coltype) else: coltype = sqltypes.NULLTYPE return coltype @classmethod def import_schema(self, schema, srv_options, options, restriction_type, restricts): """ Reflects the remote schema. """ metadata = MetaData() url = _parse_url_from_options(srv_options) engine = create_engine(url) dialect = PGDialect() if restriction_type == 'limit': only = restricts elif restriction_type == 'except': only = lambda t, _: t not in restricts else: only = None metadata.reflect(bind=engine, schema=schema, only=only) to_import = [] for _, table in sorted(metadata.tables.items()): ftable = TableDefinition(table.name) ftable.options['schema'] = schema ftable.options['tablename'] = table.name for c in table.c: # Force collation to None to prevent imcompatibilities setattr(c.type, "collation", None) # If the type is specialized, call the generic # superclass method if type(c.type) in CONVERSION_MAP: class_name = CONVERSION_MAP[type(c.type)] old_args = c.type.__dict__ c.type = class_name() c.type.__dict__.update(old_args) if c.primary_key: ftable.options['primary_key'] = c.name ftable.columns.append(ColumnDefinition( c.name, type_name=c.type.compile(dialect))) to_import.append(ftable) return to_import Multicorn-1.3.4/python/multicorn/statefdw.py000066400000000000000000000011601320447423600212260ustar00rootroot00000000000000"""A dummy foreign data wrapper""" from . import ForeignDataWrapper from .utils import log_to_postgres from logging import ERROR, DEBUG, INFO, WARNING class StateFdw(ForeignDataWrapper): """A dummy foreign data wrapper. This dummy foreign data wrapper is intended as a proof of concept of state keeping foreign data wrappers. It keeps an internal state as an integer, auto-incremented at each request. """ def __init__(self, *args): super(StateFdw, self).__init__(*args) self.state = 0 def execute(self, quals, columns): self.state += 1 yield [self.state] Multicorn-1.3.4/python/multicorn/testfdw.py000066400000000000000000000175451320447423600211030ustar00rootroot00000000000000# -*- coding: utf-8 -*- from multicorn import ForeignDataWrapper, TableDefinition, ColumnDefinition from multicorn.compat import unicode_ from .utils import log_to_postgres, WARNING, ERROR from itertools import cycle from datetime import datetime from operator import itemgetter class TestForeignDataWrapper(ForeignDataWrapper): _startup_cost = 10 def __init__(self, options, columns): super(TestForeignDataWrapper, self).__init__(options, columns) self.columns = columns self.test_type = options.get('test_type', None) self.tx_hook = options.get('tx_hook', False) self._row_id_column = options.get('row_id_column', list(self.columns.keys())[0]) log_to_postgres(str(sorted(options.items()))) log_to_postgres(str(sorted([(key, column.type_name) for key, column in columns.items()]))) for column in columns.values(): if column.options: log_to_postgres('Column %s options: %s' % (column.column_name, column.options)) if self.test_type == 'logger': log_to_postgres("An error is about to occur", WARNING) log_to_postgres("An error occured", ERROR) def _as_generator(self, quals, columns): random_thing = cycle([1, 2, 3]) for index in range(20): if self.test_type == 'sequence': line = [] for column_name in self.columns: line.append('%s %s %s' % (column_name, next(random_thing), index)) else: line = {} for column_name, column in self.columns.items(): if self.test_type == 'list': line[column_name] = [ column_name, next(random_thing), index, '%s,"%s"' % (column_name, index), '{some value, \\" \' 2}'] elif self.test_type == 'dict': line[column_name] = { "column_name": column_name, "repeater": next(random_thing), "index": index, "maybe_hstore": "a => b"} elif self.test_type == 'date': line[column_name] = datetime(2011, (index % 12) + 1, next(random_thing), 14, 30, 25) elif self.test_type == 'int': line[column_name] = index elif self.test_type == 'encoding': line[column_name] = (b'\xc3\xa9\xc3\xa0\xc2\xa4' .decode('utf-8')) elif self.test_type == 'nested_list': line[column_name] = [ [column_name, column_name], [next(random_thing), '{some value, \\" 2}'], [index, '%s,"%s"' % (column_name, index)]] elif self.test_type == 'float': line[column_name] = 1. / float(next(random_thing)) else: line[column_name] = '%s %s %s' % (column_name, next(random_thing), index) yield line def execute(self, quals, columns, sortkeys=None): sortkeys = sortkeys or [] log_to_postgres(str(sorted(quals))) log_to_postgres(str(sorted(columns))) if (len(sortkeys)) > 0: log_to_postgres("requested sort(s): ") for k in sortkeys: log_to_postgres(k) if self.test_type == 'None': return None elif self.test_type == 'iter_none': return [None, None] else: if (len(sortkeys) > 0): # testfdw don't have tables with more than 2 fields, without # duplicates, so we only need to worry about sorting on 1st # asked column k = sortkeys[0]; res = self._as_generator(quals, columns) if (self.test_type == 'sequence'): return sorted(res, key=itemgetter(k.attnum - 1), reverse=k.is_reversed) else: return sorted(res, key=itemgetter(k.attname), reverse=k.is_reversed) return self._as_generator(quals, columns) def get_rel_size(self, quals, columns): if self.test_type == 'planner': return (10000000, len(columns) * 10) return (20, len(columns) * 10) def get_path_keys(self): if self.test_type == 'planner': return [(('test1',), 1)] return [] def can_sort(self, sortkeys): # assume sort pushdown ok for all cols, in any order, any collation return sortkeys def update(self, rowid, newvalues): if self.test_type == 'nowrite': super(TestForeignDataWrapper, self).update(rowid, newvalues) log_to_postgres("UPDATING: %s with %s" % ( rowid, sorted(newvalues.items()))) if self.test_type == 'returning': for key in newvalues: newvalues[key] = "UPDATED: %s" % newvalues[key] return newvalues def delete(self, rowid): if self.test_type == 'nowrite': super(TestForeignDataWrapper, self).delete(rowid) log_to_postgres("DELETING: %s" % rowid) def insert(self, values): if self.test_type == 'nowrite': super(TestForeignDataWrapper, self).insert(values) log_to_postgres("INSERTING: %s" % sorted(values.items())) if self.test_type == 'returning': for key in self.columns: values[key] = "INSERTED: %s" % values.get(key, None) return values @property def rowid_column(self): return self._row_id_column def begin(self, serializable): if self.tx_hook: log_to_postgres('BEGIN') def sub_begin(self, level): if self.tx_hook: log_to_postgres('SUBBEGIN') def sub_rollback(self, level): if self.tx_hook: log_to_postgres('SUBROLLBACK') def sub_commit(self, level): if self.tx_hook: log_to_postgres('SUBCOMMIT') def commit(self): if self.tx_hook: log_to_postgres('COMMIT') def pre_commit(self): if self.tx_hook: log_to_postgres('PRECOMMIT') def rollback(self): if self.tx_hook: log_to_postgres('ROLLBACK') @classmethod def import_schema(self, schema, srv_options, options, restriction_type, restricts): log_to_postgres("IMPORT %s FROM srv %s OPTIONS %s RESTRICTION: %s %s" % (schema, srv_options, options, restriction_type, restricts)) tables = set([unicode_("imported_table_1"), unicode_("imported_table_2"), unicode_("imported_table_3")]) if restriction_type == 'limit': tables = tables.intersection(set(restricts)) elif restriction_type == 'except': tables = tables - set(restricts) rv = [] for tname in sorted(list(tables)): table = TableDefinition(tname) nb_col = options.get('nb_col', 3) for col in range(nb_col): table.columns.append( ColumnDefinition("col%s" % col, type_name="text", options={"option1": "value1"})) rv.append(table) return rv Multicorn-1.3.4/python/multicorn/utils.py000066400000000000000000000013171320447423600205510ustar00rootroot00000000000000from logging import ERROR, INFO, DEBUG, WARNING, CRITICAL try: from ._utils import _log_to_postgres from ._utils import check_interrupts except ImportError as e: from warnings import warn warn("Not executed in a postgresql server," " disabling log_to_postgres", ImportWarning) def _log_to_postgres(message, level=0, hint=None, detail=None): pass REPORT_CODES = { DEBUG: 0, INFO: 1, WARNING: 2, ERROR: 3, CRITICAL: 4 } def log_to_postgres(message, level=INFO, hint=None, detail=None): code = REPORT_CODES.get(level, None) if code is None: raise KeyError("Not a valid log level") _log_to_postgres(message, code, hint=hint, detail=detail) Multicorn-1.3.4/python/multicorn/xmlfdw.py000066400000000000000000000045421320447423600207150ustar00rootroot00000000000000""" An XML Foreign Data Wrapper. """ from . import ForeignDataWrapper from xml.sax import ContentHandler, make_parser class MulticornXMLHandler(ContentHandler): def __init__(self, elem_tag, columns): self.elem_tag = elem_tag self.columns = columns self.reset() def reset(self): self.parsed_rows = [] self.current_row = {} self.tag = None self.root_seen = 0 self.nested = False def startElement(self, name, attrs): if name == self.elem_tag: # Keep track of nested "elem_tag" self.root_seen += 1 elif self.root_seen == 1: # Ignore nested tag. if name in self.columns: self.tag = name self.current_row[name] = '' def characters(self, content): if self.tag is not None: self.current_row[self.tag] += content def get_rows(self): """Return the parsed_rows, and forget about it.""" result, self.parsed_rows = self.parsed_rows, [] return result def endElement(self, name): if name == self.elem_tag: self.root_seen -= 1 self.parsed_rows.append(self.current_row) self.current_row = {} elif name in self.columns: self.tag = None class XMLFdw(ForeignDataWrapper): """A foreign data wrapper for accessing xml files. Valid options: - filename: full path to the xml file. - elem_tag: a tagname acting as a root for a tag. Child tag will be mapped to corresponding columns. """ def __init__(self, fdw_options, fdw_columns): super(XMLFdw, self).__init__(fdw_options, fdw_columns) self.filename = fdw_options['filename'] self.elem_tag = fdw_options['elem_tag'] self.buffer_size = fdw_options.get('buffer_size', 4096) self.columns = fdw_columns def execute(self, quals, columns): parser = make_parser() handler = MulticornXMLHandler(self.elem_tag, self.columns) parser.setContentHandler(handler) with open(self.filename) as stream: while(True): a = stream.read(self.buffer_size) if not a: break parser.feed(a) for row in handler.get_rows(): yield row parser.close() Multicorn-1.3.4/setup.cfg000066400000000000000000000000371320447423600153210ustar00rootroot00000000000000[pytest] python_files=test*.py Multicorn-1.3.4/setup.py000077500000000000000000000020401320447423600152110ustar00rootroot00000000000000import subprocess import sys from setuptools import setup, find_packages, Extension # hum... borrowed from psycopg2 def get_pg_config(kind, pg_config="pg_config"): p = subprocess.Popen([pg_config, '--%s' % kind], stdout=subprocess.PIPE) r = p.communicate() r = r[0].strip().decode('utf8') if not r: raise Warning(p[2].readline()) return r include_dirs = [get_pg_config(d) for d in ("includedir", "pkgincludedir", "includedir-server")] multicorn_utils_module = Extension('multicorn._utils', include_dirs=include_dirs, extra_compile_args = ['-shared'], sources=['src/utils.c']) requires=[] if sys.version_info[0] == 2: if sys.version_info[1] == 6: requires.append("ordereddict") elif sys.version_info[1] < 6: sys.exit("Sorry, you need at least python 2.6 for Multicorn") setup( name='multicorn', version='__VERSION__', author='Kozea', license='Postgresql', package_dir={'': 'python'}, packages=['multicorn', 'multicorn.fsfdw'], ext_modules = [multicorn_utils_module] ) Multicorn-1.3.4/sql/000077500000000000000000000000001320447423600142775ustar00rootroot00000000000000Multicorn-1.3.4/sql/multicorn.sql000077500000000000000000000005541320447423600170430ustar00rootroot00000000000000-- create wrapper with validator and handler CREATE OR REPLACE FUNCTION multicorn_validator (text[], oid) RETURNS bool AS 'MODULE_PATHNAME' LANGUAGE C STRICT; CREATE OR REPLACE FUNCTION multicorn_handler () RETURNS fdw_handler AS 'MODULE_PATHNAME' LANGUAGE C STRICT; CREATE FOREIGN DATA WRAPPER multicorn VALIDATOR multicorn_validator HANDLER multicorn_handler; Multicorn-1.3.4/src/000077500000000000000000000000001320447423600142675ustar00rootroot00000000000000Multicorn-1.3.4/src/errors.c000066400000000000000000000043301320447423600157470ustar00rootroot00000000000000/*------------------------------------------------------------------------- * * The Multicorn Foreign Data Wrapper allows you to fetch foreign data in * Python in your PostgreSQL server. * * This module contains error handling functions. * * This software is released under the postgresql licence * * author: Kozea * * *------------------------------------------------------------------------- */ #include "multicorn.h" #include "bytesobject.h" #include "access/xact.h" void reportException(PyObject *pErrType, PyObject *pErrValue, PyObject *pErrTraceback); void errorCheck() { PyObject *pErrType, *pErrValue, *pErrTraceback; PyErr_Fetch(&pErrType, &pErrValue, &pErrTraceback); if (pErrType) { reportException(pErrType, pErrValue, pErrTraceback); } } void reportException(PyObject *pErrType, PyObject *pErrValue, PyObject *pErrTraceback) { char *errName, *errValue, *errTraceback = ""; PyObject *traceback_list; PyObject *pTemp; PyObject *tracebackModule = PyImport_ImportModule("traceback"); PyObject *format_exception = PyObject_GetAttrString(tracebackModule, "format_exception"); PyObject *newline = PyString_FromString("\n"); int severity; PyErr_NormalizeException(&pErrType, &pErrValue, &pErrTraceback); pTemp = PyObject_GetAttrString(pErrType, "__name__"); errName = PyString_AsString(pTemp); errValue = PyString_AsString(PyObject_Str(pErrValue)); if (pErrTraceback != NULL) { traceback_list = PyObject_CallFunction(format_exception, "(O,O,O)", pErrType, pErrValue, pErrTraceback); errTraceback = PyString_AsString(PyObject_CallMethod(newline, "join", "(O)", traceback_list)); Py_DECREF(pErrTraceback); Py_DECREF(traceback_list); } if (IsAbortedTransactionBlockState()) { severity = WARNING; } else { severity = ERROR; } if (errstart(severity, __FILE__, __LINE__, PG_FUNCNAME_MACRO, TEXTDOMAIN)) { if (errstart(severity, __FILE__, __LINE__, PG_FUNCNAME_MACRO, TEXTDOMAIN)) errmsg("Error in python: %s", errName); errdetail("%s", errValue); errdetail_log("%s", errTraceback); } Py_DECREF(pErrType); Py_DECREF(pErrValue); Py_DECREF(format_exception); Py_DECREF(tracebackModule); Py_DECREF(newline); Py_DECREF(pTemp); errfinish(0); } Multicorn-1.3.4/src/multicorn.c000066400000000000000000000673171320447423600164650ustar00rootroot00000000000000/* * The Multicorn Foreign Data Wrapper allows you to fetch foreign data in * Python in your PostgreSQL server * * This software is released under the postgresql licence * * author: Kozea */ #include "multicorn.h" #include "optimizer/paths.h" #include "optimizer/pathnode.h" #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" #include "optimizer/clauses.h" #include "optimizer/var.h" #include "access/reloptions.h" #include "access/relscan.h" #include "access/sysattr.h" #include "access/xact.h" #include "nodes/makefuncs.h" #include "catalog/pg_type.h" #include "utils/memutils.h" #include "miscadmin.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "parser/parsetree.h" PG_MODULE_MAGIC; extern Datum multicorn_handler(PG_FUNCTION_ARGS); extern Datum multicorn_validator(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(multicorn_handler); PG_FUNCTION_INFO_V1(multicorn_validator); void _PG_init(void); void _PG_fini(void); /* * FDW functions declarations */ static void multicornGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); static void multicornGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); static ForeignScan *multicornGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, ForeignPath *best_path, List *tlist, List *scan_clauses #if PG_VERSION_NUM >= 90500 , Plan *outer_plan #endif ); static void multicornExplainForeignScan(ForeignScanState *node, ExplainState *es); static void multicornBeginForeignScan(ForeignScanState *node, int eflags); static TupleTableSlot *multicornIterateForeignScan(ForeignScanState *node); static void multicornReScanForeignScan(ForeignScanState *node); static void multicornEndForeignScan(ForeignScanState *node); #if PG_VERSION_NUM >= 90300 static void multicornAddForeignUpdateTargets(Query *parsetree, RangeTblEntry *target_rte, Relation target_relation); static List *multicornPlanForeignModify(PlannerInfo *root, ModifyTable *plan, Index resultRelation, int subplan_index); static void multicornBeginForeignModify(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo, List *fdw_private, int subplan_index, int eflags); static TupleTableSlot *multicornExecForeignInsert(EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planslot); static TupleTableSlot *multicornExecForeignDelete(EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot); static TupleTableSlot *multicornExecForeignUpdate(EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot); static void multicornEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo); static void multicorn_subxact_callback(SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg); #endif #if PG_VERSION_NUM >= 90500 static List *multicornImportForeignSchema(ImportForeignSchemaStmt * stmt, Oid serverOid); #endif static void multicorn_xact_callback(XactEvent event, void *arg); /* Helpers functions */ void *serializePlanState(MulticornPlanState * planstate); MulticornExecState *initializeExecState(void *internal_plan_state); /* Hash table mapping oid to fdw instances */ HTAB *InstancesHash; void _PG_init() { HASHCTL ctl; MemoryContext oldctx = MemoryContextSwitchTo(CacheMemoryContext); Py_Initialize(); RegisterXactCallback(multicorn_xact_callback, NULL); #if PG_VERSION_NUM >= 90300 RegisterSubXactCallback(multicorn_subxact_callback, NULL); #endif /* Initialize the global oid -> python instances hash */ MemSet(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(CacheEntry); ctl.hash = oid_hash; ctl.hcxt = CacheMemoryContext; InstancesHash = hash_create("multicorn instances", 32, &ctl, HASH_ELEM | HASH_FUNCTION); MemoryContextSwitchTo(oldctx); } void _PG_fini() { Py_Finalize(); } Datum multicorn_handler(PG_FUNCTION_ARGS) { FdwRoutine *fdw_routine = makeNode(FdwRoutine); /* Plan phase */ fdw_routine->GetForeignRelSize = multicornGetForeignRelSize; fdw_routine->GetForeignPaths = multicornGetForeignPaths; fdw_routine->GetForeignPlan = multicornGetForeignPlan; fdw_routine->ExplainForeignScan = multicornExplainForeignScan; /* Scan phase */ fdw_routine->BeginForeignScan = multicornBeginForeignScan; fdw_routine->IterateForeignScan = multicornIterateForeignScan; fdw_routine->ReScanForeignScan = multicornReScanForeignScan; fdw_routine->EndForeignScan = multicornEndForeignScan; #if PG_VERSION_NUM >= 90300 /* Code for 9.3 */ fdw_routine->AddForeignUpdateTargets = multicornAddForeignUpdateTargets; /* Writable API */ fdw_routine->PlanForeignModify = multicornPlanForeignModify; fdw_routine->BeginForeignModify = multicornBeginForeignModify; fdw_routine->ExecForeignInsert = multicornExecForeignInsert; fdw_routine->ExecForeignDelete = multicornExecForeignDelete; fdw_routine->ExecForeignUpdate = multicornExecForeignUpdate; fdw_routine->EndForeignModify = multicornEndForeignModify; #endif #if PG_VERSION_NUM >= 90500 fdw_routine->ImportForeignSchema = multicornImportForeignSchema; #endif PG_RETURN_POINTER(fdw_routine); } Datum multicorn_validator(PG_FUNCTION_ARGS) { List *options_list = untransformRelOptions(PG_GETARG_DATUM(0)); Oid catalog = PG_GETARG_OID(1); char *className = NULL; ListCell *cell; PyObject *p_class; foreach(cell, options_list) { DefElem *def = (DefElem *) lfirst(cell); if (strcmp(def->defname, "wrapper") == 0) { /* Only at server creation can we set the wrapper, */ /* for security issues. */ if (catalog == ForeignTableRelationId) { ereport(ERROR, (errmsg("%s", "Cannot set the wrapper class on the table"), errhint("%s", "Set it on the server"))); } else { className = (char *) defGetString(def); } } } if (catalog == ForeignServerRelationId) { if (className == NULL) { ereport(ERROR, (errmsg("%s", "The wrapper parameter is mandatory, specify a valid class name"))); } /* Try to import the class. */ p_class = getClassString(className); errorCheck(); Py_DECREF(p_class); } PG_RETURN_VOID(); } /* * multicornGetForeignRelSize * Obtain relation size estimates for a foreign table. * This is done by calling the */ static void multicornGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) { MulticornPlanState *planstate = palloc0(sizeof(MulticornPlanState)); ForeignTable *ftable = GetForeignTable(foreigntableid); ListCell *lc; bool needWholeRow = false; TupleDesc desc; baserel->fdw_private = planstate; planstate->fdw_instance = getInstance(foreigntableid); planstate->foreigntableid = foreigntableid; /* Initialize the conversion info array */ { Relation rel = RelationIdGetRelation(ftable->relid); AttInMetadata *attinmeta; desc = RelationGetDescr(rel); attinmeta = TupleDescGetAttInMetadata(desc); planstate->numattrs = RelationGetNumberOfAttributes(rel); planstate->cinfos = palloc0(sizeof(ConversionInfo *) * planstate->numattrs); initConversioninfo(planstate->cinfos, attinmeta); needWholeRow = rel->trigdesc && rel->trigdesc->trig_insert_after_row; RelationClose(rel); } if (needWholeRow) { int i; for (i = 0; i < desc->natts; i++) { Form_pg_attribute att = desc->attrs[i]; if (!att->attisdropped) { planstate->target_list = lappend(planstate->target_list, makeString(NameStr(att->attname))); } } } else { /* Pull "var" clauses to build an appropriate target list */ #if PG_VERSION_NUM >= 90600 foreach(lc, extractColumns(baserel->reltarget->exprs, baserel->baserestrictinfo)) #else foreach(lc, extractColumns(baserel->reltargetlist, baserel->baserestrictinfo)) #endif { Var *var = (Var *) lfirst(lc); Value *colname; /* * Store only a Value node containing the string name of the * column. */ colname = colnameFromVar(var, root, planstate); if (colname != NULL && strVal(colname) != NULL) { planstate->target_list = lappend(planstate->target_list, colname); } } } /* Extract the restrictions from the plan. */ foreach(lc, baserel->baserestrictinfo) { extractRestrictions(baserel->relids, ((RestrictInfo *) lfirst(lc))->clause, &planstate->qual_list); } /* Inject the "rows" and "width" attribute into the baserel */ #if PG_VERSION_NUM >= 90600 getRelSize(planstate, root, &baserel->rows, &baserel->reltarget->width); #else getRelSize(planstate, root, &baserel->rows, &baserel->width); #endif } /* * multicornGetForeignPaths * Create possible access paths for a scan on the foreign table. * This is done by calling the "get_path_keys method on the python side, * and parsing its result to build parameterized paths according to the * equivalence classes found in the plan. */ static void multicornGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) { List *pathes; /* List of ForeignPath */ MulticornPlanState *planstate = baserel->fdw_private; ListCell *lc; /* These lists are used to handle sort pushdown */ List *apply_pathkeys = NULL; List *deparsed_pathkeys = NULL; /* Extract a friendly version of the pathkeys. */ List *possiblePaths = pathKeys(planstate); /* Try to find parameterized paths */ pathes = findPaths(root, baserel, possiblePaths, planstate->startupCost, planstate, apply_pathkeys, deparsed_pathkeys); /* Add a simple default path */ pathes = lappend(pathes, create_foreignscan_path(root, baserel, #if PG_VERSION_NUM >= 90600 NULL, /* default pathtarget */ #endif baserel->rows, planstate->startupCost, #if PG_VERSION_NUM >= 90600 baserel->rows * baserel->reltarget->width, #else baserel->rows * baserel->width, #endif NIL, /* no pathkeys */ NULL, #if PG_VERSION_NUM >= 90500 NULL, #endif NULL)); /* Handle sort pushdown */ if (root->query_pathkeys) { List *deparsed = deparse_sortgroup(root, foreigntableid, baserel); if (deparsed) { /* Update the sort_*_pathkeys lists if needed */ computeDeparsedSortGroup(deparsed, planstate, &apply_pathkeys, &deparsed_pathkeys); } } /* Add each ForeignPath previously found */ foreach(lc, pathes) { ForeignPath *path = (ForeignPath *) lfirst(lc); /* Add the path without modification */ add_path(baserel, (Path *) path); /* Add the path with sort pusdown if possible */ if (apply_pathkeys && deparsed_pathkeys) { ForeignPath *newpath; newpath = create_foreignscan_path(root, baserel, #if PG_VERSION_NUM >= 90600 NULL, /* default pathtarget */ #endif path->path.rows, path->path.startup_cost, path->path.total_cost, apply_pathkeys, NULL, #if PG_VERSION_NUM >= 90500 NULL, #endif (void *) deparsed_pathkeys); newpath->path.param_info = path->path.param_info; add_path(baserel, (Path *) newpath); } } errorCheck(); } /* * multicornGetForeignPlan * Create a ForeignScan plan node for scanning the foreign table */ static ForeignScan * multicornGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, ForeignPath *best_path, List *tlist, List *scan_clauses #if PG_VERSION_NUM >= 90500 , Plan *outer_plan #endif ) { Index scan_relid = baserel->relid; MulticornPlanState *planstate = (MulticornPlanState *) baserel->fdw_private; ListCell *lc; scan_clauses = extract_actual_clauses(scan_clauses, false); /* Extract the quals coming from a parameterized path, if any */ if (best_path->path.param_info) { foreach(lc, scan_clauses) { extractRestrictions(baserel->relids, (Expr *) lfirst(lc), &planstate->qual_list); } } planstate->pathkeys = (List *) best_path->fdw_private; return make_foreignscan(tlist, scan_clauses, scan_relid, scan_clauses, /* no expressions to evaluate */ serializePlanState(planstate) #if PG_VERSION_NUM >= 90500 , NULL , NULL /* All quals are meant to be rechecked */ , NULL #endif ); } /* * multicornExplainForeignScan * Placeholder for additional "EXPLAIN" information. * This should (at least) output the python class name, as well * as information that was taken into account for the choice of a path. */ static void multicornExplainForeignScan(ForeignScanState *node, ExplainState *es) { PyObject *p_iterable = execute(node, es), *p_item, *p_str; Py_INCREF(p_iterable); while((p_item = PyIter_Next(p_iterable))){ p_str = PyObject_Str(p_item); ExplainPropertyText("Multicorn", PyString_AsString(p_str), es); Py_DECREF(p_str); } Py_DECREF(p_iterable); errorCheck(); } /* * multicornBeginForeignScan * Initialize the foreign scan. * This (primarily) involves : * - retrieving cached info from the plan phase * - initializing various buffers */ static void multicornBeginForeignScan(ForeignScanState *node, int eflags) { ForeignScan *fscan = (ForeignScan *) node->ss.ps.plan; MulticornExecState *execstate; TupleDesc tupdesc = RelationGetDescr(node->ss.ss_currentRelation); ListCell *lc; execstate = initializeExecState(fscan->fdw_private); execstate->values = palloc(sizeof(Datum) * tupdesc->natts); execstate->nulls = palloc(sizeof(bool) * tupdesc->natts); execstate->qual_list = NULL; foreach(lc, fscan->fdw_exprs) { extractRestrictions(bms_make_singleton(fscan->scan.scanrelid), ((Expr *) lfirst(lc)), &execstate->qual_list); } initConversioninfo(execstate->cinfos, TupleDescGetAttInMetadata(tupdesc)); node->fdw_state = execstate; } /* * multicornIterateForeignScan * Retrieve next row from the result set, or clear tuple slot to indicate * EOF. * * This is done by iterating over the result from the "execute" python * method. */ static TupleTableSlot * multicornIterateForeignScan(ForeignScanState *node) { TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; MulticornExecState *execstate = node->fdw_state; PyObject *p_value; if (execstate->p_iterator == NULL) { execute(node, NULL); } ExecClearTuple(slot); if (execstate->p_iterator == Py_None) { /* No iterator returned from get_iterator */ Py_DECREF(execstate->p_iterator); return slot; } p_value = PyIter_Next(execstate->p_iterator); errorCheck(); /* A none value results in an empty slot. */ if (p_value == NULL || p_value == Py_None) { Py_XDECREF(p_value); return slot; } slot->tts_values = execstate->values; slot->tts_isnull = execstate->nulls; pythonResultToTuple(p_value, slot, execstate->cinfos, execstate->buffer); ExecStoreVirtualTuple(slot); Py_DECREF(p_value); return slot; } /* * multicornReScanForeignScan * Restart the scan */ static void multicornReScanForeignScan(ForeignScanState *node) { MulticornExecState *state = node->fdw_state; if (state->p_iterator) { Py_DECREF(state->p_iterator); state->p_iterator = NULL; } } /* * multicornEndForeignScan * Finish scanning foreign table and dispose objects used for this scan. */ static void multicornEndForeignScan(ForeignScanState *node) { MulticornExecState *state = node->fdw_state; PyObject *result = PyObject_CallMethod(state->fdw_instance, "end_scan", "()"); errorCheck(); Py_DECREF(result); Py_DECREF(state->fdw_instance); Py_XDECREF(state->p_iterator); state->p_iterator = NULL; } #if PG_VERSION_NUM >= 90300 /* * multicornAddForeigUpdateTargets * Add resjunk columns needed for update/delete. */ static void multicornAddForeignUpdateTargets(Query *parsetree, RangeTblEntry *target_rte, Relation target_relation) { Var *var = NULL; TargetEntry *tle, *returningTle; PyObject *instance = getInstance(target_relation->rd_id); const char *attrname = getRowIdColumn(instance); TupleDesc desc = target_relation->rd_att; int i; ListCell *cell; foreach(cell, parsetree->returningList) { returningTle = lfirst(cell); tle = copyObject(returningTle); tle->resjunk = true; parsetree->targetList = lappend(parsetree->targetList, tle); } for (i = 0; i < desc->natts; i++) { Form_pg_attribute att = desc->attrs[i]; if (!att->attisdropped) { if (strcmp(NameStr(att->attname), attrname) == 0) { var = makeVar(parsetree->resultRelation, att->attnum, att->atttypid, att->atttypmod, att->attcollation, 0); break; } } } if (var == NULL) { ereport(ERROR, (errmsg("%s", "The rowid attribute does not exist"))); } tle = makeTargetEntry((Expr *) var, list_length(parsetree->targetList) + 1, strdup(attrname), true); parsetree->targetList = lappend(parsetree->targetList, tle); Py_DECREF(instance); } /* * multicornPlanForeignModify * Plan a foreign write operation. * This is done by checking the "supported operations" attribute * on the python class. */ static List * multicornPlanForeignModify(PlannerInfo *root, ModifyTable *plan, Index resultRelation, int subplan_index) { return NULL; } /* * multicornBeginForeignModify * Initialize a foreign write operation. */ static void multicornBeginForeignModify(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo, List *fdw_private, int subplan_index, int eflags) { MulticornModifyState *modstate = palloc0(sizeof(MulticornModifyState)); Relation rel = resultRelInfo->ri_RelationDesc; TupleDesc desc = RelationGetDescr(rel); PlanState *ps = mtstate->mt_plans[subplan_index]; Plan *subplan = ps->plan; MemoryContext oldcontext; int i; modstate->cinfos = palloc0(sizeof(ConversionInfo *) * desc->natts); modstate->buffer = makeStringInfo(); modstate->fdw_instance = getInstance(rel->rd_id); modstate->rowidAttrName = getRowIdColumn(modstate->fdw_instance); initConversioninfo(modstate->cinfos, TupleDescGetAttInMetadata(desc)); oldcontext = MemoryContextSwitchTo(TopMemoryContext); MemoryContextSwitchTo(oldcontext); if (ps->ps_ResultTupleSlot) { TupleDesc resultTupleDesc = ps->ps_ResultTupleSlot->tts_tupleDescriptor; modstate->resultCinfos = palloc0(sizeof(ConversionInfo *) * resultTupleDesc->natts); initConversioninfo(modstate->resultCinfos, TupleDescGetAttInMetadata(resultTupleDesc)); } for (i = 0; i < desc->natts; i++) { Form_pg_attribute att = desc->attrs[i]; if (!att->attisdropped) { if (strcmp(NameStr(att->attname), modstate->rowidAttrName) == 0) { modstate->rowidCinfo = modstate->cinfos[i]; break; } } } modstate->rowidAttno = ExecFindJunkAttributeInTlist(subplan->targetlist, modstate->rowidAttrName); resultRelInfo->ri_FdwState = modstate; } /* * multicornExecForeignInsert * Execute a foreign insert operation * This is done by calling the python "insert" method. */ static TupleTableSlot * multicornExecForeignInsert(EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot) { MulticornModifyState *modstate = resultRelInfo->ri_FdwState; PyObject *fdw_instance = modstate->fdw_instance; PyObject *values = tupleTableSlotToPyObject(slot, modstate->cinfos); PyObject *p_new_value = PyObject_CallMethod(fdw_instance, "insert", "(O)", values); errorCheck(); if (p_new_value && p_new_value != Py_None) { ExecClearTuple(slot); pythonResultToTuple(p_new_value, slot, modstate->cinfos, modstate->buffer); ExecStoreVirtualTuple(slot); } Py_XDECREF(p_new_value); Py_DECREF(values); errorCheck(); return slot; } /* * multicornExecForeignDelete * Execute a foreign delete operation * This is done by calling the python "delete" method, with the opaque * rowid that was supplied. */ static TupleTableSlot * multicornExecForeignDelete(EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot) { MulticornModifyState *modstate = resultRelInfo->ri_FdwState; PyObject *fdw_instance = modstate->fdw_instance, *p_row_id, *p_new_value; bool is_null; ConversionInfo *cinfo = modstate->rowidCinfo; Datum value = ExecGetJunkAttribute(planSlot, modstate->rowidAttno, &is_null); p_row_id = datumToPython(value, cinfo->atttypoid, cinfo); p_new_value = PyObject_CallMethod(fdw_instance, "delete", "(O)", p_row_id); errorCheck(); if (p_new_value == NULL || p_new_value == Py_None) { Py_XDECREF(p_new_value); p_new_value = tupleTableSlotToPyObject(planSlot, modstate->resultCinfos); } ExecClearTuple(slot); pythonResultToTuple(p_new_value, slot, modstate->cinfos, modstate->buffer); ExecStoreVirtualTuple(slot); Py_DECREF(p_new_value); Py_DECREF(p_row_id); errorCheck(); return slot; } /* * multicornExecForeignUpdate * Execute a foreign update operation * This is done by calling the python "update" method, with the opaque * rowid that was supplied. */ static TupleTableSlot * multicornExecForeignUpdate(EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, TupleTableSlot *planSlot) { MulticornModifyState *modstate = resultRelInfo->ri_FdwState; PyObject *fdw_instance = modstate->fdw_instance, *p_row_id, *p_new_value, *p_value = tupleTableSlotToPyObject(slot, modstate->cinfos); bool is_null; ConversionInfo *cinfo = modstate->rowidCinfo; Datum value = ExecGetJunkAttribute(planSlot, modstate->rowidAttno, &is_null); p_row_id = datumToPython(value, cinfo->atttypoid, cinfo); p_new_value = PyObject_CallMethod(fdw_instance, "update", "(O,O)", p_row_id, p_value); errorCheck(); if (p_new_value != NULL && p_new_value != Py_None) { ExecClearTuple(slot); pythonResultToTuple(p_new_value, slot, modstate->cinfos, modstate->buffer); ExecStoreVirtualTuple(slot); } Py_XDECREF(p_new_value); Py_DECREF(p_row_id); errorCheck(); return slot; } /* * multicornEndForeignModify * Clean internal state after a modify operation. */ static void multicornEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo) { MulticornModifyState *modstate = resultRelInfo->ri_FdwState; PyObject *result = PyObject_CallMethod(modstate->fdw_instance, "end_modify", "()"); errorCheck(); Py_DECREF(modstate->fdw_instance); Py_DECREF(result); } /* * Callback used to propagate a subtransaction end. */ static void multicorn_subxact_callback(SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg) { PyObject *instance; int curlevel; HASH_SEQ_STATUS status; CacheEntry *entry; /* Nothing to do after commit or subtransaction start. */ if (event == SUBXACT_EVENT_COMMIT_SUB || event == SUBXACT_EVENT_START_SUB) return; curlevel = GetCurrentTransactionNestLevel(); hash_seq_init(&status, InstancesHash); while ((entry = (CacheEntry *) hash_seq_search(&status)) != NULL) { if (entry->xact_depth < curlevel) continue; instance = entry->value; if (event == SUBXACT_EVENT_PRE_COMMIT_SUB) { PyObject_CallMethod(instance, "sub_commit", "(i)", curlevel); } else { PyObject_CallMethod(instance, "sub_rollback", "(i)", curlevel); } errorCheck(); entry->xact_depth--; } } #endif /* * Callback used to propagate pre-commit / commit / rollback. */ static void multicorn_xact_callback(XactEvent event, void *arg) { PyObject *instance; HASH_SEQ_STATUS status; CacheEntry *entry; hash_seq_init(&status, InstancesHash); while ((entry = (CacheEntry *) hash_seq_search(&status)) != NULL) { instance = entry->value; if (entry->xact_depth == 0) continue; switch (event) { #if PG_VERSION_NUM >= 90300 case XACT_EVENT_PRE_COMMIT: PyObject_CallMethod(instance, "pre_commit", "()"); break; #endif case XACT_EVENT_COMMIT: PyObject_CallMethod(instance, "commit", "()"); entry->xact_depth = 0; break; case XACT_EVENT_ABORT: PyObject_CallMethod(instance, "rollback", "()"); entry->xact_depth = 0; break; default: break; } errorCheck(); } } #if PG_VERSION_NUM >= 90500 static List * multicornImportForeignSchema(ImportForeignSchemaStmt * stmt, Oid serverOid) { List *cmds = NULL; List *options = NULL; UserMapping *mapping; ForeignServer *f_server; char *restrict_type = NULL; PyObject *p_class = NULL; PyObject *p_tables, *p_srv_options, *p_options, *p_restrict_list, *p_iter, *p_item; ListCell *lc; f_server = GetForeignServer(serverOid); foreach(lc, f_server->options) { DefElem *option = (DefElem *) lfirst(lc); if (strcmp(option->defname, "wrapper") == 0) { p_class = getClassString(defGetString(option)); errorCheck(); } else { options = lappend(options, option); } } mapping = multicorn_GetUserMapping(GetUserId(), serverOid); if (mapping) options = list_concat(options, mapping->options); if (p_class == NULL) { /* * This should never happen, since we validate the wrapper parameter * at */ /* object creation time. */ ereport(ERROR, (errmsg("%s", "The wrapper parameter is mandatory, specify a valid class name"))); } switch (stmt->list_type) { case FDW_IMPORT_SCHEMA_LIMIT_TO: restrict_type = "limit"; break; case FDW_IMPORT_SCHEMA_EXCEPT: restrict_type = "except"; break; case FDW_IMPORT_SCHEMA_ALL: break; } p_srv_options = optionsListToPyDict(options); p_options = optionsListToPyDict(stmt->options); p_restrict_list = PyList_New(0); foreach(lc, stmt->table_list) { RangeVar *rv = (RangeVar *) lfirst(lc); PyObject *p_tablename = PyUnicode_Decode( rv->relname, strlen(rv->relname), getPythonEncodingName(), NULL); errorCheck(); PyList_Append(p_restrict_list, p_tablename); Py_DECREF(p_tablename); } errorCheck(); p_tables = PyObject_CallMethod(p_class, "import_schema", "(s, O, O, s, O)", stmt->remote_schema, p_srv_options, p_options, restrict_type, p_restrict_list); errorCheck(); Py_DECREF(p_class); Py_DECREF(p_options); Py_DECREF(p_srv_options); Py_DECREF(p_restrict_list); errorCheck(); p_iter = PyObject_GetIter(p_tables); while ((p_item = PyIter_Next(p_iter))) { PyObject *p_string; char *value; p_string = PyObject_CallMethod(p_item, "to_statement", "(s,s)", stmt->local_schema, f_server->servername); errorCheck(); value = PyString_AsString(p_string); errorCheck(); cmds = lappend(cmds, pstrdup(value)); Py_DECREF(p_string); Py_DECREF(p_item); } errorCheck(); Py_DECREF(p_iter); Py_DECREF(p_tables); return cmds; } #endif /* * "Serialize" a MulticornPlanState, so that it is safe to be carried * between the plan and the execution safe. */ void * serializePlanState(MulticornPlanState * state) { List *result = NULL; result = lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(state->numattrs), false, true)); result = lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(state->foreigntableid), false, true)); result = lappend(result, state->target_list); result = lappend(result, serializeDeparsedSortGroup(state->pathkeys)); return result; } /* * "Deserialize" an internal state and inject it in an * MulticornExecState */ MulticornExecState * initializeExecState(void *internalstate) { MulticornExecState *execstate = palloc0(sizeof(MulticornExecState)); List *values = (List *) internalstate; AttrNumber attnum = ((Const *) linitial(values))->constvalue; Oid foreigntableid = ((Const *) lsecond(values))->constvalue; List *pathkeys; /* Those list must be copied, because their memory context can become */ /* invalid during the execution (in particular with the cursor interface) */ execstate->target_list = copyObject(lthird(values)); pathkeys = lfourth(values); execstate->pathkeys = deserializeDeparsedSortGroup(pathkeys); execstate->fdw_instance = getInstance(foreigntableid); execstate->buffer = makeStringInfo(); execstate->cinfos = palloc0(sizeof(ConversionInfo *) * attnum); execstate->values = palloc(attnum * sizeof(Datum)); execstate->nulls = palloc(attnum * sizeof(bool)); return execstate; } Multicorn-1.3.4/src/multicorn.h000066400000000000000000000120471320447423600164600ustar00rootroot00000000000000#include "Python.h" #include "postgres.h" #include "access/relscan.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/explain.h" #include "foreign/fdwapi.h" #include "foreign/foreign.h" #include "funcapi.h" #include "lib/stringinfo.h" #include "nodes/bitmapset.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" #include "nodes/relation.h" #include "utils/builtins.h" #include "utils/syscache.h" #ifndef PG_MULTICORN_H #define PG_MULTICORN_H /* Data structures */ typedef struct CacheEntry { Oid hashkey; PyObject *value; List *options; List *columns; int xact_depth; /* Keep the "options" and "columns" in a specific context to avoid leaks. */ MemoryContext cacheContext; } CacheEntry; typedef struct ConversionInfo { char *attrname; FmgrInfo *attinfunc; FmgrInfo *attoutfunc; Oid atttypoid; Oid attioparam; int32 atttypmod; int attnum; bool is_array; int attndims; bool need_quote; } ConversionInfo; typedef struct MulticornPlanState { Oid foreigntableid; AttrNumber numattrs; PyObject *fdw_instance; List *target_list; List *qual_list; int startupCost; ConversionInfo **cinfos; List *pathkeys; /* list of MulticornDeparsedSortGroup) */ } MulticornPlanState; typedef struct MulticornExecState { /* instance and iterator */ PyObject *fdw_instance; PyObject *p_iterator; /* Information carried from the plan phase. */ List *target_list; List *qual_list; Datum *values; bool *nulls; ConversionInfo **cinfos; /* Common buffer to avoid repeated allocations */ StringInfo buffer; AttrNumber rowidAttno; char *rowidAttrName; List *pathkeys; /* list of MulticornDeparsedSortGroup) */ } MulticornExecState; typedef struct MulticornModifyState { ConversionInfo **cinfos; ConversionInfo **resultCinfos; PyObject *fdw_instance; StringInfo buffer; AttrNumber rowidAttno; char *rowidAttrName; ConversionInfo *rowidCinfo; } MulticornModifyState; typedef struct MulticornBaseQual { AttrNumber varattno; NodeTag right_type; Oid typeoid; char *opname; bool isArray; bool useOr; } MulticornBaseQual; typedef struct MulticornConstQual { MulticornBaseQual base; Datum value; bool isnull; } MulticornConstQual; typedef struct MulticornVarQual { MulticornBaseQual base; AttrNumber rightvarattno; } MulticornVarQual; typedef struct MulticornParamQual { MulticornBaseQual base; Expr *expr; } MulticornParamQual; typedef struct MulticornDeparsedSortGroup { Name attname; int attnum; bool reversed; bool nulls_first; Name collate; PathKey *key; } MulticornDeparsedSortGroup; /* errors.c */ void errorCheck(void); /* python.c */ PyObject *pgstringToPyUnicode(const char *string); char **pyUnicodeToPgString(PyObject *pyobject); PyObject *getInstance(Oid foreigntableid); PyObject *qualToPyObject(Expr *expr, PlannerInfo *root); PyObject *getClassString(const char *className); PyObject *execute(ForeignScanState *state, ExplainState *es); void pythonResultToTuple(PyObject *p_value, TupleTableSlot *slot, ConversionInfo ** cinfos, StringInfo buffer); PyObject *tupleTableSlotToPyObject(TupleTableSlot *slot, ConversionInfo ** cinfos); char *getRowIdColumn(PyObject *fdw_instance); PyObject *optionsListToPyDict(List *options); const char *getPythonEncodingName(void); void getRelSize(MulticornPlanState * state, PlannerInfo *root, double *rows, int *width); List *pathKeys(MulticornPlanState * state); List *canSort(MulticornPlanState * state, List *deparsed); CacheEntry *getCacheEntry(Oid foreigntableid); UserMapping *multicorn_GetUserMapping(Oid userid, Oid serverid); /* Hash table mapping oid to fdw instances */ extern PGDLLIMPORT HTAB *InstancesHash; /* query.c */ void extractRestrictions(Relids base_relids, Expr *node, List **quals); List *extractColumns(List *reltargetlist, List *restrictinfolist); void initConversioninfo(ConversionInfo ** cinfo, AttInMetadata *attinmeta); Value *colnameFromVar(Var *var, PlannerInfo *root, MulticornPlanState * state); void computeDeparsedSortGroup(List *deparsed, MulticornPlanState *planstate, List **apply_pathkeys, List **deparsed_pathkeys); List *findPaths(PlannerInfo *root, RelOptInfo *baserel, List *possiblePaths, int startupCost, MulticornPlanState *state, List *apply_pathkeys, List *deparsed_pathkeys); List *deparse_sortgroup(PlannerInfo *root, Oid foreigntableid, RelOptInfo *rel); PyObject *datumToPython(Datum node, Oid typeoid, ConversionInfo * cinfo); List *serializeDeparsedSortGroup(List *pathkeys); List *deserializeDeparsedSortGroup(List *items); #endif /* PG_MULTICORN_H */ char *PyUnicode_AsPgString(PyObject *p_unicode); #if PY_MAJOR_VERSION >= 3 PyObject *PyString_FromString(const char *s); PyObject *PyString_FromStringAndSize(const char *s, Py_ssize_t size); char *PyString_AsString(PyObject *unicode); int PyString_AsStringAndSize(PyObject *unicode, char **tempbuffer, Py_ssize_t *length); #endif Multicorn-1.3.4/src/python.c000066400000000000000000001204341320447423600157600ustar00rootroot00000000000000#include #include "datetime.h" #include "postgres.h" #include "multicorn.h" #include "catalog/pg_user_mapping.h" #include "access/reloptions.h" #include "miscadmin.h" #include "utils/numeric.h" #include "utils/date.h" #include "utils/timestamp.h" #include "utils/array.h" #include "utils/catcache.h" #include "utils/memutils.h" #include "utils/resowner.h" #include "utils/rel.h" #include "utils/rel.h" #include "executor/nodeSubplan.h" #include "bytesobject.h" #include "mb/pg_wchar.h" #include "access/xact.h" #include "utils/lsyscache.h" List *getOptions(Oid foreigntableid); bool compareOptions(List *options1, List *options2); void getColumnsFromTable(TupleDesc desc, PyObject **p_columns, List **columns); bool compareColumns(List *columns1, List *columns2); PyObject *getClass(PyObject *className); PyObject *valuesToPySet(List *targetlist); PyObject *qualDefsToPyList(List *quallist, ConversionInfo ** cinfo); PyObject *pythonQual(char *operatorname, PyObject *value, ConversionInfo * cinfo, bool is_array, bool use_or, Oid typeoid); PyObject *getSortKey(MulticornDeparsedSortGroup *key); MulticornDeparsedSortGroup *getDeparsedSortGroup(PyObject *key); Datum pyobjectToDatum(PyObject *object, StringInfo buffer, ConversionInfo * cinfo); PyObject *qualdefToPython(MulticornConstQual * qualdef, ConversionInfo ** cinfo); PyObject *paramDefToPython(List *paramdef, ConversionInfo ** cinfos, Oid typeoid, Datum value); PyObject *datumToPython(Datum node, Oid typeoid, ConversionInfo * cinfo); PyObject *datumStringToPython(Datum node, ConversionInfo * cinfo); PyObject *datumNumberToPython(Datum node, ConversionInfo * cinfo); PyObject *datumDateToPython(Datum datum, ConversionInfo * cinfo); PyObject *datumTimestampToPython(Datum datum, ConversionInfo * cinfo); PyObject *datumIntToPython(Datum datum, ConversionInfo * cinfo); PyObject *datumArrayToPython(Datum datum, Oid type, ConversionInfo * cinfo); PyObject *datumByteaToPython(Datum datum, ConversionInfo * cinfo); PyObject *datumUnknownToPython(Datum datum, ConversionInfo * cinfo, Oid type); void pythonDictToTuple(PyObject *p_value, TupleTableSlot *slot, ConversionInfo ** cinfos, StringInfo buffer); void pythonSequenceToTuple(PyObject *p_value, TupleTableSlot *slot, ConversionInfo ** cinfos, StringInfo buffer); /* Python to cstring functions */ void pyobjectToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pynumberToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pyunicodeToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pystringToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pysequenceToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pymappingToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pydateToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void pyunknownToCstring(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo); void appendBinaryStringInfoQuote(StringInfo buffer, char *tempbuffer, Py_ssize_t strlength, bool need_quote); static void begin_remote_xact(CacheEntry * entry); /* * Get a (python) encoding name for an attribute. */ const char * getPythonEncodingName() { const char *encoding_name = GetDatabaseEncodingName(); if (strcmp(encoding_name, "SQL_ASCII") == 0) { encoding_name = "ascii"; } return encoding_name; } char * PyUnicode_AsPgString(PyObject *p_unicode) { Py_ssize_t unicode_size; char *message = NULL; PyObject *pTempStr; if (p_unicode == NULL) { elog(ERROR, "Received a null pointer in pyunicode_aspgstring"); } unicode_size = PyUnicode_GET_SIZE(p_unicode); pTempStr = PyUnicode_Encode(PyUnicode_AsUnicode(p_unicode), unicode_size, getPythonEncodingName(), NULL); errorCheck(); message = strdup(PyBytes_AsString(pTempStr)); errorCheck(); Py_DECREF(pTempStr); return message; } #if PY_MAJOR_VERSION >= 3 /* * Convert a C string in the PostgreSQL server encoding to a Python * unicode object. Reference ownership is passed to the caller. */ PyObject * PyString_FromStringAndSize(const char *s, Py_ssize_t size) { char *utf8string; PyObject *o; utf8string = (char *) pg_do_encoding_conversion((unsigned char *) s, strlen(s), GetDatabaseEncoding(), PG_UTF8); if (size < 0) { o = PyUnicode_FromString(utf8string); } else { o = PyUnicode_FromStringAndSize(utf8string, size); } if (utf8string != s) pfree(utf8string); return o; } PyObject * PyString_FromString(const char *s) { return PyString_FromStringAndSize(s, -1); } char * PyString_AsString(PyObject *unicode) { char *rv; PyObject *o = PyUnicode_AsEncodedString(unicode, GetDatabaseEncodingName(), NULL); errorCheck(); rv = pstrdup(PyBytes_AsString(o)); Py_XDECREF(o); return rv; } int PyString_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length) { PyObject *o; int rv; char *tempbuffer; if (PyUnicode_Check(obj)) { o = PyUnicode_AsEncodedString(obj, GetDatabaseEncodingName(), NULL); errorCheck(); rv = PyBytes_AsStringAndSize(o, &tempbuffer, length); *buffer = pstrdup(tempbuffer); Py_XDECREF(o); return rv; } return PyBytes_AsStringAndSize(obj, buffer, length); } #endif /* PY_MAJOR_VERSION >= 3 */ /* * Utility function responsible for importing, and returning, a class by name * * Returns a new reference to the class. */ PyObject * getClass(PyObject *className) { PyObject *p_multicorn = PyImport_ImportModule("multicorn"), *p_class = PyObject_CallMethod(p_multicorn, "get_class", "(O)", className); errorCheck(); Py_DECREF(p_multicorn); return p_class; } void appendBinaryStringInfoQuote(StringInfo buffer, char *tempbuffer, Py_ssize_t strlength, bool need_quote) { if (need_quote) { char *c; int i; appendStringInfoChar(buffer, '"'); for (c = tempbuffer, i = 0; i < strlength; ++i, ++c) { if (*c == '"') { appendBinaryStringInfo(buffer, "\\\"", 2); } else if (*c == '\\') { appendBinaryStringInfo(buffer, "\\\\", 2); } else { appendStringInfoChar(buffer, *c); } } appendStringInfoChar(buffer, '"'); } else { appendBinaryStringInfo(buffer, tempbuffer, strlength); } } /* * Convert a list of Value nodes containing the column name as a string to a * pyset of python unicode strings. */ PyObject * valuesToPySet(List *targetlist) { PyObject *result = PySet_New(0); ListCell *lc; foreach(lc, targetlist) { Value *value = (Value *) lfirst(lc); PyObject *pyString = PyString_FromString(strVal(value)); PySet_Add(result, pyString); Py_DECREF(pyString); } return result; } PyObject * qualDefsToPyList(List *qual_list, ConversionInfo ** cinfos) { ListCell *lc; PyObject *p_quals = PyList_New(0); foreach(lc, qual_list) { MulticornBaseQual *qual_def = (MulticornBaseQual *) lfirst(lc); if (qual_def->right_type == T_Const) { PyObject *python_qual = qualdefToPython((MulticornConstQual *) qual_def, cinfos); if (python_qual != NULL) { PyList_Append(p_quals, python_qual); Py_DECREF(python_qual); } } } return p_quals; } /* * Same as getClass, but accepts a C-String argument instead of a python * string. * * Returns a new reference to the class. */ PyObject * getClassString(const char *className) { PyObject *p_classname = PyString_FromString(className), *p_class = getClass(p_classname); Py_DECREF(p_classname); return p_class; } List * getOptions(Oid foreigntableid) { ForeignTable *f_table; ForeignServer *f_server; UserMapping *mapping; List *options; f_table = GetForeignTable(foreigntableid); f_server = GetForeignServer(f_table->serverid); options = NIL; options = list_concat(options, f_table->options); options = list_concat(options, f_server->options); /* An error might occur if no user mapping is defined. */ /* In that case, just ignore it */ mapping = multicorn_GetUserMapping(GetUserId(), f_table->serverid); if (mapping) options = list_concat(options, mapping->options); return options; } /* * Reimplementation of GetUserMapping, which returns NULL instead of throwing an * error when the mapping is not found. */ UserMapping * multicorn_GetUserMapping(Oid userid, Oid serverid) { Datum datum; HeapTuple tp; bool isnull; UserMapping *um; tp = SearchSysCache2(USERMAPPINGUSERSERVER, ObjectIdGetDatum(userid), ObjectIdGetDatum(serverid)); if (!HeapTupleIsValid(tp)) { /* Not found for the specific user -- try PUBLIC */ tp = SearchSysCache2(USERMAPPINGUSERSERVER, ObjectIdGetDatum(InvalidOid), ObjectIdGetDatum(serverid)); } if (!HeapTupleIsValid(tp)) return NULL; um = (UserMapping *) palloc(sizeof(UserMapping)); um->userid = userid; um->serverid = serverid; /* Extract the umoptions */ datum = SysCacheGetAttr(USERMAPPINGUSERSERVER, tp, Anum_pg_user_mapping_umoptions, &isnull); if (isnull) um->options = NIL; else um->options = untransformRelOptions(datum); ReleaseSysCache(tp); return um; } /* * Collect and validate options. * Only one option is required "wrapper". * * Returns a new reference to a dictionary. * * */ PyObject * optionsListToPyDict(List *options) { ListCell *lc; PyObject *p_options_dict = PyDict_New(); foreach(lc, options) { DefElem *def = (DefElem *) lfirst(lc); PyObject *pStr = PyString_FromString((char *) defGetString(def)); PyDict_SetItemString(p_options_dict, def->defname, pStr); Py_DECREF(pStr); } return p_options_dict; } bool compareOptions(List *options1, List *options2) { ListCell *lc1, *lc2; if (list_length(options1) != list_length(options2)) { return false; } forboth(lc1, options1, lc2, options2) { DefElem *def1 = (DefElem *) lfirst(lc1); DefElem *def2 = (DefElem *) lfirst(lc2); if (def1 == NULL || def2 == NULL || strcmp(def1->defname, def2->defname) != 0) { return false; } if (strcmp(defGetString(def1), defGetString(def2)) != 0) { return false; } } return true; } void getColumnsFromTable(TupleDesc desc, PyObject **p_columns, List **columns) { PyObject *columns_dict = *p_columns; List *columns_list = *columns; if ((columns_dict != NULL) && (columns_list != NULL)) { return; } else { int i; PyObject *p_columnclass = getClassString("multicorn." "ColumnDefinition"), *p_collections = PyImport_ImportModule("collections"), *p_dictclass = PyObject_GetAttrString(p_collections, "OrderedDict"); columns_dict = PyObject_CallFunction(p_dictclass, "()"); for (i = 0; i < desc->natts; i++) { Form_pg_attribute att = desc->attrs[i]; if (!att->attisdropped) { Oid typOid = att->atttypid; char *key = NameStr(att->attname); int32 typmod = att->atttypmod; char *base_type = format_type_be(typOid); char *modded_type = format_type_with_typemod(typOid, typmod); List *options = GetForeignColumnOptions(att->attrelid, att->attnum); PyObject *p_options = optionsListToPyDict(options); PyObject *column = PyObject_CallFunction(p_columnclass, "(s,i,i,s,s,O)", key, typOid, typmod, modded_type, base_type, p_options); List *columnDef = NULL; errorCheck(); columnDef = lappend(columnDef, makeString(key)); columnDef = lappend(columnDef, makeConst(TYPEOID, -1, InvalidOid, 4, ObjectIdGetDatum(typOid), false, true)); columnDef = lappend(columnDef, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(typmod), false, true)); columnDef = lappend(columnDef, options); columns_list = lappend(columns_list, columnDef); PyMapping_SetItemString(columns_dict, key, column); Py_DECREF(p_options); Py_DECREF(column); } } Py_DECREF(p_columnclass); Py_DECREF(p_collections); Py_DECREF(p_dictclass); errorCheck(); *p_columns = columns_dict; *columns = columns_list; } } bool compareColumns(List *columns1, List *columns2) { ListCell *lc1, *lc2; if (columns1->length != columns2->length) { return false; } forboth(lc1, columns1, lc2, columns2) { List *coldef1 = lfirst(lc1); List *coldef2 = lfirst(lc2); ListCell *cell1 = list_head(coldef1), *cell2 = list_head(coldef2); /* Compare column name */ if (strcmp(strVal(lfirst(cell1)), strVal(lfirst(cell2))) != 0) { return false; } cell1 = lnext(cell1); cell2 = lnext(cell2); /* Compare typoid */ if (((Const *) (lfirst(cell1)))->constvalue != ((Const *) lfirst(cell2))->constvalue) { return false; } cell1 = lnext(cell1); cell2 = lnext(cell2); /* Compare typmod */ if (((Const *) (lfirst(cell1)))->constvalue != ((Const *) lfirst(cell2))->constvalue) { return false; } cell1 = lnext(cell1); cell2 = lnext(cell2); /* Compare column options */ if (!compareOptions(lfirst(cell1), lfirst(cell2))) { return false; } } return true; } CacheEntry * getCacheEntry(Oid foreigntableid) { /* * create a temporary context. If we have to (re)create the python * instance, it will be promoted to a cachememorycontext. Otherwise, it * will be freed before returning the instance */ MemoryContext tempContext = AllocSetContextCreate(CurrentMemoryContext, "multicorn temporary data", ALLOCSET_SMALL_MINSIZE, ALLOCSET_SMALL_INITSIZE, ALLOCSET_SMALL_MAXSIZE), oldContext = MemoryContextSwitchTo(tempContext); CacheEntry *entry = NULL; bool found = false; List *options = getOptions(foreigntableid); List *columns = NULL; PyObject *p_columns = NULL; ForeignTable *ftable = GetForeignTable(foreigntableid); Relation rel = RelationIdGetRelation(ftable->relid); TupleDesc desc = rel->rd_att; bool needInitialization = false; entry = hash_search(InstancesHash, &foreigntableid, HASH_ENTER, &found); if (!found || entry->value == NULL) { entry->options = NULL; entry->columns = NULL; entry->cacheContext = NULL; entry->xact_depth = 0; needInitialization = true; } else { /* Even if found, we have to check several things */ if (!compareOptions(entry->options, options)) { /* Options have changed, we must purge the cache. */ Py_XDECREF(entry->value); needInitialization = true; } else { /* Options have not changed, we should look at columns. */ getColumnsFromTable(desc, &p_columns, &columns); if (!compareColumns(columns, entry->columns)) { Py_XDECREF(entry->value); needInitialization = true; } else { Py_XDECREF(p_columns); } } } if (needInitialization) { PyObject *p_options = optionsListToPyDict(options), *p_class = getClass(PyDict_GetItemString(p_options, "wrapper")), *p_instance; entry->value = NULL; getColumnsFromTable(desc, &p_columns, &columns); PyDict_DelItemString(p_options, "wrapper"); p_instance = PyObject_CallFunction(p_class, "(O,O)", p_options, p_columns); errorCheck(); /* Cleanup the old context, containing the old columns and options */ /* values */ if (entry->cacheContext != NULL) { MemoryContextDelete(entry->cacheContext); } /* Promote this tempcontext. */ MemoryContextSetParent(tempContext, CacheMemoryContext); entry->cacheContext = tempContext; entry->options = options; entry->columns = columns; entry->xact_depth = 0; Py_DECREF(p_class); Py_DECREF(p_options); Py_DECREF(p_columns); errorCheck(); entry->value = p_instance; MemoryContextSwitchTo(oldContext); } else { MemoryContextSwitchTo(oldContext); MemoryContextDelete(tempContext); } RelationClose(rel); Py_INCREF(entry->value); /* * Start a new transaction or subtransaction if needed. */ begin_remote_xact(entry); return entry; } /* * Returns the fdw_instance associated with the foreigntableid. * * For performance reasons, it is cached in hash table. */ PyObject * getInstance(Oid foreigntableid) { return getCacheEntry(foreigntableid)->value; } static void begin_remote_xact(CacheEntry * entry) { int curlevel = GetCurrentTransactionNestLevel(); PyObject *rv; /* Start main transaction if we haven't yet */ if (entry->xact_depth <= 0) { rv = PyObject_CallMethod(entry->value, "begin", "(i)", IsolationIsSerializable()); Py_XDECREF(rv); errorCheck(); entry->xact_depth = 1; } while (entry->xact_depth < curlevel) { entry->xact_depth++; rv = PyObject_CallMethod(entry->value, "sub_begin", "(i)", entry->xact_depth); Py_XDECREF(rv); errorCheck(); } } /* * Returns the relation estimated size, in term of number of rows and width. * This is done by calling the getRelSize python method. * */ void getRelSize(MulticornPlanState * state, PlannerInfo *root, double *rows, int *width) { PyObject *p_targets_set, *p_quals, *p_rows_and_width, *p_rows, *p_width, *p_startup_cost; p_targets_set = valuesToPySet(state->target_list); p_quals = qualDefsToPyList(state->qual_list, state->cinfos); p_rows_and_width = PyObject_CallMethod(state->fdw_instance, "get_rel_size", "(O,O)", p_quals, p_targets_set); errorCheck(); Py_DECREF(p_targets_set); Py_DECREF(p_quals); if ((p_rows_and_width == Py_None) || PyTuple_Size(p_rows_and_width) != 2) { Py_DECREF(p_rows_and_width); elog(ERROR, "The get_rel_size python method should return a tuple of length 2"); } p_rows = PyNumber_Long(PyTuple_GetItem(p_rows_and_width, 0)); p_width = PyNumber_Long(PyTuple_GetItem(p_rows_and_width, 1)); p_startup_cost = PyNumber_Long( PyObject_GetAttrString(state->fdw_instance, "_startup_cost")); *rows = PyLong_AsDouble(p_rows); *width = (int) PyLong_AsLong(p_width); state->startupCost = (int) PyLong_AsLong(p_startup_cost); Py_DECREF(p_rows); Py_DECREF(p_width); Py_DECREF(p_rows_and_width); } PyObject * qualdefToPython(MulticornConstQual * qualdef, ConversionInfo ** cinfos) { int arrayindex = qualdef->base.varattno - 1; char *operatorname = qualdef->base.opname; ConversionInfo *cinfo = cinfos[arrayindex]; bool is_array = qualdef->base.isArray, use_or = qualdef->base.useOr; Oid typeoid = qualdef->base.typeoid; Datum value = qualdef->value; PyObject *p_value; if (qualdef->isnull) { p_value = Py_None; Py_INCREF(Py_None); } else { if (typeoid == InvalidOid) { typeoid = cinfo->atttypoid; } p_value = datumToPython(value, typeoid, cinfo); if (p_value == NULL) { return NULL; } } if (typeoid <= 0) { typeoid = cinfo->atttypoid; } return pythonQual(operatorname, p_value, cinfo, is_array, use_or, typeoid); } PyObject * pythonQual(char *operatorname, PyObject *value, ConversionInfo * cinfo, bool is_array, bool use_or, Oid typeoid) { PyObject *qualClass = getClassString("multicorn.Qual"), *qualInstance, *p_operatorname, *operator, *columnName; p_operatorname = PyUnicode_Decode(operatorname, strlen(operatorname), getPythonEncodingName(), NULL); errorCheck(); if (is_array) { PyObject *arrayOpType; if (use_or) { arrayOpType = Py_True; } else { arrayOpType = Py_False; } operator = Py_BuildValue("(O, O)", p_operatorname, arrayOpType); Py_DECREF(p_operatorname); errorCheck(); } else { operator = p_operatorname; } columnName = PyUnicode_Decode(cinfo->attrname, strlen(cinfo->attrname), getPythonEncodingName(), NULL); qualInstance = PyObject_CallFunction(qualClass, "(O,O,O)", columnName, operator, value); errorCheck(); Py_DECREF(value); Py_DECREF(operator); Py_DECREF(qualClass); Py_DECREF(columnName); return qualInstance; } PyObject * getSortKey(MulticornDeparsedSortGroup *key) { PyObject *SortKeyClass = getClassString("multicorn.SortKey"), *SortKeyInstance, *p_attname, *p_reversed, *p_nulls_first, *p_collate; p_attname = PyUnicode_Decode(NameStr(*(key->attname)), strlen(NameStr(*(key->attname))), getPythonEncodingName(), NULL); if (key->reversed) p_reversed = Py_True; else p_reversed = Py_False; if (key->nulls_first) p_nulls_first = Py_True; else p_nulls_first = Py_False; if(key->collate == NULL){ p_collate = Py_None; Py_INCREF(p_collate); } else p_collate = PyUnicode_Decode(NameStr(*(key->collate)), strlen(NameStr(*(key->collate))), getPythonEncodingName(), NULL); SortKeyInstance = PyObject_CallFunction(SortKeyClass, "(O,i,O,O,O)", p_attname, key->attnum, p_reversed, p_nulls_first, p_collate); errorCheck(); Py_DECREF(p_attname); Py_DECREF(p_collate); Py_DECREF(SortKeyClass); return SortKeyInstance; } MulticornDeparsedSortGroup * getDeparsedSortGroup(PyObject *sortKey) { MulticornDeparsedSortGroup *md = palloc0(sizeof(MulticornDeparsedSortGroup)); PyObject * p_temp; p_temp = PyObject_GetAttrString(sortKey, "attname"); md->attname = (Name) strdup(PyUnicode_AS_DATA(p_temp)); Py_DECREF(p_temp); p_temp = PyObject_GetAttrString(sortKey, "attnum"); md->attnum = (int) PyLong_AsLong(p_temp); Py_DECREF(p_temp); p_temp = PyObject_GetAttrString(sortKey, "is_reversed"); md->reversed = PyObject_IsTrue(p_temp); Py_DECREF(p_temp); p_temp = PyObject_GetAttrString(sortKey, "nulls_first"); md->nulls_first = PyObject_IsTrue(PyObject_GetAttrString(sortKey, "nulls_first")); Py_DECREF(p_temp); p_temp = PyObject_GetAttrString(sortKey, "collate"); if(p_temp == Py_None) md->collate = 0; else md->collate = (Name) strdup(PyUnicode_AS_DATA(p_temp)); Py_DECREF(p_temp); return md; } /* * Execute the query in the python fdw, and returns an iterator. */ PyObject * execute(ForeignScanState *node, ExplainState *es) { MulticornExecState *state = node->fdw_state; PyObject *p_targets_set, *p_quals = PyList_New(0), *p_pathkeys = PyList_New(0), *p_iterable, *p_method; ListCell *lc; ExprContext *econtext = node->ss.ps.ps_ExprContext; foreach(lc, state->qual_list) { MulticornBaseQual *qual = lfirst(lc); MulticornConstQual *newqual = NULL; bool isNull; ExprState *expr_state = NULL; switch (qual->right_type) { case T_Param: expr_state = ExecInitExpr(((MulticornParamQual *) qual)->expr, (PlanState *) node); newqual = palloc0(sizeof(MulticornConstQual)); newqual->base.right_type = T_Const; newqual->base.varattno = qual->varattno; newqual->base.opname = qual->opname; newqual->base.isArray = qual->isArray; newqual->base.useOr = qual->useOr; #if PG_VERSION_NUM >= 100000 newqual->value = ExecEvalExpr(expr_state, econtext, &isNull); #else newqual->value = ExecEvalExpr(expr_state, econtext, &isNull, NULL); #endif newqual->base.typeoid = ((Param*) ((MulticornParamQual *) qual)->expr)->paramtype; newqual->isnull = isNull; break; case T_Const: newqual = (MulticornConstQual *) qual; break; default: break; } if (newqual != NULL) { PyObject *python_qual = qualdefToPython((MulticornConstQual *) newqual, state->cinfos); if (python_qual != NULL) { PyList_Append(p_quals, python_qual); Py_DECREF(python_qual); } } } /* Transform every object to a suitable python representation */ p_targets_set = valuesToPySet(state->target_list); foreach(lc, state->pathkeys) { MulticornDeparsedSortGroup *pathkey = (MulticornDeparsedSortGroup *) lfirst(lc); PyObject *python_sortkey = getSortKey(pathkey); PyList_Append(p_pathkeys, python_sortkey); Py_DECREF(python_sortkey); } { PyObject * args, * kwargs = PyDict_New(); if(PyList_Size(p_pathkeys) > 0){ PyDict_SetItemString(kwargs, "sortkeys", p_pathkeys); } if(es != NULL){ PyObject * verbose; if(es->verbose){ verbose = Py_True; } else { verbose = Py_False; } p_method = PyObject_GetAttrString(state->fdw_instance, "explain"); args = PyTuple_Pack(2, p_quals, p_targets_set); PyDict_SetItemString(kwargs, "verbose", verbose); errorCheck(); } else { p_method = PyObject_GetAttrString(state->fdw_instance, "execute"); errorCheck(); args = PyTuple_Pack(2, p_quals, p_targets_set); errorCheck(); } p_iterable = PyObject_Call(p_method, args, kwargs); errorCheck(); Py_DECREF(p_method); Py_DECREF(args); Py_DECREF(kwargs); } errorCheck(); if (p_iterable == Py_None){ state->p_iterator = p_iterable; } else { state->p_iterator = PyObject_GetIter(p_iterable); } Py_DECREF(p_quals); Py_DECREF(p_targets_set); Py_DECREF(p_pathkeys); Py_DECREF(p_iterable); errorCheck(); return state->p_iterator; } void pynumberToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { PyObject *pTempStr; char *tempbuffer; Py_ssize_t strlength = 0; pTempStr = PyObject_Str(pyobject); PyString_AsStringAndSize(pTempStr, &tempbuffer, &strlength); appendBinaryStringInfo(buffer, tempbuffer, strlength); Py_DECREF(pTempStr); } void pyunicodeToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { Py_ssize_t unicode_size; char *tempbuffer; Py_ssize_t strlength = 0; PyObject *pTempStr; unicode_size = PyUnicode_GET_SIZE(pyobject); pTempStr = PyUnicode_Encode(PyUnicode_AsUnicode(pyobject), unicode_size, getPythonEncodingName(), NULL); errorCheck(); PyBytes_AsStringAndSize(pTempStr, &tempbuffer, &strlength); appendBinaryStringInfoQuote(buffer, tempbuffer, strlength, cinfo->need_quote); Py_DECREF(pTempStr); } void pystringToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { char *tempbuffer; Py_ssize_t strlength = 0; if (PyString_AsStringAndSize(pyobject, &tempbuffer, &strlength) < 0) { ereport(WARNING, (errmsg("An error occured while decoding the %s column", cinfo->attrname), errhint("You should maybe return unicode instead?"))); } appendBinaryStringInfoQuote(buffer, tempbuffer, strlength, cinfo->need_quote); } void pysequenceToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { /* Its an array */ Py_ssize_t i, size = PySequence_Size(pyobject); PyObject *p_item; int previous_dims = cinfo->attndims, previous_needquote = cinfo->need_quote; if (cinfo->attndims == 0) { /* We are not supposed to be converted to an array. */ pyunknownToCstring(pyobject, buffer, cinfo); return; } appendStringInfoChar(buffer, '{'); /* We are an array, so we need to quote stuff */ cinfo->need_quote = true; cinfo->attndims = cinfo->attndims - 1; for (i = 0; i < size; i++) { p_item = PySequence_GetItem(pyobject, i); pyobjectToCString(p_item, buffer, cinfo); Py_DECREF(p_item); if (i != size - 1) { appendBinaryStringInfo(buffer, ", ", 2); } } appendStringInfoChar(buffer, '}'); cinfo->attndims = previous_dims; cinfo->need_quote = previous_needquote; } void pymappingToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { PyObject *items = PyMapping_Items(pyobject); PyObject *current_tuple; Py_ssize_t i; Py_ssize_t size = PyList_Size(items); bool need_quote = cinfo->need_quote; cinfo->need_quote = true; for (i = 0; i < size; i++) { current_tuple = PySequence_GetItem(items, i); pyobjectToCString(PyTuple_GetItem(current_tuple, 0), buffer, cinfo); appendBinaryStringInfo(buffer, "=>", 2); pyobjectToCString(PyTuple_GetItem(current_tuple, 1), buffer, cinfo); if (i != size - 1) { appendBinaryStringInfo(buffer, ", ", 2); } Py_DECREF(current_tuple); } Py_DECREF(items); cinfo->need_quote = need_quote; } void pydateToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { char *tempbuffer; Py_ssize_t strlength = 0; PyObject *formatted_date; formatted_date = PyObject_CallMethod(pyobject, "isoformat", "()"); PyString_AsStringAndSize(formatted_date, &tempbuffer, &strlength); appendBinaryStringInfo(buffer, tempbuffer, strlength); Py_DECREF(formatted_date); } void pyobjectToCString(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { if (pyobject == NULL || pyobject == Py_None) { return; } if (PyNumber_Check(pyobject)) { pynumberToCString(pyobject, buffer, cinfo); return; } if (PyUnicode_Check(pyobject)) { pyunicodeToCString(pyobject, buffer, cinfo); return; } if (PyBytes_Check(pyobject)) { pystringToCString(pyobject, buffer, cinfo); return; } if (PySequence_Check(pyobject)) { pysequenceToCString(pyobject, buffer, cinfo); return; } if (PyMapping_Check(pyobject)) { pymappingToCString(pyobject, buffer, cinfo); return; } PyDateTime_IMPORT; if (PyDate_Check(pyobject)) { pydateToCString(pyobject, buffer, cinfo); return; } pyunknownToCstring(pyobject, buffer, cinfo); } void pyunknownToCstring(PyObject *pyobject, StringInfo buffer, ConversionInfo * cinfo) { PyObject *pTempStr = PyObject_Str(pyobject); char *tempbuffer; Py_ssize_t strlength; PyString_AsStringAndSize(pTempStr, &tempbuffer, &strlength); errorCheck(); appendBinaryStringInfoQuote(buffer, tempbuffer, strlength, cinfo->need_quote); Py_DECREF(pTempStr); return; } void pythonDictToTuple(PyObject *p_value, TupleTableSlot *slot, ConversionInfo ** cinfos, StringInfo buffer) { int i; PyObject *p_object; Datum *values = slot->tts_values; bool *nulls = slot->tts_isnull; for (i = 0; i < slot->tts_tupleDescriptor->natts; i++) { char *key; Form_pg_attribute attr = slot->tts_tupleDescriptor->attrs[i]; AttrNumber cinfo_idx = attr->attnum - 1; if (cinfos[cinfo_idx] == NULL) { continue; } key = cinfos[cinfo_idx]->attrname; p_object = PyMapping_GetItemString(p_value, key); if (p_object != NULL && p_object != Py_None) { resetStringInfo(buffer); values[i] = pyobjectToDatum(p_object, buffer, cinfos[cinfo_idx]); if (buffer->data == NULL) { nulls[i] = true; } else { nulls[i] = false; } } else { /* "KeyError", doesnt matter. */ PyErr_Clear(); values[i] = (Datum) NULL; nulls[i] = true; } Py_XDECREF(p_object); } } void pythonSequenceToTuple(PyObject *p_value, TupleTableSlot *slot, ConversionInfo ** cinfos, StringInfo buffer) { int i, j; Datum *values = slot->tts_values; bool *nulls = slot->tts_isnull; for (i = 0, j = 0; i < slot->tts_tupleDescriptor->natts; i++) { PyObject *p_object; Form_pg_attribute attr = slot->tts_tupleDescriptor->attrs[i]; AttrNumber cinfo_idx = attr->attnum - 1; if (cinfos[cinfo_idx] == NULL) { continue; } p_object = PySequence_GetItem(p_value, j); if(p_object == NULL || p_object == Py_None){ nulls[i] = true; values[i] = 0; continue; } resetStringInfo(buffer); values[i] = pyobjectToDatum(p_object, buffer, cinfos[cinfo_idx]); if (buffer->data == NULL) { nulls[i] = true; } else { nulls[i] = false; } errorCheck(); Py_DECREF(p_object); j++; } } /* * Convert a python result (a sequence or a dictionary) to a tupletableslot. */ void pythonResultToTuple(PyObject *p_value, TupleTableSlot *slot, ConversionInfo ** cinfos, StringInfo buffer) { if (PySequence_Check(p_value)) { pythonSequenceToTuple(p_value, slot, cinfos, buffer); } else { if (PyMapping_Check(p_value)) { pythonDictToTuple(p_value, slot, cinfos, buffer); } else { elog(ERROR, "Cannot transform anything else than mappings and" "sequences to rows"); } } } Datum pyobjectToDatum(PyObject *object, StringInfo buffer, ConversionInfo * cinfo) { Datum value = 0; pyobjectToCString(object, buffer, cinfo); if (buffer->len >= 0) { if (cinfo->atttypoid == BYTEAOID || cinfo->atttypoid == TEXTOID || cinfo->atttypoid == VARCHAROID) { /* * Special case, since the value is already a byte string. */ value = PointerGetDatum(cstring_to_text_with_len(buffer->data, buffer->len)); } else { value = InputFunctionCall(cinfo->attinfunc, buffer->data, cinfo->attioparam, cinfo->atttypmod); } } return value; } PyObject * datumStringToPython(Datum datum, ConversionInfo * cinfo) { char *temp; ssize_t size; PyObject *result; temp = datum == 0 ? "?" : TextDatumGetCString(datum); size = strlen(temp); result = PyUnicode_Decode(temp, size, getPythonEncodingName(), NULL); return result; } PyObject * datumUnknownToPython(Datum datum, ConversionInfo * cinfo, Oid type) { char *temp; ssize_t size; PyObject *result; Oid outfuncoid; bool isvarlena; FmgrInfo *fmout = palloc0(sizeof(FmgrInfo)); getTypeOutputInfo(type, &outfuncoid, &isvarlena); fmgr_info(outfuncoid, fmout); temp = OutputFunctionCall(fmout, datum); size = strlen(temp); result = PyUnicode_Decode(temp, size, getPythonEncodingName(), NULL); pfree(fmout); return result; } PyObject * datumNumberToPython(Datum datum, ConversionInfo * cinfo) { ssize_t numvalue = (ssize_t) DatumGetNumeric(datum); char *tempvalue = (char *) DirectFunctionCall1(numeric_out, numvalue); PyObject *buffer = PyString_FromString(tempvalue), #if PY_MAJOR_VERSION >= 3 *value = PyFloat_FromString(buffer); #else *value = PyFloat_FromString(buffer, NULL); #endif Py_DECREF(buffer); return value; } PyObject * datumDateToPython(Datum datum, ConversionInfo * cinfo) { struct pg_tm *pg_tm_value = palloc(sizeof(struct pg_tm)); PyObject *result; fsec_t fsec; PyDateTime_IMPORT; datum = DirectFunctionCall1(date_timestamp, datum); timestamp2tm(DatumGetTimestamp(datum), NULL, pg_tm_value, &fsec, NULL, NULL); result = PyDate_FromDate(pg_tm_value->tm_year, pg_tm_value->tm_mon, pg_tm_value->tm_mday); pfree(pg_tm_value); return result; } PyObject * datumTimestampToPython(Datum datum, ConversionInfo * cinfo) { struct pg_tm *pg_tm_value = palloc(sizeof(struct pg_tm)); PyObject *result; fsec_t fsec; PyDateTime_IMPORT; timestamp2tm(DatumGetTimestamp(datum), NULL, pg_tm_value, &fsec, NULL, NULL); result = PyDateTime_FromDateAndTime(pg_tm_value->tm_year, pg_tm_value->tm_mon, pg_tm_value->tm_mday, pg_tm_value->tm_hour, pg_tm_value->tm_min, pg_tm_value->tm_sec, 0); pfree(pg_tm_value); return result; } PyObject * datumIntToPython(Datum datum, ConversionInfo * cinfo) { return PyLong_FromLong(DatumGetInt32(datum)); } PyObject * datumArrayToPython(Datum datum, Oid type, ConversionInfo * cinfo) { #if PG_VERSION_NUM >= 90500 ArrayIterator iterator = array_create_iterator(DatumGetArrayTypeP(datum), 0, NULL); # else ArrayIterator iterator = array_create_iterator(DatumGetArrayTypeP(datum), 0); # endif Datum elem = (Datum) NULL; bool isnull; PyObject *result = PyList_New(0), *pyitem; while (array_iterate(iterator, &elem, &isnull)) { if (isnull) { PyList_Append(result, Py_None); } else { HeapTuple tuple; Form_pg_type typeStruct; tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type)); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "lookup failed for type %u", type); } typeStruct = (Form_pg_type) GETSTRUCT(tuple); pyitem = datumToPython(elem, typeStruct->typelem, cinfo); ReleaseSysCache(tuple); PyList_Append(result, pyitem); Py_DECREF(pyitem); } } return result; } PyObject * datumByteaToPython(Datum datum, ConversionInfo * cinfo) { text *txt = DatumGetByteaP(datum); char *str = txt == NULL ? "?" : VARDATA(txt); size_t size = VARSIZE(txt) - VARHDRSZ; #if PY_MAJOR_VERSION >= 3 return PyBytes_FromStringAndSize(str, size); #else return PyString_FromStringAndSize(str, size); #endif } PyObject * datumToPython(Datum datum, Oid type, ConversionInfo * cinfo) { HeapTuple tuple; Form_pg_type typeStruct; switch (type) { case BYTEAOID: return datumByteaToPython(datum, cinfo); case TEXTOID: case VARCHAROID: return datumStringToPython(datum, cinfo); case NUMERICOID: return datumNumberToPython(datum, cinfo); case DATEOID: return datumDateToPython(datum, cinfo); case TIMESTAMPOID: return datumTimestampToPython(datum, cinfo); case INT4OID: return datumIntToPython(datum, cinfo); default: /* Case for the array ? */ tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type)); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "lookup failed for type %u", type); } typeStruct = (Form_pg_type) GETSTRUCT(tuple); ReleaseSysCache(tuple); if ((typeStruct->typelem != 0) && (typeStruct->typlen == -1)) { /* Its an array. */ return datumArrayToPython(datum, type, cinfo); } return datumUnknownToPython(datum, cinfo, type); } } /* * Call the path_keys method from the python implementation, and convert the * result to a list of "tuples" (list) of the form: * * - Bitmapset of attnums - Cost (integer) */ List * pathKeys(MulticornPlanState * state) { List *result = NULL; Py_ssize_t i; PyObject *fdw_instance = state->fdw_instance, *p_pathkeys; p_pathkeys = PyObject_CallMethod(fdw_instance, "get_path_keys", "()"); errorCheck(); for (i = 0; i < PySequence_Length(p_pathkeys); i++) { PyObject *p_item = PySequence_GetItem(p_pathkeys, i), *p_keys = PySequence_GetItem(p_item, 0), *p_cost = PySequence_GetItem(p_item, 1), *p_cost_long = PyNumber_Long(p_cost); double rows = PyLong_AsDouble(p_cost_long); ssize_t j; List *attnums = NULL; List *item = NULL; for (j = 0; j < PySequence_Length(p_keys); j++) { PyObject *p_key = PySequence_GetItem(p_keys, j); ssize_t k; /* Lookup the attribute number by its key. */ for (k = 0; k < state->numattrs; k++) { ConversionInfo *cinfo = state->cinfos[k]; if (cinfo == NULL) { continue; } if (p_key != Py_None && strcmp(cinfo->attrname, PyString_AsString(p_key)) == 0) { attnums = list_append_unique_int(attnums, cinfo->attnum); break; } } Py_DECREF(p_key); } item = lappend(item, attnums); item = lappend(item, makeConst(INT4OID, -1, InvalidOid, 4, rows, false, true)); result = lappend(result, item); Py_DECREF(p_keys); Py_DECREF(p_cost); Py_DECREF(p_cost_long); Py_DECREF(p_item); } Py_DECREF(p_pathkeys); return result; } /* * Call the can_sort method from the python implementation. We provide a deparsed * version of the requested fields to sort with all detail as needed (nulls, * collate...), and convert the result to a list of "tuples" (list) of the form: * * - Bitmapset of attnums * * representing the fields that the foreign data wrapper can be sort as * we requested. */ List * canSort(MulticornPlanState * state, List *deparsed) { List *result = NULL; ListCell *lc; Py_ssize_t i; PyObject *fdw_instance = state->fdw_instance, *p_pathkeys = PyList_New(0), *p_sortable; foreach(lc, deparsed) { MulticornDeparsedSortGroup *pathkey = (MulticornDeparsedSortGroup *) lfirst(lc); PyObject *python_sortkey = getSortKey(pathkey); PyList_Append(p_pathkeys, python_sortkey); Py_DECREF(python_sortkey); } p_sortable = PyObject_CallMethod(fdw_instance, "can_sort", "(O)", p_pathkeys); errorCheck(); for (i = 0; i < PySequence_Length(p_sortable); i++) { PyObject *p_key = PySequence_GetItem(p_sortable, i); MulticornDeparsedSortGroup *md = getDeparsedSortGroup(p_key); result = lappend(result, md); Py_DECREF(p_key); } Py_DECREF(p_pathkeys); Py_DECREF(p_sortable); return result; } PyObject * tupleTableSlotToPyObject(TupleTableSlot *slot, ConversionInfo ** cinfos) { PyObject *result = PyDict_New(); TupleDesc tupdesc = slot->tts_tupleDescriptor; int i; for (i = 0; i < tupdesc->natts; i++) { Form_pg_attribute attr = tupdesc->attrs[i]; bool isnull; Datum value; PyObject *item; AttrNumber cinfo_idx = attr->attnum - 1; if (attr->attisdropped || cinfos[cinfo_idx] == NULL) { continue; } value = slot_getattr(slot, i + 1, &isnull); if (isnull) { item = Py_None; Py_INCREF(item); } else { item = datumToPython(value, cinfos[cinfo_idx]->atttypoid, cinfos[cinfo_idx]); errorCheck(); } PyDict_SetItemString(result, cinfos[cinfo_idx]->attrname, item); Py_DECREF(item); } return result; } /* * Get the rowid column name */ char * getRowIdColumn(PyObject *fdw_instance) { PyObject *value = PyObject_GetAttrString(fdw_instance, "rowid_column"); char *result; errorCheck(); if (value == Py_None) { Py_DECREF(value); elog(ERROR, "This FDW does not support the writable API"); } result = PyString_AsString(value); Py_DECREF(value); return result; } Multicorn-1.3.4/src/query.c000066400000000000000000000501251320447423600156030ustar00rootroot00000000000000#include "multicorn.h" #include "optimizer/var.h" #include "optimizer/clauses.h" #include "optimizer/pathnode.h" #include "optimizer/subselect.h" #include "catalog/pg_collation.h" #include "catalog/pg_database.h" #include "catalog/pg_operator.h" #include "mb/pg_wchar.h" #include "utils/lsyscache.h" #include "miscadmin.h" #include "parser/parsetree.h" void extractClauseFromOpExpr(Relids base_relids, OpExpr *node, List **quals); void extractClauseFromNullTest(Relids base_relids, NullTest *node, List **quals); void extractClauseFromScalarArrayOpExpr(Relids base_relids, ScalarArrayOpExpr *node, List **quals); char *getOperatorString(Oid opoid); MulticornBaseQual *makeQual(AttrNumber varattno, char *opname, Expr *value, bool isarray, bool useOr); Node *unnestClause(Node *node); void swapOperandsAsNeeded(Node **left, Node **right, Oid *opoid, Relids base_relids); OpExpr *canonicalOpExpr(OpExpr *opExpr, Relids base_relids); ScalarArrayOpExpr *canonicalScalarArrayOpExpr(ScalarArrayOpExpr *opExpr, Relids base_relids); bool isAttrInRestrictInfo(Index relid, AttrNumber attno, RestrictInfo *restrictinfo); List *clausesInvolvingAttr(Index relid, AttrNumber attnum, EquivalenceClass *eq_class); Expr *multicorn_get_em_expr(EquivalenceClass *ec, RelOptInfo *rel); /* * The list of needed columns (represented by their respective vars) * is pulled from: * - the targetcolumns * - the restrictinfo */ List * extractColumns(List *reltargetlist, List *restrictinfolist) { ListCell *lc; List *columns = NULL; int i = 0; foreach(lc, reltargetlist) { List *targetcolumns; Node *node = (Node *) lfirst(lc); targetcolumns = pull_var_clause(node, #if PG_VERSION_NUM >= 90600 PVC_RECURSE_AGGREGATES| PVC_RECURSE_PLACEHOLDERS); #else PVC_RECURSE_AGGREGATES, PVC_RECURSE_PLACEHOLDERS); #endif columns = list_union(columns, targetcolumns); i++; } foreach(lc, restrictinfolist) { List *targetcolumns; RestrictInfo *node = (RestrictInfo *) lfirst(lc); targetcolumns = pull_var_clause((Node *) node->clause, #if PG_VERSION_NUM >= 90600 PVC_RECURSE_AGGREGATES| PVC_RECURSE_PLACEHOLDERS); #else PVC_RECURSE_AGGREGATES, PVC_RECURSE_PLACEHOLDERS); #endif columns = list_union(columns, targetcolumns); } return columns; } /* * Initialize the array of "ConversionInfo" elements, needed to convert python * objects back to suitable postgresql data structures. */ void initConversioninfo(ConversionInfo ** cinfos, AttInMetadata *attinmeta) { int i; for (i = 0; i < attinmeta->tupdesc->natts; i++) { Form_pg_attribute attr = attinmeta->tupdesc->attrs[i]; Oid outfuncoid; bool typIsVarlena; if (!attr->attisdropped) { ConversionInfo *cinfo = palloc0(sizeof(ConversionInfo)); cinfo->attoutfunc = (FmgrInfo *) palloc0(sizeof(FmgrInfo)); getTypeOutputInfo(attr->atttypid, &outfuncoid, &typIsVarlena); fmgr_info(outfuncoid, cinfo->attoutfunc); cinfo->atttypoid = attr->atttypid; cinfo->atttypmod = attinmeta->atttypmods[i]; cinfo->attioparam = attinmeta->attioparams[i]; cinfo->attinfunc = &attinmeta->attinfuncs[i]; cinfo->attrname = NameStr(attr->attname); cinfo->attnum = i + 1; cinfo->attndims = attr->attndims; cinfo->need_quote = false; cinfos[i] = cinfo; } else { cinfos[i] = NULL; } } } char * getOperatorString(Oid opoid) { HeapTuple tp; Form_pg_operator operator; tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opoid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for operator %u", opoid); operator = (Form_pg_operator) GETSTRUCT(tp); ReleaseSysCache(tp); return NameStr(operator->oprname); } /* * Returns the node of interest from a node. */ Node * unnestClause(Node *node) { switch (node->type) { case T_RelabelType: return (Node *) ((RelabelType *) node)->arg; case T_ArrayCoerceExpr: return (Node *) ((ArrayCoerceExpr *) node)->arg; default: return node; } } void swapOperandsAsNeeded(Node **left, Node **right, Oid *opoid, Relids base_relids) { HeapTuple tp; Form_pg_operator op; Node *l = *left, *r = *right; tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(*opoid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for operator %u", *opoid); op = (Form_pg_operator) GETSTRUCT(tp); ReleaseSysCache(tp); /* Right is already a var. */ /* If "left" is a Var from another rel, and right is a Var from the */ /* target rel, swap them. */ /* Same thing is left is not a var at all. */ /* To swap them, we have to lookup the commutator operator. */ if (IsA(r, Var)) { Var *rvar = (Var *) r; if (!IsA(l, Var) || (!bms_is_member(((Var *) l)->varno, base_relids) && bms_is_member(rvar->varno, base_relids))) { /* If the operator has no commutator operator, */ /* bail out. */ if (op->oprcom == 0) { return; } { *left = r; *right = l; *opoid = op->oprcom; } } } } /* * Swaps the operands if needed / possible, so that left is always a node * belonging to the baserel and right is either: * - a Const * - a Param * - a Var from another relation */ OpExpr * canonicalOpExpr(OpExpr *opExpr, Relids base_relids) { Oid operatorid = opExpr->opno; Node *l, *r; OpExpr *result = NULL; /* Only treat binary operators for now. */ if (list_length(opExpr->args) == 2) { l = unnestClause(list_nth(opExpr->args, 0)); r = unnestClause(list_nth(opExpr->args, 1)); swapOperandsAsNeeded(&l, &r, &operatorid, base_relids); if (IsA(l, Var) &&bms_is_member(((Var *) l)->varno, base_relids) && ((Var *) l)->varattno >= 1) { result = (OpExpr *) make_opclause(operatorid, opExpr->opresulttype, opExpr->opretset, (Expr *) l, (Expr *) r, opExpr->opcollid, opExpr->inputcollid); } } return result; } /* * Swaps the operands if needed / possible, so that left is always a node * belonging to the baserel and right is either: * - a Const * - a Param * - a Var from another relation */ ScalarArrayOpExpr * canonicalScalarArrayOpExpr(ScalarArrayOpExpr *opExpr, Relids base_relids) { Oid operatorid = opExpr->opno; Node *l, *r; ScalarArrayOpExpr *result = NULL; HeapTuple tp; Form_pg_operator op; /* Only treat binary operators for now. */ if (list_length(opExpr->args) == 2) { l = unnestClause(list_nth(opExpr->args, 0)); r = unnestClause(list_nth(opExpr->args, 1)); tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(operatorid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for operator %u", operatorid); op = (Form_pg_operator) GETSTRUCT(tp); ReleaseSysCache(tp); if (IsA(l, Var) &&bms_is_member(((Var *) l)->varno, base_relids) && ((Var *) l)->varattno >= 1) { result = makeNode(ScalarArrayOpExpr); result->opno = operatorid; result->opfuncid = op->oprcode; result->useOr = opExpr->useOr; result->args = lappend(result->args, l); result->args = lappend(result->args, r); result->location = opExpr->location; } } return result; } /* * Extract conditions that can be pushed down, as well as the parameters. * */ void extractRestrictions(Relids base_relids, Expr *node, List **quals) { switch (nodeTag(node)) { case T_OpExpr: extractClauseFromOpExpr(base_relids, (OpExpr *) node, quals); break; case T_NullTest: extractClauseFromNullTest(base_relids, (NullTest *) node, quals); break; case T_ScalarArrayOpExpr: extractClauseFromScalarArrayOpExpr(base_relids, (ScalarArrayOpExpr *) node, quals); break; default: { ereport(WARNING, (errmsg("unsupported expression for " "extractClauseFrom"), errdetail("%s", nodeToString(node)))); } break; } } /* * Build an intermediate value representation for an OpExpr, * and append it to the corresponding list (quals, or params). * * The quals list consist of list of the form: * * - Const key: the column index in the cinfo array * - Const operator: the operator representation * - Var or Const value: the value. */ void extractClauseFromOpExpr(Relids base_relids, OpExpr *op, List **quals) { Var *left; Expr *right; /* Use a "canonical" version of the op expression, to ensure that the */ /* left operand is a Var on our relation. */ op = canonicalOpExpr(op, base_relids); if (op) { left = list_nth(op->args, 0); right = list_nth(op->args, 1); /* Do not add it if it either contains a mutable function, or makes */ /* self references in the right hand side. */ if (!(contain_volatile_functions((Node *) right) || bms_is_subset(base_relids, pull_varnos((Node *) right)))) { *quals = lappend(*quals, makeQual(left->varattno, getOperatorString(op->opno), right, false, false)); } } } void extractClauseFromScalarArrayOpExpr(Relids base_relids, ScalarArrayOpExpr *op, List **quals) { Var *left; Expr *right; op = canonicalScalarArrayOpExpr(op, base_relids); if (op) { left = list_nth(op->args, 0); right = list_nth(op->args, 1); if (!(contain_volatile_functions((Node *) right) || bms_is_subset(base_relids, pull_varnos((Node *) right)))) { *quals = lappend(*quals, makeQual(left->varattno, getOperatorString(op->opno), right, true, op->useOr)); } } } /* * Convert a "NullTest" (IS NULL, or IS NOT NULL) * to a suitable intermediate representation. */ void extractClauseFromNullTest(Relids base_relids, NullTest *node, List **quals) { if (IsA(node->arg, Var)) { Var *var = (Var *) node->arg; MulticornBaseQual *result; char *opname = NULL; if (var->varattno < 1) { return; } if (node->nulltesttype == IS_NULL) { opname = "="; } else { opname = "<>"; } result = makeQual(var->varattno, opname, (Expr *) makeNullConst(INT4OID, -1, InvalidOid), false, false); *quals = lappend(*quals, result); } } /* * Returns a "Value" node containing the string name of the column from a var. */ Value * colnameFromVar(Var *var, PlannerInfo *root, MulticornPlanState * planstate) { RangeTblEntry *rte = rte = planner_rt_fetch(var->varno, root); char *attname = get_attname(rte->relid, var->varattno); if (attname == NULL) { return NULL; } else { return makeString(attname); } } /* * Build an opaque "qual" object. */ MulticornBaseQual * makeQual(AttrNumber varattno, char *opname, Expr *value, bool isarray, bool useOr) { MulticornBaseQual *qual; switch (value->type) { case T_Const: qual = palloc0(sizeof(MulticornConstQual)); qual->right_type = T_Const; qual->typeoid = ((Const *) value)->consttype; ((MulticornConstQual *) qual)->value = ((Const *) value)->constvalue; ((MulticornConstQual *) qual)->isnull = ((Const *) value)->constisnull; break; case T_Var: qual = palloc0(sizeof(MulticornVarQual)); qual->right_type = T_Var; ((MulticornVarQual *) qual)->rightvarattno = ((Var *) value)->varattno; break; default: qual = palloc0(sizeof(MulticornParamQual)); qual->right_type = T_Param; ((MulticornParamQual *) qual)->expr = value; qual->typeoid = InvalidOid; break; } qual->varattno = varattno; qual->opname = opname; qual->isArray = isarray; qual->useOr = useOr; return qual; } /* * Test wheter an attribute identified by its relid and attno * is present in a list of restrictinfo */ bool isAttrInRestrictInfo(Index relid, AttrNumber attno, RestrictInfo *restrictinfo) { List *vars = pull_var_clause((Node *) restrictinfo->clause, #if PG_VERSION_NUM >= 90600 PVC_RECURSE_AGGREGATES| PVC_RECURSE_PLACEHOLDERS); #else PVC_RECURSE_AGGREGATES, PVC_RECURSE_PLACEHOLDERS); #endif ListCell *lc; foreach(lc, vars) { Var *var = (Var *) lfirst(lc); if (var->varno == relid && var->varattno == attno) { return true; } } return false; } List * clausesInvolvingAttr(Index relid, AttrNumber attnum, EquivalenceClass *ec) { List *clauses = NULL; /* * If there is only one member, then the equivalence class is either for * an outer join, or a desired sort order. So we better leave it * untouched. */ if (ec->ec_members->length > 1) { ListCell *ri_lc; foreach(ri_lc, ec->ec_sources) { RestrictInfo *ri = (RestrictInfo *) lfirst(ri_lc); if (isAttrInRestrictInfo(relid, attnum, ri)) { clauses = lappend(clauses, ri); } } } return clauses; } /* * Given a list of MulticornDeparsedSortGroup and a MulticornPlanState, * construct a list of PathKey and MulticornDeparsedSortGroup that belongs to * the FDW and that the FDW say it can enforce. */ void computeDeparsedSortGroup(List *deparsed, MulticornPlanState *planstate, List **apply_pathkeys, List **deparsed_pathkeys) { List *sortable_fields = NULL; ListCell *lc, *lc2; /* Both lists should be empty */ Assert(*apply_pathkeys == NIL); Assert(*deparsed_pathkeys == NIL); /* Don't ask FDW if nothing to sort */ if (deparsed == NIL) return; sortable_fields = canSort(planstate, deparsed); /* Don't go further if FDW can't enforce any sort */ if (sortable_fields == NIL) return; foreach(lc, sortable_fields) { MulticornDeparsedSortGroup *sortable_md = (MulticornDeparsedSortGroup *) lfirst(lc); foreach(lc2, deparsed) { MulticornDeparsedSortGroup *wanted_md = lfirst(lc2); if (sortable_md->attnum == wanted_md->attnum) { *apply_pathkeys = lappend(*apply_pathkeys, wanted_md->key); *deparsed_pathkeys = lappend(*deparsed_pathkeys, wanted_md); } } } } List * findPaths(PlannerInfo *root, RelOptInfo *baserel, List *possiblePaths, int startupCost, MulticornPlanState *state, List *apply_pathkeys, List *deparsed_pathkeys) { List *result = NULL; ListCell *lc; foreach(lc, possiblePaths) { List *item = lfirst(lc); List *attrnos = linitial(item); ListCell *attno_lc; int nbrows = ((Const *) lsecond(item))->constvalue; List *allclauses = NULL; Bitmapset *outer_relids = NULL; /* Armed with this knowledge, look for a join condition */ /* matching the path list. */ /* Every key must be present in either, a join clause or an */ /* equivalence_class. */ foreach(attno_lc, attrnos) { AttrNumber attnum = lfirst_int(attno_lc); ListCell *lc; List *clauses = NULL; /* Look in the equivalence classes. */ foreach(lc, root->eq_classes) { EquivalenceClass *ec = (EquivalenceClass *) lfirst(lc); List *ec_clauses = clausesInvolvingAttr(baserel->relid, attnum, ec); clauses = list_concat(clauses, ec_clauses); if (ec_clauses != NIL) { outer_relids = bms_union(outer_relids, ec->ec_relids); } } /* Do the same thing for the outer joins */ foreach(lc, list_union(root->left_join_clauses, root->right_join_clauses)) { RestrictInfo *ri = (RestrictInfo *) lfirst(lc); if (isAttrInRestrictInfo(baserel->relid, attnum, ri)) { clauses = lappend(clauses, ri); outer_relids = bms_union(outer_relids, ri->outer_relids); } } /* We did NOT find anything for this key, bail out */ if (clauses == NIL) { allclauses = NULL; break; } else { allclauses = list_concat(allclauses, clauses); } } /* Every key has a corresponding restriction, we can build */ /* the parameterized path and add it to the plan. */ if (allclauses != NIL) { Bitmapset *req_outer = bms_difference(outer_relids, bms_make_singleton(baserel->relid)); ParamPathInfo *ppi; ForeignPath *foreignPath; if (!bms_is_empty(req_outer)) { ppi = makeNode(ParamPathInfo); ppi->ppi_req_outer = req_outer; ppi->ppi_rows = nbrows; ppi->ppi_clauses = list_concat(ppi->ppi_clauses, allclauses); /* Add a simple parameterized path */ foreignPath = create_foreignscan_path( root, baserel, #if PG_VERSION_NUM >= 90600 NULL, /* default pathtarget */ #endif nbrows, startupCost, #if PG_VERSION_NUM >= 90600 nbrows * baserel->reltarget->width, #else nbrows * baserel->width, #endif NIL, /* no pathkeys */ NULL, #if PG_VERSION_NUM >= 90500 NULL, #endif NULL); foreignPath->path.param_info = ppi; result = lappend(result, foreignPath); } } } return result; } /* * Deparse a list of PathKey and return a list of MulticornDeparsedSortGroup. * This function will return data iif all the PathKey belong to the current * foreign table. */ List * deparse_sortgroup(PlannerInfo *root, Oid foreigntableid, RelOptInfo *rel) { List *result = NULL; ListCell *lc; /* return empty list if no pathkeys for the PlannerInfo */ if (! root->query_pathkeys) return NIL; foreach(lc,root->query_pathkeys) { PathKey *key = (PathKey *) lfirst(lc); MulticornDeparsedSortGroup *md = palloc0(sizeof(MulticornDeparsedSortGroup)); EquivalenceClass *ec = key->pk_eclass; Expr *expr; bool found = false; if ((expr = multicorn_get_em_expr(ec, rel))) { md->reversed = (key->pk_strategy == BTGreaterStrategyNumber); md->nulls_first = key->pk_nulls_first; md->key = key; if (IsA(expr, Var)) { Var *var = (Var *) expr; md->attname = (Name) strdup(get_attname(foreigntableid, var->varattno)); md->attnum = var->varattno; found = true; } /* ORDER BY clauses having a COLLATE option will be RelabelType */ else if (IsA(expr, RelabelType) && IsA(((RelabelType *) expr)->arg, Var)) { Var *var = (Var *)((RelabelType *) expr)->arg; Oid collid = ((RelabelType *) expr)->resultcollid; if (collid == DEFAULT_COLLATION_OID) md->collate = NULL; else md->collate = (Name) strdup(get_collation_name(collid)); md->attname = (Name) strdup(get_attname(foreigntableid, var->varattno)); md->attnum = var->varattno; found = true; } } if (found) result = lappend(result, md); else { /* pfree() current entry */ pfree(md); /* pfree() all previous entries */ while ((lc = list_head(result)) != NULL) { md = (MulticornDeparsedSortGroup *) lfirst(lc); result = list_delete_ptr(result, md); pfree(md); } break; } } return result; } Expr * multicorn_get_em_expr(EquivalenceClass *ec, RelOptInfo *rel) { ListCell *lc_em; foreach(lc_em, ec->ec_members) { EquivalenceMember *em = lfirst(lc_em); if (bms_equal(em->em_relids, rel->relids)) { /* * If there is more than one equivalence member whose Vars are * taken entirely from this relation, we'll be content to choose * any one of those. */ return em->em_expr; } } /* We didn't find any suitable equivalence class expression */ return NULL; } List * serializeDeparsedSortGroup(List *pathkeys) { List *result = NIL; ListCell *lc; foreach(lc, pathkeys) { List *item = NIL; MulticornDeparsedSortGroup *key = (MulticornDeparsedSortGroup *) lfirst(lc); item = lappend(item, makeString(NameStr(*(key->attname)))); item = lappend(item, makeInteger(key->attnum)); item = lappend(item, makeInteger(key->reversed)); item = lappend(item, makeInteger(key->nulls_first)); if(key->collate != NULL) item = lappend(item, makeString(NameStr(*(key->collate)))); else item = lappend(item, NULL); item = lappend(item, key->key); result = lappend(result, item); } return result; } List * deserializeDeparsedSortGroup(List *items) { List *result = NIL; ListCell *k; foreach(k, items) { ListCell *lc; MulticornDeparsedSortGroup *key = palloc0(sizeof(MulticornDeparsedSortGroup)); lc = list_head(lfirst(k)); key->attname = (Name) strdup(strVal(lfirst(lc))); lc = lnext(lc); key->attnum = (int) intVal(lfirst(lc)); lc = lnext(lc); key->reversed = (bool) intVal(lfirst(lc)); lc = lnext(lc); key->nulls_first = (bool) intVal(lfirst(lc)); lc = lnext(lc); if(lfirst(lc) != NULL) key->collate = (Name) strdup(strVal(lfirst(lc))); else key->collate = NULL; lc = lnext(lc); key->key = (PathKey *) lfirst(lc); result = lappend(result, key); } return result; } Multicorn-1.3.4/src/utils.c000066400000000000000000000064441320447423600156030ustar00rootroot00000000000000/*------------------------------------------------------------------------- * * The Multicorn Foreign Data Wrapper allows you to fetch foreign data in * Python in your PostgreSQL. * * This module contains helpers meant to be called from python code. * * This software is released under the postgresql licence * * author: Kozea * * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "multicorn.h" #include "miscadmin.h" struct module_state { PyObject *error; }; #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif static PyObject * log_to_postgres(PyObject *self, PyObject *args, PyObject *kwargs) { char *message = NULL; char *hintstr = NULL, *detailstr = NULL; int level = 1; int severity; PyObject *hint, *p_message, *detail; if (!PyArg_ParseTuple(args, "O|i", &p_message, &level)) { errorCheck(); Py_INCREF(Py_None); return Py_None; } if (PyBytes_Check(p_message)) { message = PyBytes_AsString(p_message); } else if (PyUnicode_Check(p_message)) { message = strdup(PyUnicode_AsPgString(p_message)); } else { PyObject *temp = PyObject_Str(p_message); errorCheck(); message = strdup(PyString_AsString(temp)); errorCheck(); Py_DECREF(temp); } switch (level) { case 0: severity = DEBUG1; break; case 1: severity = NOTICE; break; case 2: severity = WARNING; break; case 3: severity = ERROR; break; case 4: severity = FATAL; break; default: severity = INFO; break; } hint = PyDict_GetItemString(kwargs, "hint"); detail = PyDict_GetItemString(kwargs, "detail"); if (errstart(severity, __FILE__, __LINE__, PG_FUNCNAME_MACRO, TEXTDOMAIN)) { errmsg("%s", message); if (hint != NULL && hint != Py_None) { hintstr = PyString_AsString(hint); errhint("%s", hintstr); } if (detail != NULL && detail != Py_None) { detailstr = PyString_AsString(detail); errdetail("%s", detailstr); } Py_DECREF(args); Py_DECREF(kwargs); errfinish(0); } else { Py_DECREF(args); Py_DECREF(kwargs); } Py_INCREF(Py_None); return Py_None; } static PyObject * py_check_interrupts(PyObject *self, PyObject *args, PyObject *kwargs) { CHECK_FOR_INTERRUPTS(); Py_INCREF(Py_None); return Py_None; } static PyMethodDef UtilsMethods[] = { {"_log_to_postgres", (PyCFunction) log_to_postgres, METH_VARARGS | METH_KEYWORDS, "Log to postresql client"}, {"check_interrupts", (PyCFunction) py_check_interrupts, METH_VARARGS | METH_KEYWORDS, "Gives control back to PostgreSQL"}, {NULL, NULL, 0, NULL} }; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "multicorn._utils", NULL, sizeof(struct module_state), UtilsMethods, NULL, NULL, NULL, NULL }; #define INITERROR return NULL PyObject * PyInit__utils(void) #else #define INITERROR return void init_utils(void) #endif { #if PY_MAJOR_VERSION >= 3 PyObject *module = PyModule_Create(&moduledef); #else PyObject *module = Py_InitModule("multicorn._utils", UtilsMethods); #endif struct module_state *st; if (module == NULL) INITERROR; st = GETSTATE(module); #if PY_MAJOR_VERSION >= 3 return module; #endif } Multicorn-1.3.4/test-2.6000077700000000000000000000000001320447423600161062test-2.7ustar00rootroot00000000000000Multicorn-1.3.4/test-2.7/000077500000000000000000000000001320447423600147635ustar00rootroot00000000000000Multicorn-1.3.4/test-2.7/expected/000077500000000000000000000000001320447423600165645ustar00rootroot00000000000000Multicorn-1.3.4/test-2.7/expected/import_sqlalchemy.out000066400000000000000000000075351320447423600230630ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create_foreign_server ----------------------- (1 row) CREATE SCHEMA local_schema; CREATE TABLE local_schema.t1 ( c1 int primary key, c2 text, c3 timestamp, c4 numeric ); CREATE TABLE local_schema.t2 ( c1 int, c2 text, c3 timestamp, c4 numeric ); CREATE TABLE local_schema.t3 ( c1 int, c2 text, c3 timestamp, c4 numeric ); CREATE SCHEMA remote_schema; IMPORT FOREIGN SCHEMA local_schema FROM SERVER multicorn_srv INTO remote_schema ; \d remote_schema.t1 Foreign table "remote_schema.t1" Column | Type | Modifiers | FDW Options --------+-----------------------------+-----------+------------- c1 | integer | | c2 | text | | c3 | timestamp without time zone | | c4 | numeric | | Server: multicorn_srv FDW Options: (primary_key 'c1', schema 'local_schema', tablename 't1') \d remote_schema.t2 Foreign table "remote_schema.t2" Column | Type | Modifiers | FDW Options --------+-----------------------------+-----------+------------- c1 | integer | | c2 | text | | c3 | timestamp without time zone | | c4 | numeric | | Server: multicorn_srv FDW Options: (schema 'local_schema', tablename 't2') \d remote_schema.t3 Foreign table "remote_schema.t3" Column | Type | Modifiers | FDW Options --------+-----------------------------+-----------+------------- c1 | integer | | c2 | text | | c3 | timestamp without time zone | | c4 | numeric | | Server: multicorn_srv FDW Options: (schema 'local_schema', tablename 't3') SELECT * FROM remote_schema.t1; c1 | c2 | c3 | c4 ----+----+----+---- (0 rows) INSERT INTO remote_schema.t1 VALUES (1, '2', NULL, NULL); SELECT * FROM remote_schema.t1; c1 | c2 | c3 | c4 ----+----+----+---- 1 | 2 | | (1 row) DROP SCHEMA remote_schema CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table remote_schema.t1 drop cascades to foreign table remote_schema.t2 drop cascades to foreign table remote_schema.t3 CREATE SCHEMA remote_schema; IMPORT FOREIGN SCHEMA local_schema LIMIT TO (t1) FROM SERVER multicorn_srv INTO remote_schema ; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'remote_schema'; relname --------- t1 (1 row) IMPORT FOREIGN SCHEMA local_schema EXCEPT (t1, t3) FROM SERVER multicorn_srv INTO remote_schema ; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'remote_schema'; relname --------- t1 t2 (2 rows) DROP EXTENSION multicorn CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table remote_schema.t1 drop cascades to foreign table remote_schema.t2 DROP SCHEMA local_schema CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to table local_schema.t1 drop cascades to table local_schema.t2 drop cascades to table local_schema.t3 DROP SCHEMA remote_schema CASCADE; Multicorn-1.3.4/test-2.7/expected/import_test.out000066400000000000000000000036451320447423600216760ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source FROM SERVER multicorn_srv INTO import_dest1; NOTICE: IMPORT import_source FROM srv {} OPTIONS {} RESTRICTION: None [] SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; relname ------------------ imported_table_1 imported_table_2 imported_table_3 (3 rows) DROP SCHEMA import_dest1 CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table import_dest1.imported_table_1 drop cascades to foreign table import_dest1.imported_table_2 drop cascades to foreign table import_dest1.imported_table_3 CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source EXCEPT (imported_table_1, imported_table_3) FROM SERVER multicorn_srv INTO import_dest1; NOTICE: IMPORT import_source FROM srv {} OPTIONS {} RESTRICTION: except [u'imported_table_1', u'imported_table_3'] SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; relname ------------------ imported_table_2 (1 row) IMPORT FOREIGN SCHEMA import_source LIMIT TO (imported_table_1) FROM SERVER multicorn_srv INTO import_dest1; NOTICE: IMPORT import_source FROM srv {} OPTIONS {} RESTRICTION: limit [u'imported_table_1'] SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; relname ------------------ imported_table_2 imported_table_1 (2 rows) DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table import_dest1.imported_table_2 drop cascades to foreign table import_dest1.imported_table_1 Multicorn-1.3.4/test-2.7/expected/multicorn_alchemy_test.out000066400000000000000000000220371320447423600240760ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create_foreign_server ----------------------- (1 row) create foreign table testalchemy ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ) server multicorn_srv options ( tablename 'basetable' ); create table basetable ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ); insert into basetable (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); select * from testalchemy; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select id, adate from testalchemy; id | adate ----+------------ 1 | 01-01-1980 2 | 03-05-1990 3 | 01-02-1972 4 | 11-02-1922 (4 rows) select * from testalchemy where avarchar is null; id | adate | atimestamp | anumeric | avarchar ----+------------+--------------------------+----------+---------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (1 row) select * from testalchemy where avarchar is not null; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where adate > '1970-01-02'::date; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where adate between '1970-01-01' and '1980-01-01'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (2 rows) select * from testalchemy where anumeric > 0; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where avarchar not like '%test'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where avarchar like 'Another%'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test (1 row) select * from testalchemy where avarchar ilike 'Another%'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (2 rows) select * from testalchemy where avarchar not ilike 'Another%'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+---------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test (1 row) select * from testalchemy where id in (1,2); id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test (2 rows) select * from testalchemy where id not in (1, 2); id | adate | atimestamp | anumeric | avarchar ----+------------+--------------------------+----------+-------------- 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (2 rows) select * from testalchemy order by avarchar; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select * from testalchemy order by avarchar desc; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (4 rows) select * from testalchemy order by avarchar desc nulls first; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (4 rows) select * from testalchemy order by avarchar desc nulls last; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select * from testalchemy order by avarchar nulls first; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test (4 rows) select * from testalchemy order by avarchar nulls last; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select count(*) from testalchemy; count ------- 4 (1 row) DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testalchemy DROP table basetable; Multicorn-1.3.4/test-2.7/expected/multicorn_cache_invalidation.out000066400000000000000000000242221320447423600252170ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying(20), test2 character varying ) server multicorn_srv options ( option1 'option1' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying(20)'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) ALTER foreign table testmulticorn drop column test1; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test2', 'character varying')] NOTICE: [] NOTICE: ['test2'] test2 ------------ test2 1 0 test2 2 1 test2 3 2 test2 1 3 test2 2 4 test2 3 5 test2 1 6 test2 2 7 test2 3 8 test2 1 9 test2 2 10 test2 3 11 test2 1 12 test2 2 13 test2 3 14 test2 1 15 test2 2 16 test2 3 17 test2 1 18 test2 2 19 (20 rows) ALTER foreign table testmulticorn add column test1 varchar; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 ------------+------------ test2 1 0 | test1 2 0 test2 3 1 | test1 1 1 test2 2 2 | test1 3 2 test2 1 3 | test1 2 3 test2 3 4 | test1 1 4 test2 2 5 | test1 3 5 test2 1 6 | test1 2 6 test2 3 7 | test1 1 7 test2 2 8 | test1 3 8 test2 1 9 | test1 2 9 test2 3 10 | test1 1 10 test2 2 11 | test1 3 11 test2 1 12 | test1 2 12 test2 3 13 | test1 1 13 test2 2 14 | test1 3 14 test2 1 15 | test1 2 15 test2 3 16 | test1 1 16 test2 2 17 | test1 3 17 test2 1 18 | test1 2 18 test2 3 19 | test1 1 19 (20 rows) ALTER foreign table testmulticorn add column test3 varchar; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn options (SET option1 'option1_update'); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn options (ADD option2 'option2'); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('option2', 'option2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn options (DROP option2); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) -- Test dropping column when returning sequences (issue #15) ALTER foreign table testmulticorn options (ADD test_type 'sequence'); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn drop test3; select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 ------------+------------ test2 1 0 | test1 2 0 test2 3 1 | test1 1 1 test2 2 2 | test1 3 2 test2 1 3 | test1 2 3 test2 3 4 | test1 1 4 test2 2 5 | test1 3 5 test2 1 6 | test1 2 6 test2 3 7 | test1 1 7 test2 2 8 | test1 3 8 test2 1 9 | test1 2 9 test2 3 10 | test1 1 10 test2 2 11 | test1 3 11 test2 1 12 | test1 2 12 test2 3 13 | test1 1 13 test2 2 14 | test1 3 14 test2 1 15 | test1 2 15 test2 3 16 | test1 1 16 test2 2 17 | test1 3 17 test2 1 18 | test1 2 18 test2 3 19 | test1 1 19 (20 rows) ALTER foreign table testmulticorn alter test1 type varchar(30); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying(30)'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 -----------+----------- test2 1 0 | test1 2 0 (1 row) ALTER foreign table testmulticorn alter test1 type text; select * from testmulticorn limit 1; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'text'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 -----------+----------- test2 1 0 | test1 2 0 (1 row) ALTER foreign table testmulticorn rename test1 to testnew; select * from testmulticorn limit 1; NOTICE: [] NOTICE: ['test2', 'testnew'] test2 | testnew -----------+----------- test2 1 0 | test1 2 0 (1 row) DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_column_options_test.out000066400000000000000000000041711320447423600255230ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying options (prefix 'test'), test2 character varying ) server multicorn_srv options ( option1 'option1' ); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: Column test1 options: {'prefix': 'test'} NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ALTER foreign table testmulticorn alter test1 options (set prefix 'test2'); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: Column test1 options: {'prefix': 'test2'} NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ALTER foreign table testmulticorn alter test1 options (drop prefix); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ALTER foreign table testmulticorn alter test1 options (add prefix 'test3'); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: Column test1 options: {'prefix': 'test3'} NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_error_test.out000066400000000000000000000020471320447423600236040ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; -- Test that the wrapper option is required on the server. CREATE server multicorn_srv foreign data wrapper multicorn; ERROR: The wrapper parameter is mandatory, specify a valid class name -- Test that the wrapper option cannot be altered on the table CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', wrapper 'multicorn.evilwrapper.EvilDataWrapper' ); ERROR: Cannot set the wrapper class on the table HINT: Set it on the server ALTER server multicorn_srv options (DROP wrapper); ERROR: The wrapper parameter is mandatory, specify a valid class name CREATE server multicorn_empty_srv foreign data wrapper multicorn; ERROR: The wrapper parameter is mandatory, specify a valid class name DROP EXTENSION multicorn cascade; NOTICE: drop cascades to server multicorn_srv Multicorn-1.3.4/test-2.7/expected/multicorn_logger_test.out000066400000000000000000000014121320447423600237250ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'logger' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'logger')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] WARNING: An error is about to occur ERROR: An error occured DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_planner_test.out000066400000000000000000000102711320447423600241100ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); -- Test for two thing: first, that when a low total row count, -- a full seq scan is used on a join. CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1' ); explain select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] QUERY PLAN ---------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=10.00..400.00 rows=20 width=20) (1 row) explain select * from testmulticorn m1 inner join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------- Nested Loop (cost=20.00..806.05 rows=2 width=128) Join Filter: ((m1.test1)::text = (m2.test1)::text) -> Foreign Scan on testmulticorn m1 (cost=10.00..400.00 rows=20 width=20) -> Materialize (cost=10.00..400.10 rows=20 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..400.00 rows=20 width=20) (5 rows) explain select * from testmulticorn m1 left outer join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------- Nested Loop Left Join (cost=20.00..806.05 rows=20 width=128) Join Filter: ((m1.test1)::text = (m2.test1)::text) -> Foreign Scan on testmulticorn m1 (cost=10.00..400.00 rows=20 width=20) -> Materialize (cost=10.00..400.10 rows=20 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..400.00 rows=20 width=20) (5 rows) DROP foreign table testmulticorn; -- Second, when a total row count is high -- a parameterized path is used on the test1 attribute. CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'planner' ); explain select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'planner'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] QUERY PLAN ---------------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=10.00..200000000.00 rows=10000000 width=20) (1 row) explain select * from testmulticorn m1 inner join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------------- Nested Loop (cost=20.00..400100000.00 rows=500000000000 width=128) -> Foreign Scan on testmulticorn m1 (cost=10.00..200000000.00 rows=10000000 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..20.00 rows=1 width=20) Filter: ((m1.test1)::text = (test1)::text) (4 rows) explain select * from testmulticorn m1 left outer join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------------- Nested Loop Left Join (cost=20.00..400100000.00 rows=500000000000 width=128) -> Foreign Scan on testmulticorn m1 (cost=10.00..200000000.00 rows=10000000 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..20.00 rows=1 width=20) Filter: ((m1.test1)::text = (test1)::text) (4 rows) DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_regression_test.out000066400000000000000000000267151320447423600246430ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) -- Test quals select * from testmulticorn where test1 like '%0'; NOTICE: [test1 ~~ %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) select * from testmulticorn where test1 ilike '%0'; NOTICE: [test1 ~~* %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) -- Test columns select test2 from testmulticorn; NOTICE: [] NOTICE: ['test2'] test2 ------------ test2 2 0 test2 1 1 test2 3 2 test2 2 3 test2 1 4 test2 3 5 test2 2 6 test2 1 7 test2 3 8 test2 2 9 test2 1 10 test2 3 11 test2 2 12 test2 1 13 test2 3 14 test2 2 15 test2 1 16 test2 3 17 test2 2 18 test2 1 19 (20 rows) -- Test subquery plan select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where substr(t2.test1, 7, 1)::int = substr(t1.test1, 7, 1)::int) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] test1 | max ------------+----- test1 1 12 | 9 test1 1 15 | 9 test1 1 6 | 9 test1 1 18 | 9 test1 1 0 | 9 test1 1 3 | 9 test1 1 9 | 9 test1 2 14 | 8 test1 2 11 | 8 test1 2 8 | 8 test1 2 17 | 8 test1 2 5 | 8 test1 2 2 | 8 test1 3 19 | 7 test1 3 1 | 7 test1 3 4 | 7 test1 3 7 | 7 test1 3 10 | 7 test1 3 13 | 7 test1 3 16 | 7 (20 rows) select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where t2.test1 = t1.test1) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [test1 = test1 1 0] NOTICE: ['test1'] NOTICE: [test1 = test1 3 1] NOTICE: ['test1'] NOTICE: [test1 = test1 2 2] NOTICE: ['test1'] NOTICE: [test1 = test1 1 3] NOTICE: ['test1'] NOTICE: [test1 = test1 3 4] NOTICE: ['test1'] NOTICE: [test1 = test1 2 5] NOTICE: ['test1'] NOTICE: [test1 = test1 1 6] NOTICE: ['test1'] NOTICE: [test1 = test1 3 7] NOTICE: ['test1'] NOTICE: [test1 = test1 2 8] NOTICE: ['test1'] NOTICE: [test1 = test1 1 9] NOTICE: ['test1'] NOTICE: [test1 = test1 3 10] NOTICE: ['test1'] NOTICE: [test1 = test1 2 11] NOTICE: ['test1'] NOTICE: [test1 = test1 1 12] NOTICE: ['test1'] NOTICE: [test1 = test1 3 13] NOTICE: ['test1'] NOTICE: [test1 = test1 2 14] NOTICE: ['test1'] NOTICE: [test1 = test1 1 15] NOTICE: ['test1'] NOTICE: [test1 = test1 3 16] NOTICE: ['test1'] NOTICE: [test1 = test1 2 17] NOTICE: ['test1'] NOTICE: [test1 = test1 1 18] NOTICE: ['test1'] NOTICE: [test1 = test1 3 19] NOTICE: ['test1'] test1 | max ------------+----- test1 1 9 | 9 test1 2 8 | 8 test1 3 7 | 7 test1 1 6 | 6 test1 2 5 | 5 test1 3 4 | 4 test1 1 3 | 3 test1 2 2 | 2 test1 3 16 | 1 test1 2 17 | 1 test1 1 18 | 1 test1 3 19 | 1 test1 3 1 | 1 test1 3 10 | 1 test1 2 11 | 1 test1 1 12 | 1 test1 3 13 | 1 test1 2 14 | 1 test1 1 15 | 1 test1 1 0 | 0 (20 rows) select * from testmulticorn where test1 is null; NOTICE: [test1 = None] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 is not null; NOTICE: [test1 <> None] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) select * from testmulticorn where 'grou' > test1; NOTICE: [test1 < grou] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 < ANY(ARRAY['grou', 'MACHIN']); NOTICE: [test1 < ANY([u'grou', u'MACHIN'])] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) CREATE foreign table testmulticorn2 ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option2' ); select * from testmulticorn union all select * from testmulticorn2; NOTICE: [('option1', 'option2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (40 rows) create function test_function_immutable () returns varchar as $$ BEGIN RETURN 'test'; END $$ immutable language plpgsql; create function test_function_stable () returns varchar as $$ BEGIN RETURN 'test'; END $$ stable language plpgsql; create function test_function_volatile () returns varchar as $$ BEGIN RETURN 'test'; END $$ volatile language plpgsql; select * from testmulticorn where test1 like test_function_immutable(); NOTICE: [test1 ~~ test] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 like test_function_stable(); NOTICE: [test1 ~~ test] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 like test_function_volatile(); NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 like length(test2)::varchar; NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) \set FETCH_COUNT 1000 select * from testmulticorn; NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) -- Test that zero values are converted to zero and not null ALTER FOREIGN TABLE testmulticorn options (add test_type 'int'); ALTER FOREIGN TABLE testmulticorn alter test1 type integer; select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('test_type', 'int'), ('usermapping', 'test')] NOTICE: [('test1', 'integer'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- 0 | 0 (1 row) select * from testmulticorn where test1 = 0; NOTICE: [test1 = 0] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- 0 | 0 (1 row) ALTER FOREIGN TABLE testmulticorn options (drop test_type); -- Test operations with bytea ALTER FOREIGN TABLE testmulticorn alter test2 type bytea; ALTER FOREIGN TABLE testmulticorn alter test1 type bytea; select encode(test1, 'escape') from testmulticorn where test2 = 'test2 1 19'::bytea; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea')] NOTICE: [test2 = test2 1 19] NOTICE: ['test1', 'test2'] encode ------------ test1 3 19 (1 row) -- Test operations with None ALTER FOREIGN TABLE testmulticorn options (add test_type 'None'); select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'None'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) ALTER FOREIGN TABLE testmulticorn options (set test_type 'iter_none'); select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'iter_none'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) ALTER FOREIGN TABLE testmulticorn add test3 money; SELECT * from testmulticorn where test3 = 12::money; NOTICE: [('option1', 'option1'), ('test_type', 'iter_none'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea'), ('test3', 'money')] NOTICE: [test3 = $12.00] NOTICE: ['test1', 'test2', 'test3'] test1 | test2 | test3 -------+-------+------- (0 rows) SELECT * from testmulticorn where test1 = '12 €'; ERROR: Error in python: UnicodeDecodeError DETAIL: 'ascii' codec can't decode byte 0xe2 in position 3: ordinal not in range(128) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn drop cascades to foreign table testmulticorn2 Multicorn-1.3.4/test-2.7/expected/multicorn_sequence_test.out000066400000000000000000000171221320447423600242630ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'sequence' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) -- Test quals select * from testmulticorn where test1 like '%0'; NOTICE: [test1 ~~ %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) select * from testmulticorn where test1 ilike '%0'; NOTICE: [test1 ~~* %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) -- Test columns select test2 from testmulticorn; NOTICE: [] NOTICE: ['test2'] test2 ------------ test2 2 0 test2 1 1 test2 3 2 test2 2 3 test2 1 4 test2 3 5 test2 2 6 test2 1 7 test2 3 8 test2 2 9 test2 1 10 test2 3 11 test2 2 12 test2 1 13 test2 3 14 test2 2 15 test2 1 16 test2 3 17 test2 2 18 test2 1 19 (20 rows) -- Test subquery plan select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where substr(t2.test1, 7, 1)::int = substr(t1.test1, 7, 1)::int) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] test1 | max ------------+----- test1 1 12 | 9 test1 1 15 | 9 test1 1 6 | 9 test1 1 18 | 9 test1 1 0 | 9 test1 1 3 | 9 test1 1 9 | 9 test1 2 14 | 8 test1 2 11 | 8 test1 2 8 | 8 test1 2 17 | 8 test1 2 5 | 8 test1 2 2 | 8 test1 3 19 | 7 test1 3 1 | 7 test1 3 4 | 7 test1 3 7 | 7 test1 3 10 | 7 test1 3 13 | 7 test1 3 16 | 7 (20 rows) select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where t2.test1 = t1.test1) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [test1 = test1 1 0] NOTICE: ['test1'] NOTICE: [test1 = test1 3 1] NOTICE: ['test1'] NOTICE: [test1 = test1 2 2] NOTICE: ['test1'] NOTICE: [test1 = test1 1 3] NOTICE: ['test1'] NOTICE: [test1 = test1 3 4] NOTICE: ['test1'] NOTICE: [test1 = test1 2 5] NOTICE: ['test1'] NOTICE: [test1 = test1 1 6] NOTICE: ['test1'] NOTICE: [test1 = test1 3 7] NOTICE: ['test1'] NOTICE: [test1 = test1 2 8] NOTICE: ['test1'] NOTICE: [test1 = test1 1 9] NOTICE: ['test1'] NOTICE: [test1 = test1 3 10] NOTICE: ['test1'] NOTICE: [test1 = test1 2 11] NOTICE: ['test1'] NOTICE: [test1 = test1 1 12] NOTICE: ['test1'] NOTICE: [test1 = test1 3 13] NOTICE: ['test1'] NOTICE: [test1 = test1 2 14] NOTICE: ['test1'] NOTICE: [test1 = test1 1 15] NOTICE: ['test1'] NOTICE: [test1 = test1 3 16] NOTICE: ['test1'] NOTICE: [test1 = test1 2 17] NOTICE: ['test1'] NOTICE: [test1 = test1 1 18] NOTICE: ['test1'] NOTICE: [test1 = test1 3 19] NOTICE: ['test1'] test1 | max ------------+----- test1 1 9 | 9 test1 2 8 | 8 test1 3 7 | 7 test1 1 6 | 6 test1 2 5 | 5 test1 3 4 | 4 test1 1 3 | 3 test1 2 2 | 2 test1 3 16 | 1 test1 2 17 | 1 test1 1 18 | 1 test1 3 19 | 1 test1 3 1 | 1 test1 3 10 | 1 test1 2 11 | 1 test1 1 12 | 1 test1 3 13 | 1 test1 2 14 | 1 test1 1 15 | 1 test1 1 0 | 0 (20 rows) select * from testmulticorn where test1 is null; NOTICE: [test1 = None] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 is not null; NOTICE: [test1 <> None] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) select * from testmulticorn where 'grou' > test1; NOTICE: [test1 < grou] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 < ANY(ARRAY['grou', 'MACHIN']); NOTICE: [test1 < ANY([u'grou', u'MACHIN'])] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) CREATE foreign table testmulticorn2 ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option2' ); select * from testmulticorn union all select * from testmulticorn2; NOTICE: [('option1', 'option2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (40 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn drop cascades to foreign table testmulticorn2 Multicorn-1.3.4/test-2.7/expected/multicorn_test_date.out000066400000000000000000000056311320447423600233720ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', test_type 'date' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+-------------------------- 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 09-02-2011 | Sat Sep 03 14:30:25 2011 10-01-2011 | Sun Oct 02 14:30:25 2011 11-03-2011 | Tue Nov 01 14:30:25 2011 12-02-2011 | Sat Dec 03 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 (20 rows) select * from testmulticorn where test1 < '2011-06-01'; NOTICE: [test1 < 2011-06-01] NOTICE: ['test1', 'test2'] test1 | test2 ------------+-------------------------- 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 (10 rows) select * from testmulticorn where test2 < '2011-06-01 00:00:00'; NOTICE: [test2 < 2011-06-01 00:00:00] NOTICE: ['test1', 'test2'] test1 | test2 ------------+-------------------------- 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 (10 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_test_dict.out000066400000000000000000000111351320447423600233740ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE EXTENSION hstore; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 hstore, test2 hstore ) server multicorn_srv options ( option1 'option1', test_type 'dict' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'dict'), ('usermapping', 'test')] NOTICE: [('test1', 'hstore'), ('test2', 'hstore')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ----------------------------------------------------------------------------------+---------------------------------------------------------------------------------- "index"=>"0", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"0", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"1", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"1", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"2", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"2", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"3", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"3", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"4", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"4", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"5", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"5", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"6", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"6", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"7", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"7", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"8", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"8", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"9", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"9", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"10", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"10", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"11", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"11", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"12", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"12", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"13", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"13", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"14", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"14", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"15", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"15", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"16", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"16", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"17", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"17", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"18", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"18", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"19", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"19", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" (20 rows) select test1 -> 'repeater' as r from testmulticorn order by r; NOTICE: [] NOTICE: ['test1'] r --- 1 1 1 1 1 1 1 2 2 2 2 2 2 3 3 3 3 3 3 3 (20 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_test_list.out000066400000000000000000000173061320447423600234320ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying[], test2 character varying[] ) server multicorn_srv options ( option1 'option1', test_type 'list' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying[]'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------------------------------------------------+------------------------------------------------------ {test1,1,0,"test1,\"0\"","{some value, \\\" ' 2}"} | {test2,2,0,"test2,\"0\"","{some value, \\\" ' 2}"} {test1,3,1,"test1,\"1\"","{some value, \\\" ' 2}"} | {test2,1,1,"test2,\"1\"","{some value, \\\" ' 2}"} {test1,2,2,"test1,\"2\"","{some value, \\\" ' 2}"} | {test2,3,2,"test2,\"2\"","{some value, \\\" ' 2}"} {test1,1,3,"test1,\"3\"","{some value, \\\" ' 2}"} | {test2,2,3,"test2,\"3\"","{some value, \\\" ' 2}"} {test1,3,4,"test1,\"4\"","{some value, \\\" ' 2}"} | {test2,1,4,"test2,\"4\"","{some value, \\\" ' 2}"} {test1,2,5,"test1,\"5\"","{some value, \\\" ' 2}"} | {test2,3,5,"test2,\"5\"","{some value, \\\" ' 2}"} {test1,1,6,"test1,\"6\"","{some value, \\\" ' 2}"} | {test2,2,6,"test2,\"6\"","{some value, \\\" ' 2}"} {test1,3,7,"test1,\"7\"","{some value, \\\" ' 2}"} | {test2,1,7,"test2,\"7\"","{some value, \\\" ' 2}"} {test1,2,8,"test1,\"8\"","{some value, \\\" ' 2}"} | {test2,3,8,"test2,\"8\"","{some value, \\\" ' 2}"} {test1,1,9,"test1,\"9\"","{some value, \\\" ' 2}"} | {test2,2,9,"test2,\"9\"","{some value, \\\" ' 2}"} {test1,3,10,"test1,\"10\"","{some value, \\\" ' 2}"} | {test2,1,10,"test2,\"10\"","{some value, \\\" ' 2}"} {test1,2,11,"test1,\"11\"","{some value, \\\" ' 2}"} | {test2,3,11,"test2,\"11\"","{some value, \\\" ' 2}"} {test1,1,12,"test1,\"12\"","{some value, \\\" ' 2}"} | {test2,2,12,"test2,\"12\"","{some value, \\\" ' 2}"} {test1,3,13,"test1,\"13\"","{some value, \\\" ' 2}"} | {test2,1,13,"test2,\"13\"","{some value, \\\" ' 2}"} {test1,2,14,"test1,\"14\"","{some value, \\\" ' 2}"} | {test2,3,14,"test2,\"14\"","{some value, \\\" ' 2}"} {test1,1,15,"test1,\"15\"","{some value, \\\" ' 2}"} | {test2,2,15,"test2,\"15\"","{some value, \\\" ' 2}"} {test1,3,16,"test1,\"16\"","{some value, \\\" ' 2}"} | {test2,1,16,"test2,\"16\"","{some value, \\\" ' 2}"} {test1,2,17,"test1,\"17\"","{some value, \\\" ' 2}"} | {test2,3,17,"test2,\"17\"","{some value, \\\" ' 2}"} {test1,1,18,"test1,\"18\"","{some value, \\\" ' 2}"} | {test2,2,18,"test2,\"18\"","{some value, \\\" ' 2}"} {test1,3,19,"test1,\"19\"","{some value, \\\" ' 2}"} | {test2,1,19,"test2,\"19\"","{some value, \\\" ' 2}"} (20 rows) select test1[2] as c from testmulticorn order by c; NOTICE: [] NOTICE: ['test1'] c --- 1 1 1 1 1 1 1 2 2 2 2 2 2 3 3 3 3 3 3 3 (20 rows) alter foreign table testmulticorn alter test1 type varchar; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ----------------------------------------------------------+------------------------------------------------------ ['test1', 1, 0, 'test1,"0"', '{some value, \\" \' 2}'] | {test2,2,0,"test2,\"0\"","{some value, \\\" ' 2}"} ['test1', 3, 1, 'test1,"1"', '{some value, \\" \' 2}'] | {test2,1,1,"test2,\"1\"","{some value, \\\" ' 2}"} ['test1', 2, 2, 'test1,"2"', '{some value, \\" \' 2}'] | {test2,3,2,"test2,\"2\"","{some value, \\\" ' 2}"} ['test1', 1, 3, 'test1,"3"', '{some value, \\" \' 2}'] | {test2,2,3,"test2,\"3\"","{some value, \\\" ' 2}"} ['test1', 3, 4, 'test1,"4"', '{some value, \\" \' 2}'] | {test2,1,4,"test2,\"4\"","{some value, \\\" ' 2}"} ['test1', 2, 5, 'test1,"5"', '{some value, \\" \' 2}'] | {test2,3,5,"test2,\"5\"","{some value, \\\" ' 2}"} ['test1', 1, 6, 'test1,"6"', '{some value, \\" \' 2}'] | {test2,2,6,"test2,\"6\"","{some value, \\\" ' 2}"} ['test1', 3, 7, 'test1,"7"', '{some value, \\" \' 2}'] | {test2,1,7,"test2,\"7\"","{some value, \\\" ' 2}"} ['test1', 2, 8, 'test1,"8"', '{some value, \\" \' 2}'] | {test2,3,8,"test2,\"8\"","{some value, \\\" ' 2}"} ['test1', 1, 9, 'test1,"9"', '{some value, \\" \' 2}'] | {test2,2,9,"test2,\"9\"","{some value, \\\" ' 2}"} ['test1', 3, 10, 'test1,"10"', '{some value, \\" \' 2}'] | {test2,1,10,"test2,\"10\"","{some value, \\\" ' 2}"} ['test1', 2, 11, 'test1,"11"', '{some value, \\" \' 2}'] | {test2,3,11,"test2,\"11\"","{some value, \\\" ' 2}"} ['test1', 1, 12, 'test1,"12"', '{some value, \\" \' 2}'] | {test2,2,12,"test2,\"12\"","{some value, \\\" ' 2}"} ['test1', 3, 13, 'test1,"13"', '{some value, \\" \' 2}'] | {test2,1,13,"test2,\"13\"","{some value, \\\" ' 2}"} ['test1', 2, 14, 'test1,"14"', '{some value, \\" \' 2}'] | {test2,3,14,"test2,\"14\"","{some value, \\\" ' 2}"} ['test1', 1, 15, 'test1,"15"', '{some value, \\" \' 2}'] | {test2,2,15,"test2,\"15\"","{some value, \\\" ' 2}"} ['test1', 3, 16, 'test1,"16"', '{some value, \\" \' 2}'] | {test2,1,16,"test2,\"16\"","{some value, \\\" ' 2}"} ['test1', 2, 17, 'test1,"17"', '{some value, \\" \' 2}'] | {test2,3,17,"test2,\"17\"","{some value, \\\" ' 2}"} ['test1', 1, 18, 'test1,"18"', '{some value, \\" \' 2}'] | {test2,2,18,"test2,\"18\"","{some value, \\\" ' 2}"} ['test1', 3, 19, 'test1,"19"', '{some value, \\" \' 2}'] | {test2,1,19,"test2,\"19\"","{some value, \\\" ' 2}"} (20 rows) alter foreign table testmulticorn options (set test_type 'nested_list'); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('test_type', 'nested_list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 --------------------------------------------------------------------+----------------------------------------------------------------------------- [['test1', 'test1'], [1, '{some value, \\" 2}'], [0, 'test1,"0"']] | {"['test2', 'test2']","[2, '{some value, \\\\\" 2}']","[0, 'test2,\"0\"']"} (1 row) alter foreign table testmulticorn alter test1 type varchar[]; alter foreign table testmulticorn alter test2 type varchar[][]; select test1[2], test2[2][2], array_length(test1, 1), array_length(test2, 1), array_length(test2, 2) from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('test_type', 'nested_list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying[]'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 | array_length | array_length | array_length ----------------------------+--------------------+--------------+--------------+-------------- [1, '{some value, \\" 2}'] | {some value, \" 2} | 3 | 3 | 2 (1 row) select length(test1[2]) from testmulticorn limit 1; NOTICE: [] NOTICE: ['test1'] length -------- 26 (1 row) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/multicorn_test_sort.out000066400000000000000000000042451320447423600234440ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', test_type 'date' ); -- Test sort pushdown asked EXPLAIN SELECT * FROM testmulticorn ORDER BY test1 DESC; NOTICE: [('option1', 'option1'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] QUERY PLAN ---------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=10.00..400.00 rows=20 width=20) (1 row) -- Data should be sorted SELECT * FROM testmulticorn ORDER BY test1 DESC; NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: requested sort(s): NOTICE: SortKey(attname=u'test1', attnum=1, is_reversed=True, nulls_first=True, collate=None) test1 | test2 ------------+-------------------------- 12-02-2011 | Sat Dec 03 14:30:25 2011 11-03-2011 | Tue Nov 01 14:30:25 2011 10-01-2011 | Sun Oct 02 14:30:25 2011 09-02-2011 | Sat Sep 03 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 (20 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/write_filesystem.out000066400000000000000000000577371320447423600227360ustar00rootroot00000000000000-- Setup the test CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.fsfdw.FilesystemFdw' ); CREATE language plpythonu; CREATE TABLE temp_dir (dirname varchar); -- Create a table with the filesystem fdw in a temporary directory, -- and store the dirname in the temp_dir table. CREATE OR REPLACE FUNCTION create_table() RETURNS VOID AS $$ import plpy import tempfile import os dir = tempfile.mkdtemp() plpy.execute(""" INSERT INTO temp_dir(dirname) VALUES ('%s') """ % str(dir)) plpy.execute(""" CREATE foreign table testmulticorn ( color varchar, size varchar, name varchar, ext varchar, filename varchar, data varchar ) server multicorn_srv options ( filename_column 'filename', content_column 'data', pattern '{color}/{size}/{name}.{ext}', root_dir '%s' ); """ % dir) for color in ('blue', 'red'): for size in ('big', 'small'): dirname = os.path.join(dir, color, size) os.makedirs(dirname) for name, ext in (('square', 'txt'), ('round', 'ini')): with open(os.path.join(dirname, '.'.join([name, ext])), 'a') as fd: fd.write('Im a %s %s %s\n' % (size, color, name)) $$ language plpythonu; select create_table(); create_table -------------- (1 row) -- End of Setup \i test-common/multicorn_testfilesystem.include -- Should have 8 lines. SELECT * from testmulticorn ORDER BY filename; color | size | name | ext | filename | data -------+-------+--------+-----+-----------------------+------------------------ blue | big | round | ini | blue/big/round.ini | Im a big blue round + | | | | | blue | big | square | txt | blue/big/square.txt | Im a big blue square + | | | | | blue | small | round | ini | blue/small/round.ini | Im a small blue round + | | | | | blue | small | square | txt | blue/small/square.txt | Im a small blue square+ | | | | | red | big | round | ini | red/big/round.ini | Im a big red round + | | | | | red | big | square | txt | red/big/square.txt | Im a big red square + | | | | | red | small | round | ini | red/small/round.ini | Im a small red round + | | | | | red | small | square | txt | red/small/square.txt | Im a small red square + | | | | | (8 rows) -- Test the cost analysis EXPLAIN select color, size from testmulticorn where color = 'blue' and size = 'big' and name = 'square' and ext = 'txt'; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..120.00 rows=1 width=120) Filter: (((color)::text = 'blue'::text) AND ((size)::text = 'big'::text) AND ((name)::text = 'square'::text) AND ((ext)::text = 'txt'::text)) (2 rows) EXPLAIN select color, size from testmulticorn where color = 'blue' and size = 'big'; QUERY PLAN ----------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..6000.00 rows=100 width=60) Filter: (((color)::text = 'blue'::text) AND ((size)::text = 'big'::text)) (2 rows) EXPLAIN select color, size from testmulticorn where color = 'blue'; QUERY PLAN ---------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..600000.00 rows=10000 width=60) Filter: ((color)::text = 'blue'::text) (2 rows) EXPLAIN select color, size, data from testmulticorn where color = 'blue' and size = 'big' and name = 'square' and ext = 'txt'; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..1000150.00 rows=1 width=1000150) Filter: (((color)::text = 'blue'::text) AND ((size)::text = 'big'::text) AND ((name)::text = 'square'::text) AND ((ext)::text = 'txt'::text)) (2 rows) -- Test insertion -- Normal insertion INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('yellow', 'big', 'square', 'text', 'Im a big yellow square') RETURNING filename; filename ------------------------ yellow/big/square.text (1 row) -- Insertion with redundant filename/properties INSERT INTO testmulticorn (color, size, name, ext, data, filename) VALUES ('yellow', 'small', 'square', 'txt', 'Im a small yellow square', 'yellow/small/square.txt'); -- Insertion with just a filename INSERT INTO testmulticorn (data, filename) VALUES ('Im a big blue triangle', 'blue/big/triangle.txt') RETURNING color, size, name, ext; color | size | name | ext -------+------+----------+----- blue | big | triangle | txt (1 row) -- Should have 11 lines by now. SELECT * from testmulticorn ORDER BY filename; color | size | name | ext | filename | data --------+-------+----------+------+-------------------------+-------------------------- blue | big | round | ini | blue/big/round.ini | Im a big blue round + | | | | | blue | big | square | txt | blue/big/square.txt | Im a big blue square + | | | | | blue | big | triangle | txt | blue/big/triangle.txt | Im a big blue triangle blue | small | round | ini | blue/small/round.ini | Im a small blue round + | | | | | blue | small | square | txt | blue/small/square.txt | Im a small blue square + | | | | | red | big | round | ini | red/big/round.ini | Im a big red round + | | | | | red | big | square | txt | red/big/square.txt | Im a big red square + | | | | | red | small | round | ini | red/small/round.ini | Im a small red round + | | | | | red | small | square | txt | red/small/square.txt | Im a small red square + | | | | | yellow | big | square | text | yellow/big/square.text | Im a big yellow square yellow | small | square | txt | yellow/small/square.txt | Im a small yellow square (11 rows) -- Insertion with incoherent filename/properties (should fail) INSERT INTO testmulticorn (color, size, name, ext, data, filename) VALUES ('blue', 'big', 'triangle', 'txt', 'Im a big blue triangle', 'blue/small/triangle.txt'); psql:test-common/multicorn_testfilesystem.include:28: ERROR: The columns inferred from the filename do not match the supplied columns. HINT: Remove either the filename column or the properties column from your statement, or ensure they match -- Insertion with missing keys (should fail) INSERT INTO testmulticorn (color, size, name) VALUES ('blue', 'small', 'triangle'); psql:test-common/multicorn_testfilesystem.include:31: ERROR: The following columns are necessary: set([u'ext']) HINT: You can also insert an item by providing only the filename and content columns -- Insertion with missing keys and filename (should fail) INSERT INTO testmulticorn (color, size, name, filename) VALUES ('blue', 'small', 'triangle', 'blue/small/triangle.txt'); psql:test-common/multicorn_testfilesystem.include:34: ERROR: The following columns are necessary: set([u'ext']) HINT: You can also insert an item by providing only the filename and content columns -- Insertion which would overwrite a file. -- Normal insertion INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('yellow', 'big', 'square', 'text', 'Im a duplicate big square'); psql:test-common/multicorn_testfilesystem.include:38: ERROR: Duplicate key value violates filesystem integrity. DETAIL: Key (color, ext, name, size)=(yellow, text, square, big) already exists -- Should still have 11 lines by now. SELECT * from testmulticorn ORDER BY filename; color | size | name | ext | filename | data --------+-------+----------+------+-------------------------+-------------------------- blue | big | round | ini | blue/big/round.ini | Im a big blue round + | | | | | blue | big | square | txt | blue/big/square.txt | Im a big blue square + | | | | | blue | big | triangle | txt | blue/big/triangle.txt | Im a big blue triangle blue | small | round | ini | blue/small/round.ini | Im a small blue round + | | | | | blue | small | square | txt | blue/small/square.txt | Im a small blue square + | | | | | red | big | round | ini | red/big/round.ini | Im a big red round + | | | | | red | big | square | txt | red/big/square.txt | Im a big red square + | | | | | red | small | round | ini | red/small/round.ini | Im a small red round + | | | | | red | small | square | txt | red/small/square.txt | Im a small red square + | | | | | yellow | big | square | text | yellow/big/square.text | Im a big yellow square yellow | small | square | txt | yellow/small/square.txt | Im a small yellow square (11 rows) -- Test insertion in transaction BEGIN; INSERT INTO testmulticorn (data, filename) VALUES ('Im a big red triangle', 'red/big/triangle.txt'); SELECT * from testmulticorn where name = 'triangle' and color = 'red' ORDER BY filename; color | size | name | ext | filename | data -------+------+----------+-----+----------------------+----------------------- red | big | triangle | txt | red/big/triangle.txt | Im a big red triangle (1 row) ROLLBACK; -- The file should not be persisted. SELECT * from testmulticorn where name = 'triangle' and color = 'red' ORDER BY filename; color | size | name | ext | filename | data -------+------+------+-----+----------+------ (0 rows) -- Test Update WITH t as ( UPDATE testmulticorn set name = 'rectangle' where name = 'square' RETURNING filename ) SELECT * from t order by filename; filename ---------------------------- blue/big/rectangle.txt blue/small/rectangle.txt red/big/rectangle.txt red/small/rectangle.txt yellow/big/rectangle.text yellow/small/rectangle.txt (6 rows) -- O lines SELECT count(1) from testmulticorn where name = 'square'; count ------- 0 (1 row) -- 6 lines SELECT count(1) from testmulticorn where name = 'rectangle'; count ------- 6 (1 row) -- Update should not work if it would override an existing file. UPDATE testmulticorn set filename = 'blue/big/triangle.txt' where filename = 'blue/big/rectangle.txt'; psql:test-common/multicorn_testfilesystem.include:65: ERROR: Duplicate key value violates filesystem integrity. DETAIL: Key (color, ext, name, size)=(blue, txt, triangle, big) already exists -- Update should not work when setting filename column to NULL UPDATE testmulticorn set filename = NULL where filename = 'blue/big/rectangle.txt'; psql:test-common/multicorn_testfilesystem.include:68: ERROR: The filename, or all pattern columns are needed. -- Update should not work when setting a property column to NULL WITH t as ( UPDATE testmulticorn set color = NULL where filename = 'blue/big/rectangle.txt' RETURNING color ) SELECT * from t ORDER BY color; psql:test-common/multicorn_testfilesystem.include:73: ERROR: Null value in columns (color) are not allowed DETAIL: Failing row contains (NULL, big, rectangle, txt) -- Content column update. UPDATE testmulticorn set data = 'Im an updated rectangle' where filename = 'blue/big/rectangle.txt' RETURNING data; data ------------------------- Im an updated rectangle (1 row) SELECT * from testmulticorn where filename = 'blue/big/rectangle.txt'; color | size | name | ext | filename | data -------+------+-----------+-----+------------------------+------------------------- blue | big | rectangle | txt | blue/big/rectangle.txt | Im an updated rectangle (1 row) -- Update in transactions BEGIN; UPDATE testmulticorn set name = 'square' where name = 'rectangle'; -- O lines SELECT count(1) from testmulticorn where name = 'rectangle'; count ------- 0 (1 row) -- 6 lines SELECT count(1) from testmulticorn where name = 'square'; count ------- 6 (1 row) ROLLBACK; -- O lines SELECT count(1) from testmulticorn where name = 'square'; count ------- 0 (1 row) -- 6 lines SELECT count(1) from testmulticorn where name = 'rectangle'; count ------- 6 (1 row) BEGIN; UPDATE testmulticorn set data = data || ' UPDATED!'; -- 11 lines SELECT count(1) from testmulticorn where data ilike '% UPDATED!'; count ------- 11 (1 row) SELECT data from testmulticorn where data ilike '% UPDATED!' order by filename limit 1; data ---------------------------------- Im an updated rectangle UPDATED! (1 row) ROLLBACK; -- 0 lines SELECT count(1) from testmulticorn where data ilike '% UPDATED!'; count ------- 0 (1 row) BEGIN; UPDATE testmulticorn set data = data || ' UPDATED!'; UPDATE testmulticorn set data = data || ' TWICE!'; SELECT data from testmulticorn order by filename; data ------------------------------------------ Im an updated rectangle UPDATED! TWICE! Im a big blue round + UPDATED! TWICE! Im a big blue triangle UPDATED! TWICE! Im a small blue square + UPDATED! TWICE! Im a small blue round + UPDATED! TWICE! Im a big red square + UPDATED! TWICE! Im a big red round + UPDATED! TWICE! Im a small red square + UPDATED! TWICE! Im a small red round + UPDATED! TWICE! Im a big yellow square UPDATED! TWICE! Im a small yellow square UPDATED! TWICE! (11 rows) ROLLBACK; -- No 'UPDATED! or 'TWICE!' SELECT data from testmulticorn order by filename; data -------------------------- Im an updated rectangle Im a big blue round + Im a big blue triangle Im a small blue square + Im a small blue round + Im a big red square + Im a big red round + Im a small red square + Im a small red round + Im a big yellow square Im a small yellow square (11 rows) -- Test successive update to the same files. BEGIN; UPDATE testmulticorn set color = 'cyan' where filename = 'blue/big/rectangle.txt'; -- There should be one line with cyan color, 0 with the old filename SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ------------------------+------------------------- cyan/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ----------+------ (0 rows) -- There should be one line with magenta, and 0 with cyan and the old -- filename UPDATE testmulticorn set color = 'magenta' where color = 'cyan'; SELECT filename, data from testmulticorn where color = 'magenta' order by filename; filename | data ---------------------------+------------------------- magenta/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ----------+------ (0 rows) SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ----------+------ (0 rows) UPDATE testmulticorn set color = 'blue' where color = 'magenta'; -- There should be one line with the old filename, and zero with the rest SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ------------------------+------------------------- blue/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where color = 'magenta' order by filename; filename | data ----------+------ (0 rows) SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ----------+------ (0 rows) COMMIT; -- Result should be the same than pre-commit SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ------------------------+------------------------- blue/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where color = 'magenta' order by filename; filename | data ----------+------ (0 rows) SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ----------+------ (0 rows) -- DELETE test WITH t as ( DELETE from testmulticorn where color = 'yellow' returning filename ) SELECT * from t order by filename; filename ---------------------------- yellow/big/rectangle.text yellow/small/rectangle.txt (2 rows) -- Should have no rows select count(1) from testmulticorn where color = 'yellow'; count ------- 0 (1 row) -- DELETE in transaction BEGIN; WITH t as ( DELETE from testmulticorn where color = 'red' returning filename ) SELECT * from t order by filename; filename ------------------------- red/big/rectangle.txt red/big/round.ini red/small/rectangle.txt red/small/round.ini (4 rows) select count(1) from testmulticorn where color = 'red'; count ------- 0 (1 row) ROLLBACK; -- Should have 4 rows select count(1) from testmulticorn where color = 'red'; count ------- 4 (1 row) -- Test various combinations of INSERT/UPDATE/DELETE BEGIN; INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'medium', 'triangle', 'jpg', 'Im a triangle'); INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'large', 'triangle', 'jpg', 'Im a triangle'); -- 2 lines SELECT * from testmulticorn where color = 'cyan' order by filename; color | size | name | ext | filename | data -------+--------+----------+-----+--------------------------+--------------- cyan | large | triangle | jpg | cyan/large/triangle.jpg | Im a triangle cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set color = 'magenta' where size = 'large' and color = 'cyan' returning filename; filename ---------------------------- magenta/large/triangle.jpg (1 row) -- 2 lines, one cyan, one magenta SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+--------+----------+-----+----------------------------+--------------- cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set data = 'Im magenta' where color = 'magenta'; WITH t as ( DELETE from testmulticorn where color = 'cyan' returning filename ) SELECT * from t order by filename; filename -------------------------- cyan/medium/triangle.jpg (1 row) -- One magenta line SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+-------+----------+-----+----------------------------+------------ magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im magenta (1 row) COMMIT; -- Result should be the same as precommit SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+-------+----------+-----+----------------------------+------------ magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im magenta (1 row) DELETE from testmulticorn where color = 'magenta'; -- Same as before, but rollbacking BEGIN; INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'medium', 'triangle', 'jpg', 'Im a triangle'); INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'large', 'triangle', 'jpg', 'Im a triangle'); -- 2 lines SELECT * from testmulticorn where color = 'cyan' order by filename; color | size | name | ext | filename | data -------+--------+----------+-----+--------------------------+--------------- cyan | large | triangle | jpg | cyan/large/triangle.jpg | Im a triangle cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set color = 'magenta' where size = 'large' and color = 'cyan' returning filename; filename ---------------------------- magenta/large/triangle.jpg (1 row) -- 2 lines, one cyan, one magenta SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+--------+----------+-----+----------------------------+--------------- cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set data = 'Im magenta' where color = 'magenta'; DELETE FROM testmulticorn where color = 'cyan' RETURNING filename; filename -------------------------- cyan/medium/triangle.jpg (1 row) -- One magenta line SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+-------+----------+-----+----------------------------+------------ magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im magenta (1 row) ROLLBACK; SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data -------+------+------+-----+----------+------ (0 rows) -- Cleanup everything we've done CREATE OR REPLACE FUNCTION cleanup_dir() RETURNS VOID AS $$ import shutil root_dir = plpy.execute("""SELECT dirname from temp_dir;""")[0]['dirname'] shutil.rmtree(root_dir) $$ language plpythonu; select cleanup_dir(); cleanup_dir ------------- (1 row) DROP FUNCTION cleanup_dir(); DROP TABLE temp_dir; DROP FUNCTION create_table(); DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn DROP LANGUAGE plpythonu; Multicorn-1.3.4/test-2.7/expected/write_savepoints.out000066400000000000000000000114161320447423600227250ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'nowrite' ); -- No savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', u'0'), ('test2', u'A')] update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] DROP foreign table testmulticorn_write; ROLLBACK; -- One savepoint BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', u'0'), ('test2', u'A')] update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] ROLLBACK TO A; RELEASE A; DROP foreign table testmulticorn_write; COMMIT; -- Multiple sequential savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', u'0'), ('test2', u'A')] select * from testmulticorn LIMIT 1; NOTICE: [('option1', 'option1'), ('test_type', 'nowrite'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ROLLBACK TO A; RELEASE A; SAVEPOINT B; update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] RELEASE B; update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] DROP foreign table testmulticorn_write; ROLLBACK; -- Multiple nested savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', u'0'), ('test2', u'A')] select * from testmulticorn LIMIT 1; NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) SAVEPOINT B; update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] RELEASE B; update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] ROLLBACK TO A; RELEASE A; DROP foreign table testmulticorn_write; ROLLBACK; -- Clean up DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-2.7/expected/write_sqlalchemy.out000066400000000000000000000056451320447423600227030ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create_foreign_server ----------------------- (1 row) create foreign table testalchemy ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ) server multicorn_srv options ( tablename 'basetable' ); create table basetable ( id integer primary key, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ); insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'); NOTICE: You need to declare a primary key option in order to use the write features ERROR: This FDW does not support the writable API ALTER FOREIGN TABLE testalchemy OPTIONS (ADD primary_key 'id'); BEGIN; insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); select * from basetable; id | adate | atimestamp | anumeric | avarchar ----+-------+------------+----------+---------- (0 rows) ROLLBACK; BEGIN; insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); update testalchemy set avarchar = avarchar || ' UPDATED!'; COMMIT; SELECT * from basetable; id | adate | atimestamp | anumeric | avarchar ----+------------+--------------------------+----------+----------------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21 1980 | 3.4 | Test UPDATED! 2 | 03-05-1990 | Mon Mar 02 10:40:18 1998 | 12.2 | Another Test UPDATED! 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000.0 | another Test UPDATED! 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000.0 | (4 rows) DELETE from testalchemy; SELECT * from basetable; id | adate | atimestamp | anumeric | avarchar ----+-------+------------+----------+---------- (0 rows) DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testalchemy DROP TABLE basetable; Multicorn-1.3.4/test-2.7/expected/write_test.out000066400000000000000000000171721320447423600215160ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'nowrite', tx_hook 'true' ); insert into testmulticorn(test1, test2) VALUES ('test', 'test2'); NOTICE: [('option1', 'option1'), ('test_type', 'nowrite'), ('tx_hook', 'true'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: BEGIN NOTICE: ROLLBACK ERROR: Error in python: NotImplementedError DETAIL: This FDW does not support the writable API update testmulticorn set test1 = 'test'; NOTICE: BEGIN NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: ROLLBACK ERROR: Error in python: NotImplementedError DETAIL: This FDW does not support the writable API delete from testmulticorn where test2 = 'test2 2 0'; NOTICE: BEGIN NOTICE: [test2 = test2 2 0] NOTICE: ['test1', 'test2'] NOTICE: ROLLBACK ERROR: Error in python: NotImplementedError DETAIL: This FDW does not support the writable API CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning', tx_hook 'true' ); insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('tx_hook', 'true'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: BEGIN NOTICE: INSERTING: [('test1', u'test'), ('test2', u'test2')] NOTICE: PRECOMMIT NOTICE: COMMIT update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%'; NOTICE: BEGIN NOTICE: [test1 ~~* test1 3%] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test1 3 1 with [('test1', u'test'), ('test2', u'test2 1 1')] NOTICE: UPDATING: test1 3 4 with [('test1', u'test'), ('test2', u'test2 1 4')] NOTICE: UPDATING: test1 3 7 with [('test1', u'test'), ('test2', u'test2 1 7')] NOTICE: UPDATING: test1 3 10 with [('test1', u'test'), ('test2', u'test2 1 10')] NOTICE: UPDATING: test1 3 13 with [('test1', u'test'), ('test2', u'test2 1 13')] NOTICE: UPDATING: test1 3 16 with [('test1', u'test'), ('test2', u'test2 1 16')] NOTICE: UPDATING: test1 3 19 with [('test1', u'test'), ('test2', u'test2 1 19')] NOTICE: PRECOMMIT NOTICE: COMMIT delete from testmulticorn_write where test2 = 'test2 2 0'; NOTICE: BEGIN NOTICE: [test2 = test2 2 0] NOTICE: ['test1', 'test2'] NOTICE: DELETING: test1 1 0 NOTICE: PRECOMMIT NOTICE: COMMIT -- Test returning insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2') RETURNING test1; NOTICE: BEGIN NOTICE: INSERTING: [('test1', u'test'), ('test2', u'test2')] NOTICE: PRECOMMIT NOTICE: COMMIT test1 ---------------- INSERTED: test (1 row) update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%' RETURNING test1; NOTICE: BEGIN NOTICE: [test1 ~~* test1 3%] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test1 3 1 with [('test1', u'test'), ('test2', u'test2 1 1')] NOTICE: UPDATING: test1 3 4 with [('test1', u'test'), ('test2', u'test2 1 4')] NOTICE: UPDATING: test1 3 7 with [('test1', u'test'), ('test2', u'test2 1 7')] NOTICE: UPDATING: test1 3 10 with [('test1', u'test'), ('test2', u'test2 1 10')] NOTICE: UPDATING: test1 3 13 with [('test1', u'test'), ('test2', u'test2 1 13')] NOTICE: UPDATING: test1 3 16 with [('test1', u'test'), ('test2', u'test2 1 16')] NOTICE: UPDATING: test1 3 19 with [('test1', u'test'), ('test2', u'test2 1 19')] NOTICE: PRECOMMIT NOTICE: COMMIT test1 --------------- UPDATED: test UPDATED: test UPDATED: test UPDATED: test UPDATED: test UPDATED: test UPDATED: test (7 rows) delete from testmulticorn_write where test1 = 'test1 1 0' returning test2, test1; NOTICE: BEGIN NOTICE: [test1 = test1 1 0] NOTICE: ['test1', 'test2'] NOTICE: DELETING: test1 1 0 NOTICE: PRECOMMIT NOTICE: COMMIT test2 | test1 -----------+----------- test2 2 0 | test1 1 0 (1 row) DROP foreign table testmulticorn_write; -- Now test with another column CREATE foreign table testmulticorn_write( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test2' ); insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', u'test'), ('test2', u'test2')] update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%'; NOTICE: [test1 ~~* test1 3%] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test2 1 1 with [('test1', u'test'), ('test2', u'test2 1 1')] NOTICE: UPDATING: test2 1 4 with [('test1', u'test'), ('test2', u'test2 1 4')] NOTICE: UPDATING: test2 1 7 with [('test1', u'test'), ('test2', u'test2 1 7')] NOTICE: UPDATING: test2 1 10 with [('test1', u'test'), ('test2', u'test2 1 10')] NOTICE: UPDATING: test2 1 13 with [('test1', u'test'), ('test2', u'test2 1 13')] NOTICE: UPDATING: test2 1 16 with [('test1', u'test'), ('test2', u'test2 1 16')] NOTICE: UPDATING: test2 1 19 with [('test1', u'test'), ('test2', u'test2 1 19')] delete from testmulticorn_write where test2 = 'test2 2 0'; NOTICE: [test2 = test2 2 0] NOTICE: ['test2'] NOTICE: DELETING: test2 2 0 update testmulticorn_write set test2 = 'test' where test2 = 'test2 1 1'; NOTICE: [test2 = test2 1 1] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test2 1 1 with [('test1', u'test1 3 1'), ('test2', u'test')] DROP foreign table testmulticorn_write; -- Now test with other types CREATE foreign table testmulticorn_write( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', row_id_column 'test2', test_type 'date' ); insert into testmulticorn_write(test1, test2) VALUES ('2012-01-01', '2012-01-01 00:00:00'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test2'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] NOTICE: INSERTING: [('test1', datetime.date(2012, 1, 1)), ('test2', datetime.datetime(2012, 1, 1, 0, 0))] delete from testmulticorn_write where test2 > '2011-12-03'; NOTICE: [test2 > 2011-12-03 00:00:00] NOTICE: ['test2'] NOTICE: DELETING: 2011-12-03 14:30:25 update testmulticorn_write set test1 = date_trunc('day', test1) where test2 = '2011-09-03 14:30:25'; NOTICE: [test2 = 2011-09-03 14:30:25] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: 2011-09-03 14:30:25 with [('test1', datetime.date(2011, 9, 2)), ('test2', datetime.datetime(2011, 9, 3, 14, 30, 25))] DROP foreign table testmulticorn_write; -- Test with unknown column CREATE foreign table testmulticorn_write( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', row_id_column 'teststuff', test_type 'date' ); delete from testmulticorn_write; NOTICE: [('option1', 'option1'), ('row_id_column', 'teststuff'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] ERROR: The rowid attribute does not exist DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn drop cascades to foreign table testmulticorn_write Multicorn-1.3.4/test-2.7/sql/000077500000000000000000000000001320447423600155625ustar00rootroot00000000000000Multicorn-1.3.4/test-2.7/sql/import_sqlalchemy.sql000066400000000000000000000032741320447423600220450ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); CREATE SCHEMA local_schema; CREATE TABLE local_schema.t1 ( c1 int primary key, c2 text, c3 timestamp, c4 numeric ); CREATE TABLE local_schema.t2 ( c1 int, c2 text, c3 timestamp, c4 numeric ); CREATE TABLE local_schema.t3 ( c1 int, c2 text, c3 timestamp, c4 numeric ); CREATE SCHEMA remote_schema; IMPORT FOREIGN SCHEMA local_schema FROM SERVER multicorn_srv INTO remote_schema ; \d remote_schema.t1 \d remote_schema.t2 \d remote_schema.t3 SELECT * FROM remote_schema.t1; INSERT INTO remote_schema.t1 VALUES (1, '2', NULL, NULL); SELECT * FROM remote_schema.t1; DROP SCHEMA remote_schema CASCADE; CREATE SCHEMA remote_schema; IMPORT FOREIGN SCHEMA local_schema LIMIT TO (t1) FROM SERVER multicorn_srv INTO remote_schema ; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'remote_schema'; IMPORT FOREIGN SCHEMA local_schema EXCEPT (t1, t3) FROM SERVER multicorn_srv INTO remote_schema ; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'remote_schema'; DROP EXTENSION multicorn CASCADE; DROP SCHEMA local_schema CASCADE; DROP SCHEMA remote_schema CASCADE; Multicorn-1.3.4/test-2.7/sql/import_test.sql000066400000000000000000000017211320447423600206550ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source FROM SERVER multicorn_srv INTO import_dest1; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; DROP SCHEMA import_dest1 CASCADE; CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source EXCEPT (imported_table_1, imported_table_3) FROM SERVER multicorn_srv INTO import_dest1; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; IMPORT FOREIGN SCHEMA import_source LIMIT TO (imported_table_1) FROM SERVER multicorn_srv INTO import_dest1; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_alchemy_test.sql000066400000000000000000000044041320447423600230620ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create foreign table testalchemy ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ) server multicorn_srv options ( tablename 'basetable' ); create table basetable ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ); insert into basetable (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); select * from testalchemy; select id, adate from testalchemy; select * from testalchemy where avarchar is null; select * from testalchemy where avarchar is not null; select * from testalchemy where adate > '1970-01-02'::date; select * from testalchemy where adate between '1970-01-01' and '1980-01-01'; select * from testalchemy where anumeric > 0; select * from testalchemy where avarchar not like '%test'; select * from testalchemy where avarchar like 'Another%'; select * from testalchemy where avarchar ilike 'Another%'; select * from testalchemy where avarchar not ilike 'Another%'; select * from testalchemy where id in (1,2); select * from testalchemy where id not in (1, 2); select * from testalchemy order by avarchar; select * from testalchemy order by avarchar desc; select * from testalchemy order by avarchar desc nulls first; select * from testalchemy order by avarchar desc nulls last; select * from testalchemy order by avarchar nulls first; select * from testalchemy order by avarchar nulls last; select count(*) from testalchemy; DROP EXTENSION multicorn cascade; DROP table basetable; Multicorn-1.3.4/test-2.7/sql/multicorn_cache_invalidation.sql000066400000000000000000000031371320447423600242070ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying(20), test2 character varying ) server multicorn_srv options ( option1 'option1' ); -- Test "normal" usage select * from testmulticorn; ALTER foreign table testmulticorn drop column test1; select * from testmulticorn; ALTER foreign table testmulticorn add column test1 varchar; select * from testmulticorn; ALTER foreign table testmulticorn add column test3 varchar; select * from testmulticorn; ALTER foreign table testmulticorn options (SET option1 'option1_update'); select * from testmulticorn; ALTER foreign table testmulticorn options (ADD option2 'option2'); select * from testmulticorn; ALTER foreign table testmulticorn options (DROP option2); select * from testmulticorn; -- Test dropping column when returning sequences (issue #15) ALTER foreign table testmulticorn options (ADD test_type 'sequence'); select * from testmulticorn; ALTER foreign table testmulticorn drop test3; select * from testmulticorn; ALTER foreign table testmulticorn alter test1 type varchar(30); select * from testmulticorn limit 1; ALTER foreign table testmulticorn alter test1 type text; select * from testmulticorn limit 1; ALTER foreign table testmulticorn rename test1 to testnew; select * from testmulticorn limit 1; DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_column_options_test.sql000066400000000000000000000015571320447423600245160ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying options (prefix 'test'), test2 character varying ) server multicorn_srv options ( option1 'option1' ); select * from testmulticorn limit 1; ALTER foreign table testmulticorn alter test1 options (set prefix 'test2'); select * from testmulticorn limit 1; ALTER foreign table testmulticorn alter test1 options (drop prefix); select * from testmulticorn limit 1; ALTER foreign table testmulticorn alter test1 options (add prefix 'test3'); select * from testmulticorn limit 1; DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_error_test.sql000066400000000000000000000013301320447423600225640ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; -- Test that the wrapper option is required on the server. CREATE server multicorn_srv foreign data wrapper multicorn; -- Test that the wrapper option cannot be altered on the table CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', wrapper 'multicorn.evilwrapper.EvilDataWrapper' ); ALTER server multicorn_srv options (DROP wrapper); CREATE server multicorn_empty_srv foreign data wrapper multicorn; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_logger_test.sql000066400000000000000000000007041320447423600227160ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'logger' ); -- Test "normal" usage select * from testmulticorn; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_planner_test.sql000066400000000000000000000025441320447423600231020ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); -- Test for two thing: first, that when a low total row count, -- a full seq scan is used on a join. CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1' ); explain select * from testmulticorn; explain select * from testmulticorn m1 inner join testmulticorn m2 on m1.test1 = m2.test1; explain select * from testmulticorn m1 left outer join testmulticorn m2 on m1.test1 = m2.test1; DROP foreign table testmulticorn; -- Second, when a total row count is high -- a parameterized path is used on the test1 attribute. CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'planner' ); explain select * from testmulticorn; explain select * from testmulticorn m1 inner join testmulticorn m2 on m1.test1 = m2.test1; explain select * from testmulticorn m1 left outer join testmulticorn m2 on m1.test1 = m2.test1; DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_regression_test.sql000066400000000000000000000062671320447423600236310ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1' ); -- Test "normal" usage select * from testmulticorn; -- Test quals select * from testmulticorn where test1 like '%0'; select * from testmulticorn where test1 ilike '%0'; -- Test columns select test2 from testmulticorn; -- Test subquery plan select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where substr(t2.test1, 7, 1)::int = substr(t1.test1, 7, 1)::int) as max from testmulticorn t1 order by max desc; select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where t2.test1 = t1.test1) as max from testmulticorn t1 order by max desc; select * from testmulticorn where test1 is null; select * from testmulticorn where test1 is not null; select * from testmulticorn where 'grou' > test1; select * from testmulticorn where test1 < ANY(ARRAY['grou', 'MACHIN']); CREATE foreign table testmulticorn2 ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option2' ); select * from testmulticorn union all select * from testmulticorn2; create function test_function_immutable () returns varchar as $$ BEGIN RETURN 'test'; END $$ immutable language plpgsql; create function test_function_stable () returns varchar as $$ BEGIN RETURN 'test'; END $$ stable language plpgsql; create function test_function_volatile () returns varchar as $$ BEGIN RETURN 'test'; END $$ volatile language plpgsql; select * from testmulticorn where test1 like test_function_immutable(); select * from testmulticorn where test1 like test_function_stable(); select * from testmulticorn where test1 like test_function_volatile(); select * from testmulticorn where test1 like length(test2)::varchar; \set FETCH_COUNT 1000 select * from testmulticorn; -- Test that zero values are converted to zero and not null ALTER FOREIGN TABLE testmulticorn options (add test_type 'int'); ALTER FOREIGN TABLE testmulticorn alter test1 type integer; select * from testmulticorn limit 1; select * from testmulticorn where test1 = 0; ALTER FOREIGN TABLE testmulticorn options (drop test_type); -- Test operations with bytea ALTER FOREIGN TABLE testmulticorn alter test2 type bytea; ALTER FOREIGN TABLE testmulticorn alter test1 type bytea; select encode(test1, 'escape') from testmulticorn where test2 = 'test2 1 19'::bytea; -- Test operations with None ALTER FOREIGN TABLE testmulticorn options (add test_type 'None'); select * from testmulticorn; ALTER FOREIGN TABLE testmulticorn options (set test_type 'iter_none'); select * from testmulticorn; ALTER FOREIGN TABLE testmulticorn add test3 money; SELECT * from testmulticorn where test3 = 12::money; SELECT * from testmulticorn where test1 = '12 €'; DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_sequence_test.sql000066400000000000000000000030601320447423600232450ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'sequence' ); -- Test "normal" usage select * from testmulticorn; -- Test quals select * from testmulticorn where test1 like '%0'; select * from testmulticorn where test1 ilike '%0'; -- Test columns select test2 from testmulticorn; -- Test subquery plan select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where substr(t2.test1, 7, 1)::int = substr(t1.test1, 7, 1)::int) as max from testmulticorn t1 order by max desc; select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where t2.test1 = t1.test1) as max from testmulticorn t1 order by max desc; select * from testmulticorn where test1 is null; select * from testmulticorn where test1 is not null; select * from testmulticorn where 'grou' > test1; select * from testmulticorn where test1 < ANY(ARRAY['grou', 'MACHIN']); CREATE foreign table testmulticorn2 ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option2' ); select * from testmulticorn union all select * from testmulticorn2; DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_test_date.sql000066400000000000000000000012601320447423600223520ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', test_type 'date' ); -- Test "normal" usage select * from testmulticorn; select * from testmulticorn where test1 < '2011-06-01'; select * from testmulticorn where test2 < '2011-06-01 00:00:00'; DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_test_dict.sql000066400000000000000000000012151320447423600223600ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE EXTENSION hstore; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 hstore, test2 hstore ) server multicorn_srv options ( option1 'option1', test_type 'dict' ); -- Test "normal" usage select * from testmulticorn; select test1 -> 'repeater' as r from testmulticorn order by r; DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_test_list.sql000066400000000000000000000022041320447423600224070ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying[], test2 character varying[] ) server multicorn_srv options ( option1 'option1', test_type 'list' ); -- Test "normal" usage select * from testmulticorn; select test1[2] as c from testmulticorn order by c; alter foreign table testmulticorn alter test1 type varchar; select * from testmulticorn; alter foreign table testmulticorn options (set test_type 'nested_list'); select * from testmulticorn limit 1; alter foreign table testmulticorn alter test1 type varchar[]; alter foreign table testmulticorn alter test2 type varchar[][]; select test1[2], test2[2][2], array_length(test1, 1), array_length(test2, 1), array_length(test2, 2) from testmulticorn limit 1; select length(test1[2]) from testmulticorn limit 1; DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/multicorn_test_sort.sql000066400000000000000000000012421320447423600224240ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', test_type 'date' ); -- Test sort pushdown asked EXPLAIN SELECT * FROM testmulticorn ORDER BY test1 DESC; -- Data should be sorted SELECT * FROM testmulticorn ORDER BY test1 DESC; DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/write_filesystem.sql000066400000000000000000000035231320447423600217040ustar00rootroot00000000000000-- Setup the test CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.fsfdw.FilesystemFdw' ); CREATE language plpythonu; CREATE TABLE temp_dir (dirname varchar); -- Create a table with the filesystem fdw in a temporary directory, -- and store the dirname in the temp_dir table. CREATE OR REPLACE FUNCTION create_table() RETURNS VOID AS $$ import plpy import tempfile import os dir = tempfile.mkdtemp() plpy.execute(""" INSERT INTO temp_dir(dirname) VALUES ('%s') """ % str(dir)) plpy.execute(""" CREATE foreign table testmulticorn ( color varchar, size varchar, name varchar, ext varchar, filename varchar, data varchar ) server multicorn_srv options ( filename_column 'filename', content_column 'data', pattern '{color}/{size}/{name}.{ext}', root_dir '%s' ); """ % dir) for color in ('blue', 'red'): for size in ('big', 'small'): dirname = os.path.join(dir, color, size) os.makedirs(dirname) for name, ext in (('square', 'txt'), ('round', 'ini')): with open(os.path.join(dirname, '.'.join([name, ext])), 'a') as fd: fd.write('Im a %s %s %s\n' % (size, color, name)) $$ language plpythonu; select create_table(); -- End of Setup \i test-common/multicorn_testfilesystem.include -- Cleanup everything we've done CREATE OR REPLACE FUNCTION cleanup_dir() RETURNS VOID AS $$ import shutil root_dir = plpy.execute("""SELECT dirname from temp_dir;""")[0]['dirname'] shutil.rmtree(root_dir) $$ language plpythonu; select cleanup_dir(); DROP FUNCTION cleanup_dir(); DROP TABLE temp_dir; DROP FUNCTION create_table(); DROP EXTENSION multicorn cascade; DROP LANGUAGE plpythonu; Multicorn-1.3.4/test-2.7/sql/write_savepoints.sql000066400000000000000000000055721320447423600217210ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'nowrite' ); -- No savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); update testmulticorn_write set test2 = 'B' where test1 = '0'; update testmulticorn_write set test2 = 'C' where test1 = '1'; delete from testmulticorn_write where test1 = '1'; DROP foreign table testmulticorn_write; ROLLBACK; -- One savepoint BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); update testmulticorn_write set test2 = 'B' where test1 = '0'; update testmulticorn_write set test2 = 'C' where test1 = '1'; delete from testmulticorn_write where test1 = '1'; ROLLBACK TO A; RELEASE A; DROP foreign table testmulticorn_write; COMMIT; -- Multiple sequential savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); select * from testmulticorn LIMIT 1; ROLLBACK TO A; RELEASE A; SAVEPOINT B; update testmulticorn_write set test2 = 'B' where test1 = '0'; RELEASE B; update testmulticorn_write set test2 = 'C' where test1 = '1'; delete from testmulticorn_write where test1 = '1'; DROP foreign table testmulticorn_write; ROLLBACK; -- Multiple nested savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); select * from testmulticorn LIMIT 1; SAVEPOINT B; update testmulticorn_write set test2 = 'B' where test1 = '0'; RELEASE B; update testmulticorn_write set test2 = 'C' where test1 = '1'; delete from testmulticorn_write where test1 = '1'; ROLLBACK TO A; RELEASE A; DROP foreign table testmulticorn_write; ROLLBACK; -- Clean up DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-2.7/sql/write_sqlalchemy.sql000066400000000000000000000037241320447423600216650ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create foreign table testalchemy ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ) server multicorn_srv options ( tablename 'basetable' ); create table basetable ( id integer primary key, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ); insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'); ALTER FOREIGN TABLE testalchemy OPTIONS (ADD primary_key 'id'); BEGIN; insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); select * from basetable; ROLLBACK; BEGIN; insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); update testalchemy set avarchar = avarchar || ' UPDATED!'; COMMIT; SELECT * from basetable; DELETE from testalchemy; SELECT * from basetable; DROP EXTENSION multicorn cascade; DROP TABLE basetable; Multicorn-1.3.4/test-2.7/sql/write_test.sql000066400000000000000000000052731320447423600205030ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'nowrite', tx_hook 'true' ); insert into testmulticorn(test1, test2) VALUES ('test', 'test2'); update testmulticorn set test1 = 'test'; delete from testmulticorn where test2 = 'test2 2 0'; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning', tx_hook 'true' ); insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2'); update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%'; delete from testmulticorn_write where test2 = 'test2 2 0'; -- Test returning insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2') RETURNING test1; update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%' RETURNING test1; delete from testmulticorn_write where test1 = 'test1 1 0' returning test2, test1; DROP foreign table testmulticorn_write; -- Now test with another column CREATE foreign table testmulticorn_write( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test2' ); insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2'); update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%'; delete from testmulticorn_write where test2 = 'test2 2 0'; update testmulticorn_write set test2 = 'test' where test2 = 'test2 1 1'; DROP foreign table testmulticorn_write; -- Now test with other types CREATE foreign table testmulticorn_write( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', row_id_column 'test2', test_type 'date' ); insert into testmulticorn_write(test1, test2) VALUES ('2012-01-01', '2012-01-01 00:00:00'); delete from testmulticorn_write where test2 > '2011-12-03'; update testmulticorn_write set test1 = date_trunc('day', test1) where test2 = '2011-09-03 14:30:25'; DROP foreign table testmulticorn_write; -- Test with unknown column CREATE foreign table testmulticorn_write( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', row_id_column 'teststuff', test_type 'date' ); delete from testmulticorn_write; DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; Multicorn-1.3.4/test-3.3/000077500000000000000000000000001320447423600147605ustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/expected/000077500000000000000000000000001320447423600165615ustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/expected/import_sqlalchemy.out000066400000000000000000000075351320447423600230600ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create_foreign_server ----------------------- (1 row) CREATE SCHEMA local_schema; CREATE TABLE local_schema.t1 ( c1 int primary key, c2 text, c3 timestamp, c4 numeric ); CREATE TABLE local_schema.t2 ( c1 int, c2 text, c3 timestamp, c4 numeric ); CREATE TABLE local_schema.t3 ( c1 int, c2 text, c3 timestamp, c4 numeric ); CREATE SCHEMA remote_schema; IMPORT FOREIGN SCHEMA local_schema FROM SERVER multicorn_srv INTO remote_schema ; \d remote_schema.t1 Foreign table "remote_schema.t1" Column | Type | Modifiers | FDW Options --------+-----------------------------+-----------+------------- c1 | integer | | c2 | text | | c3 | timestamp without time zone | | c4 | numeric | | Server: multicorn_srv FDW Options: (primary_key 'c1', schema 'local_schema', tablename 't1') \d remote_schema.t2 Foreign table "remote_schema.t2" Column | Type | Modifiers | FDW Options --------+-----------------------------+-----------+------------- c1 | integer | | c2 | text | | c3 | timestamp without time zone | | c4 | numeric | | Server: multicorn_srv FDW Options: (schema 'local_schema', tablename 't2') \d remote_schema.t3 Foreign table "remote_schema.t3" Column | Type | Modifiers | FDW Options --------+-----------------------------+-----------+------------- c1 | integer | | c2 | text | | c3 | timestamp without time zone | | c4 | numeric | | Server: multicorn_srv FDW Options: (schema 'local_schema', tablename 't3') SELECT * FROM remote_schema.t1; c1 | c2 | c3 | c4 ----+----+----+---- (0 rows) INSERT INTO remote_schema.t1 VALUES (1, '2', NULL, NULL); SELECT * FROM remote_schema.t1; c1 | c2 | c3 | c4 ----+----+----+---- 1 | 2 | | (1 row) DROP SCHEMA remote_schema CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table remote_schema.t1 drop cascades to foreign table remote_schema.t2 drop cascades to foreign table remote_schema.t3 CREATE SCHEMA remote_schema; IMPORT FOREIGN SCHEMA local_schema LIMIT TO (t1) FROM SERVER multicorn_srv INTO remote_schema ; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'remote_schema'; relname --------- t1 (1 row) IMPORT FOREIGN SCHEMA local_schema EXCEPT (t1, t3) FROM SERVER multicorn_srv INTO remote_schema ; SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'remote_schema'; relname --------- t1 t2 (2 rows) DROP EXTENSION multicorn CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table remote_schema.t1 drop cascades to foreign table remote_schema.t2 DROP SCHEMA local_schema CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to table local_schema.t1 drop cascades to table local_schema.t2 drop cascades to table local_schema.t3 DROP SCHEMA remote_schema CASCADE; Multicorn-1.3.4/test-3.3/expected/import_test.out000066400000000000000000000036421320447423600216700ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source FROM SERVER multicorn_srv INTO import_dest1; NOTICE: IMPORT import_source FROM srv {} OPTIONS {} RESTRICTION: None [] SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; relname ------------------ imported_table_1 imported_table_2 imported_table_3 (3 rows) DROP SCHEMA import_dest1 CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to foreign table import_dest1.imported_table_1 drop cascades to foreign table import_dest1.imported_table_2 drop cascades to foreign table import_dest1.imported_table_3 CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source EXCEPT (imported_table_1, imported_table_3) FROM SERVER multicorn_srv INTO import_dest1; NOTICE: IMPORT import_source FROM srv {} OPTIONS {} RESTRICTION: except ['imported_table_1', 'imported_table_3'] SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; relname ------------------ imported_table_2 (1 row) IMPORT FOREIGN SCHEMA import_source LIMIT TO (imported_table_1) FROM SERVER multicorn_srv INTO import_dest1; NOTICE: IMPORT import_source FROM srv {} OPTIONS {} RESTRICTION: limit ['imported_table_1'] SELECT relname FROM pg_class c INNER JOIN pg_namespace n on c.relnamespace = n.oid WHERE n.nspname = 'import_dest1'; relname ------------------ imported_table_2 imported_table_1 (2 rows) DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table import_dest1.imported_table_2 drop cascades to foreign table import_dest1.imported_table_1 Multicorn-1.3.4/test-3.3/expected/multicorn_alchemy_test.out000066400000000000000000000220371320447423600240730ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create_foreign_server ----------------------- (1 row) create foreign table testalchemy ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ) server multicorn_srv options ( tablename 'basetable' ); create table basetable ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ); insert into basetable (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); select * from testalchemy; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select id, adate from testalchemy; id | adate ----+------------ 1 | 01-01-1980 2 | 03-05-1990 3 | 01-02-1972 4 | 11-02-1922 (4 rows) select * from testalchemy where avarchar is null; id | adate | atimestamp | anumeric | avarchar ----+------------+--------------------------+----------+---------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (1 row) select * from testalchemy where avarchar is not null; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where adate > '1970-01-02'::date; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where adate between '1970-01-01' and '1980-01-01'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (2 rows) select * from testalchemy where anumeric > 0; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where avarchar not like '%test'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (3 rows) select * from testalchemy where avarchar like 'Another%'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test (1 row) select * from testalchemy where avarchar ilike 'Another%'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (2 rows) select * from testalchemy where avarchar not ilike 'Another%'; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+---------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test (1 row) select * from testalchemy where id in (1,2); id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test (2 rows) select * from testalchemy where id not in (1, 2); id | adate | atimestamp | anumeric | avarchar ----+------------+--------------------------+----------+-------------- 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (2 rows) select * from testalchemy order by avarchar; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select * from testalchemy order by avarchar desc; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (4 rows) select * from testalchemy order by avarchar desc nulls first; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test (4 rows) select * from testalchemy order by avarchar desc nulls last; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select * from testalchemy order by avarchar nulls first; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test (4 rows) select * from testalchemy order by avarchar nulls last; id | adate | atimestamp | anumeric | avarchar ----+------------+---------------------------------+----------+-------------- 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000 | another Test 2 | 03-05-1990 | Mon Mar 02 10:40:18.321023 1998 | 12.2 | Another Test 1 | 01-01-1980 | Tue Jan 01 11:01:21.132912 1980 | 3.4 | Test 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000 | (4 rows) select count(*) from testalchemy; count ------- 4 (1 row) DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testalchemy DROP table basetable; Multicorn-1.3.4/test-3.3/expected/multicorn_cache_invalidation.out000066400000000000000000000242221320447423600252140ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying(20), test2 character varying ) server multicorn_srv options ( option1 'option1' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying(20)'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) ALTER foreign table testmulticorn drop column test1; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test2', 'character varying')] NOTICE: [] NOTICE: ['test2'] test2 ------------ test2 1 0 test2 2 1 test2 3 2 test2 1 3 test2 2 4 test2 3 5 test2 1 6 test2 2 7 test2 3 8 test2 1 9 test2 2 10 test2 3 11 test2 1 12 test2 2 13 test2 3 14 test2 1 15 test2 2 16 test2 3 17 test2 1 18 test2 2 19 (20 rows) ALTER foreign table testmulticorn add column test1 varchar; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 ------------+------------ test2 1 0 | test1 2 0 test2 3 1 | test1 1 1 test2 2 2 | test1 3 2 test2 1 3 | test1 2 3 test2 3 4 | test1 1 4 test2 2 5 | test1 3 5 test2 1 6 | test1 2 6 test2 3 7 | test1 1 7 test2 2 8 | test1 3 8 test2 1 9 | test1 2 9 test2 3 10 | test1 1 10 test2 2 11 | test1 3 11 test2 1 12 | test1 2 12 test2 3 13 | test1 1 13 test2 2 14 | test1 3 14 test2 1 15 | test1 2 15 test2 3 16 | test1 1 16 test2 2 17 | test1 3 17 test2 1 18 | test1 2 18 test2 3 19 | test1 1 19 (20 rows) ALTER foreign table testmulticorn add column test3 varchar; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn options (SET option1 'option1_update'); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn options (ADD option2 'option2'); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('option2', 'option2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn options (DROP option2); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) -- Test dropping column when returning sequences (issue #15) ALTER foreign table testmulticorn options (ADD test_type 'sequence'); select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying'), ('test3', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2', 'test3'] test2 | test1 | test3 ------------+------------+------------ test2 1 0 | test1 2 0 | test3 3 0 test2 1 1 | test1 2 1 | test3 3 1 test2 1 2 | test1 2 2 | test3 3 2 test2 1 3 | test1 2 3 | test3 3 3 test2 1 4 | test1 2 4 | test3 3 4 test2 1 5 | test1 2 5 | test3 3 5 test2 1 6 | test1 2 6 | test3 3 6 test2 1 7 | test1 2 7 | test3 3 7 test2 1 8 | test1 2 8 | test3 3 8 test2 1 9 | test1 2 9 | test3 3 9 test2 1 10 | test1 2 10 | test3 3 10 test2 1 11 | test1 2 11 | test3 3 11 test2 1 12 | test1 2 12 | test3 3 12 test2 1 13 | test1 2 13 | test3 3 13 test2 1 14 | test1 2 14 | test3 3 14 test2 1 15 | test1 2 15 | test3 3 15 test2 1 16 | test1 2 16 | test3 3 16 test2 1 17 | test1 2 17 | test3 3 17 test2 1 18 | test1 2 18 | test3 3 18 test2 1 19 | test1 2 19 | test3 3 19 (20 rows) ALTER foreign table testmulticorn drop test3; select * from testmulticorn; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 ------------+------------ test2 1 0 | test1 2 0 test2 3 1 | test1 1 1 test2 2 2 | test1 3 2 test2 1 3 | test1 2 3 test2 3 4 | test1 1 4 test2 2 5 | test1 3 5 test2 1 6 | test1 2 6 test2 3 7 | test1 1 7 test2 2 8 | test1 3 8 test2 1 9 | test1 2 9 test2 3 10 | test1 1 10 test2 2 11 | test1 3 11 test2 1 12 | test1 2 12 test2 3 13 | test1 1 13 test2 2 14 | test1 3 14 test2 1 15 | test1 2 15 test2 3 16 | test1 1 16 test2 2 17 | test1 3 17 test2 1 18 | test1 2 18 test2 3 19 | test1 1 19 (20 rows) ALTER foreign table testmulticorn alter test1 type varchar(30); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying(30)'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 -----------+----------- test2 1 0 | test1 2 0 (1 row) ALTER foreign table testmulticorn alter test1 type text; select * from testmulticorn limit 1; NOTICE: [('option1', 'option1_update'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'text'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test2 | test1 -----------+----------- test2 1 0 | test1 2 0 (1 row) ALTER foreign table testmulticorn rename test1 to testnew; select * from testmulticorn limit 1; NOTICE: [] NOTICE: ['test2', 'testnew'] test2 | testnew -----------+----------- test2 1 0 | test1 2 0 (1 row) DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_column_options_test.out000066400000000000000000000041711320447423600255200ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying options (prefix 'test'), test2 character varying ) server multicorn_srv options ( option1 'option1' ); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: Column test1 options: {'prefix': 'test'} NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ALTER foreign table testmulticorn alter test1 options (set prefix 'test2'); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: Column test1 options: {'prefix': 'test2'} NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ALTER foreign table testmulticorn alter test1 options (drop prefix); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ALTER foreign table testmulticorn alter test1 options (add prefix 'test3'); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: Column test1 options: {'prefix': 'test3'} NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_error_test.out000066400000000000000000000020471320447423600236010ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; -- Test that the wrapper option is required on the server. CREATE server multicorn_srv foreign data wrapper multicorn; ERROR: The wrapper parameter is mandatory, specify a valid class name -- Test that the wrapper option cannot be altered on the table CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', wrapper 'multicorn.evilwrapper.EvilDataWrapper' ); ERROR: Cannot set the wrapper class on the table HINT: Set it on the server ALTER server multicorn_srv options (DROP wrapper); ERROR: The wrapper parameter is mandatory, specify a valid class name CREATE server multicorn_empty_srv foreign data wrapper multicorn; ERROR: The wrapper parameter is mandatory, specify a valid class name DROP EXTENSION multicorn cascade; NOTICE: drop cascades to server multicorn_srv Multicorn-1.3.4/test-3.3/expected/multicorn_logger_test.out000066400000000000000000000014121320447423600237220ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'logger' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'logger')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] WARNING: An error is about to occur ERROR: An error occured DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_planner_test.out000066400000000000000000000102711320447423600241050ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); -- Test for two thing: first, that when a low total row count, -- a full seq scan is used on a join. CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1' ); explain select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] QUERY PLAN ---------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=10.00..400.00 rows=20 width=20) (1 row) explain select * from testmulticorn m1 inner join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------- Nested Loop (cost=20.00..806.05 rows=2 width=128) Join Filter: ((m1.test1)::text = (m2.test1)::text) -> Foreign Scan on testmulticorn m1 (cost=10.00..400.00 rows=20 width=20) -> Materialize (cost=10.00..400.10 rows=20 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..400.00 rows=20 width=20) (5 rows) explain select * from testmulticorn m1 left outer join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------- Nested Loop Left Join (cost=20.00..806.05 rows=20 width=128) Join Filter: ((m1.test1)::text = (m2.test1)::text) -> Foreign Scan on testmulticorn m1 (cost=10.00..400.00 rows=20 width=20) -> Materialize (cost=10.00..400.10 rows=20 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..400.00 rows=20 width=20) (5 rows) DROP foreign table testmulticorn; -- Second, when a total row count is high -- a parameterized path is used on the test1 attribute. CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'planner' ); explain select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'planner'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] QUERY PLAN ---------------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=10.00..200000000.00 rows=10000000 width=20) (1 row) explain select * from testmulticorn m1 inner join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------------- Nested Loop (cost=20.00..400100000.00 rows=500000000000 width=128) -> Foreign Scan on testmulticorn m1 (cost=10.00..200000000.00 rows=10000000 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..20.00 rows=1 width=20) Filter: ((m1.test1)::text = (test1)::text) (4 rows) explain select * from testmulticorn m1 left outer join testmulticorn m2 on m1.test1 = m2.test1; QUERY PLAN ------------------------------------------------------------------------------------------- Nested Loop Left Join (cost=20.00..400100000.00 rows=500000000000 width=128) -> Foreign Scan on testmulticorn m1 (cost=10.00..200000000.00 rows=10000000 width=20) -> Foreign Scan on testmulticorn m2 (cost=10.00..20.00 rows=1 width=20) Filter: ((m1.test1)::text = (test1)::text) (4 rows) DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_regression_test.out000066400000000000000000000267201320447423600246340ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) -- Test quals select * from testmulticorn where test1 like '%0'; NOTICE: [test1 ~~ %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) select * from testmulticorn where test1 ilike '%0'; NOTICE: [test1 ~~* %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) -- Test columns select test2 from testmulticorn; NOTICE: [] NOTICE: ['test2'] test2 ------------ test2 2 0 test2 1 1 test2 3 2 test2 2 3 test2 1 4 test2 3 5 test2 2 6 test2 1 7 test2 3 8 test2 2 9 test2 1 10 test2 3 11 test2 2 12 test2 1 13 test2 3 14 test2 2 15 test2 1 16 test2 3 17 test2 2 18 test2 1 19 (20 rows) -- Test subquery plan select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where substr(t2.test1, 7, 1)::int = substr(t1.test1, 7, 1)::int) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] test1 | max ------------+----- test1 1 12 | 9 test1 1 15 | 9 test1 1 6 | 9 test1 1 18 | 9 test1 1 0 | 9 test1 1 3 | 9 test1 1 9 | 9 test1 2 14 | 8 test1 2 11 | 8 test1 2 8 | 8 test1 2 17 | 8 test1 2 5 | 8 test1 2 2 | 8 test1 3 19 | 7 test1 3 1 | 7 test1 3 4 | 7 test1 3 7 | 7 test1 3 10 | 7 test1 3 13 | 7 test1 3 16 | 7 (20 rows) select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where t2.test1 = t1.test1) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [test1 = test1 1 0] NOTICE: ['test1'] NOTICE: [test1 = test1 3 1] NOTICE: ['test1'] NOTICE: [test1 = test1 2 2] NOTICE: ['test1'] NOTICE: [test1 = test1 1 3] NOTICE: ['test1'] NOTICE: [test1 = test1 3 4] NOTICE: ['test1'] NOTICE: [test1 = test1 2 5] NOTICE: ['test1'] NOTICE: [test1 = test1 1 6] NOTICE: ['test1'] NOTICE: [test1 = test1 3 7] NOTICE: ['test1'] NOTICE: [test1 = test1 2 8] NOTICE: ['test1'] NOTICE: [test1 = test1 1 9] NOTICE: ['test1'] NOTICE: [test1 = test1 3 10] NOTICE: ['test1'] NOTICE: [test1 = test1 2 11] NOTICE: ['test1'] NOTICE: [test1 = test1 1 12] NOTICE: ['test1'] NOTICE: [test1 = test1 3 13] NOTICE: ['test1'] NOTICE: [test1 = test1 2 14] NOTICE: ['test1'] NOTICE: [test1 = test1 1 15] NOTICE: ['test1'] NOTICE: [test1 = test1 3 16] NOTICE: ['test1'] NOTICE: [test1 = test1 2 17] NOTICE: ['test1'] NOTICE: [test1 = test1 1 18] NOTICE: ['test1'] NOTICE: [test1 = test1 3 19] NOTICE: ['test1'] test1 | max ------------+----- test1 1 9 | 9 test1 2 8 | 8 test1 3 7 | 7 test1 1 6 | 6 test1 2 5 | 5 test1 3 4 | 4 test1 1 3 | 3 test1 2 2 | 2 test1 3 16 | 1 test1 2 17 | 1 test1 1 18 | 1 test1 3 19 | 1 test1 3 1 | 1 test1 3 10 | 1 test1 2 11 | 1 test1 1 12 | 1 test1 3 13 | 1 test1 2 14 | 1 test1 1 15 | 1 test1 1 0 | 0 (20 rows) select * from testmulticorn where test1 is null; NOTICE: [test1 = None] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 is not null; NOTICE: [test1 <> None] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) select * from testmulticorn where 'grou' > test1; NOTICE: [test1 < grou] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 < ANY(ARRAY['grou', 'MACHIN']); NOTICE: [test1 < ANY(['grou', 'MACHIN'])] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) CREATE foreign table testmulticorn2 ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option2' ); select * from testmulticorn union all select * from testmulticorn2; NOTICE: [('option1', 'option2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (40 rows) create function test_function_immutable () returns varchar as $$ BEGIN RETURN 'test'; END $$ immutable language plpgsql; create function test_function_stable () returns varchar as $$ BEGIN RETURN 'test'; END $$ stable language plpgsql; create function test_function_volatile () returns varchar as $$ BEGIN RETURN 'test'; END $$ volatile language plpgsql; select * from testmulticorn where test1 like test_function_immutable(); NOTICE: [test1 ~~ test] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 like test_function_stable(); NOTICE: [test1 ~~ test] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 like test_function_volatile(); NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 like length(test2)::varchar; NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) \set FETCH_COUNT 1000 select * from testmulticorn; NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) -- Test that zero values are converted to zero and not null ALTER FOREIGN TABLE testmulticorn options (add test_type 'int'); ALTER FOREIGN TABLE testmulticorn alter test1 type integer; select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('test_type', 'int'), ('usermapping', 'test')] NOTICE: [('test1', 'integer'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- 0 | 0 (1 row) select * from testmulticorn where test1 = 0; NOTICE: [test1 = 0] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- 0 | 0 (1 row) ALTER FOREIGN TABLE testmulticorn options (drop test_type); -- Test operations with bytea ALTER FOREIGN TABLE testmulticorn alter test2 type bytea; ALTER FOREIGN TABLE testmulticorn alter test1 type bytea; select encode(test1, 'escape') from testmulticorn where test2 = 'test2 1 19'::bytea; NOTICE: [('option1', 'option1'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea')] NOTICE: [test2 = b'test2 1 19'] NOTICE: ['test1', 'test2'] encode ------------ test1 3 19 (1 row) -- Test operations with None ALTER FOREIGN TABLE testmulticorn options (add test_type 'None'); select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'None'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) ALTER FOREIGN TABLE testmulticorn options (set test_type 'iter_none'); select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'iter_none'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) ALTER FOREIGN TABLE testmulticorn add test3 money; SELECT * from testmulticorn where test3 = 12::money; NOTICE: [('option1', 'option1'), ('test_type', 'iter_none'), ('usermapping', 'test')] NOTICE: [('test1', 'bytea'), ('test2', 'bytea'), ('test3', 'money')] NOTICE: [test3 = $12.00] NOTICE: ['test1', 'test2', 'test3'] test1 | test2 | test3 -------+-------+------- (0 rows) SELECT * from testmulticorn where test1 = '12 €'; NOTICE: [test1 = b'12 \xe2\x82\xac'] NOTICE: ['test1', 'test2', 'test3'] test1 | test2 | test3 -------+-------+------- (0 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn drop cascades to foreign table testmulticorn2 Multicorn-1.3.4/test-3.3/expected/multicorn_sequence_test.out000066400000000000000000000171201320447423600242560ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'sequence' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'sequence'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) -- Test quals select * from testmulticorn where test1 like '%0'; NOTICE: [test1 ~~ %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) select * from testmulticorn where test1 ilike '%0'; NOTICE: [test1 ~~* %0] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 10 | test2 1 10 (2 rows) -- Test columns select test2 from testmulticorn; NOTICE: [] NOTICE: ['test2'] test2 ------------ test2 2 0 test2 1 1 test2 3 2 test2 2 3 test2 1 4 test2 3 5 test2 2 6 test2 1 7 test2 3 8 test2 2 9 test2 1 10 test2 3 11 test2 2 12 test2 1 13 test2 3 14 test2 2 15 test2 1 16 test2 3 17 test2 2 18 test2 1 19 (20 rows) -- Test subquery plan select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where substr(t2.test1, 7, 1)::int = substr(t1.test1, 7, 1)::int) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] NOTICE: [] NOTICE: ['test1'] test1 | max ------------+----- test1 1 12 | 9 test1 1 15 | 9 test1 1 6 | 9 test1 1 18 | 9 test1 1 0 | 9 test1 1 3 | 9 test1 1 9 | 9 test1 2 14 | 8 test1 2 11 | 8 test1 2 8 | 8 test1 2 17 | 8 test1 2 5 | 8 test1 2 2 | 8 test1 3 19 | 7 test1 3 1 | 7 test1 3 4 | 7 test1 3 7 | 7 test1 3 10 | 7 test1 3 13 | 7 test1 3 16 | 7 (20 rows) select test1, (select max(substr(test1, 9, 1))::int as max from testmulticorn t2 where t2.test1 = t1.test1) as max from testmulticorn t1 order by max desc; NOTICE: [] NOTICE: ['test1'] NOTICE: [test1 = test1 1 0] NOTICE: ['test1'] NOTICE: [test1 = test1 3 1] NOTICE: ['test1'] NOTICE: [test1 = test1 2 2] NOTICE: ['test1'] NOTICE: [test1 = test1 1 3] NOTICE: ['test1'] NOTICE: [test1 = test1 3 4] NOTICE: ['test1'] NOTICE: [test1 = test1 2 5] NOTICE: ['test1'] NOTICE: [test1 = test1 1 6] NOTICE: ['test1'] NOTICE: [test1 = test1 3 7] NOTICE: ['test1'] NOTICE: [test1 = test1 2 8] NOTICE: ['test1'] NOTICE: [test1 = test1 1 9] NOTICE: ['test1'] NOTICE: [test1 = test1 3 10] NOTICE: ['test1'] NOTICE: [test1 = test1 2 11] NOTICE: ['test1'] NOTICE: [test1 = test1 1 12] NOTICE: ['test1'] NOTICE: [test1 = test1 3 13] NOTICE: ['test1'] NOTICE: [test1 = test1 2 14] NOTICE: ['test1'] NOTICE: [test1 = test1 1 15] NOTICE: ['test1'] NOTICE: [test1 = test1 3 16] NOTICE: ['test1'] NOTICE: [test1 = test1 2 17] NOTICE: ['test1'] NOTICE: [test1 = test1 1 18] NOTICE: ['test1'] NOTICE: [test1 = test1 3 19] NOTICE: ['test1'] test1 | max ------------+----- test1 1 9 | 9 test1 2 8 | 8 test1 3 7 | 7 test1 1 6 | 6 test1 2 5 | 5 test1 3 4 | 4 test1 1 3 | 3 test1 2 2 | 2 test1 3 16 | 1 test1 2 17 | 1 test1 1 18 | 1 test1 3 19 | 1 test1 3 1 | 1 test1 3 10 | 1 test1 2 11 | 1 test1 1 12 | 1 test1 3 13 | 1 test1 2 14 | 1 test1 1 15 | 1 test1 1 0 | 0 (20 rows) select * from testmulticorn where test1 is null; NOTICE: [test1 = None] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 is not null; NOTICE: [test1 <> None] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (20 rows) select * from testmulticorn where 'grou' > test1; NOTICE: [test1 < grou] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) select * from testmulticorn where test1 < ANY(ARRAY['grou', 'MACHIN']); NOTICE: [test1 < ANY(['grou', 'MACHIN'])] NOTICE: ['test1', 'test2'] test1 | test2 -------+------- (0 rows) CREATE foreign table testmulticorn2 ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option2' ); select * from testmulticorn union all select * from testmulticorn2; NOTICE: [('option1', 'option2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+------------ test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 test1 1 0 | test2 2 0 test1 3 1 | test2 1 1 test1 2 2 | test2 3 2 test1 1 3 | test2 2 3 test1 3 4 | test2 1 4 test1 2 5 | test2 3 5 test1 1 6 | test2 2 6 test1 3 7 | test2 1 7 test1 2 8 | test2 3 8 test1 1 9 | test2 2 9 test1 3 10 | test2 1 10 test1 2 11 | test2 3 11 test1 1 12 | test2 2 12 test1 3 13 | test2 1 13 test1 2 14 | test2 3 14 test1 1 15 | test2 2 15 test1 3 16 | test2 1 16 test1 2 17 | test2 3 17 test1 1 18 | test2 2 18 test1 3 19 | test2 1 19 (40 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn drop cascades to foreign table testmulticorn2 Multicorn-1.3.4/test-3.3/expected/multicorn_test_date.out000066400000000000000000000056311320447423600233670ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', test_type 'date' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------+-------------------------- 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 09-02-2011 | Sat Sep 03 14:30:25 2011 10-01-2011 | Sun Oct 02 14:30:25 2011 11-03-2011 | Tue Nov 01 14:30:25 2011 12-02-2011 | Sat Dec 03 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 (20 rows) select * from testmulticorn where test1 < '2011-06-01'; NOTICE: [test1 < 2011-06-01] NOTICE: ['test1', 'test2'] test1 | test2 ------------+-------------------------- 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 (10 rows) select * from testmulticorn where test2 < '2011-06-01 00:00:00'; NOTICE: [test2 < 2011-06-01 00:00:00] NOTICE: ['test1', 'test2'] test1 | test2 ------------+-------------------------- 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 (10 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_test_dict.out000066400000000000000000000111351320447423600233710ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE EXTENSION hstore; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 hstore, test2 hstore ) server multicorn_srv options ( option1 'option1', test_type 'dict' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'dict'), ('usermapping', 'test')] NOTICE: [('test1', 'hstore'), ('test2', 'hstore')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ----------------------------------------------------------------------------------+---------------------------------------------------------------------------------- "index"=>"0", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"0", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"1", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"1", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"2", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"2", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"3", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"3", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"4", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"4", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"5", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"5", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"6", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"6", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"7", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"7", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"8", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"8", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"9", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"9", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"10", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"10", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"11", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"11", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"12", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"12", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"13", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"13", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"14", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"14", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"15", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"15", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"16", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"16", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"17", "repeater"=>"2", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"17", "repeater"=>"3", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"18", "repeater"=>"1", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"18", "repeater"=>"2", "column_name"=>"test2", "maybe_hstore"=>"a => b" "index"=>"19", "repeater"=>"3", "column_name"=>"test1", "maybe_hstore"=>"a => b" | "index"=>"19", "repeater"=>"1", "column_name"=>"test2", "maybe_hstore"=>"a => b" (20 rows) select test1 -> 'repeater' as r from testmulticorn order by r; NOTICE: [] NOTICE: ['test1'] r --- 1 1 1 1 1 1 1 2 2 2 2 2 2 3 3 3 3 3 3 3 (20 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_test_list.out000066400000000000000000000173061320447423600234270ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying[], test2 character varying[] ) server multicorn_srv options ( option1 'option1', test_type 'list' ); -- Test "normal" usage select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying[]'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ------------------------------------------------------+------------------------------------------------------ {test1,1,0,"test1,\"0\"","{some value, \\\" ' 2}"} | {test2,2,0,"test2,\"0\"","{some value, \\\" ' 2}"} {test1,3,1,"test1,\"1\"","{some value, \\\" ' 2}"} | {test2,1,1,"test2,\"1\"","{some value, \\\" ' 2}"} {test1,2,2,"test1,\"2\"","{some value, \\\" ' 2}"} | {test2,3,2,"test2,\"2\"","{some value, \\\" ' 2}"} {test1,1,3,"test1,\"3\"","{some value, \\\" ' 2}"} | {test2,2,3,"test2,\"3\"","{some value, \\\" ' 2}"} {test1,3,4,"test1,\"4\"","{some value, \\\" ' 2}"} | {test2,1,4,"test2,\"4\"","{some value, \\\" ' 2}"} {test1,2,5,"test1,\"5\"","{some value, \\\" ' 2}"} | {test2,3,5,"test2,\"5\"","{some value, \\\" ' 2}"} {test1,1,6,"test1,\"6\"","{some value, \\\" ' 2}"} | {test2,2,6,"test2,\"6\"","{some value, \\\" ' 2}"} {test1,3,7,"test1,\"7\"","{some value, \\\" ' 2}"} | {test2,1,7,"test2,\"7\"","{some value, \\\" ' 2}"} {test1,2,8,"test1,\"8\"","{some value, \\\" ' 2}"} | {test2,3,8,"test2,\"8\"","{some value, \\\" ' 2}"} {test1,1,9,"test1,\"9\"","{some value, \\\" ' 2}"} | {test2,2,9,"test2,\"9\"","{some value, \\\" ' 2}"} {test1,3,10,"test1,\"10\"","{some value, \\\" ' 2}"} | {test2,1,10,"test2,\"10\"","{some value, \\\" ' 2}"} {test1,2,11,"test1,\"11\"","{some value, \\\" ' 2}"} | {test2,3,11,"test2,\"11\"","{some value, \\\" ' 2}"} {test1,1,12,"test1,\"12\"","{some value, \\\" ' 2}"} | {test2,2,12,"test2,\"12\"","{some value, \\\" ' 2}"} {test1,3,13,"test1,\"13\"","{some value, \\\" ' 2}"} | {test2,1,13,"test2,\"13\"","{some value, \\\" ' 2}"} {test1,2,14,"test1,\"14\"","{some value, \\\" ' 2}"} | {test2,3,14,"test2,\"14\"","{some value, \\\" ' 2}"} {test1,1,15,"test1,\"15\"","{some value, \\\" ' 2}"} | {test2,2,15,"test2,\"15\"","{some value, \\\" ' 2}"} {test1,3,16,"test1,\"16\"","{some value, \\\" ' 2}"} | {test2,1,16,"test2,\"16\"","{some value, \\\" ' 2}"} {test1,2,17,"test1,\"17\"","{some value, \\\" ' 2}"} | {test2,3,17,"test2,\"17\"","{some value, \\\" ' 2}"} {test1,1,18,"test1,\"18\"","{some value, \\\" ' 2}"} | {test2,2,18,"test2,\"18\"","{some value, \\\" ' 2}"} {test1,3,19,"test1,\"19\"","{some value, \\\" ' 2}"} | {test2,1,19,"test2,\"19\"","{some value, \\\" ' 2}"} (20 rows) select test1[2] as c from testmulticorn order by c; NOTICE: [] NOTICE: ['test1'] c --- 1 1 1 1 1 1 1 2 2 2 2 2 2 3 3 3 3 3 3 3 (20 rows) alter foreign table testmulticorn alter test1 type varchar; select * from testmulticorn; NOTICE: [('option1', 'option1'), ('test_type', 'list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 ----------------------------------------------------------+------------------------------------------------------ ['test1', 1, 0, 'test1,"0"', '{some value, \\" \' 2}'] | {test2,2,0,"test2,\"0\"","{some value, \\\" ' 2}"} ['test1', 3, 1, 'test1,"1"', '{some value, \\" \' 2}'] | {test2,1,1,"test2,\"1\"","{some value, \\\" ' 2}"} ['test1', 2, 2, 'test1,"2"', '{some value, \\" \' 2}'] | {test2,3,2,"test2,\"2\"","{some value, \\\" ' 2}"} ['test1', 1, 3, 'test1,"3"', '{some value, \\" \' 2}'] | {test2,2,3,"test2,\"3\"","{some value, \\\" ' 2}"} ['test1', 3, 4, 'test1,"4"', '{some value, \\" \' 2}'] | {test2,1,4,"test2,\"4\"","{some value, \\\" ' 2}"} ['test1', 2, 5, 'test1,"5"', '{some value, \\" \' 2}'] | {test2,3,5,"test2,\"5\"","{some value, \\\" ' 2}"} ['test1', 1, 6, 'test1,"6"', '{some value, \\" \' 2}'] | {test2,2,6,"test2,\"6\"","{some value, \\\" ' 2}"} ['test1', 3, 7, 'test1,"7"', '{some value, \\" \' 2}'] | {test2,1,7,"test2,\"7\"","{some value, \\\" ' 2}"} ['test1', 2, 8, 'test1,"8"', '{some value, \\" \' 2}'] | {test2,3,8,"test2,\"8\"","{some value, \\\" ' 2}"} ['test1', 1, 9, 'test1,"9"', '{some value, \\" \' 2}'] | {test2,2,9,"test2,\"9\"","{some value, \\\" ' 2}"} ['test1', 3, 10, 'test1,"10"', '{some value, \\" \' 2}'] | {test2,1,10,"test2,\"10\"","{some value, \\\" ' 2}"} ['test1', 2, 11, 'test1,"11"', '{some value, \\" \' 2}'] | {test2,3,11,"test2,\"11\"","{some value, \\\" ' 2}"} ['test1', 1, 12, 'test1,"12"', '{some value, \\" \' 2}'] | {test2,2,12,"test2,\"12\"","{some value, \\\" ' 2}"} ['test1', 3, 13, 'test1,"13"', '{some value, \\" \' 2}'] | {test2,1,13,"test2,\"13\"","{some value, \\\" ' 2}"} ['test1', 2, 14, 'test1,"14"', '{some value, \\" \' 2}'] | {test2,3,14,"test2,\"14\"","{some value, \\\" ' 2}"} ['test1', 1, 15, 'test1,"15"', '{some value, \\" \' 2}'] | {test2,2,15,"test2,\"15\"","{some value, \\\" ' 2}"} ['test1', 3, 16, 'test1,"16"', '{some value, \\" \' 2}'] | {test2,1,16,"test2,\"16\"","{some value, \\\" ' 2}"} ['test1', 2, 17, 'test1,"17"', '{some value, \\" \' 2}'] | {test2,3,17,"test2,\"17\"","{some value, \\\" ' 2}"} ['test1', 1, 18, 'test1,"18"', '{some value, \\" \' 2}'] | {test2,2,18,"test2,\"18\"","{some value, \\\" ' 2}"} ['test1', 3, 19, 'test1,"19"', '{some value, \\" \' 2}'] | {test2,1,19,"test2,\"19\"","{some value, \\\" ' 2}"} (20 rows) alter foreign table testmulticorn options (set test_type 'nested_list'); select * from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('test_type', 'nested_list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 --------------------------------------------------------------------+----------------------------------------------------------------------------- [['test1', 'test1'], [1, '{some value, \\" 2}'], [0, 'test1,"0"']] | {"['test2', 'test2']","[2, '{some value, \\\\\" 2}']","[0, 'test2,\"0\"']"} (1 row) alter foreign table testmulticorn alter test1 type varchar[]; alter foreign table testmulticorn alter test2 type varchar[][]; select test1[2], test2[2][2], array_length(test1, 1), array_length(test2, 1), array_length(test2, 2) from testmulticorn limit 1; NOTICE: [('option1', 'option1'), ('test_type', 'nested_list'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying[]'), ('test2', 'character varying[]')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 | array_length | array_length | array_length ----------------------------+--------------------+--------------+--------------+-------------- [1, '{some value, \\" 2}'] | {some value, \" 2} | 3 | 3 | 2 (1 row) select length(test1[2]) from testmulticorn limit 1; NOTICE: [] NOTICE: ['test1'] length -------- 26 (1 row) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/multicorn_test_sort.out000066400000000000000000000042441320447423600234400ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', test_type 'date' ); -- Test sort pushdown asked EXPLAIN SELECT * FROM testmulticorn ORDER BY test1 DESC; NOTICE: [('option1', 'option1'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] QUERY PLAN ---------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=10.00..400.00 rows=20 width=20) (1 row) -- Data should be sorted SELECT * FROM testmulticorn ORDER BY test1 DESC; NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: requested sort(s): NOTICE: SortKey(attname='test1', attnum=1, is_reversed=True, nulls_first=True, collate=None) test1 | test2 ------------+-------------------------- 12-02-2011 | Sat Dec 03 14:30:25 2011 11-03-2011 | Tue Nov 01 14:30:25 2011 10-01-2011 | Sun Oct 02 14:30:25 2011 09-02-2011 | Sat Sep 03 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 08-03-2011 | Mon Aug 01 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 07-01-2011 | Sat Jul 02 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 06-02-2011 | Fri Jun 03 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 05-03-2011 | Sun May 01 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 04-01-2011 | Sat Apr 02 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 03-02-2011 | Thu Mar 03 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 02-03-2011 | Tue Feb 01 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 01-01-2011 | Sun Jan 02 14:30:25 2011 (20 rows) DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/write_filesystem.out000066400000000000000000000577271320447423600227320ustar00rootroot00000000000000-- Setup the test CREATE language plpython3u; CREATE OR REPLACE FUNCTION create_table() RETURNS VOID AS $$ import plpy import tempfile import os dir = tempfile.mkdtemp() plpy.execute(""" INSERT INTO temp_dir(dirname) VALUES ('%s') """ % str(dir)) plpy.execute(""" CREATE foreign table testmulticorn ( color varchar, size varchar, name varchar, ext varchar, filename varchar, data varchar ) server multicorn_srv options ( filename_column 'filename', content_column 'data', pattern '{color}/{size}/{name}.{ext}', root_dir '%s' ); """ % dir) for color in ('blue', 'red'): for size in ('big', 'small'): dirname = os.path.join(dir, color, size) os.makedirs(dirname) for name, ext in (('square', 'txt'), ('round', 'ini')): with open(os.path.join(dirname, '.'.join([name, ext])), 'a') as fd: fd.write('Im a %s %s %s\n' % (size, color, name)) $$ language plpython3u; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.fsfdw.FilesystemFdw' ); CREATE TABLE temp_dir (dirname varchar); -- Create a table with the filesystem fdw in a temporary directory, -- and store the dirname in the temp_dir table. select create_table(); create_table -------------- (1 row) -- End of Setup \i test-common/multicorn_testfilesystem.include -- Should have 8 lines. SELECT * from testmulticorn ORDER BY filename; color | size | name | ext | filename | data -------+-------+--------+-----+-----------------------+------------------------ blue | big | round | ini | blue/big/round.ini | Im a big blue round + | | | | | blue | big | square | txt | blue/big/square.txt | Im a big blue square + | | | | | blue | small | round | ini | blue/small/round.ini | Im a small blue round + | | | | | blue | small | square | txt | blue/small/square.txt | Im a small blue square+ | | | | | red | big | round | ini | red/big/round.ini | Im a big red round + | | | | | red | big | square | txt | red/big/square.txt | Im a big red square + | | | | | red | small | round | ini | red/small/round.ini | Im a small red round + | | | | | red | small | square | txt | red/small/square.txt | Im a small red square + | | | | | (8 rows) -- Test the cost analysis EXPLAIN select color, size from testmulticorn where color = 'blue' and size = 'big' and name = 'square' and ext = 'txt'; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..120.00 rows=1 width=120) Filter: (((color)::text = 'blue'::text) AND ((size)::text = 'big'::text) AND ((name)::text = 'square'::text) AND ((ext)::text = 'txt'::text)) (2 rows) EXPLAIN select color, size from testmulticorn where color = 'blue' and size = 'big'; QUERY PLAN ----------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..6000.00 rows=100 width=60) Filter: (((color)::text = 'blue'::text) AND ((size)::text = 'big'::text)) (2 rows) EXPLAIN select color, size from testmulticorn where color = 'blue'; QUERY PLAN ---------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..600000.00 rows=10000 width=60) Filter: ((color)::text = 'blue'::text) (2 rows) EXPLAIN select color, size, data from testmulticorn where color = 'blue' and size = 'big' and name = 'square' and ext = 'txt'; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on testmulticorn (cost=20.00..1000150.00 rows=1 width=1000150) Filter: (((color)::text = 'blue'::text) AND ((size)::text = 'big'::text) AND ((name)::text = 'square'::text) AND ((ext)::text = 'txt'::text)) (2 rows) -- Test insertion -- Normal insertion INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('yellow', 'big', 'square', 'text', 'Im a big yellow square') RETURNING filename; filename ------------------------ yellow/big/square.text (1 row) -- Insertion with redundant filename/properties INSERT INTO testmulticorn (color, size, name, ext, data, filename) VALUES ('yellow', 'small', 'square', 'txt', 'Im a small yellow square', 'yellow/small/square.txt'); -- Insertion with just a filename INSERT INTO testmulticorn (data, filename) VALUES ('Im a big blue triangle', 'blue/big/triangle.txt') RETURNING color, size, name, ext; color | size | name | ext -------+------+----------+----- blue | big | triangle | txt (1 row) -- Should have 11 lines by now. SELECT * from testmulticorn ORDER BY filename; color | size | name | ext | filename | data --------+-------+----------+------+-------------------------+-------------------------- blue | big | round | ini | blue/big/round.ini | Im a big blue round + | | | | | blue | big | square | txt | blue/big/square.txt | Im a big blue square + | | | | | blue | big | triangle | txt | blue/big/triangle.txt | Im a big blue triangle blue | small | round | ini | blue/small/round.ini | Im a small blue round + | | | | | blue | small | square | txt | blue/small/square.txt | Im a small blue square + | | | | | red | big | round | ini | red/big/round.ini | Im a big red round + | | | | | red | big | square | txt | red/big/square.txt | Im a big red square + | | | | | red | small | round | ini | red/small/round.ini | Im a small red round + | | | | | red | small | square | txt | red/small/square.txt | Im a small red square + | | | | | yellow | big | square | text | yellow/big/square.text | Im a big yellow square yellow | small | square | txt | yellow/small/square.txt | Im a small yellow square (11 rows) -- Insertion with incoherent filename/properties (should fail) INSERT INTO testmulticorn (color, size, name, ext, data, filename) VALUES ('blue', 'big', 'triangle', 'txt', 'Im a big blue triangle', 'blue/small/triangle.txt'); psql:test-common/multicorn_testfilesystem.include:28: ERROR: The columns inferred from the filename do not match the supplied columns. HINT: Remove either the filename column or the properties column from your statement, or ensure they match -- Insertion with missing keys (should fail) INSERT INTO testmulticorn (color, size, name) VALUES ('blue', 'small', 'triangle'); psql:test-common/multicorn_testfilesystem.include:31: ERROR: The following columns are necessary: {'ext'} HINT: You can also insert an item by providing only the filename and content columns -- Insertion with missing keys and filename (should fail) INSERT INTO testmulticorn (color, size, name, filename) VALUES ('blue', 'small', 'triangle', 'blue/small/triangle.txt'); psql:test-common/multicorn_testfilesystem.include:34: ERROR: The following columns are necessary: {'ext'} HINT: You can also insert an item by providing only the filename and content columns -- Insertion which would overwrite a file. -- Normal insertion INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('yellow', 'big', 'square', 'text', 'Im a duplicate big square'); psql:test-common/multicorn_testfilesystem.include:38: ERROR: Duplicate key value violates filesystem integrity. DETAIL: Key (color, ext, name, size)=(yellow, text, square, big) already exists -- Should still have 11 lines by now. SELECT * from testmulticorn ORDER BY filename; color | size | name | ext | filename | data --------+-------+----------+------+-------------------------+-------------------------- blue | big | round | ini | blue/big/round.ini | Im a big blue round + | | | | | blue | big | square | txt | blue/big/square.txt | Im a big blue square + | | | | | blue | big | triangle | txt | blue/big/triangle.txt | Im a big blue triangle blue | small | round | ini | blue/small/round.ini | Im a small blue round + | | | | | blue | small | square | txt | blue/small/square.txt | Im a small blue square + | | | | | red | big | round | ini | red/big/round.ini | Im a big red round + | | | | | red | big | square | txt | red/big/square.txt | Im a big red square + | | | | | red | small | round | ini | red/small/round.ini | Im a small red round + | | | | | red | small | square | txt | red/small/square.txt | Im a small red square + | | | | | yellow | big | square | text | yellow/big/square.text | Im a big yellow square yellow | small | square | txt | yellow/small/square.txt | Im a small yellow square (11 rows) -- Test insertion in transaction BEGIN; INSERT INTO testmulticorn (data, filename) VALUES ('Im a big red triangle', 'red/big/triangle.txt'); SELECT * from testmulticorn where name = 'triangle' and color = 'red' ORDER BY filename; color | size | name | ext | filename | data -------+------+----------+-----+----------------------+----------------------- red | big | triangle | txt | red/big/triangle.txt | Im a big red triangle (1 row) ROLLBACK; -- The file should not be persisted. SELECT * from testmulticorn where name = 'triangle' and color = 'red' ORDER BY filename; color | size | name | ext | filename | data -------+------+------+-----+----------+------ (0 rows) -- Test Update WITH t as ( UPDATE testmulticorn set name = 'rectangle' where name = 'square' RETURNING filename ) SELECT * from t order by filename; filename ---------------------------- blue/big/rectangle.txt blue/small/rectangle.txt red/big/rectangle.txt red/small/rectangle.txt yellow/big/rectangle.text yellow/small/rectangle.txt (6 rows) -- O lines SELECT count(1) from testmulticorn where name = 'square'; count ------- 0 (1 row) -- 6 lines SELECT count(1) from testmulticorn where name = 'rectangle'; count ------- 6 (1 row) -- Update should not work if it would override an existing file. UPDATE testmulticorn set filename = 'blue/big/triangle.txt' where filename = 'blue/big/rectangle.txt'; psql:test-common/multicorn_testfilesystem.include:65: ERROR: Duplicate key value violates filesystem integrity. DETAIL: Key (color, ext, name, size)=(blue, txt, triangle, big) already exists -- Update should not work when setting filename column to NULL UPDATE testmulticorn set filename = NULL where filename = 'blue/big/rectangle.txt'; psql:test-common/multicorn_testfilesystem.include:68: ERROR: The filename, or all pattern columns are needed. -- Update should not work when setting a property column to NULL WITH t as ( UPDATE testmulticorn set color = NULL where filename = 'blue/big/rectangle.txt' RETURNING color ) SELECT * from t ORDER BY color; psql:test-common/multicorn_testfilesystem.include:73: ERROR: Null value in columns (color) are not allowed DETAIL: Failing row contains (NULL, big, rectangle, txt) -- Content column update. UPDATE testmulticorn set data = 'Im an updated rectangle' where filename = 'blue/big/rectangle.txt' RETURNING data; data ------------------------- Im an updated rectangle (1 row) SELECT * from testmulticorn where filename = 'blue/big/rectangle.txt'; color | size | name | ext | filename | data -------+------+-----------+-----+------------------------+------------------------- blue | big | rectangle | txt | blue/big/rectangle.txt | Im an updated rectangle (1 row) -- Update in transactions BEGIN; UPDATE testmulticorn set name = 'square' where name = 'rectangle'; -- O lines SELECT count(1) from testmulticorn where name = 'rectangle'; count ------- 0 (1 row) -- 6 lines SELECT count(1) from testmulticorn where name = 'square'; count ------- 6 (1 row) ROLLBACK; -- O lines SELECT count(1) from testmulticorn where name = 'square'; count ------- 0 (1 row) -- 6 lines SELECT count(1) from testmulticorn where name = 'rectangle'; count ------- 6 (1 row) BEGIN; UPDATE testmulticorn set data = data || ' UPDATED!'; -- 11 lines SELECT count(1) from testmulticorn where data ilike '% UPDATED!'; count ------- 11 (1 row) SELECT data from testmulticorn where data ilike '% UPDATED!' order by filename limit 1; data ---------------------------------- Im an updated rectangle UPDATED! (1 row) ROLLBACK; -- 0 lines SELECT count(1) from testmulticorn where data ilike '% UPDATED!'; count ------- 0 (1 row) BEGIN; UPDATE testmulticorn set data = data || ' UPDATED!'; UPDATE testmulticorn set data = data || ' TWICE!'; SELECT data from testmulticorn order by filename; data ------------------------------------------ Im an updated rectangle UPDATED! TWICE! Im a big blue round + UPDATED! TWICE! Im a big blue triangle UPDATED! TWICE! Im a small blue square + UPDATED! TWICE! Im a small blue round + UPDATED! TWICE! Im a big red square + UPDATED! TWICE! Im a big red round + UPDATED! TWICE! Im a small red square + UPDATED! TWICE! Im a small red round + UPDATED! TWICE! Im a big yellow square UPDATED! TWICE! Im a small yellow square UPDATED! TWICE! (11 rows) ROLLBACK; -- No 'UPDATED! or 'TWICE!' SELECT data from testmulticorn order by filename; data -------------------------- Im an updated rectangle Im a big blue round + Im a big blue triangle Im a small blue square + Im a small blue round + Im a big red square + Im a big red round + Im a small red square + Im a small red round + Im a big yellow square Im a small yellow square (11 rows) -- Test successive update to the same files. BEGIN; UPDATE testmulticorn set color = 'cyan' where filename = 'blue/big/rectangle.txt'; -- There should be one line with cyan color, 0 with the old filename SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ------------------------+------------------------- cyan/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ----------+------ (0 rows) -- There should be one line with magenta, and 0 with cyan and the old -- filename UPDATE testmulticorn set color = 'magenta' where color = 'cyan'; SELECT filename, data from testmulticorn where color = 'magenta' order by filename; filename | data ---------------------------+------------------------- magenta/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ----------+------ (0 rows) SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ----------+------ (0 rows) UPDATE testmulticorn set color = 'blue' where color = 'magenta'; -- There should be one line with the old filename, and zero with the rest SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ------------------------+------------------------- blue/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where color = 'magenta' order by filename; filename | data ----------+------ (0 rows) SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ----------+------ (0 rows) COMMIT; -- Result should be the same than pre-commit SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; filename | data ------------------------+------------------------- blue/big/rectangle.txt | Im an updated rectangle (1 row) SELECT filename, data from testmulticorn where color = 'magenta' order by filename; filename | data ----------+------ (0 rows) SELECT filename, data from testmulticorn where color = 'cyan' order by filename; filename | data ----------+------ (0 rows) -- DELETE test WITH t as ( DELETE from testmulticorn where color = 'yellow' returning filename ) SELECT * from t order by filename; filename ---------------------------- yellow/big/rectangle.text yellow/small/rectangle.txt (2 rows) -- Should have no rows select count(1) from testmulticorn where color = 'yellow'; count ------- 0 (1 row) -- DELETE in transaction BEGIN; WITH t as ( DELETE from testmulticorn where color = 'red' returning filename ) SELECT * from t order by filename; filename ------------------------- red/big/rectangle.txt red/big/round.ini red/small/rectangle.txt red/small/round.ini (4 rows) select count(1) from testmulticorn where color = 'red'; count ------- 0 (1 row) ROLLBACK; -- Should have 4 rows select count(1) from testmulticorn where color = 'red'; count ------- 4 (1 row) -- Test various combinations of INSERT/UPDATE/DELETE BEGIN; INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'medium', 'triangle', 'jpg', 'Im a triangle'); INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'large', 'triangle', 'jpg', 'Im a triangle'); -- 2 lines SELECT * from testmulticorn where color = 'cyan' order by filename; color | size | name | ext | filename | data -------+--------+----------+-----+--------------------------+--------------- cyan | large | triangle | jpg | cyan/large/triangle.jpg | Im a triangle cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set color = 'magenta' where size = 'large' and color = 'cyan' returning filename; filename ---------------------------- magenta/large/triangle.jpg (1 row) -- 2 lines, one cyan, one magenta SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+--------+----------+-----+----------------------------+--------------- cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set data = 'Im magenta' where color = 'magenta'; WITH t as ( DELETE from testmulticorn where color = 'cyan' returning filename ) SELECT * from t order by filename; filename -------------------------- cyan/medium/triangle.jpg (1 row) -- One magenta line SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+-------+----------+-----+----------------------------+------------ magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im magenta (1 row) COMMIT; -- Result should be the same as precommit SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+-------+----------+-----+----------------------------+------------ magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im magenta (1 row) DELETE from testmulticorn where color = 'magenta'; -- Same as before, but rollbacking BEGIN; INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'medium', 'triangle', 'jpg', 'Im a triangle'); INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'large', 'triangle', 'jpg', 'Im a triangle'); -- 2 lines SELECT * from testmulticorn where color = 'cyan' order by filename; color | size | name | ext | filename | data -------+--------+----------+-----+--------------------------+--------------- cyan | large | triangle | jpg | cyan/large/triangle.jpg | Im a triangle cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set color = 'magenta' where size = 'large' and color = 'cyan' returning filename; filename ---------------------------- magenta/large/triangle.jpg (1 row) -- 2 lines, one cyan, one magenta SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+--------+----------+-----+----------------------------+--------------- cyan | medium | triangle | jpg | cyan/medium/triangle.jpg | Im a triangle magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im a triangle (2 rows) UPDATE testmulticorn set data = 'Im magenta' where color = 'magenta'; DELETE FROM testmulticorn where color = 'cyan' RETURNING filename; filename -------------------------- cyan/medium/triangle.jpg (1 row) -- One magenta line SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data ---------+-------+----------+-----+----------------------------+------------ magenta | large | triangle | jpg | magenta/large/triangle.jpg | Im magenta (1 row) ROLLBACK; SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; color | size | name | ext | filename | data -------+------+------+-----+----------+------ (0 rows) -- Cleanup everything we've done CREATE OR REPLACE FUNCTION cleanup_dir() RETURNS VOID AS $$ import shutil root_dir = plpy.execute("""SELECT dirname from temp_dir;""")[0]['dirname'] shutil.rmtree(root_dir) $$ language plpython3u; select cleanup_dir(); cleanup_dir ------------- (1 row) DROP FUNCTION cleanup_dir(); DROP TABLE temp_dir; DROP FUNCTION create_table(); DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn DROP LANGUAGE plpython3u; Multicorn-1.3.4/test-3.3/expected/write_savepoints.out000066400000000000000000000114061320447423600227210ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'nowrite' ); -- No savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', '0'), ('test2', 'A')] update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] DROP foreign table testmulticorn_write; ROLLBACK; -- One savepoint BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', '0'), ('test2', 'A')] update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] ROLLBACK TO A; RELEASE A; DROP foreign table testmulticorn_write; COMMIT; -- Multiple sequential savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', '0'), ('test2', 'A')] select * from testmulticorn LIMIT 1; NOTICE: [('option1', 'option1'), ('test_type', 'nowrite'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) ROLLBACK TO A; RELEASE A; SAVEPOINT B; update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] RELEASE B; update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] DROP foreign table testmulticorn_write; ROLLBACK; -- Multiple nested savepoints BEGIN; CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning' ); SAVEPOINT A; insert into testmulticorn_write(test1, test2) VALUES ('0', 'A'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', '0'), ('test2', 'A')] select * from testmulticorn LIMIT 1; NOTICE: [] NOTICE: ['test1', 'test2'] test1 | test2 -----------+----------- test1 1 0 | test2 2 0 (1 row) SAVEPOINT B; update testmulticorn_write set test2 = 'B' where test1 = '0'; NOTICE: [test1 = 0] NOTICE: ['test1'] RELEASE B; update testmulticorn_write set test2 = 'C' where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] delete from testmulticorn_write where test1 = '1'; NOTICE: [test1 = 1] NOTICE: ['test1'] ROLLBACK TO A; RELEASE A; DROP foreign table testmulticorn_write; ROLLBACK; -- Clean up DROP USER MAPPING FOR postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn Multicorn-1.3.4/test-3.3/expected/write_sqlalchemy.out000066400000000000000000000056451320447423600227000ustar00rootroot00000000000000SET client_min_messages=NOTICE; CREATE EXTENSION multicorn; create or replace function create_foreign_server() returns void as $block$ DECLARE current_db varchar; BEGIN SELECT into current_db current_database(); EXECUTE $$ CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.sqlalchemyfdw.SqlAlchemyFdw', db_url 'postgresql://$$ || current_user || '@localhost/' || current_db || $$' ); $$; END; $block$ language plpgsql; select create_foreign_server(); create_foreign_server ----------------------- (1 row) create foreign table testalchemy ( id integer, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ) server multicorn_srv options ( tablename 'basetable' ); create table basetable ( id integer primary key, adate date, atimestamp timestamp, anumeric numeric, avarchar varchar ); insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'); NOTICE: You need to declare a primary key option in order to use the write features ERROR: This FDW does not support the writable API ALTER FOREIGN TABLE testalchemy OPTIONS (ADD primary_key 'id'); BEGIN; insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); select * from basetable; id | adate | atimestamp | anumeric | avarchar ----+-------+------------+----------+---------- (0 rows) ROLLBACK; BEGIN; insert into testalchemy (id, adate, atimestamp, anumeric, avarchar) values (1, '1980-01-01', '1980-01-01 11:01:21.132912', 3.4, 'Test'), (2, '1990-03-05', '1998-03-02 10:40:18.321023', 12.2, 'Another Test'), (3, '1972-01-02', '1972-01-02 16:12:54', 4000, 'another Test'), (4, '1922-11-02', '1962-01-02 23:12:54', -3000, NULL); update testalchemy set avarchar = avarchar || ' UPDATED!'; COMMIT; SELECT * from basetable; id | adate | atimestamp | anumeric | avarchar ----+------------+--------------------------+----------+----------------------- 1 | 01-01-1980 | Tue Jan 01 11:01:21 1980 | 3.4 | Test UPDATED! 2 | 03-05-1990 | Mon Mar 02 10:40:18 1998 | 12.2 | Another Test UPDATED! 3 | 01-02-1972 | Sun Jan 02 16:12:54 1972 | 4000.0 | another Test UPDATED! 4 | 11-02-1922 | Tue Jan 02 23:12:54 1962 | -3000.0 | (4 rows) DELETE from testalchemy; SELECT * from basetable; id | adate | atimestamp | anumeric | avarchar ----+-------+------------+----------+---------- (0 rows) DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testalchemy DROP TABLE basetable; Multicorn-1.3.4/test-3.3/expected/write_test.out000066400000000000000000000171101320447423600215030ustar00rootroot00000000000000CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.testfdw.TestForeignDataWrapper' ); CREATE user mapping for postgres server multicorn_srv options (usermapping 'test'); CREATE foreign table testmulticorn ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', test_type 'nowrite', tx_hook 'true' ); insert into testmulticorn(test1, test2) VALUES ('test', 'test2'); NOTICE: [('option1', 'option1'), ('test_type', 'nowrite'), ('tx_hook', 'true'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: BEGIN NOTICE: ROLLBACK ERROR: Error in python: NotImplementedError DETAIL: This FDW does not support the writable API update testmulticorn set test1 = 'test'; NOTICE: BEGIN NOTICE: [] NOTICE: ['test1', 'test2'] NOTICE: ROLLBACK ERROR: Error in python: NotImplementedError DETAIL: This FDW does not support the writable API delete from testmulticorn where test2 = 'test2 2 0'; NOTICE: BEGIN NOTICE: [test2 = test2 2 0] NOTICE: ['test1', 'test2'] NOTICE: ROLLBACK ERROR: Error in python: NotImplementedError DETAIL: This FDW does not support the writable API CREATE foreign table testmulticorn_write ( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test1', test_type 'returning', tx_hook 'true' ); insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test1'), ('test_type', 'returning'), ('tx_hook', 'true'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: BEGIN NOTICE: INSERTING: [('test1', 'test'), ('test2', 'test2')] NOTICE: PRECOMMIT NOTICE: COMMIT update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%'; NOTICE: BEGIN NOTICE: [test1 ~~* test1 3%] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test1 3 1 with [('test1', 'test'), ('test2', 'test2 1 1')] NOTICE: UPDATING: test1 3 4 with [('test1', 'test'), ('test2', 'test2 1 4')] NOTICE: UPDATING: test1 3 7 with [('test1', 'test'), ('test2', 'test2 1 7')] NOTICE: UPDATING: test1 3 10 with [('test1', 'test'), ('test2', 'test2 1 10')] NOTICE: UPDATING: test1 3 13 with [('test1', 'test'), ('test2', 'test2 1 13')] NOTICE: UPDATING: test1 3 16 with [('test1', 'test'), ('test2', 'test2 1 16')] NOTICE: UPDATING: test1 3 19 with [('test1', 'test'), ('test2', 'test2 1 19')] NOTICE: PRECOMMIT NOTICE: COMMIT delete from testmulticorn_write where test2 = 'test2 2 0'; NOTICE: BEGIN NOTICE: [test2 = test2 2 0] NOTICE: ['test1', 'test2'] NOTICE: DELETING: test1 1 0 NOTICE: PRECOMMIT NOTICE: COMMIT -- Test returning insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2') RETURNING test1; NOTICE: BEGIN NOTICE: INSERTING: [('test1', 'test'), ('test2', 'test2')] NOTICE: PRECOMMIT NOTICE: COMMIT test1 ---------------- INSERTED: test (1 row) update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%' RETURNING test1; NOTICE: BEGIN NOTICE: [test1 ~~* test1 3%] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test1 3 1 with [('test1', 'test'), ('test2', 'test2 1 1')] NOTICE: UPDATING: test1 3 4 with [('test1', 'test'), ('test2', 'test2 1 4')] NOTICE: UPDATING: test1 3 7 with [('test1', 'test'), ('test2', 'test2 1 7')] NOTICE: UPDATING: test1 3 10 with [('test1', 'test'), ('test2', 'test2 1 10')] NOTICE: UPDATING: test1 3 13 with [('test1', 'test'), ('test2', 'test2 1 13')] NOTICE: UPDATING: test1 3 16 with [('test1', 'test'), ('test2', 'test2 1 16')] NOTICE: UPDATING: test1 3 19 with [('test1', 'test'), ('test2', 'test2 1 19')] NOTICE: PRECOMMIT NOTICE: COMMIT test1 --------------- UPDATED: test UPDATED: test UPDATED: test UPDATED: test UPDATED: test UPDATED: test UPDATED: test (7 rows) delete from testmulticorn_write where test1 = 'test1 1 0' returning test2, test1; NOTICE: BEGIN NOTICE: [test1 = test1 1 0] NOTICE: ['test1', 'test2'] NOTICE: DELETING: test1 1 0 NOTICE: PRECOMMIT NOTICE: COMMIT test2 | test1 -----------+----------- test2 2 0 | test1 1 0 (1 row) DROP foreign table testmulticorn_write; -- Now test with another column CREATE foreign table testmulticorn_write( test1 character varying, test2 character varying ) server multicorn_srv options ( option1 'option1', row_id_column 'test2' ); insert into testmulticorn_write(test1, test2) VALUES ('test', 'test2'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test2'), ('usermapping', 'test')] NOTICE: [('test1', 'character varying'), ('test2', 'character varying')] NOTICE: INSERTING: [('test1', 'test'), ('test2', 'test2')] update testmulticorn_write set test1 = 'test' where test1 ilike 'test1 3%'; NOTICE: [test1 ~~* test1 3%] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test2 1 1 with [('test1', 'test'), ('test2', 'test2 1 1')] NOTICE: UPDATING: test2 1 4 with [('test1', 'test'), ('test2', 'test2 1 4')] NOTICE: UPDATING: test2 1 7 with [('test1', 'test'), ('test2', 'test2 1 7')] NOTICE: UPDATING: test2 1 10 with [('test1', 'test'), ('test2', 'test2 1 10')] NOTICE: UPDATING: test2 1 13 with [('test1', 'test'), ('test2', 'test2 1 13')] NOTICE: UPDATING: test2 1 16 with [('test1', 'test'), ('test2', 'test2 1 16')] NOTICE: UPDATING: test2 1 19 with [('test1', 'test'), ('test2', 'test2 1 19')] delete from testmulticorn_write where test2 = 'test2 2 0'; NOTICE: [test2 = test2 2 0] NOTICE: ['test2'] NOTICE: DELETING: test2 2 0 update testmulticorn_write set test2 = 'test' where test2 = 'test2 1 1'; NOTICE: [test2 = test2 1 1] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: test2 1 1 with [('test1', 'test1 3 1'), ('test2', 'test')] DROP foreign table testmulticorn_write; -- Now test with other types CREATE foreign table testmulticorn_write( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', row_id_column 'test2', test_type 'date' ); insert into testmulticorn_write(test1, test2) VALUES ('2012-01-01', '2012-01-01 00:00:00'); NOTICE: [('option1', 'option1'), ('row_id_column', 'test2'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] NOTICE: INSERTING: [('test1', datetime.date(2012, 1, 1)), ('test2', datetime.datetime(2012, 1, 1, 0, 0))] delete from testmulticorn_write where test2 > '2011-12-03'; NOTICE: [test2 > 2011-12-03 00:00:00] NOTICE: ['test2'] NOTICE: DELETING: 2011-12-03 14:30:25 update testmulticorn_write set test1 = date_trunc('day', test1) where test2 = '2011-09-03 14:30:25'; NOTICE: [test2 = 2011-09-03 14:30:25] NOTICE: ['test1', 'test2'] NOTICE: UPDATING: 2011-09-03 14:30:25 with [('test1', datetime.date(2011, 9, 2)), ('test2', datetime.datetime(2011, 9, 3, 14, 30, 25))] DROP foreign table testmulticorn_write; -- Test with unknown column CREATE foreign table testmulticorn_write( test1 date, test2 timestamp ) server multicorn_srv options ( option1 'option1', row_id_column 'teststuff', test_type 'date' ); delete from testmulticorn_write; NOTICE: [('option1', 'option1'), ('row_id_column', 'teststuff'), ('test_type', 'date'), ('usermapping', 'test')] NOTICE: [('test1', 'date'), ('test2', 'timestamp without time zone')] ERROR: The rowid attribute does not exist DROP USER MAPPING for postgres SERVER multicorn_srv; DROP EXTENSION multicorn cascade; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to server multicorn_srv drop cascades to foreign table testmulticorn drop cascades to foreign table testmulticorn_write Multicorn-1.3.4/test-3.3/sql/000077500000000000000000000000001320447423600155575ustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/import_sqlalchemy.sql000077700000000000000000000000001320447423600310142../../test-2.7/sql/import_sqlalchemy.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/import_test.sql000077700000000000000000000000001320447423600264462../../test-2.7/sql/import_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_alchemy_test.sql000077700000000000000000000000001320447423600330562../../test-2.7/sql/multicorn_alchemy_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_cache_invalidation.sql000077700000000000000000000000001320447423600353242../../test-2.7/sql/multicorn_cache_invalidation.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_column_options_test.sql000077700000000000000000000000001320447423600361322../../test-2.7/sql/multicorn_column_options_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_error_test.sql000077700000000000000000000000001320447423600322742../../test-2.7/sql/multicorn_error_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_logger_test.sql000077700000000000000000000000001320447423600325502../../test-2.7/sql/multicorn_logger_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_planner_test.sql000077700000000000000000000000001320447423600331102../../test-2.7/sql/multicorn_planner_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_regression_test.sql000077700000000000000000000000001320447423600343522../../test-2.7/sql/multicorn_regression_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_sequence_test.sql000077700000000000000000000000001320447423600334322../../test-2.7/sql/multicorn_sequence_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_test_date.sql000077700000000000000000000000001320447423600316442../../test-2.7/sql/multicorn_test_date.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_test_dict.sql000077700000000000000000000000001320447423600316602../../test-2.7/sql/multicorn_test_dict.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_test_list.sql000077700000000000000000000000001320447423600317402../../test-2.7/sql/multicorn_test_list.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/multicorn_test_sort.sql000077700000000000000000000000001320447423600317702../../test-2.7/sql/multicorn_test_sort.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/write_filesystem.sql000066400000000000000000000035301320447423600216770ustar00rootroot00000000000000-- Setup the test CREATE language plpython3u; CREATE OR REPLACE FUNCTION create_table() RETURNS VOID AS $$ import plpy import tempfile import os dir = tempfile.mkdtemp() plpy.execute(""" INSERT INTO temp_dir(dirname) VALUES ('%s') """ % str(dir)) plpy.execute(""" CREATE foreign table testmulticorn ( color varchar, size varchar, name varchar, ext varchar, filename varchar, data varchar ) server multicorn_srv options ( filename_column 'filename', content_column 'data', pattern '{color}/{size}/{name}.{ext}', root_dir '%s' ); """ % dir) for color in ('blue', 'red'): for size in ('big', 'small'): dirname = os.path.join(dir, color, size) os.makedirs(dirname) for name, ext in (('square', 'txt'), ('round', 'ini')): with open(os.path.join(dirname, '.'.join([name, ext])), 'a') as fd: fd.write('Im a %s %s %s\n' % (size, color, name)) $$ language plpython3u; CREATE EXTENSION multicorn; CREATE server multicorn_srv foreign data wrapper multicorn options ( wrapper 'multicorn.fsfdw.FilesystemFdw' ); CREATE TABLE temp_dir (dirname varchar); -- Create a table with the filesystem fdw in a temporary directory, -- and store the dirname in the temp_dir table. select create_table(); -- End of Setup \i test-common/multicorn_testfilesystem.include -- Cleanup everything we've done CREATE OR REPLACE FUNCTION cleanup_dir() RETURNS VOID AS $$ import shutil root_dir = plpy.execute("""SELECT dirname from temp_dir;""")[0]['dirname'] shutil.rmtree(root_dir) $$ language plpython3u; select cleanup_dir(); DROP FUNCTION cleanup_dir(); DROP TABLE temp_dir; DROP FUNCTION create_table(); DROP EXTENSION multicorn cascade; DROP LANGUAGE plpython3u; Multicorn-1.3.4/test-3.3/sql/write_savepoints.sql000077700000000000000000000000001320447423600305362../../test-2.7/sql/write_savepoints.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/write_sqlalchemy.sql000077700000000000000000000000001320447423600304542../../test-2.7/sql/write_sqlalchemy.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.3/sql/write_test.sql000077700000000000000000000000001320447423600261062../../test-2.7/sql/write_test.sqlustar00rootroot00000000000000Multicorn-1.3.4/test-3.4000077700000000000000000000000001320447423600161022test-3.3ustar00rootroot00000000000000Multicorn-1.3.4/test-3.5000077700000000000000000000000001320447423600161032test-3.3ustar00rootroot00000000000000Multicorn-1.3.4/test-common/000077500000000000000000000000001320447423600157455ustar00rootroot00000000000000Multicorn-1.3.4/test-common/multicorn_testfilesystem.include000066400000000000000000000207001320447423600244710ustar00rootroot00000000000000-- Should have 8 lines. SELECT * from testmulticorn ORDER BY filename; -- Test the cost analysis EXPLAIN select color, size from testmulticorn where color = 'blue' and size = 'big' and name = 'square' and ext = 'txt'; EXPLAIN select color, size from testmulticorn where color = 'blue' and size = 'big'; EXPLAIN select color, size from testmulticorn where color = 'blue'; EXPLAIN select color, size, data from testmulticorn where color = 'blue' and size = 'big' and name = 'square' and ext = 'txt'; -- Test insertion -- Normal insertion INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('yellow', 'big', 'square', 'text', 'Im a big yellow square') RETURNING filename; -- Insertion with redundant filename/properties INSERT INTO testmulticorn (color, size, name, ext, data, filename) VALUES ('yellow', 'small', 'square', 'txt', 'Im a small yellow square', 'yellow/small/square.txt'); -- Insertion with just a filename INSERT INTO testmulticorn (data, filename) VALUES ('Im a big blue triangle', 'blue/big/triangle.txt') RETURNING color, size, name, ext; -- Should have 11 lines by now. SELECT * from testmulticorn ORDER BY filename; -- Insertion with incoherent filename/properties (should fail) INSERT INTO testmulticorn (color, size, name, ext, data, filename) VALUES ('blue', 'big', 'triangle', 'txt', 'Im a big blue triangle', 'blue/small/triangle.txt'); -- Insertion with missing keys (should fail) INSERT INTO testmulticorn (color, size, name) VALUES ('blue', 'small', 'triangle'); -- Insertion with missing keys and filename (should fail) INSERT INTO testmulticorn (color, size, name, filename) VALUES ('blue', 'small', 'triangle', 'blue/small/triangle.txt'); -- Insertion which would overwrite a file. -- Normal insertion INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('yellow', 'big', 'square', 'text', 'Im a duplicate big square'); -- Should still have 11 lines by now. SELECT * from testmulticorn ORDER BY filename; -- Test insertion in transaction BEGIN; INSERT INTO testmulticorn (data, filename) VALUES ('Im a big red triangle', 'red/big/triangle.txt'); SELECT * from testmulticorn where name = 'triangle' and color = 'red' ORDER BY filename; ROLLBACK; -- The file should not be persisted. SELECT * from testmulticorn where name = 'triangle' and color = 'red' ORDER BY filename; -- Test Update WITH t as ( UPDATE testmulticorn set name = 'rectangle' where name = 'square' RETURNING filename ) SELECT * from t order by filename; -- O lines SELECT count(1) from testmulticorn where name = 'square'; -- 6 lines SELECT count(1) from testmulticorn where name = 'rectangle'; -- Update should not work if it would override an existing file. UPDATE testmulticorn set filename = 'blue/big/triangle.txt' where filename = 'blue/big/rectangle.txt'; -- Update should not work when setting filename column to NULL UPDATE testmulticorn set filename = NULL where filename = 'blue/big/rectangle.txt'; -- Update should not work when setting a property column to NULL WITH t as ( UPDATE testmulticorn set color = NULL where filename = 'blue/big/rectangle.txt' RETURNING color ) SELECT * from t ORDER BY color; -- Content column update. UPDATE testmulticorn set data = 'Im an updated rectangle' where filename = 'blue/big/rectangle.txt' RETURNING data; SELECT * from testmulticorn where filename = 'blue/big/rectangle.txt'; -- Update in transactions BEGIN; UPDATE testmulticorn set name = 'square' where name = 'rectangle'; -- O lines SELECT count(1) from testmulticorn where name = 'rectangle'; -- 6 lines SELECT count(1) from testmulticorn where name = 'square'; ROLLBACK; -- O lines SELECT count(1) from testmulticorn where name = 'square'; -- 6 lines SELECT count(1) from testmulticorn where name = 'rectangle'; BEGIN; UPDATE testmulticorn set data = data || ' UPDATED!'; -- 11 lines SELECT count(1) from testmulticorn where data ilike '% UPDATED!'; SELECT data from testmulticorn where data ilike '% UPDATED!' order by filename limit 1; ROLLBACK; -- 0 lines SELECT count(1) from testmulticorn where data ilike '% UPDATED!'; BEGIN; UPDATE testmulticorn set data = data || ' UPDATED!'; UPDATE testmulticorn set data = data || ' TWICE!'; SELECT data from testmulticorn order by filename; ROLLBACK; -- No 'UPDATED! or 'TWICE!' SELECT data from testmulticorn order by filename; -- Test successive update to the same files. BEGIN; UPDATE testmulticorn set color = 'cyan' where filename = 'blue/big/rectangle.txt'; -- There should be one line with cyan color, 0 with the old filename SELECT filename, data from testmulticorn where color = 'cyan' order by filename; SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; -- There should be one line with magenta, and 0 with cyan and the old -- filename UPDATE testmulticorn set color = 'magenta' where color = 'cyan'; SELECT filename, data from testmulticorn where color = 'magenta' order by filename; SELECT filename, data from testmulticorn where color = 'cyan' order by filename; SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; UPDATE testmulticorn set color = 'blue' where color = 'magenta'; -- There should be one line with the old filename, and zero with the rest SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; SELECT filename, data from testmulticorn where color = 'magenta' order by filename; SELECT filename, data from testmulticorn where color = 'cyan' order by filename; COMMIT; -- Result should be the same than pre-commit SELECT filename, data from testmulticorn where filename = 'blue/big/rectangle.txt' order by filename; SELECT filename, data from testmulticorn where color = 'magenta' order by filename; SELECT filename, data from testmulticorn where color = 'cyan' order by filename; -- DELETE test WITH t as ( DELETE from testmulticorn where color = 'yellow' returning filename ) SELECT * from t order by filename; -- Should have no rows select count(1) from testmulticorn where color = 'yellow'; -- DELETE in transaction BEGIN; WITH t as ( DELETE from testmulticorn where color = 'red' returning filename ) SELECT * from t order by filename; select count(1) from testmulticorn where color = 'red'; ROLLBACK; -- Should have 4 rows select count(1) from testmulticorn where color = 'red'; -- Test various combinations of INSERT/UPDATE/DELETE BEGIN; INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'medium', 'triangle', 'jpg', 'Im a triangle'); INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'large', 'triangle', 'jpg', 'Im a triangle'); -- 2 lines SELECT * from testmulticorn where color = 'cyan' order by filename; UPDATE testmulticorn set color = 'magenta' where size = 'large' and color = 'cyan' returning filename; -- 2 lines, one cyan, one magenta SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; UPDATE testmulticorn set data = 'Im magenta' where color = 'magenta'; WITH t as ( DELETE from testmulticorn where color = 'cyan' returning filename ) SELECT * from t order by filename; -- One magenta line SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; COMMIT; -- Result should be the same as precommit SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; DELETE from testmulticorn where color = 'magenta'; -- Same as before, but rollbacking BEGIN; INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'medium', 'triangle', 'jpg', 'Im a triangle'); INSERT INTO testmulticorn (color, size, name, ext, data) VALUES ('cyan', 'large', 'triangle', 'jpg', 'Im a triangle'); -- 2 lines SELECT * from testmulticorn where color = 'cyan' order by filename; UPDATE testmulticorn set color = 'magenta' where size = 'large' and color = 'cyan' returning filename; -- 2 lines, one cyan, one magenta SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; UPDATE testmulticorn set data = 'Im magenta' where color = 'magenta'; DELETE FROM testmulticorn where color = 'cyan' RETURNING filename; -- One magenta line SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename; ROLLBACK; SELECT * from testmulticorn where color in ('cyan', 'magenta') order by filename;