pax_global_header00006660000000000000000000000064137323371430014520gustar00rootroot0000000000000052 comment=f74fe2339d95cabe681cec4b6e14a4415b3a3256 pgq-3.4.1/000077500000000000000000000000001373233714300123145ustar00rootroot00000000000000pgq-3.4.1/.github/000077500000000000000000000000001373233714300136545ustar00rootroot00000000000000pgq-3.4.1/.github/workflows/000077500000000000000000000000001373233714300157115ustar00rootroot00000000000000pgq-3.4.1/.github/workflows/ci.yml000066400000000000000000000036011373233714300170270ustar00rootroot00000000000000# # https://docs.github.com/en/actions # https://github.com/actions # name: CI on: pull_request: {} push: {} jobs: linux: name: "Ubuntu 18.04 + PostgreSQL ${{matrix.PGVER}}" runs-on: ubuntu-18.04 strategy: matrix: PGVER: [10, 11, 12, 13] steps: - name: "Checkout" uses: actions/checkout@v2 - name: "InstallDB" run: | echo "::group::apt-get-update" echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main ${{matrix.PGVER}}" \ | sudo tee /etc/apt/sources.list.d/pgdg.list sudo -nH apt-get -q update echo "::endgroup::" echo "::group::apt-get-install" # disable new cluster creation sudo -nH mkdir -p /etc/postgresql-common/createcluster.d echo "create_main_cluster = false" | sudo -nH tee /etc/postgresql-common/createcluster.d/no-main.conf sudo -nH apt-get -qyu install \ postgresql-${{matrix.PGVER}} \ postgresql-server-dev-${{matrix.PGVER}} \ libpq-dev patchutils echo "::endgroup::" # tune environment echo "::add-path::/usr/lib/postgresql/${{matrix.PGVER}}/bin" echo "::set-env name=PGHOST::/tmp" dpkg -l postgres\* libpq\* bison\* flex\* gcc\* clang\* libllvm\* - name: "Build" run: make - name: "Install" run: sudo -nH bash -c "PATH='${PATH}' make install" - name: "StartDB" run: | rm -rf data log mkdir -p log LANG=C initdb data sed -ri -e "s,^[# ]*(unix_socket_directories).*,\\1='/tmp'," data/postgresql.conf pg_ctl -D data -l log/pg.log start || { cat log/pg.log ; exit 1; } - name: "Test" run: make citest - name: "StopDB" run: | pg_ctl -D data stop rm -rf data log /tmp/.s.PGSQL* pgq-3.4.1/.github/workflows/release.yml000066400000000000000000000027101373233714300200540ustar00rootroot00000000000000 name: REL on: push: tags: ["v[0-9]*"] jobs: release: name: Release runs-on: ubuntu-latest steps: - name: Checkout code id: checkout uses: actions/checkout@v2 - name: Build tarball id: build run: | make checkver make dist EXTENSION=$(grep ^EXTENSION Makefile | sed 's/.*= *//') EXT_VERSION=$(grep ^EXT_VERSION Makefile | sed 's/.*= *//') test "${{github.ref}}" = "refs/tags/v${EXT_VERSION}" || { echo "ERR: tag mismatch"; exit 1; } echo "::set-env name=EXTENSION::${EXTENSION}" echo "::set-env name=EXT_VERSION::${EXT_VERSION}" echo "::set-env name=TGZ::${EXTENSION}-${EXT_VERSION}.tar.gz" - name: Create release id: release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} with: tag_name: ${{github.ref}} release_name: ${{github.event.repository.name}} v${{env.EXT_VERSION}} body_path: docs/notes/v${{env.EXT_VERSION}}.md draft: false prerelease: false - name: Upload source id: upload uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} with: upload_url: ${{steps.release.outputs.upload_url}} asset_path: ${{env.TGZ}} asset_name: ${{env.TGZ}} asset_content_type: application/x-gzip pgq-3.4.1/.gitignore000066400000000000000000000004411373233714300143030ustar00rootroot00000000000000*.o *.so .deps *.swp *.dump *--* *.gz sql/*/*.sql sql/*/results regression.* newgrants_*.sql oldgrants_*.sql debian/control debian/tmp debian/files debian/postgresql-* debian/*-stamp *.substvars *.log *.debhelper pgq.sql pgq.upgrade.sql results/ /*.sql docs/Data docs/html docs/sql pgq-3.4.1/Makefile000066400000000000000000000044501373233714300137570ustar00rootroot00000000000000 EXTENSION = pgq EXT_VERSION = 3.4.1 EXT_OLD_VERSIONS = 3.2 3.2.3 3.2.6 3.3.1 3.4 PGQ_TESTS = pgq_core pgq_core_disabled pgq_core_tx_limit \ pgq_session_role pgq_perms \ trigger_base trigger_sess_role trigger_types trigger_trunc trigger_ignore \ trigger_pkey trigger_deny trigger_when trigger_extra_args trigger_extra_cols \ \ clean_ext pgq_init_ext \ switch_plonly \ \ pgq_core pgq_core_disabled \ pgq_session_role pgq_perms \ trigger_base trigger_sess_role trigger_types trigger_trunc trigger_ignore \ trigger_pkey trigger_deny trigger_when trigger_extra_args trigger_extra_cols \ # comment it out if not wanted #UPGRADE_TESTS = pgq_init_upgrade $(PGQ_TESTS) clean Contrib_data = structure/uninstall_pgq.sql Contrib_data_built = pgq_pl_only.sql pgq_pl_only.upgrade.sql Contrib_regress = $(UPGRADE_TESTS) pgq_init_noext $(PGQ_TESTS) Extension_regress = $(UPGRADE_TESTS) pgq_init_ext $(PGQ_TESTS) include mk/common-pgxs.mk SUBDIRS = lowlevel triggers # PGXS does not have subdir support, thus hack to recurse into lowlevel/ all: sub-all install: sub-install clean: sub-clean distclean: sub-distclean sub-all sub-install sub-clean sub-distclean: for dir in $(SUBDIRS); do \ $(MAKE) -C $$dir $(subst sub-,,$@) \ DESTDIR=$(DESTDIR) \ PG_CONFIG=$(PG_CONFIG) \ || exit 1; \ done lowlevel/pgq_lowlevel.sql: sub-all triggers/pgq_triggers.sql: sub-all PLONLY_SRCS = lowlevel_pl/insert_event.sql lowlevel_pl/logutriga.sql lowlevel_pl/sqltriga.sql pgq_pl_only.sql: $(SRCS) $(PLONLY_SRCS) $(CATSQL) structure/install_pl.sql $(GRANT_SQL) > $@ pgq_pl_only.upgrade.sql: $(SRCS) $(PLONLY_SRCS) $(CATSQL) structure/upgrade_pl.sql $(GRANT_SQL) > $@ plonly: pgq_pl_only.sql pgq_pl_only.upgrade.sql # # docs # dox: cleandox $(SRCS) mkdir -p docs/html mkdir -p docs/sql $(CATSQL) --ndoc structure/tables.sql > docs/sql/schema.sql $(CATSQL) --ndoc structure/func_public.sql > docs/sql/external.sql $(CATSQL) --ndoc structure/func_internal.sql > docs/sql/internal.sql $(CATSQL) --ndoc structure/triggers.sql > docs/sql/triggers.sql $(NDOC) $(NDOCARGS) doxsync: for m in pgq_coop pgq_node pgq_ext londiste; do \ cp docs/Topics.txt docs/Languages.txt ../$$m/docs; \ done deb: make -f debian/rules genfiles debuild -us -uc -b debclean: make -f debian/rules debclean pgq-3.4.1/README.rst000066400000000000000000000003751373233714300140100ustar00rootroot00000000000000Generic Queue for PostgreSQL ============================ PgQ is PostgreSQL extension that provides generic, high-performance lockless queue with simple API based on SQL functions. Supported PostgreSQL versions: 9.3+ Deprecated but might work: 8.4+ pgq-3.4.1/debian/000077500000000000000000000000001373233714300135365ustar00rootroot00000000000000pgq-3.4.1/debian/changelog000066400000000000000000000011131373233714300154040ustar00rootroot00000000000000pgq (3.4.1-1) unstable; urgency=low * v3.4.1 -- Marko Kreen Tue, 22 Sep 2020 11:51:41 +0300 pgq (3.4-1) unstable; urgency=low * v3.4 -- Marko Kreen Thu, 09 Jul 2020 18:43:53 +0300 pgq (3.3.1-1) unstable; urgency=low * v3.3.1 -- Marko Kreen Wed, 27 Nov 2019 15:07:17 +0200 pgq (3.3-1) unstable; urgency=low * v3.3 -- Marko Kreen Thu, 21 Nov 2019 19:42:20 +0200 pgq (3.2.6-1) unstable; urgency=low * v3.2.6 -- Marko Kreen Sat, 11 Jun 2016 16:21:25 +0300 pgq-3.4.1/debian/compat000066400000000000000000000000021373233714300147340ustar00rootroot000000000000009 pgq-3.4.1/debian/control.in000066400000000000000000000011051373233714300155430ustar00rootroot00000000000000Source: pgq Section: database Priority: extra Maintainer: Marko Kreen Build-Depends: debhelper (>= 9), python3, postgresql-server-dev-all Standards-Version: 3.9.1 Homepage: https://github.io/pgq Package: postgresql-PGVERSION-pgq Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, postgresql-PGVERSION, pgqd Conflicts: postgresql-PGVERSION-pgq3 Description: Generic queue for PostgreSQL This is extension that provides generic queue for PostgreSQL. . It allows multiple queues in one database, each queue can be consumer by multiple consumers. pgq-3.4.1/debian/copyright000066400000000000000000000016631373233714300154770ustar00rootroot00000000000000Based on Skytools 3 packaging: Copyright (C) 2011 Dimitri Fontaine Upstream Author: Marko Kreen Copyright: Copyright (C) 2007-2016 Marko Kreen License: Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. pgq-3.4.1/debian/pgversions000066400000000000000000000000041373233714300156520ustar00rootroot00000000000000all pgq-3.4.1/debian/rules000077500000000000000000000021341373233714300146160ustar00rootroot00000000000000#!/usr/bin/make -f # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 DEB_BUILD_OPTIONS := nostrip nocheck export DEB_BUILD_OPTIONS PG_BUILDEXT = pg_buildext export PG_UPDATECONTROL=1 PG_VERSIONS = $$($(PG_BUILDEXT) supported-versions . ) include /usr/share/postgresql-common/pgxs_debian_control.mk %: dh $@ override_dh_auto_clean: dh_auto_clean -- clean || exit 0 override_dh_auto_install: mkdir -p $(CURDIR)/debian/tmp for v in $(PG_VERSIONS); do \ echo "### Building for PostgreSQL $${v}" && \ $(MAKE) clean install \ PG_CONFIG=/usr/lib/postgresql/$${v}/bin/pg_config \ DESTDIR=$(CURDIR)/debian/tmp \ || exit 1 ; \ done genfiles: rm -f debian/control rm -rf debian/postgresql-* set -e; for ver in $(PG_VERSIONS); do \ xmod=pgq; \ dst="debian/postgresql-$${ver}-$${xmod}.install"; \ echo "usr/share/postgresql/$${ver}/extension" > "$${dst}"; \ echo "usr/share/postgresql/$${ver}/contrib" >> "$${dst}"; \ echo "usr/lib/postgresql/$${ver}/lib" >> "$${dst}"; \ done $(PG_BUILDEXT) updatecontrol debclean: clean rm -f debian/control rm -rf debian/postgresql-* pgq-3.4.1/debian/source/000077500000000000000000000000001373233714300150365ustar00rootroot00000000000000pgq-3.4.1/debian/source/format000066400000000000000000000000141373233714300162440ustar00rootroot000000000000003.0 (quilt) pgq-3.4.1/docs/000077500000000000000000000000001373233714300132445ustar00rootroot00000000000000pgq-3.4.1/docs/Languages.txt000066400000000000000000000120211373233714300157070ustar00rootroot00000000000000Format: 1.52 # This is the Natural Docs languages file for this project. If you change # anything here, it will apply to THIS PROJECT ONLY. If you'd like to change # something for all your projects, edit the Languages.txt in Natural Docs' # Config directory instead. Ignore Extension: sql #------------------------------------------------------------------------------- # SYNTAX: # # Unlike other Natural Docs configuration files, in this file all comments # MUST be alone on a line. Some languages deal with the # character, so you # cannot put comments on the same line as content. # # Also, all lists are separated with spaces, not commas, again because some # languages may need to use them. # # Language: [name] # Alter Language: [name] # Defines a new language or alters an existing one. Its name can use any # characters. If any of the properties below have an add/replace form, you # must use that when using Alter Language. # # The language Shebang Script is special. It's entry is only used for # extensions, and files with those extensions have their shebang (#!) lines # read to determine the real language of the file. Extensionless files are # always treated this way. # # The language Text File is also special. It's treated as one big comment # so you can put Natural Docs content in them without special symbols. Also, # if you don't specify a package separator, ignored prefixes, or enum value # behavior, it will copy those settings from the language that is used most # in the source tree. # # Extensions: [extension] [extension] ... # [Add/Replace] Extensions: [extension] [extension] ... # Defines the file extensions of the language's source files. You can # redefine extensions found in the main languages file. You can use * to # mean any undefined extension. # # Shebang Strings: [string] [string] ... # [Add/Replace] Shebang Strings: [string] [string] ... # Defines a list of strings that can appear in the shebang (#!) line to # designate that it's part of the language. You can redefine strings found # in the main languages file. # # Ignore Prefixes in Index: [prefix] [prefix] ... # [Add/Replace] Ignored Prefixes in Index: [prefix] [prefix] ... # # Ignore [Topic Type] Prefixes in Index: [prefix] [prefix] ... # [Add/Replace] Ignored [Topic Type] Prefixes in Index: [prefix] [prefix] ... # Specifies prefixes that should be ignored when sorting symbols in an # index. Can be specified in general or for a specific topic type. # #------------------------------------------------------------------------------ # For basic language support only: # # Line Comments: [symbol] [symbol] ... # Defines a space-separated list of symbols that are used for line comments, # if any. # # Block Comments: [opening sym] [closing sym] [opening sym] [closing sym] ... # Defines a space-separated list of symbol pairs that are used for block # comments, if any. # # Package Separator: [symbol] # Defines the default package separator symbol. The default is a dot. # # [Topic Type] Prototype Enders: [symbol] [symbol] ... # When defined, Natural Docs will attempt to get a prototype from the code # immediately following the topic type. It stops when it reaches one of # these symbols. Use \n for line breaks. # # Line Extender: [symbol] # Defines the symbol that allows a prototype to span multiple lines if # normally a line break would end it. # # Enum Values: [global|under type|under parent] # Defines how enum values are referenced. The default is global. # global - Values are always global, referenced as 'value'. # under type - Values are under the enum type, referenced as # 'package.enum.value'. # under parent - Values are under the enum's parent, referenced as # 'package.value'. # # Perl Package: [perl package] # Specifies the Perl package used to fine-tune the language behavior in ways # too complex to do in this file. # #------------------------------------------------------------------------------ # For full language support only: # # Full Language Support: [perl package] # Specifies the Perl package that has the parsing routines necessary for full # language support. # #------------------------------------------------------------------------------- # The following languages are defined in the main file, if you'd like to alter # them: # # Text File, Shebang Script, C/C++, C#, Java, JavaScript, Perl, Python, # PHP, SQL, Visual Basic, Pascal, Assembly, Ada, Tcl, Ruby, Makefile, # ActionScript, ColdFusion, R, Fortran # If you add a language that you think would be useful to other developers # and should be included in Natural Docs by default, please e-mail it to # languages [at] naturaldocs [dot] org. Language: PLPGSQL Extension: sql Line Comment: -- Block Comment: /* */ Enum Values: Global Function Prototype Enders: , ; ) $ ' Variable Prototype Enders: , ; ) := default Default DEFAULT Database Index Prototype Enders: , ; ) Database Trigger Prototype Enders: begin Begin BEGIN pgq-3.4.1/docs/Menu.txt000066400000000000000000000037311373233714300147150ustar00rootroot00000000000000Format: 1.52 Title: PgQ SubTitle: Database API # You can add a footer to your documentation like this: # Footer: [text] # If you want to add a copyright notice, this would be the place to do it. # You can add a timestamp to your documentation like one of these: # Timestamp: Generated on month day, year # Timestamp: Updated mm/dd/yyyy # Timestamp: Last updated mon day # # m - One or two digit month. January is "1" # mm - Always two digit month. January is "01" # mon - Short month word. January is "Jan" # month - Long month word. January is "January" # d - One or two digit day. 1 is "1" # dd - Always two digit day. 1 is "01" # day - Day with letter extension. 1 is "1st" # yy - Two digit year. 2006 is "06" # yyyy - Four digit year. 2006 is "2006" # year - Four digit year. 2006 is "2006" # -------------------------------------------------------------------------- # # Cut and paste the lines below to change the order in which your files # appear on the menu. Don't worry about adding or removing files, Natural # Docs will take care of that. # # You can further organize the menu by grouping the entries. Add a # "Group: [name] {" line to start a group, and add a "}" to end it. # # You can add text and web links to the menu by adding "Text: [text]" and # "Link: [name] ([URL])" lines, respectively. # # The formatting and comments are auto-generated, so don't worry about # neatness when editing the file. Natural Docs will clean it up the next # time it is run. When working with groups, just deal with the braces and # forget about the indentation and comments. # # -------------------------------------------------------------------------- File: Public Functions (external.sql) File: Public Triggers (triggers.sql) File: Internal Functions (internal.sql) File: Internal Tables (schema.sql) Group: Index { Index: Everything Database Table Index: Database Tables Function Index: Functions } # Group: Index pgq-3.4.1/docs/Topics.txt000066400000000000000000000065051373233714300152540ustar00rootroot00000000000000Format: 1.52 # This is the Natural Docs topics file for this project. If you change anything # here, it will apply to THIS PROJECT ONLY. If you'd like to change something # for all your projects, edit the Topics.txt in Natural Docs' Config directory # instead. # If you'd like to prevent keywords from being recognized by Natural Docs, you # can do it like this: # Ignore Keywords: [keyword], [keyword], ... # # Or you can use the list syntax like how they are defined: # Ignore Keywords: # [keyword] # [keyword], [plural keyword] # ... #------------------------------------------------------------------------------- # SYNTAX: # # Topic Type: [name] # Alter Topic Type: [name] # Creates a new topic type or alters one from the main file. Each type gets # its own index and behavior settings. Its name can have letters, numbers, # spaces, and these charaters: - / . ' # # Plural: [name] # Sets the plural name of the topic type, if different. # # Keywords: # [keyword] # [keyword], [plural keyword] # ... # Defines or adds to the list of keywords for the topic type. They may only # contain letters, numbers, and spaces and are not case sensitive. Plural # keywords are used for list topics. You can redefine keywords found in the # main topics file. # # Index: [yes|no] # Whether the topics get their own index. Defaults to yes. Everything is # included in the general index regardless of this setting. # # Scope: [normal|start|end|always global] # How the topics affects scope. Defaults to normal. # normal - Topics stay within the current scope. # start - Topics start a new scope for all the topics beneath it, # like class topics. # end - Topics reset the scope back to global for all the topics # beneath it. # always global - Topics are defined as global, but do not change the scope # for any other topics. # # Class Hierarchy: [yes|no] # Whether the topics are part of the class hierarchy. Defaults to no. # # Page Title If First: [yes|no] # Whether the topic's title becomes the page title if it's the first one in # a file. Defaults to no. # # Break Lists: [yes|no] # Whether list topics should be broken into individual topics in the output. # Defaults to no. # # Can Group With: [type], [type], ... # Defines a list of topic types that this one can possibly be grouped with. # Defaults to none. #------------------------------------------------------------------------------- # The following topics are defined in the main file, if you'd like to alter # their behavior or add keywords: # # Generic, Class, Interface, Section, File, Group, Function, Variable, # Property, Type, Constant, Enumeration, Event, Delegate, Macro, # Database, Database Table, Database View, Database Index, Database # Cursor, Database Trigger, Cookie, Build Target # If you add something that you think would be useful to other developers # and should be included in Natural Docs by default, please e-mail it to # topics [at] naturaldocs [dot] org. Topic Type: Schema Plural: Schemas Index: No Scope: Start Class Hierarchy: Yes Keywords: schema, schemas Alter Topic Type: Function Add Keywords: public function internal function Alter Topic Type: File Index: No pgq-3.4.1/docs/notes/000077500000000000000000000000001373233714300143745ustar00rootroot00000000000000pgq-3.4.1/docs/notes/v3.4.1.md000066400000000000000000000000511373233714300155430ustar00rootroot00000000000000* Fix newgrants - jsontriga was missing. pgq-3.4.1/docs/notes/v3.4.md000066400000000000000000000001051373233714300154040ustar00rootroot00000000000000* Set up Github actions * Support Postgres 13 * Fix version function pgq-3.4.1/docs/release.txt000066400000000000000000000002441373233714300154250ustar00rootroot00000000000000 * Set new version number in .control, Makefile, debian/changelog * Add release notes to docs/notes * git commit -m "vX.Y" * "make release" to tag and push it out pgq-3.4.1/expected/000077500000000000000000000000001373233714300141155ustar00rootroot00000000000000pgq-3.4.1/expected/clean.out000066400000000000000000000006761373233714300157410ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; drop schema pgq cascade; drop sequence tmptest_seq; drop table custom_expr; drop table custom_expr2; drop table custom_fields; drop table custom_fields2; drop table custom_pkey; drop table deny_test; drop table nopkey; drop table nopkey2; drop table rtest; drop table if exists trunctrg1; drop table if exists trunctrg2; drop table ucustom_pkey; drop table udata; drop table when_test; pgq-3.4.1/expected/clean_ext.out000066400000000000000000000000241373233714300166040ustar00rootroot00000000000000drop extension pgq; pgq-3.4.1/expected/pgq_core.out000066400000000000000000000242071373233714300164520ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; select * from pgq.maint_tables_to_vacuum(); maint_tables_to_vacuum ------------------------ (0 rows) select * from pgq.maint_retry_events(); maint_retry_events -------------------- 0 (1 row) select pgq.create_queue('tmpqueue'); create_queue -------------- 1 (1 row) select pgq.register_consumer('tmpqueue', 'consumer'); register_consumer ------------------- 1 (1 row) select pgq.unregister_consumer('tmpqueue', 'consumer'); unregister_consumer --------------------- 1 (1 row) select pgq.drop_queue('tmpqueue'); drop_queue ------------ 1 (1 row) select pgq.create_queue('myqueue'); create_queue -------------- 1 (1 row) select pgq.register_consumer('myqueue', 'consumer'); register_consumer ------------------- 1 (1 row) update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0'; select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ (1 row) select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ (1 row) select pgq.ticker(); ticker -------- 1 (1 row) select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ 1 (1 row) select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ 1 (1 row) select queue_name, consumer_name, prev_tick_id, tick_id, lag < '30 seconds' as lag_exists from pgq.get_batch_info(1); queue_name | consumer_name | prev_tick_id | tick_id | lag_exists ------------+---------------+--------------+---------+------------ myqueue | consumer | 1 | 2 | t (1 row) select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, queue_switch_time <= now() as switch_time_exists, queue_external_ticker, queue_ticker_max_count, queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag < '2 hours' as ticker_lag_exists, last_tick_id from pgq.get_queue_info() order by 1; queue_name | queue_ntables | queue_cur_table | queue_rotation_period | switch_time_exists | queue_external_ticker | queue_ticker_max_count | queue_ticker_max_lag | queue_ticker_idle_period | ticker_lag_exists | last_tick_id ------------+---------------+-----------------+-----------------------+--------------------+-----------------------+------------------------+----------------------+--------------------------+-------------------+-------------- myqueue | 3 | 0 | @ 2 hours | t | f | 500 | @ 0 | @ 0 | t | 2 (1 row) select queue_name, consumer_name, lag < '30 seconds' as lag_exists, last_seen < '30 seconds' as last_seen_exists, last_tick, current_batch, next_tick from pgq.get_consumer_info() order by 1, 2; queue_name | consumer_name | lag_exists | last_seen_exists | last_tick | current_batch | next_tick ------------+---------------+------------+------------------+-----------+---------------+----------- myqueue | consumer | t | t | 1 | 1 | 2 (1 row) select pgq.finish_batch(1); finish_batch -------------- 1 (1 row) select pgq.finish_batch(1); WARNING: finish_batch: batch 1 not found finish_batch -------------- 0 (1 row) select pgq.ticker(); ticker -------- 1 (1 row) select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ 2 (1 row) select * from pgq.batch_event_tables(2); batch_event_tables -------------------- pgq.event_2_0 (1 row) select * from pgq.get_batch_events(2); ev_id | ev_time | ev_txid | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 -------+---------+---------+----------+---------+---------+-----------+-----------+-----------+----------- (0 rows) select pgq.finish_batch(2); finish_batch -------------- 1 (1 row) select pgq.insert_event('myqueue', 'r1', 'data'); insert_event -------------- 1 (1 row) select pgq.insert_event('myqueue', 'r2', 'data', 'extra1', 'extra2', 'extra3', 'extra4'); insert_event -------------- 2 (1 row) select pgq.insert_event('myqueue', 'r3', 'data'); insert_event -------------- 3 (1 row) select pgq.current_event_table('myqueue'); current_event_table --------------------- pgq.event_2_0 (1 row) select pgq.ticker(); ticker -------- 1 (1 row) select * from pgq.next_batch_custom('myqueue', 'consumer', '1 hour', null, null); batch_id | cur_tick_id | prev_tick_id | cur_tick_time | prev_tick_time | cur_tick_event_seq | prev_tick_event_seq ----------+-------------+--------------+---------------+----------------+--------------------+--------------------- | | | | | | (1 row) select * from pgq.next_batch_custom('myqueue', 'consumer', null, 10000, null); batch_id | cur_tick_id | prev_tick_id | cur_tick_time | prev_tick_time | cur_tick_event_seq | prev_tick_event_seq ----------+-------------+--------------+---------------+----------------+--------------------+--------------------- | | | | | | (1 row) select * from pgq.next_batch_custom('myqueue', 'consumer', null, null, '10 minutes'); batch_id | cur_tick_id | prev_tick_id | cur_tick_time | prev_tick_time | cur_tick_event_seq | prev_tick_event_seq ----------+-------------+--------------+---------------+----------------+--------------------+--------------------- | | | | | | (1 row) select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ 3 (1 row) select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3); ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 -------+----------+---------+---------+-----------+-----------+-----------+----------- 1 | | r1 | data | | | | 2 | | r2 | data | extra1 | extra2 | extra3 | extra4 3 | | r3 | data | | | | (3 rows) begin; select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_cursor(3, 'acurs', 10); ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 -------+----------+---------+---------+-----------+-----------+-----------+----------- 1 | | r1 | data | | | | 2 | | r2 | data | extra1 | extra2 | extra3 | extra4 3 | | r3 | data | | | | (3 rows) close acurs; select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_cursor(3, 'acurs', 2); ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 -------+----------+---------+---------+-----------+-----------+-----------+----------- 1 | | r1 | data | | | | 2 | | r2 | data | extra1 | extra2 | extra3 | extra4 (2 rows) close acurs; select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_cursor(3, 'acurs', 2, 'ev_id = 1'); ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 -------+----------+---------+---------+-----------+-----------+-----------+----------- 1 | | r1 | data | | | | (1 row) close acurs; end; select pgq.event_retry(3, 2, 0); event_retry ------------- 1 (1 row) select pgq.batch_retry(3, 0); batch_retry ------------- 2 (1 row) select pgq.finish_batch(3); finish_batch -------------- 1 (1 row) select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0, 'rawtest', 'data', null, null, null, null); event_retry_raw ----------------- 666 (1 row) select pgq.ticker(); ticker -------- 1 (1 row) -- test maint update pgq.queue set queue_rotation_period = '0 seconds'; select queue_name, pgq.maint_rotate_tables_step1(queue_name) from pgq.queue; queue_name | maint_rotate_tables_step1 ------------+--------------------------- myqueue | 0 (1 row) select pgq.maint_rotate_tables_step2(); maint_rotate_tables_step2 --------------------------- 0 (1 row) -- test extra select nextval(queue_event_seq) from pgq.queue where queue_name = 'myqueue'; nextval --------- 4 (1 row) select pgq.force_tick('myqueue'); force_tick ------------ 5 (1 row) select nextval(queue_event_seq) from pgq.queue where queue_name = 'myqueue'; nextval --------- 2006 (1 row) create sequence tmptest_seq; select pgq.seq_getval('tmptest_seq'); seq_getval ------------ 1 (1 row) select pgq.seq_setval('tmptest_seq', 10); seq_setval ------------ 10 (1 row) select pgq.seq_setval('tmptest_seq', 5); seq_setval ------------ 10 (1 row) select pgq.seq_setval('tmptest_seq', 15); seq_setval ------------ 15 (1 row) select pgq.seq_getval('tmptest_seq'); seq_getval ------------ 15 (1 row) drop sequence tmptest_seq; select * from pgq.maint_operations(); func_name | func_arg -------------------------------+---------- pgq.maint_rotate_tables_step1 | myqueue pgq.maint_rotate_tables_step2 | (2 rows) update pgq.queue set queue_extra_maint = array['baz', 'foo.bar']; select * from pgq.maint_operations(); func_name | func_arg -------------------------------+---------- pgq.maint_rotate_tables_step1 | myqueue pgq.maint_rotate_tables_step2 | baz | myqueue foo.bar | myqueue (4 rows) select pgq.drop_queue('myqueue', true); drop_queue ------------ 1 (1 row) pgq-3.4.1/expected/pgq_core_disabled.out000066400000000000000000000013651373233714300203010ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; select pgq.create_queue('queue_disabled'); create_queue -------------- 1 (1 row) -- test disabled select pgq.insert_event('queue_disabled', 'test', 'event'); insert_event -------------- 1 (1 row) update pgq.queue set queue_disable_insert = true where queue_name = 'queue_disabled'; select pgq.insert_event('queue_disabled', 'test', 'event'); ERROR: Insert into queue disallowed update pgq.queue set queue_disable_insert = false where queue_name = 'queue_disabled'; select pgq.insert_event('queue_disabled', 'test', 'event'); insert_event -------------- 3 (1 row) select pgq.drop_queue('queue_disabled'); drop_queue ------------ 1 (1 row) pgq-3.4.1/expected/pgq_core_tx_limit.out000066400000000000000000000025161373233714300203620ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; select pgq.create_queue('queue_tx_limit'); create_queue -------------- 1 (1 row) -- test limit update pgq.queue set queue_per_tx_limit = 2 where queue_name = 'queue_tx_limit'; begin; select pgq.insert_event('queue_tx_limit', 'test', 'event1'); insert_event -------------- 1 (1 row) select pgq.insert_event('queue_tx_limit', 'test', 'event2'); insert_event -------------- 2 (1 row) select pgq.insert_event('queue_tx_limit', 'test', 'event3'); ERROR: Queue 'queue_tx_limit' allows max 2 events from one TX end; update pgq.queue set queue_per_tx_limit = 0 where queue_name = 'queue_tx_limit'; begin; select pgq.insert_event('queue_tx_limit', 'test', 'event1'); ERROR: Queue 'queue_tx_limit' allows max 0 events from one TX end; update pgq.queue set queue_per_tx_limit = null where queue_name = 'queue_tx_limit'; begin; select pgq.insert_event('queue_tx_limit', 'test', 'event1'); insert_event -------------- 5 (1 row) select pgq.insert_event('queue_tx_limit', 'test', 'event2'); insert_event -------------- 6 (1 row) select pgq.insert_event('queue_tx_limit', 'test', 'event3'); insert_event -------------- 7 (1 row) end; select pgq.drop_queue('queue_tx_limit'); drop_queue ------------ 1 (1 row) pgq-3.4.1/expected/pgq_init_ext.out000066400000000000000000000013241373233714300173400ustar00rootroot00000000000000\set ECHO none create_queue -------------- 1 (1 row) select array_length(extconfig, 1) from pg_catalog.pg_extension where extname = 'pgq'; array_length -------------- 7 (1 row) select pgq.create_queue('testqueue2'); create_queue -------------- 1 (1 row) --drop extension pgq; -- will fail select pgq.drop_queue('testqueue2'); drop_queue ------------ 1 (1 row) select pgq.drop_queue('testqueue1'); drop_queue ------------ 1 (1 row) -- drop extension drop extension pgq; -- create clean schema create extension pgq; select array_length(extconfig, 1) from pg_catalog.pg_extension where extname = 'pgq'; array_length -------------- 7 (1 row) pgq-3.4.1/expected/pgq_init_noext.out000066400000000000000000000001121373233714300176670ustar00rootroot00000000000000\set ECHO none upgrade_schema ---------------- 0 (1 row) pgq-3.4.1/expected/pgq_init_upgrade.out000066400000000000000000000002051373233714300201640ustar00rootroot00000000000000\set ECHO none upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 0 (1 row) pgq-3.4.1/expected/pgq_init_upgrade_1.out000066400000000000000000000002051373233714300204040ustar00rootroot00000000000000\set ECHO none upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 3 (1 row) pgq-3.4.1/expected/pgq_perms.out000066400000000000000000000025411373233714300166450ustar00rootroot00000000000000\set ECHO none drop role if exists pgq_test_producer; drop role if exists pgq_test_consumer; drop role if exists pgq_test_admin; create role pgq_test_consumer with login in role pgq_reader; create role pgq_test_producer with login in role pgq_writer; create role pgq_test_admin with login in role pgq_admin; \c - pgq_test_admin select * from pgq.create_queue('pqueue'); -- ok create_queue -------------- 1 (1 row) \c - pgq_test_producer select * from pgq.create_queue('pqueue'); -- fail ERROR: permission denied for function create_queue select * from pgq.insert_event('pqueue', 'test', 'data'); -- ok insert_event -------------- 1 (1 row) select * from pgq.register_consumer('pqueue', 'prod'); -- fail ERROR: permission denied for function register_consumer \c - pgq_test_consumer select * from pgq.create_queue('pqueue'); -- fail ERROR: permission denied for function create_queue select * from pgq.insert_event('pqueue', 'test', 'data'); -- fail ERROR: permission denied for function insert_event select * from pgq.register_consumer('pqueue', 'cons'); -- ok register_consumer ------------------- 1 (1 row) select * from pgq.next_batch('pqueue', 'cons'); -- ok next_batch ------------ (1 row) \c - pgq_test_admin select * from pgq.drop_queue('pqueue', true); drop_queue ------------ 1 (1 row) pgq-3.4.1/expected/pgq_session_role.out000066400000000000000000000037451373233714300202320ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; show session_replication_role; session_replication_role -------------------------- origin (1 row) select pgq.create_queue('role_test_enabled'); create_queue -------------- 1 (1 row) select pgq.create_queue('role_test_disabled'); create_queue -------------- 1 (1 row) update pgq.queue set queue_disable_insert=true where queue_name = 'role_test_disabled'; select pgq.insert_event('role_test_enabled', 'enabled', 'role:origin'); insert_event -------------- 1 (1 row) select pgq.insert_event('role_test_disabled', 'disabled', 'role:origin'); ERROR: Insert into queue disallowed set session_replication_role = 'replica'; show session_replication_role; session_replication_role -------------------------- replica (1 row) select pgq.insert_event('role_test_enabled', 'enabled', 'role:replica'); insert_event -------------- 2 (1 row) select pgq.insert_event('role_test_disabled', 'disabled', 'role:replica'); insert_event -------------- 2 (1 row) set session_replication_role = 'local'; show session_replication_role; session_replication_role -------------------------- local (1 row) select pgq.insert_event('role_test_enabled', 'enabled', 'role:local'); insert_event -------------- 3 (1 row) select pgq.insert_event('role_test_disabled', 'disabled', 'role:local'); ERROR: Insert into queue disallowed set session_replication_role = 'origin'; show session_replication_role; session_replication_role -------------------------- origin (1 row) select pgq.insert_event('role_test_enabled', 'enabled', 'role:origin'); insert_event -------------- 4 (1 row) select pgq.insert_event('role_test_disabled', 'disabled', 'role:origin'); ERROR: Insert into queue disallowed select pgq.drop_queue('role_test_enabled'); drop_queue ------------ 1 (1 row) select pgq.drop_queue('role_test_disabled'); drop_queue ------------ 1 (1 row) pgq-3.4.1/expected/switch_plonly.out000066400000000000000000000000171373233714300175420ustar00rootroot00000000000000\set ECHO none pgq-3.4.1/expected/trigger_base.out000066400000000000000000000136541373233714300173140ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; set bytea_output = 'hex'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_base ( id serial primary key, txt text, val float ); create trigger base_trig_0 after insert or update or delete on trigger_base for each row execute procedure pgq.jsontriga('jsontriga'); create trigger base_trig_1 after insert or update or delete on trigger_base for each row execute procedure pgq.logutriga('logutriga'); create trigger base_trig_2 after insert or update or delete on trigger_base for each row execute procedure pgq.sqltriga('sqltriga'); insert into trigger_base (txt) values ('text1'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","trigger_base"],"pkey":["id"]}], d=[{"id":1,"txt":"text1","val":null}], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[I:id], d=[id=1&txt=text1&val], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(id,txt,val) values ('1','text1',null)], 1=[public.trigger_base], 2=[], 3=[], 4=[]) insert into trigger_base (val) values (1.5); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","trigger_base"],"pkey":["id"]}], d=[{"id":2,"txt":null,"val":"1.5"}], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[I:id], d=[id=2&txt&val=1.5], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(id,txt,val) values ('2',null,'1.5')], 1=[public.trigger_base], 2=[], 3=[], 4=[]) update trigger_base set txt='text2' where id=1; WARNING: insert_event(q=[jsontriga], t=[{"op":"UPDATE","table":["public","trigger_base"],"pkey":["id"]}], d=[{"id":1,"txt":"text2","val":null}], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[U:id], d=[id=1&txt=text2&val], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[U], d=[txt='text2' where id='1'], 1=[public.trigger_base], 2=[], 3=[], 4=[]) delete from trigger_base where id=2; WARNING: insert_event(q=[jsontriga], t=[{"op":"DELETE","table":["public","trigger_base"],"pkey":["id"]}], d=[{"id":2,"txt":null,"val":"1.5"}], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[D:id], d=[id=2&txt&val=1.5], 1=[public.trigger_base], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[D], d=[id='2'], 1=[public.trigger_base], 2=[], 3=[], 4=[]) -- test missing pkey create table trigger_nopkey_jsontriga (dat text); create table trigger_nopkey_logutriga (dat text); create table trigger_nopkey_sqltriga (dat text); create trigger nopkey after insert or update or delete on trigger_nopkey_jsontriga for each row execute procedure pgq.jsontriga('jsontriga'); create trigger nopkey after insert or update or delete on trigger_nopkey_logutriga for each row execute procedure pgq.logutriga('logutriga'); create trigger nopkey after insert or update or delete on trigger_nopkey_sqltriga for each row execute procedure pgq.sqltriga('sqltriga'); insert into trigger_nopkey_jsontriga values ('foo'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","trigger_nopkey_jsontriga"],"pkey":[]}], d=[{"dat":"foo"}], 1=[public.trigger_nopkey_jsontriga], 2=[], 3=[], 4=[]) insert into trigger_nopkey_logutriga values ('foo'); WARNING: insert_event(q=[logutriga], t=[I:], d=[dat=foo], 1=[public.trigger_nopkey_logutriga], 2=[], 3=[], 4=[]) insert into trigger_nopkey_sqltriga values ('foo'); WARNING: insert_event(q=[sqltriga], t=[I], d=[(dat) values ('foo')], 1=[public.trigger_nopkey_sqltriga], 2=[], 3=[], 4=[]) update trigger_nopkey_jsontriga set dat = 'bat'; ERROR: Update/Delete on table without pkey update trigger_nopkey_logutriga set dat = 'bat'; ERROR: Update/Delete on table without pkey update trigger_nopkey_sqltriga set dat = 'bat'; ERROR: Update/Delete on table without pkey delete from trigger_nopkey_jsontriga; ERROR: Update/Delete on table without pkey delete from trigger_nopkey_logutriga; ERROR: Update/Delete on table without pkey delete from trigger_nopkey_sqltriga; ERROR: Update/Delete on table without pkey -- test invalid pk update create table trigger_pkey_jsontriga (id int4 primary key); create table trigger_pkey_logutriga (id int4 primary key); create table trigger_pkey_sqltriga (id int4 primary key); insert into trigger_pkey_jsontriga values (1); insert into trigger_pkey_logutriga values (1); insert into trigger_pkey_sqltriga values (1); create trigger nopkey after insert or update or delete on trigger_pkey_jsontriga for each row execute procedure pgq.jsontriga('jsontriga'); create trigger nopkey after insert or update or delete on trigger_pkey_logutriga for each row execute procedure pgq.logutriga('logutriga'); create trigger nopkey after insert or update or delete on trigger_pkey_sqltriga for each row execute procedure pgq.sqltriga('sqltriga'); update trigger_pkey_jsontriga set id = 6; ERROR: primary key update not allowed update trigger_pkey_logutriga set id = 6; ERROR: primary key update not allowed update trigger_pkey_sqltriga set id = 6; ERROR: primary key update not allowed -- restore drop table trigger_base; drop table trigger_nopkey_jsontriga; drop table trigger_nopkey_logutriga; drop table trigger_nopkey_sqltriga; drop table trigger_pkey_jsontriga; drop table trigger_pkey_logutriga; drop table trigger_pkey_sqltriga; \set ECHO none pgq-3.4.1/expected/trigger_deny.out000066400000000000000000000045731373233714300173410ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; -- create tables with data create table jsontriga_deny (dat1 text primary key); create table logutriga_deny (dat1 text primary key); create table sqltriga_deny (dat1 text primary key); insert into jsontriga_deny values ('a'); insert into logutriga_deny values ('a'); insert into sqltriga_deny values ('a'); -- create triggers create trigger deny_trig after insert or update or delete on jsontriga_deny for each row execute procedure pgq.jsontriga('jsontriga', 'deny'); create trigger deny_trig after insert or update or delete on logutriga_deny for each row execute procedure pgq.logutriga('logutriga', 'deny'); create trigger deny_trig after insert or update or delete on sqltriga_deny for each row execute procedure pgq.sqltriga('sqltriga', 'deny'); -- see what happens insert into jsontriga_deny values ('b'); ERROR: Table 'public.jsontriga_deny' to queue 'jsontriga': change not allowed (INSERT) insert into logutriga_deny values ('b'); ERROR: Table 'public.logutriga_deny' to queue 'logutriga': change not allowed (INSERT) insert into sqltriga_deny values ('b'); ERROR: Table 'public.sqltriga_deny' to queue 'sqltriga': change not allowed (INSERT) update jsontriga_deny set dat1 = 'c'; ERROR: Table 'public.jsontriga_deny' to queue 'jsontriga': change not allowed (UPDATE) update logutriga_deny set dat1 = 'c'; ERROR: Table 'public.logutriga_deny' to queue 'logutriga': change not allowed (UPDATE) update sqltriga_deny set dat1 = 'c'; ERROR: Table 'public.sqltriga_deny' to queue 'sqltriga': change not allowed (UPDATE) delete from jsontriga_deny; ERROR: Table 'public.jsontriga_deny' to queue 'jsontriga': change not allowed (DELETE) delete from logutriga_deny; ERROR: Table 'public.logutriga_deny' to queue 'logutriga': change not allowed (DELETE) delete from sqltriga_deny; ERROR: Table 'public.sqltriga_deny' to queue 'sqltriga': change not allowed (DELETE) -- restore drop table jsontriga_deny; drop table logutriga_deny; drop table sqltriga_deny; \set ECHO none pgq-3.4.1/expected/trigger_extra_args.out000066400000000000000000000052601373233714300205330ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_extra_args (nr int4 primary key, col1 text, col2 text); create trigger extra_trig_0 after insert or update or delete on trigger_extra_args for each row execute procedure pgq.jsontriga('jsontriga', 'ev_extra1=(nr+3)::text', 'ev_extra2=col1||col2', 'ev_extra3=$$333$$', 'ev_extra4=$$444$$', 'ev_type=$$badidea$$'); create trigger extra_trig_1 after insert or update or delete on trigger_extra_args for each row execute procedure pgq.logutriga('logutriga', 'ev_extra1=(nr+3)::text', 'ev_extra2=col1||col2', 'ev_extra3=$$333$$', 'ev_extra4=$$444$$', 'ev_type=$$badidea$$'); create trigger extra_trig_2 after insert or update or delete on trigger_extra_args for each row execute procedure pgq.sqltriga('sqltriga', 'ev_extra1=(nr+3)::text', 'ev_extra2=col1||col2', 'ev_extra3=$$333$$', 'ev_extra4=$$444$$', 'ev_type=$$badidea$$'); -- test insert insert into trigger_extra_args values (1, 'col1', 'col2'); WARNING: insert_event(q=[jsontriga], t=[badidea], d=[{"nr":1,"col1":"col1","col2":"col2"}], 1=[4], 2=[col1col2], 3=[333], 4=[444]) WARNING: insert_event(q=[logutriga], t=[badidea], d=[nr=1&col1=col1&col2=col2], 1=[4], 2=[col1col2], 3=[333], 4=[444]) WARNING: insert_event(q=[sqltriga], t=[badidea], d=[(nr,col1,col2) values ('1','col1','col2')], 1=[4], 2=[col1col2], 3=[333], 4=[444]) -- test update update trigger_extra_args set col1 = 'col1x', col2='col2x' where nr=1; WARNING: insert_event(q=[jsontriga], t=[badidea], d=[{"nr":1,"col1":"col1x","col2":"col2x"}], 1=[4], 2=[col1xcol2x], 3=[333], 4=[444]) WARNING: insert_event(q=[logutriga], t=[badidea], d=[nr=1&col1=col1x&col2=col2x], 1=[4], 2=[col1xcol2x], 3=[333], 4=[444]) WARNING: insert_event(q=[sqltriga], t=[badidea], d=[col1='col1x',col2='col2x' where nr='1'], 1=[4], 2=[col1xcol2x], 3=[333], 4=[444]) -- test delete delete from trigger_extra_args where nr=1; WARNING: insert_event(q=[jsontriga], t=[badidea], d=[{"nr":1,"col1":"col1x","col2":"col2x"}], 1=[4], 2=[col1xcol2x], 3=[333], 4=[444]) WARNING: insert_event(q=[logutriga], t=[badidea], d=[nr=1&col1=col1x&col2=col2x], 1=[4], 2=[col1xcol2x], 3=[333], 4=[444]) WARNING: insert_event(q=[sqltriga], t=[badidea], d=[nr='1'], 1=[4], 2=[col1xcol2x], 3=[333], 4=[444]) -- restore drop table trigger_extra_args; \set ECHO none pgq-3.4.1/expected/trigger_extra_cols.out000066400000000000000000000047521373233714300205440ustar00rootroot00000000000000-- this functionality is deprecated \set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_extra_columns (nr int4 primary key, col1 text, col2 text, _pgq_ev_type text, _pgq_ev_extra1 text, _pgq_ev_extra2 text default 'e2', _pgq_ev_extra3 text default 'e3', _pgq_ev_extra4 text default 'e4'); create trigger extra_trig_0 after insert or update or delete on trigger_extra_columns for each row execute procedure pgq.jsontriga('jsontriga'); create trigger extra_trig_1 after insert or update or delete on trigger_extra_columns for each row execute procedure pgq.logutriga('logutriga'); create trigger extra_trig_2 after insert or update or delete on trigger_extra_columns for each row execute procedure pgq.sqltriga('sqltriga'); -- test insert insert into trigger_extra_columns (nr, col1, col2, _pgq_ev_type, _pgq_ev_extra1) values (1, 'col1', 'col2', 'xt', 'E1'); WARNING: insert_event(q=[jsontriga], t=[xt], d=[{"nr":1,"col1":"col1","col2":"col2"}], 1=[E1], 2=[e2], 3=[e3], 4=[e4]) WARNING: insert_event(q=[logutriga], t=[xt], d=[nr=1&col1=col1&col2=col2], 1=[E1], 2=[e2], 3=[e3], 4=[e4]) WARNING: insert_event(q=[sqltriga], t=[xt], d=[(nr,col1,col2) values ('1','col1','col2')], 1=[E1], 2=[e2], 3=[e3], 4=[e4]) -- test update update trigger_extra_columns set col1 = 'col1x', col2='col2x', _pgq_ev_extra1='X1' where nr=1; WARNING: insert_event(q=[jsontriga], t=[xt], d=[{"nr":1,"col1":"col1x","col2":"col2x"}], 1=[X1], 2=[e2], 3=[e3], 4=[e4]) WARNING: insert_event(q=[logutriga], t=[xt], d=[nr=1&col1=col1x&col2=col2x], 1=[X1], 2=[e2], 3=[e3], 4=[e4]) WARNING: insert_event(q=[sqltriga], t=[xt], d=[col1='col1x',col2='col2x' where nr='1'], 1=[X1], 2=[e2], 3=[e3], 4=[e4]) -- test delete delete from trigger_extra_columns where nr=1; WARNING: insert_event(q=[jsontriga], t=[xt], d=[{"nr":1,"col1":"col1x","col2":"col2x"}], 1=[X1], 2=[e2], 3=[e3], 4=[e4]) WARNING: insert_event(q=[logutriga], t=[xt], d=[nr=1&col1=col1x&col2=col2x], 1=[X1], 2=[e2], 3=[e3], 4=[e4]) WARNING: insert_event(q=[sqltriga], t=[xt], d=[nr='1'], 1=[X1], 2=[e2], 3=[e3], 4=[e4]) -- restore drop table trigger_extra_columns; \set ECHO none pgq-3.4.1/expected/trigger_ignore.out000066400000000000000000000064541373233714300176650ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_ignore (dat1 text primary key, col1 text, col2 text); create trigger ignore_trig_0 after insert or update or delete on trigger_ignore for each row execute procedure pgq.jsontriga('jsontriga', 'ignore=col2'); create trigger ignore_trig_1 after insert or update or delete on trigger_ignore for each row execute procedure pgq.logutriga('logutriga', 'ignore=col2'); create trigger ignore_trig_2 after insert or update or delete on trigger_ignore for each row execute procedure pgq.sqltriga('sqltriga', 'ignore=col2'); -- test insert insert into trigger_ignore values ('a', 'col1', 'col2'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","trigger_ignore"],"pkey":["dat1"]}], d=[{"dat1":"a","col1":"col1"}], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[I:dat1], d=[dat1=a&col1=col1], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(dat1,col1) values ('a','col1')], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) -- test update of non-ignored column update trigger_ignore set col1 = 'col1x' where dat1 = 'a'; WARNING: insert_event(q=[jsontriga], t=[{"op":"UPDATE","table":["public","trigger_ignore"],"pkey":["dat1"]}], d=[{"dat1":"a","col1":"col1x"}], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[U:dat1], d=[dat1=a&col1=col1x], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[U], d=[col1='col1x' where dat1='a'], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) update trigger_ignore set col1 = 'col1y', col2='col2y' where dat1 = 'a'; WARNING: insert_event(q=[jsontriga], t=[{"op":"UPDATE","table":["public","trigger_ignore"],"pkey":["dat1"]}], d=[{"dat1":"a","col1":"col1y"}], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[U:dat1], d=[dat1=a&col1=col1y], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[U], d=[col1='col1y' where dat1='a'], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) -- test update of ignored column update trigger_ignore set col2 = 'col2z' where dat1 = 'a'; -- test null update update trigger_ignore set col2 = col2 where dat1 = 'a'; WARNING: insert_event(q=[jsontriga], t=[{"op":"UPDATE","table":["public","trigger_ignore"],"pkey":["dat1"]}], d=[{"dat1":"a","col1":"col1y"}], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[U:dat1], d=[dat1=a&col1=col1y], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[U], d=[dat1='a' where dat1='a'], 1=[public.trigger_ignore], 2=[], 3=[], 4=[]) -- restore drop table trigger_ignore; \set ECHO none pgq-3.4.1/expected/trigger_pkey.out000066400000000000000000000051551373233714300173470ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_pkey (nr int4, col1 text, col2 text); create trigger pkey_trig_0 after insert or update or delete on trigger_pkey for each row execute procedure pgq.jsontriga('jsontriga', 'pkey=nr,col1'); create trigger pkey_trig_1 after insert or update or delete on trigger_pkey for each row execute procedure pgq.logutriga('logutriga', 'pkey=nr,col1'); create trigger pkey_trig_2 after insert or update or delete on trigger_pkey for each row execute procedure pgq.sqltriga('sqltriga', 'pkey=nr,col1'); insert into trigger_pkey values (1, 'col1', 'col2'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","trigger_pkey"],"pkey":["nr","col1"]}], d=[{"nr":1,"col1":"col1","col2":"col2"}], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[I:nr,col1], d=[nr=1&col1=col1&col2=col2], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,col1,col2) values ('1','col1','col2')], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) update trigger_pkey set col2='col2x' where nr=1; WARNING: insert_event(q=[jsontriga], t=[{"op":"UPDATE","table":["public","trigger_pkey"],"pkey":["nr","col1"]}], d=[{"nr":1,"col1":"col1","col2":"col2x"}], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[U:nr,col1], d=[nr=1&col1=col1&col2=col2x], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[U], d=[col2='col2x' where nr='1' and col1='col1'], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) delete from trigger_pkey where nr=1; WARNING: insert_event(q=[jsontriga], t=[{"op":"DELETE","table":["public","trigger_pkey"],"pkey":["nr","col1"]}], d=[{"nr":1,"col1":"col1","col2":"col2x"}], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[D:nr,col1], d=[nr=1&col1=col1&col2=col2x], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[D], d=[nr='1' and col1='col1'], 1=[public.trigger_pkey], 2=[], 3=[], 4=[]) -- restore drop table trigger_pkey; \set ECHO none pgq-3.4.1/expected/trigger_sess_role.out000066400000000000000000000066531373233714300204010ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ declare ev_id int8; begin raise warning 'calling insert_event_raw'; ev_id := pgq.insert_event_raw(queue_name, null, now(), null, null, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%]) = %', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4, ev_id; return ev_id; end; $$ language plpgsql; select pgq.create_queue('jsontriga_role'); create_queue -------------- 1 (1 row) select pgq.create_queue('logutriga_role'); create_queue -------------- 1 (1 row) select pgq.create_queue('sqltriga_role'); create_queue -------------- 1 (1 row) update pgq.queue set queue_disable_insert = true where queue_name = 'jsontriga_role'; update pgq.queue set queue_disable_insert = true where queue_name = 'logutriga_role'; update pgq.queue set queue_disable_insert = true where queue_name = 'sqltriga_role'; -- create tables create table jsontriga_role (dat1 text primary key); create table logutriga_role (dat1 text primary key); create table sqltriga_role (dat1 text primary key); create trigger trig after insert or update or delete on jsontriga_role for each row execute procedure pgq.jsontriga('jsontriga_role'); create trigger trig after insert or update or delete on logutriga_role for each row execute procedure pgq.logutriga('logutriga_role'); create trigger trig after insert or update or delete on sqltriga_role for each row execute procedure pgq.sqltriga('sqltriga_role'); -- origin: expect insert_event error show session_replication_role; session_replication_role -------------------------- origin (1 row) insert into jsontriga_role values ('a'); WARNING: calling insert_event_raw ERROR: Insert into queue disallowed insert into logutriga_role values ('a'); WARNING: calling insert_event_raw ERROR: Insert into queue disallowed insert into sqltriga_role values ('a'); WARNING: calling insert_event_raw ERROR: Insert into queue disallowed -- local: silence, trigger does not call insert_event set session_replication_role = 'local'; show session_replication_role; session_replication_role -------------------------- local (1 row) insert into jsontriga_role values ('b'); insert into logutriga_role values ('b'); insert into sqltriga_role values ('b'); -- replica: silence, trigger does not call insert_event set session_replication_role = 'replica'; show session_replication_role; session_replication_role -------------------------- replica (1 row) insert into jsontriga_role values ('c'); insert into logutriga_role values ('c'); insert into sqltriga_role values ('c'); select * from jsontriga_role; dat1 ------ b c (2 rows) select * from logutriga_role; dat1 ------ b c (2 rows) select * from sqltriga_role; dat1 ------ b c (2 rows) -- restore set session_replication_role = 'origin'; drop table jsontriga_role; drop table logutriga_role; drop table sqltriga_role; select pgq.drop_queue('jsontriga_role'); drop_queue ------------ 1 (1 row) select pgq.drop_queue('logutriga_role'); drop_queue ------------ 1 (1 row) select pgq.drop_queue('sqltriga_role'); drop_queue ------------ 1 (1 row) \set ECHO none pgq-3.4.1/expected/trigger_trunc.out000066400000000000000000000051521373233714300175270ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table jsontriga_trunc (dat1 text primary key); create table logutriga_trunc (dat1 text primary key); create table sqltriga_trunc (dat1 text primary key); -- test successful truncate create trigger trunc1_trig after truncate on jsontriga_trunc for each statement execute procedure pgq.jsontriga('jsontriga'); create trigger trunc1_trig after truncate on logutriga_trunc for each statement execute procedure pgq.logutriga('logutriga'); create trigger trunc1_trig after truncate on sqltriga_trunc for each statement execute procedure pgq.sqltriga('sqltriga'); truncate jsontriga_trunc; WARNING: insert_event(q=[jsontriga], t=[{"op":"TRUNCATE","table":["public","jsontriga_trunc"],"pkey":["dat1"]}], d=[{}], 1=[public.jsontriga_trunc], 2=[], 3=[], 4=[]) truncate logutriga_trunc; WARNING: insert_event(q=[logutriga], t=[R], d=[], 1=[public.logutriga_trunc], 2=[], 3=[], 4=[]) truncate sqltriga_trunc; WARNING: insert_event(q=[sqltriga], t=[R], d=[], 1=[public.sqltriga_trunc], 2=[], 3=[], 4=[]) -- test deny create table jsontriga_trunc2 (dat1 text primary key); create table logutriga_trunc2 (dat1 text primary key); create table sqltriga_trunc2 (dat1 text primary key); create trigger trunc_trig after truncate on jsontriga_trunc2 for each statement execute procedure pgq.jsontriga('jsontriga_trunc2', 'deny'); create trigger trunc_trig after truncate on logutriga_trunc2 for each statement execute procedure pgq.sqltriga('logutriga_trunc2', 'deny'); create trigger trunc_trig after truncate on sqltriga_trunc2 for each statement execute procedure pgq.sqltriga('sqltriga_trunc2', 'deny'); truncate jsontriga_trunc2; ERROR: Table 'public.jsontriga_trunc2' to queue 'jsontriga_trunc2': change not allowed (TRUNCATE) truncate logutriga_trunc2; ERROR: Table 'public.logutriga_trunc2' to queue 'logutriga_trunc2': change not allowed (TRUNCATE) truncate sqltriga_trunc2; ERROR: Table 'public.sqltriga_trunc2' to queue 'sqltriga_trunc2': change not allowed (TRUNCATE) -- restore drop table jsontriga_trunc; drop table logutriga_trunc; drop table sqltriga_trunc; drop table jsontriga_trunc2; drop table logutriga_trunc2; drop table sqltriga_trunc2; \set ECHO none pgq-3.4.1/expected/trigger_types.out000066400000000000000000000306021373233714300175360ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; set DateStyle = 'ISO, YMD'; set timezone = 'UTC'; set bytea_output = 'hex'; set extra_float_digits = 0; \set ECHO none select typetest('text', null); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":null,"arr":"{NULL,NULL}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val&arr=%7bNULL%2cNULL%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1',null,'{NULL,NULL}')]) typetest ---------- (1 row) select typetest('text', $$'"quoted\string$%,@"'$$); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"'\"quoted\\string$%,@\"'","arr":"{\"'\\\"quoted\\\\string$%,@\\\"'\",\"'\\\"quoted\\\\string$%,@\\\"'\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=%27%22quoted%5cstring%24%25%2c%40%22%27&arr=%7b%22%27%5c%22quoted%5c%5cstring%24%25%2c%40%5c%22%27%22%2c%22%27%5c%22quoted%5c%5cstring%24%25%2c%40%5c%22%27%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1',E'''"quoted\\string$%,@"''',E'{"''\\"quoted\\\\string$%,@\\"''","''\\"quoted\\\\string$%,@\\"''"}')]) typetest ---------- (1 row) select typetest('bytea', $$\x00FF01$$); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"\\x00ff01","arr":"{\"\\\\x00ff01\",\"\\\\x00ff01\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=%5cx00ff01&arr=%7b%22%5c%5cx00ff01%22%2c%22%5c%5cx00ff01%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1',E'\\x00ff01',E'{"\\\\x00ff01","\\\\x00ff01"}')]) typetest ---------- (1 row) select typetest('bool', 'true'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":true,"arr":"{t,t}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=t&arr=%7bt%2ct%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','t','{t,t}')]) typetest ---------- (1 row) select typetest('bool', 'false'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":false,"arr":"{f,f}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=f&arr=%7bf%2cf%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','f','{f,f}')]) typetest ---------- (1 row) select typetest('timestamptz', '2009-09-19 11:59:48.599'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"2009-09-19T11:59:48.599+00:00","arr":"{\"2009-09-19 11:59:48.599+00\",\"2009-09-19 11:59:48.599+00\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=2009-09-19+11%3a59%3a48.599%2b00&arr=%7b%222009-09-19+11%3a59%3a48.599%2b00%22%2c%222009-09-19+11%3a59%3a48.599%2b00%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','2009-09-19 11:59:48.599+00','{"2009-09-19 11:59:48.599+00","2009-09-19 11:59:48.599+00"}')]) typetest ---------- (1 row) select typetest('timestamp', '2009-09-19 11:59:48.599'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"2009-09-19T11:59:48.599","arr":"{\"2009-09-19 11:59:48.599\",\"2009-09-19 11:59:48.599\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=2009-09-19+11%3a59%3a48.599&arr=%7b%222009-09-19+11%3a59%3a48.599%22%2c%222009-09-19+11%3a59%3a48.599%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','2009-09-19 11:59:48.599','{"2009-09-19 11:59:48.599","2009-09-19 11:59:48.599"}')]) typetest ---------- (1 row) select typetest('date', '2009-09-19'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"2009-09-19","arr":"{2009-09-19,2009-09-19}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=2009-09-19&arr=%7b2009-09-19%2c2009-09-19%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','2009-09-19','{2009-09-19,2009-09-19}')]) typetest ---------- (1 row) select typetest('time', '11:59:48.599'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"11:59:48.599","arr":"{11:59:48.599,11:59:48.599}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=11%3a59%3a48.599&arr=%7b11%3a59%3a48.599%2c11%3a59%3a48.599%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','11:59:48.599','{11:59:48.599,11:59:48.599}')]) typetest ---------- (1 row) select typetest('interval', '2 minutes'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"@ 2 mins","arr":"{\"@ 2 mins\",\"@ 2 mins\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=%40+2+mins&arr=%7b%22%40+2+mins%22%2c%22%40+2+mins%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','@ 2 mins','{"@ 2 mins","@ 2 mins"}')]) typetest ---------- (1 row) select typetest('int2', '10010'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":10010,"arr":"{10010,10010}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=10010&arr=%7b10010%2c10010%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','10010','{10010,10010}')]) typetest ---------- (1 row) select typetest('int4', '100100100'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":100100100,"arr":"{100100100,100100100}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100100100&arr=%7b100100100%2c100100100%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100100100','{100100100,100100100}')]) typetest ---------- (1 row) select typetest('int8', '100200300400500600'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":100200300400500600,"arr":"{100200300400500600,100200300400500600}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100200300400500600&arr=%7b100200300400500600%2c100200300400500600%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100200300400500600','{100200300400500600,100200300400500600}')]) typetest ---------- (1 row) select typetest('int8', '9223372036854775807'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":9223372036854775807,"arr":"{9223372036854775807,9223372036854775807}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=9223372036854775807&arr=%7b9223372036854775807%2c9223372036854775807%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','9223372036854775807','{9223372036854775807,9223372036854775807}')]) typetest ---------- (1 row) select typetest('int8', '-9223372036854775808'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":-9223372036854775808,"arr":"{-9223372036854775808,-9223372036854775808}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=-9223372036854775808&arr=%7b-9223372036854775808%2c-9223372036854775808%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','-9223372036854775808','{-9223372036854775808,-9223372036854775808}')]) typetest ---------- (1 row) select typetest('oid', '100200300'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"100200300","arr":"{100200300,100200300}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100200300&arr=%7b100200300%2c100200300%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100200300','{100200300,100200300}')]) typetest ---------- (1 row) select typetest('xid', '100200300'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"100200300","arr":"{100200300,100200300}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100200300&arr=%7b100200300%2c100200300%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100200300','{100200300,100200300}')]) typetest ---------- (1 row) select typetest('tid', '100200300'); ERROR: invalid input syntax for type tid: "100200300" select typetest('real', '100100.666'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"100101","arr":"{100101,100101}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100101&arr=%7b100101%2c100101%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100101','{100101,100101}')]) typetest ---------- (1 row) select typetest('float', '100100.6005005665'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"100100.600500567","arr":"{100100.600500567,100100.600500567}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100100.600500567&arr=%7b100100.600500567%2c100100.600500567%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100100.600500567','{100100.600500567,100100.600500567}')]) typetest ---------- (1 row) select typetest('numeric(40,15)', '100100.600500566501811'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"100100.600500566501811","arr":"{100100.600500566501811,100100.600500566501811}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=100100.600500566501811&arr=%7b100100.600500566501811%2c100100.600500566501811%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','100100.600500566501811','{100100.600500566501811,100100.600500566501811}')]) typetest ---------- (1 row) select typetest('box', '((1.1, 2.1), (5.6, 5.7))'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"(5.6,5.7),(1.1,2.1)","arr":"{(5.6,5.7),(1.1,2.1);(5.6,5.7),(1.1,2.1)}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=%285.6%2c5.7%29%2c%281.1%2c2.1%29&arr=%7b%285.6%2c5.7%29%2c%281.1%2c2.1%29%3b%285.6%2c5.7%29%2c%281.1%2c2.1%29%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','(5.6,5.7),(1.1,2.1)','{(5.6,5.7),(1.1,2.1);(5.6,5.7),(1.1,2.1)}')]) typetest ---------- (1 row) select typetest('uuid', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","arr":"{a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11&arr=%7ba0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11%2ca0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11','{a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11}')]) typetest ---------- (1 row) select typetest('json', '{"a": [false, null, true]}'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"{\"a\": [false, null, true]}","arr":"{\"{\\\"a\\\": [false, null, true]}\",\"{\\\"a\\\": [false, null, true]}\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=%7b%22a%22%3a+%5bfalse%2c+null%2c+true%5d%7d&arr=%7b%22%7b%5c%22a%5c%22%3a+%5bfalse%2c+null%2c+true%5d%7d%22%2c%22%7b%5c%22a%5c%22%3a+%5bfalse%2c+null%2c+true%5d%7d%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','{"a": [false, null, true]}',E'{"{\\"a\\": [false, null, true]}","{\\"a\\": [false, null, true]}"}')]) typetest ---------- (1 row) select typetest('json', '[1,2,3]'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","ttest"],"pkey":[]}], d=[{"nr":1,"val":"[1,2,3]","arr":"{\"[1,2,3]\",\"[1,2,3]\"}"}]) WARNING: insert_event(q=[logutriga], t=[I:], d=[nr=1&val=%5b1%2c2%2c3%5d&arr=%7b%22%5b1%2c2%2c3%5d%22%2c%22%5b1%2c2%2c3%5d%22%7d]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,val,arr) values ('1','[1,2,3]','{"[1,2,3]","[1,2,3]"}')]) typetest ---------- (1 row) -- restore drop function typetest(text,text); \set ECHO none pgq-3.4.1/expected/trigger_when.out000066400000000000000000000031751373233714300173400ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_when (nr int4 primary key, col1 text, col2 text); create trigger when_trig_0 after insert or update or delete on trigger_when for each row execute procedure pgq.jsontriga('jsontriga', 'when=col2=''foo'''); create trigger when_trig_1 after insert or update or delete on trigger_when for each row execute procedure pgq.logutriga('logutriga', 'when=col2=''foo'''); create trigger when_trig_2 after insert or update or delete on trigger_when for each row execute procedure pgq.sqltriga('sqltriga', 'when=col2=''foo'''); -- test insert with when insert into trigger_when values (1, 'col1', 'col2'); insert into trigger_when values (2, 'col1', 'foo'); WARNING: insert_event(q=[jsontriga], t=[{"op":"INSERT","table":["public","trigger_when"],"pkey":["nr"]}], d=[{"nr":2,"col1":"col1","col2":"foo"}], 1=[public.trigger_when], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[logutriga], t=[I:nr], d=[nr=2&col1=col1&col2=foo], 1=[public.trigger_when], 2=[], 3=[], 4=[]) WARNING: insert_event(q=[sqltriga], t=[I], d=[(nr,col1,col2) values ('2','col1','foo')], 1=[public.trigger_when], 2=[], 3=[], 4=[]) -- restore drop table trigger_when; \set ECHO none pgq-3.4.1/functions/000077500000000000000000000000001373233714300143245ustar00rootroot00000000000000pgq-3.4.1/functions/pgq.batch_event_sql.sql000066400000000000000000000123071373233714300207770ustar00rootroot00000000000000create or replace function pgq.batch_event_sql(x_batch_id bigint) returns text as $$ -- ---------------------------------------------------------------------- -- Function: pgq.batch_event_sql(1) -- Creates SELECT statement that fetches events for this batch. -- -- Parameters: -- x_batch_id - ID of a active batch. -- -- Returns: -- SQL statement. -- ---------------------------------------------------------------------- -- ---------------------------------------------------------------------- -- Algorithm description: -- Given 2 snapshots, sn1 and sn2 with sn1 having xmin1, xmax1 -- and sn2 having xmin2, xmax2 create expression that filters -- right txid's from event table. -- -- Simplest solution would be -- > WHERE ev_txid >= xmin1 AND ev_txid <= xmax2 -- > AND NOT txid_visible_in_snapshot(ev_txid, sn1) -- > AND txid_visible_in_snapshot(ev_txid, sn2) -- -- The simple solution has a problem with long transactions (xmin1 very low). -- All the batches that happen when the long tx is active will need -- to scan all events in that range. Here is 2 optimizations used: -- -- 1) Use [xmax1..xmax2] for range scan. That limits the range to -- txids that actually happened between two snapshots. For txids -- in the range [xmin1..xmax1] look which ones were actually -- committed between snapshots and search for them using exact -- values using IN (..) list. -- -- 2) As most TX are short, there could be lot of them that were -- just below xmax1, but were committed before xmax2. So look -- if there are ID's near xmax1 and lower the range to include -- them, thus decresing size of IN (..) list. -- ---------------------------------------------------------------------- declare rec record; sql text; tbl text; arr text; part text; select_fields text; retry_expr text; batch record; begin select s.sub_last_tick, s.sub_next_tick, s.sub_id, s.sub_queue, txid_snapshot_xmax(last.tick_snapshot) as tx_start, txid_snapshot_xmax(cur.tick_snapshot) as tx_end, last.tick_snapshot as last_snapshot, cur.tick_snapshot as cur_snapshot into batch from pgq.subscription s, pgq.tick last, pgq.tick cur where s.sub_batch = x_batch_id and last.tick_queue = s.sub_queue and last.tick_id = s.sub_last_tick and cur.tick_queue = s.sub_queue and cur.tick_id = s.sub_next_tick; if not found then raise exception 'batch not found'; end if; -- load older transactions arr := ''; for rec in -- active tx-es in prev_snapshot that were committed in cur_snapshot select id1 from txid_snapshot_xip(batch.last_snapshot) id1 left join txid_snapshot_xip(batch.cur_snapshot) id2 on (id1 = id2) where id2 is null order by 1 desc loop -- try to avoid big IN expression, so try to include nearby -- tx'es into range if batch.tx_start - 100 <= rec.id1 then batch.tx_start := rec.id1; else if arr = '' then arr := rec.id1::text; else arr := arr || ',' || rec.id1::text; end if; end if; end loop; -- must match pgq.event_template select_fields := 'select ev_id, ev_time, ev_txid, ev_retry, ev_type,' || ' ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4'; retry_expr := ' and (ev_owner is null or ev_owner = ' || batch.sub_id::text || ')'; -- now generate query that goes over all potential tables sql := ''; for rec in select xtbl from pgq.batch_event_tables(x_batch_id) xtbl loop tbl := pgq.quote_fqname(rec.xtbl); -- this gets newer queries that definitely are not in prev_snapshot part := select_fields || ' from pgq.tick cur, pgq.tick last, ' || tbl || ' ev ' || ' where cur.tick_id = ' || batch.sub_next_tick::text || ' and cur.tick_queue = ' || batch.sub_queue::text || ' and last.tick_id = ' || batch.sub_last_tick::text || ' and last.tick_queue = ' || batch.sub_queue::text || ' and ev.ev_txid >= ' || batch.tx_start::text || ' and ev.ev_txid <= ' || batch.tx_end::text || ' and txid_visible_in_snapshot(ev.ev_txid, cur.tick_snapshot)' || ' and not txid_visible_in_snapshot(ev.ev_txid, last.tick_snapshot)' || retry_expr; -- now include older tx-es, that were ongoing -- at the time of prev_snapshot if arr <> '' then part := part || ' union all ' || select_fields || ' from ' || tbl || ' ev ' || ' where ev.ev_txid in (' || arr || ')' || retry_expr; end if; if sql = '' then sql := part; else sql := sql || ' union all ' || part; end if; end loop; if sql = '' then raise exception 'could not construct sql for batch %', x_batch_id; end if; return sql || ' order by 1'; end; $$ language plpgsql; -- no perms needed pgq-3.4.1/functions/pgq.batch_event_tables.sql000066400000000000000000000041541373233714300214530ustar00rootroot00000000000000create or replace function pgq.batch_event_tables(x_batch_id bigint) returns setof text as $$ -- ---------------------------------------------------------------------- -- Function: pgq.batch_event_tables(1) -- -- Returns set of table names where this batch events may reside. -- -- Parameters: -- x_batch_id - ID of a active batch. -- ---------------------------------------------------------------------- declare nr integer; tbl text; use_prev integer; use_next integer; batch record; begin select txid_snapshot_xmin(last.tick_snapshot) as tx_min, -- absolute minimum txid_snapshot_xmax(cur.tick_snapshot) as tx_max, -- absolute maximum q.queue_data_pfx, q.queue_ntables, q.queue_cur_table, q.queue_switch_step1, q.queue_switch_step2 into batch from pgq.tick last, pgq.tick cur, pgq.subscription s, pgq.queue q where cur.tick_id = s.sub_next_tick and cur.tick_queue = s.sub_queue and last.tick_id = s.sub_last_tick and last.tick_queue = s.sub_queue and s.sub_batch = x_batch_id and q.queue_id = s.sub_queue; if not found then raise exception 'Cannot find data for batch %', x_batch_id; end if; -- if its definitely not in one or other, look into both if batch.tx_max < batch.queue_switch_step1 then use_prev := 1; use_next := 0; elsif batch.queue_switch_step2 is not null and (batch.tx_min > batch.queue_switch_step2) then use_prev := 0; use_next := 1; else use_prev := 1; use_next := 1; end if; if use_prev then nr := batch.queue_cur_table - 1; if nr < 0 then nr := batch.queue_ntables - 1; end if; tbl := batch.queue_data_pfx || '_' || nr::text; return next tbl; end if; if use_next then tbl := batch.queue_data_pfx || '_' || batch.queue_cur_table::text; return next tbl; end if; return; end; $$ language plpgsql; -- no perms needed pgq-3.4.1/functions/pgq.batch_retry.sql000066400000000000000000000033271373233714300201460ustar00rootroot00000000000000create or replace function pgq.batch_retry( i_batch_id bigint, i_retry_seconds integer) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.batch_retry(2) -- -- Put whole batch into retry queue, to be processed again later. -- -- Parameters: -- i_batch_id - ID of active batch. -- i_retry_time - Time when the event should be put back into queue -- -- Returns: -- number of events inserted -- Calls: -- None -- Tables directly manipulated: -- pgq.retry_queue -- ---------------------------------------------------------------------- declare _retry timestamptz; _cnt integer; _s record; begin _retry := current_timestamp + ((i_retry_seconds::text || ' seconds')::interval); select * into _s from pgq.subscription where sub_batch = i_batch_id; if not found then raise exception 'batch_retry: batch % not found', i_batch_id; end if; insert into pgq.retry_queue (ev_retry_after, ev_queue, ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4) select distinct _retry, _s.sub_queue, b.ev_id, b.ev_time, NULL::int8, _s.sub_id, coalesce(b.ev_retry, 0) + 1, b.ev_type, b.ev_data, b.ev_extra1, b.ev_extra2, b.ev_extra3, b.ev_extra4 from pgq.get_batch_events(i_batch_id) b left join pgq.retry_queue rq on (rq.ev_id = b.ev_id and rq.ev_owner = _s.sub_id and rq.ev_queue = _s.sub_queue) where rq.ev_id is null; GET DIAGNOSTICS _cnt = ROW_COUNT; return _cnt; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.create_queue.sql000066400000000000000000000051731373233714300203100ustar00rootroot00000000000000create or replace function pgq.create_queue(i_queue_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.create_queue(1) -- -- Creates new queue with given name. -- -- Returns: -- 0 - queue already exists -- 1 - queue created -- Calls: -- pgq.grant_perms(i_queue_name); -- pgq.ticker(i_queue_name); -- pgq.tune_storage(i_queue_name); -- Tables directly manipulated: -- insert - pgq.queue -- create - pgq.event_N () inherits (pgq.event_template) -- create - pgq.event_N_0 .. pgq.event_N_M () inherits (pgq.event_N) -- ---------------------------------------------------------------------- declare tblpfx text; tblname text; idxpfx text; idxname text; sql text; id integer; tick_seq text; ev_seq text; n_tables integer; begin if i_queue_name is null then raise exception 'Invalid NULL value'; end if; -- check if exists perform 1 from pgq.queue where queue_name = i_queue_name; if found then return 0; end if; -- insert event id := nextval('pgq.queue_queue_id_seq'); tblpfx := 'pgq.event_' || id::text; idxpfx := 'event_' || id::text; tick_seq := 'pgq.event_' || id::text || '_tick_seq'; ev_seq := 'pgq.event_' || id::text || '_id_seq'; insert into pgq.queue (queue_id, queue_name, queue_data_pfx, queue_event_seq, queue_tick_seq) values (id, i_queue_name, tblpfx, ev_seq, tick_seq); select queue_ntables into n_tables from pgq.queue where queue_id = id; -- create seqs execute 'CREATE SEQUENCE ' || pgq.quote_fqname(tick_seq); execute 'CREATE SEQUENCE ' || pgq.quote_fqname(ev_seq); -- create data tables execute 'CREATE TABLE ' || pgq.quote_fqname(tblpfx) || ' () ' || ' INHERITS (pgq.event_template)'; for i in 0 .. (n_tables - 1) loop tblname := tblpfx || '_' || i::text; idxname := idxpfx || '_' || i::text || '_txid_idx'; execute 'CREATE TABLE ' || pgq.quote_fqname(tblname) || ' () ' || ' INHERITS (' || pgq.quote_fqname(tblpfx) || ')'; execute 'ALTER TABLE ' || pgq.quote_fqname(tblname) || ' ALTER COLUMN ev_id ' || ' SET DEFAULT nextval(' || quote_literal(ev_seq) || ')'; execute 'create index ' || quote_ident(idxname) || ' on ' || pgq.quote_fqname(tblname) || ' (ev_txid)'; end loop; perform pgq.grant_perms(i_queue_name); perform pgq.ticker(i_queue_name); perform pgq.tune_storage(i_queue_name); return 1; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.current_event_table.sql000066400000000000000000000025601373233714300216700ustar00rootroot00000000000000create or replace function pgq.current_event_table(x_queue_name text) returns text as $$ -- ---------------------------------------------------------------------- -- Function: pgq.current_event_table(1) -- -- Return active event table for particular queue. -- Event can be added to it without going via functions, -- e.g. by COPY. -- -- If queue is disabled and GUC session_replication_role <> 'replica' -- then raises exception. -- -- or expressed in a different way - an even table of a disabled queue -- is returned only on replica -- -- Note: -- The result is valid only during current transaction. -- -- Permissions: -- Actual insertion requires superuser access. -- -- Parameters: -- x_queue_name - Queue name. -- ---------------------------------------------------------------------- declare res text; disabled boolean; begin select queue_data_pfx || '_' || queue_cur_table::text, queue_disable_insert into res, disabled from pgq.queue where queue_name = x_queue_name; if not found then raise exception 'Event queue not found'; end if; if disabled then if current_setting('session_replication_role') <> 'replica' then raise exception 'Writing to queue disabled'; end if; end if; return res; end; $$ language plpgsql; -- no perms needed pgq-3.4.1/functions/pgq.drop_queue.sql000066400000000000000000000047731373233714300200160ustar00rootroot00000000000000create or replace function pgq.drop_queue(x_queue_name text, x_force bool) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.drop_queue(2) -- -- Drop queue and all associated tables. -- -- Parameters: -- x_queue_name - queue name -- x_force - ignore (drop) existing consumers -- Returns: -- 1 - success -- Calls: -- pgq.unregister_consumer(queue_name, consumer_name) -- perform pgq.ticker(i_queue_name); -- perform pgq.tune_storage(i_queue_name); -- Tables directly manipulated: -- delete - pgq.queue -- drop - pgq.event_N (), pgq.event_N_0 .. pgq.event_N_M -- ---------------------------------------------------------------------- declare tblname text; q record; num integer; begin -- check if exists select * into q from pgq.queue where queue_name = x_queue_name for update; if not found then raise exception 'No such event queue'; end if; if x_force then perform pgq.unregister_consumer(queue_name, consumer_name) from pgq.get_consumer_info(x_queue_name); else -- check if no consumers select count(*) into num from pgq.subscription where sub_queue = q.queue_id; if num > 0 then raise exception 'cannot drop queue, consumers still attached'; end if; end if; -- drop data tables for i in 0 .. (q.queue_ntables - 1) loop tblname := q.queue_data_pfx || '_' || i::text; execute 'DROP TABLE ' || pgq.quote_fqname(tblname); end loop; execute 'DROP TABLE ' || pgq.quote_fqname(q.queue_data_pfx); -- delete ticks delete from pgq.tick where tick_queue = q.queue_id; -- drop seqs -- FIXME: any checks needed here? execute 'DROP SEQUENCE ' || pgq.quote_fqname(q.queue_tick_seq); execute 'DROP SEQUENCE ' || pgq.quote_fqname(q.queue_event_seq); -- delete event delete from pgq.queue where queue_name = x_queue_name; return 1; end; $$ language plpgsql security definer; create or replace function pgq.drop_queue(x_queue_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.drop_queue(1) -- -- Drop queue and all associated tables. -- No consumers must be listening on the queue. -- -- ---------------------------------------------------------------------- begin return pgq.drop_queue(x_queue_name, false); end; $$ language plpgsql strict; pgq-3.4.1/functions/pgq.event_retry.sql000066400000000000000000000046061373233714300202070ustar00rootroot00000000000000create or replace function pgq.event_retry( x_batch_id bigint, x_event_id bigint, x_retry_time timestamptz) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.event_retry(3a) -- -- Put the event into retry queue, to be processed again later. -- -- Parameters: -- x_batch_id - ID of active batch. -- x_event_id - event id -- x_retry_time - Time when the event should be put back into queue -- -- Returns: -- 1 - success -- 0 - event already in retry queue -- Calls: -- None -- Tables directly manipulated: -- insert - pgq.retry_queue -- ---------------------------------------------------------------------- begin insert into pgq.retry_queue (ev_retry_after, ev_queue, ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4) select x_retry_time, sub_queue, ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0) + 1, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 from pgq.get_batch_events(x_batch_id), pgq.subscription where sub_batch = x_batch_id and ev_id = x_event_id; if not found then raise exception 'event not found'; end if; return 1; -- dont worry if the event is already in queue exception when unique_violation then return 0; end; $$ language plpgsql security definer; create or replace function pgq.event_retry( x_batch_id bigint, x_event_id bigint, x_retry_seconds integer) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.event_retry(3b) -- -- Put the event into retry queue, to be processed later again. -- -- Parameters: -- x_batch_id - ID of active batch. -- x_event_id - event id -- x_retry_seconds - Time when the event should be put back into queue -- -- Returns: -- 1 - success -- 0 - event already in retry queue -- Calls: -- pgq.event_retry(3a) -- Tables directly manipulated: -- None -- ---------------------------------------------------------------------- declare new_retry timestamptz; begin new_retry := current_timestamp + ((x_retry_seconds::text || ' seconds')::interval); return pgq.event_retry(x_batch_id, x_event_id, new_retry); end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.event_retry_raw.sql000066400000000000000000000037351373233714300210620ustar00rootroot00000000000000create or replace function pgq.event_retry_raw( x_queue text, x_consumer text, x_retry_after timestamptz, x_ev_id bigint, x_ev_time timestamptz, x_ev_retry integer, x_ev_type text, x_ev_data text, x_ev_extra1 text, x_ev_extra2 text, x_ev_extra3 text, x_ev_extra4 text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.event_retry_raw(12) -- -- Allows full control over what goes to retry queue. -- -- Parameters: -- x_queue - name of the queue -- x_consumer - name of the consumer -- x_retry_after - when the event should be processed again -- x_ev_id - event id -- x_ev_time - creation time -- x_ev_retry - retry count -- x_ev_type - user data -- x_ev_data - user data -- x_ev_extra1 - user data -- x_ev_extra2 - user data -- x_ev_extra3 - user data -- x_ev_extra4 - user data -- -- Returns: -- Event ID. -- ---------------------------------------------------------------------- declare q record; id bigint; begin select sub_id, queue_event_seq, sub_queue into q from pgq.consumer, pgq.queue, pgq.subscription where queue_name = x_queue and co_name = x_consumer and sub_consumer = co_id and sub_queue = queue_id; if not found then raise exception 'consumer not registered'; end if; id := x_ev_id; if id is null then id := nextval(q.queue_event_seq); end if; insert into pgq.retry_queue (ev_retry_after, ev_queue, ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4) values (x_retry_after, q.sub_queue, id, x_ev_time, q.sub_id, x_ev_retry, x_ev_type, x_ev_data, x_ev_extra1, x_ev_extra2, x_ev_extra3, x_ev_extra4); return id; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.find_tick_helper.sql000066400000000000000000000045541373233714300211340ustar00rootroot00000000000000create or replace function pgq.find_tick_helper( in i_queue_id int4, in i_prev_tick_id int8, in i_prev_tick_time timestamptz, in i_prev_tick_seq int8, in i_min_count int8, in i_min_interval interval, out next_tick_id int8, out next_tick_time timestamptz, out next_tick_seq int8) as $$ -- ---------------------------------------------------------------------- -- Function: pgq.find_tick_helper(6) -- -- Helper function for pgq.next_batch_custom() to do extended tick search. -- ---------------------------------------------------------------------- declare sure boolean; can_set boolean; t record; cnt int8; ival interval; begin -- first, fetch last tick of the queue select tick_id, tick_time, tick_event_seq into t from pgq.tick where tick_queue = i_queue_id and tick_id > i_prev_tick_id order by tick_queue desc, tick_id desc limit 1; if not found then return; end if; -- check whether batch would end up within reasonable limits sure := true; can_set := false; if i_min_count is not null then cnt = t.tick_event_seq - i_prev_tick_seq; if cnt >= i_min_count then can_set := true; end if; if cnt > i_min_count * 2 then sure := false; end if; end if; if i_min_interval is not null then ival = t.tick_time - i_prev_tick_time; if ival >= i_min_interval then can_set := true; end if; if ival > i_min_interval * 2 then sure := false; end if; end if; -- if last tick too far away, do large scan if not sure then select tick_id, tick_time, tick_event_seq into t from pgq.tick where tick_queue = i_queue_id and tick_id > i_prev_tick_id and ((i_min_count is not null and (tick_event_seq - i_prev_tick_seq) >= i_min_count) or (i_min_interval is not null and (tick_time - i_prev_tick_time) >= i_min_interval)) order by tick_queue asc, tick_id asc limit 1; can_set := true; end if; if can_set then next_tick_id := t.tick_id; next_tick_time := t.tick_time; next_tick_seq := t.tick_event_seq; end if; return; end; $$ language plpgsql stable; pgq-3.4.1/functions/pgq.finish_batch.sql000066400000000000000000000017001373233714300202520ustar00rootroot00000000000000 create or replace function pgq.finish_batch( x_batch_id bigint) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.finish_batch(1) -- -- Closes a batch. No more operations can be done with events -- of this batch. -- -- Parameters: -- x_batch_id - id of batch. -- -- Returns: -- 1 if batch was found, 0 otherwise. -- Calls: -- None -- Tables directly manipulated: -- update - pgq.subscription -- ---------------------------------------------------------------------- begin update pgq.subscription set sub_active = now(), sub_last_tick = sub_next_tick, sub_next_tick = null, sub_batch = null where sub_batch = x_batch_id; if not found then raise warning 'finish_batch: batch % not found', x_batch_id; return 0; end if; return 1; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.force_tick.sql000066400000000000000000000027061373233714300177500ustar00rootroot00000000000000 create or replace function pgq.force_tick(i_queue_name text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.force_tick(2) -- -- Simulate lots of events happening to force ticker to tick. -- -- Should be called in loop, with some delay until last tick -- changes or too much time is passed. -- -- Such function is needed because paraller calls of pgq.ticker() are -- dangerous, and cannot be protected with locks as snapshot -- is taken before locking. -- -- Parameters: -- i_queue_name - Name of the queue -- -- Returns: -- Currently last tick id. -- ---------------------------------------------------------------------- declare q record; t record; begin -- bump seq and get queue id select queue_id, setval(queue_event_seq, nextval(queue_event_seq) + queue_ticker_max_count * 2 + 1000) as tmp into q from pgq.queue where queue_name = i_queue_name and not queue_external_ticker and not queue_ticker_paused; --if not found then -- raise notice 'queue not found or ticks not allowed'; --end if; -- return last tick id select tick_id into t from pgq.tick, pgq.queue where tick_queue = queue_id and queue_name = i_queue_name order by tick_queue desc, tick_id desc limit 1; return t.tick_id; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.get_batch_cursor.sql000066400000000000000000000065261373233714300211610ustar00rootroot00000000000000 create or replace function pgq.get_batch_cursor( in i_batch_id bigint, in i_cursor_name text, in i_quick_limit int4, in i_extra_where text, out ev_id bigint, out ev_time timestamptz, out ev_txid bigint, out ev_retry int4, out ev_type text, out ev_data text, out ev_extra1 text, out ev_extra2 text, out ev_extra3 text, out ev_extra4 text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_batch_cursor(4) -- -- Get events in batch using a cursor. -- -- Parameters: -- i_batch_id - ID of active batch. -- i_cursor_name - Name for new cursor -- i_quick_limit - Number of events to return immediately -- i_extra_where - optional where clause to filter events -- -- Returns: -- List of events. -- Calls: -- pgq.batch_event_sql(i_batch_id) - internal function which generates SQL optimised specially for getting events in this batch -- ---------------------------------------------------------------------- declare _cname text; _sql text; begin if i_batch_id is null or i_cursor_name is null or i_quick_limit is null then return; end if; _cname := quote_ident(i_cursor_name); _sql := pgq.batch_event_sql(i_batch_id); -- apply extra where if i_extra_where is not null then _sql := replace(_sql, ' order by 1', ''); _sql := 'select * from (' || _sql || ') _evs where ' || i_extra_where || ' order by 1'; end if; -- create cursor execute 'declare ' || _cname || ' no scroll cursor for ' || _sql; -- if no events wanted, don't bother with execute if i_quick_limit <= 0 then return; end if; -- return first block of events for ev_id, ev_time, ev_txid, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 in execute 'fetch ' || i_quick_limit::text || ' from ' || _cname loop return next; end loop; return; end; $$ language plpgsql; -- no perms needed create or replace function pgq.get_batch_cursor( in i_batch_id bigint, in i_cursor_name text, in i_quick_limit int4, out ev_id bigint, out ev_time timestamptz, out ev_txid bigint, out ev_retry int4, out ev_type text, out ev_data text, out ev_extra1 text, out ev_extra2 text, out ev_extra3 text, out ev_extra4 text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_batch_cursor(3) -- -- Get events in batch using a cursor. -- -- Parameters: -- i_batch_id - ID of active batch. -- i_cursor_name - Name for new cursor -- i_quick_limit - Number of events to return immediately -- -- Returns: -- List of events. -- Calls: -- pgq.get_batch_cursor(4) -- ---------------------------------------------------------------------- begin for ev_id, ev_time, ev_txid, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 in select * from pgq.get_batch_cursor(i_batch_id, i_cursor_name, i_quick_limit, null) loop return next; end loop; return; end; $$ language plpgsql strict; -- no perms needed pgq-3.4.1/functions/pgq.get_batch_events.sql000066400000000000000000000017611373233714300211440ustar00rootroot00000000000000create or replace function pgq.get_batch_events( in x_batch_id bigint, out ev_id bigint, out ev_time timestamptz, out ev_txid bigint, out ev_retry int4, out ev_type text, out ev_data text, out ev_extra1 text, out ev_extra2 text, out ev_extra3 text, out ev_extra4 text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_batch_events(1) -- -- Get all events in batch. -- -- Parameters: -- x_batch_id - ID of active batch. -- -- Returns: -- List of events. -- ---------------------------------------------------------------------- declare sql text; begin sql := pgq.batch_event_sql(x_batch_id); for ev_id, ev_time, ev_txid, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 in execute sql loop return next; end loop; return; end; $$ language plpgsql; -- no perms needed pgq-3.4.1/functions/pgq.get_batch_info.sql000066400000000000000000000036431373233714300205740ustar00rootroot00000000000000 create or replace function pgq.get_batch_info( in x_batch_id bigint, out queue_name text, out consumer_name text, out batch_start timestamptz, out batch_end timestamptz, out prev_tick_id bigint, out tick_id bigint, out lag interval, out seq_start bigint, out seq_end bigint) as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_batch_info(1) -- -- Returns detailed info about a batch. -- -- Parameters: -- x_batch_id - id of a active batch. -- -- Returns: ??? pls check -- queue_name - which queue this batch came from -- consumer_name - batch processed by -- batch_start - start time of batch -- batch_end - end time of batch -- prev_tick_id - start tick for this batch -- tick_id - end tick for this batch -- lag - now() - tick_id.time -- seq_start - start event id for batch -- seq_end - end event id for batch -- ---------------------------------------------------------------------- begin select q.queue_name, c.co_name, prev.tick_time, cur.tick_time, s.sub_last_tick, s.sub_next_tick, current_timestamp - cur.tick_time, prev.tick_event_seq, cur.tick_event_seq into queue_name, consumer_name, batch_start, batch_end, prev_tick_id, tick_id, lag, seq_start, seq_end from pgq.subscription s, pgq.tick cur, pgq.tick prev, pgq.queue q, pgq.consumer c where s.sub_batch = x_batch_id and prev.tick_id = s.sub_last_tick and prev.tick_queue = s.sub_queue and cur.tick_id = s.sub_next_tick and cur.tick_queue = s.sub_queue and q.queue_id = s.sub_queue and c.co_id = s.sub_consumer; return; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.get_consumer_info.sql000066400000000000000000000107531373233714300213460ustar00rootroot00000000000000 create or replace function pgq.get_consumer_info( out queue_name text, out consumer_name text, out lag interval, out last_seen interval, out last_tick bigint, out current_batch bigint, out next_tick bigint, out pending_events bigint) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_consumer_info(0) -- -- Returns info about all consumers on all queues. -- -- Returns: -- See pgq.get_consumer_info(2) -- ---------------------------------------------------------------------- begin for queue_name, consumer_name, lag, last_seen, last_tick, current_batch, next_tick, pending_events in select f.queue_name, f.consumer_name, f.lag, f.last_seen, f.last_tick, f.current_batch, f.next_tick, f.pending_events from pgq.get_consumer_info(null, null) f loop return next; end loop; return; end; $$ language plpgsql security definer; create or replace function pgq.get_consumer_info( in i_queue_name text, out queue_name text, out consumer_name text, out lag interval, out last_seen interval, out last_tick bigint, out current_batch bigint, out next_tick bigint, out pending_events bigint) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_consumer_info(1) -- -- Returns info about all consumers on single queue. -- -- Returns: -- See pgq.get_consumer_info(2) -- ---------------------------------------------------------------------- begin for queue_name, consumer_name, lag, last_seen, last_tick, current_batch, next_tick, pending_events in select f.queue_name, f.consumer_name, f.lag, f.last_seen, f.last_tick, f.current_batch, f.next_tick, f.pending_events from pgq.get_consumer_info(i_queue_name, null) f loop return next; end loop; return; end; $$ language plpgsql security definer; create or replace function pgq.get_consumer_info( in i_queue_name text, in i_consumer_name text, out queue_name text, out consumer_name text, out lag interval, out last_seen interval, out last_tick bigint, out current_batch bigint, out next_tick bigint, out pending_events bigint) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_consumer_info(2) -- -- Get info about particular consumer on particular queue. -- -- Parameters: -- i_queue_name - name of a queue. (null = all) -- i_consumer_name - name of a consumer (null = all) -- -- Returns: -- queue_name - Queue name -- consumer_name - Consumer name -- lag - How old are events the consumer is processing -- last_seen - When the consumer seen by pgq -- last_tick - Tick ID of last processed tick -- current_batch - Current batch ID, if one is active or NULL -- next_tick - If batch is active, then its final tick. -- ---------------------------------------------------------------------- declare _pending_events bigint; _queue_id bigint; begin for queue_name, consumer_name, lag, last_seen, last_tick, current_batch, next_tick, _pending_events, _queue_id in select q.queue_name, c.co_name, current_timestamp - t.tick_time, current_timestamp - s.sub_active, s.sub_last_tick, s.sub_batch, s.sub_next_tick, t.tick_event_seq, q.queue_id from pgq.queue q, pgq.consumer c, pgq.subscription s left join pgq.tick t on (t.tick_queue = s.sub_queue and t.tick_id = s.sub_last_tick) where q.queue_id = s.sub_queue and c.co_id = s.sub_consumer and (i_queue_name is null or q.queue_name = i_queue_name) and (i_consumer_name is null or c.co_name = i_consumer_name) order by 1,2 loop select t.tick_event_seq - _pending_events into pending_events from pgq.tick t where t.tick_queue = _queue_id order by t.tick_queue desc, t.tick_id desc limit 1; return next; end loop; return; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.get_queue_info.sql000066400000000000000000000134151373233714300206350ustar00rootroot00000000000000drop function if exists pgq.get_queue_info(); drop function if exists pgq.get_queue_info(text); create or replace function pgq.get_queue_info( out queue_name text, out queue_ntables integer, out queue_cur_table integer, out queue_rotation_period interval, out queue_switch_time timestamptz, out queue_external_ticker boolean, out queue_ticker_paused boolean, out queue_ticker_max_count integer, out queue_ticker_max_lag interval, out queue_ticker_idle_period interval, out ticker_lag interval, out ev_per_sec float8, out ev_new bigint, out last_tick_id bigint) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_queue_info(0) -- -- Get info about all queues. -- -- Returns: -- List of pgq.ret_queue_info records. -- queue_name - queue name -- queue_ntables - number of tables in this queue -- queue_cur_table - ??? -- queue_rotation_period - how often the event_N_M tables in this queue are rotated -- queue_switch_time - ??? when was this queue last rotated -- queue_external_ticker - ??? -- queue_ticker_paused - ??? is ticker paused in this queue -- queue_ticker_max_count - max number of events before a tick is issued -- queue_ticker_max_lag - maks time without a tick -- queue_ticker_idle_period - how often the ticker should check this queue -- ticker_lag - time from last tick -- ev_per_sec - how many events per second this queue serves -- ev_new - ??? -- last_tick_id - last tick id for this queue -- -- ---------------------------------------------------------------------- begin for queue_name, queue_ntables, queue_cur_table, queue_rotation_period, queue_switch_time, queue_external_ticker, queue_ticker_paused, queue_ticker_max_count, queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag, ev_per_sec, ev_new, last_tick_id in select f.queue_name, f.queue_ntables, f.queue_cur_table, f.queue_rotation_period, f.queue_switch_time, f.queue_external_ticker, f.queue_ticker_paused, f.queue_ticker_max_count, f.queue_ticker_max_lag, f.queue_ticker_idle_period, f.ticker_lag, f.ev_per_sec, f.ev_new, f.last_tick_id from pgq.get_queue_info(null) f loop return next; end loop; return; end; $$ language plpgsql; create or replace function pgq.get_queue_info( in i_queue_name text, out queue_name text, out queue_ntables integer, out queue_cur_table integer, out queue_rotation_period interval, out queue_switch_time timestamptz, out queue_external_ticker boolean, out queue_ticker_paused boolean, out queue_ticker_max_count integer, out queue_ticker_max_lag interval, out queue_ticker_idle_period interval, out ticker_lag interval, out ev_per_sec float8, out ev_new bigint, out last_tick_id bigint) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_queue_info(1) -- -- Get info about particular queue. -- -- Returns: -- One pgq.ret_queue_info record. -- contente same as forpgq.get_queue_info() -- ---------------------------------------------------------------------- declare _ticker_lag interval; _top_tick_id bigint; _ht_tick_id bigint; _top_tick_time timestamptz; _top_tick_event_seq bigint; _ht_tick_time timestamptz; _ht_tick_event_seq bigint; _queue_id integer; _queue_event_seq text; begin for queue_name, queue_ntables, queue_cur_table, queue_rotation_period, queue_switch_time, queue_external_ticker, queue_ticker_paused, queue_ticker_max_count, queue_ticker_max_lag, queue_ticker_idle_period, _queue_id, _queue_event_seq in select q.queue_name, q.queue_ntables, q.queue_cur_table, q.queue_rotation_period, q.queue_switch_time, q.queue_external_ticker, q.queue_ticker_paused, q.queue_ticker_max_count, q.queue_ticker_max_lag, q.queue_ticker_idle_period, q.queue_id, q.queue_event_seq from pgq.queue q where (i_queue_name is null or q.queue_name = i_queue_name) order by q.queue_name loop -- most recent tick select (current_timestamp - t.tick_time), tick_id, t.tick_time, t.tick_event_seq into ticker_lag, _top_tick_id, _top_tick_time, _top_tick_event_seq from pgq.tick t where t.tick_queue = _queue_id order by t.tick_queue desc, t.tick_id desc limit 1; -- slightly older tick select ht.tick_id, ht.tick_time, ht.tick_event_seq into _ht_tick_id, _ht_tick_time, _ht_tick_event_seq from pgq.tick ht where ht.tick_queue = _queue_id and ht.tick_id >= _top_tick_id - 20 order by ht.tick_queue asc, ht.tick_id asc limit 1; if _ht_tick_time < _top_tick_time then ev_per_sec = (_top_tick_event_seq - _ht_tick_event_seq) / extract(epoch from (_top_tick_time - _ht_tick_time)); else ev_per_sec = null; end if; ev_new = pgq.seq_getval(_queue_event_seq) - _top_tick_event_seq; last_tick_id = _top_tick_id; return next; end loop; return; end; $$ language plpgsql; pgq-3.4.1/functions/pgq.grant_perms.sql000066400000000000000000000060311373233714300201540ustar00rootroot00000000000000create or replace function pgq.grant_perms(x_queue_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.grant_perms(1) -- -- Make event tables readable by public. -- -- Parameters: -- x_queue_name - Name of the queue. -- -- Returns: -- nothing -- ---------------------------------------------------------------------- declare q record; i integer; pos integer; tbl_perms text; seq_perms text; dst_schema text; dst_table text; part_table text; begin select * from pgq.queue into q where queue_name = x_queue_name; if not found then raise exception 'Queue not found'; end if; -- split data table name to components pos := position('.' in q.queue_data_pfx); if pos > 0 then dst_schema := substring(q.queue_data_pfx for pos - 1); dst_table := substring(q.queue_data_pfx from pos + 1); else dst_schema := 'public'; dst_table := q.queue_data_pfx; end if; -- tick seq, normal users don't need to modify it execute 'grant select on ' || pgq.quote_fqname(q.queue_tick_seq) || ' to public'; -- event seq execute 'grant select on ' || pgq.quote_fqname(q.queue_event_seq) || ' to public'; execute 'grant usage on ' || pgq.quote_fqname(q.queue_event_seq) || ' to pgq_admin'; -- set grants on parent table perform pgq._grant_perms_from('pgq', 'event_template', dst_schema, dst_table); -- set grants on real event tables for i in 0 .. q.queue_ntables - 1 loop part_table := dst_table || '_' || i::text; perform pgq._grant_perms_from('pgq', 'event_template', dst_schema, part_table); end loop; return 1; end; $$ language plpgsql security definer; create or replace function pgq._grant_perms_from(src_schema text, src_table text, dst_schema text, dst_table text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.grant_perms_from(1) -- -- Copy grants from one table to another. -- Workaround for missing GRANTS option for CREATE TABLE LIKE. -- ---------------------------------------------------------------------- declare fq_table text; sql text; g record; q_grantee text; begin fq_table := quote_ident(dst_schema) || '.' || quote_ident(dst_table); for g in select grantor, grantee, privilege_type, is_grantable from information_schema.table_privileges where table_schema = src_schema and table_name = src_table loop if g.grantee = 'PUBLIC' then q_grantee = 'public'; else q_grantee = quote_ident(g.grantee); end if; sql := 'grant ' || g.privilege_type || ' on ' || fq_table || ' to ' || q_grantee; if g.is_grantable = 'YES' then sql := sql || ' with grant option'; end if; execute sql; end loop; return 1; end; $$ language plpgsql strict; pgq-3.4.1/functions/pgq.insert_event.sql000066400000000000000000000035701373233714300203450ustar00rootroot00000000000000create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.insert_event(3) -- -- Insert a event into queue. -- -- Parameters: -- queue_name - Name of the queue -- ev_type - User-specified type for the event -- ev_data - User data for the event -- -- Returns: -- Event ID -- Calls: -- pgq.insert_event(7) -- ---------------------------------------------------------------------- begin return pgq.insert_event(queue_name, ev_type, ev_data, null, null, null, null); end; $$ language plpgsql; create or replace function pgq.insert_event( queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.insert_event(7) -- -- Insert a event into queue with all the extra fields. -- -- Parameters: -- queue_name - Name of the queue -- ev_type - User-specified type for the event -- ev_data - User data for the event -- ev_extra1 - Extra data field for the event -- ev_extra2 - Extra data field for the event -- ev_extra3 - Extra data field for the event -- ev_extra4 - Extra data field for the event -- -- Returns: -- Event ID -- Calls: -- pgq.insert_event_raw(11) -- Tables directly manipulated: -- insert - pgq.insert_event_raw(11), a C function, inserts into current event_N_M table -- ---------------------------------------------------------------------- begin return pgq.insert_event_raw(queue_name, null, now(), null, null, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.maint_operations.sql000066400000000000000000000072441373233714300212150ustar00rootroot00000000000000create or replace function pgq.maint_operations(out func_name text, out func_arg text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.maint_operations(0) -- -- Returns list of functions to call for maintenance. -- -- The goal is to avoid hardcoding them into maintenance process. -- -- Function signature: -- Function should take either 1 or 0 arguments and return 1 if it wants -- to be called immediately again, 0 if not. -- -- Returns: -- func_name - Function to call -- func_arg - Optional argument to function (queue name) -- ---------------------------------------------------------------------- declare ops text[]; nrot int4; begin -- rotate step 1 nrot := 0; func_name := 'pgq.maint_rotate_tables_step1'; for func_arg in select queue_name from pgq.queue where queue_rotation_period is not null and queue_switch_step2 is not null and queue_switch_time + queue_rotation_period < current_timestamp order by 1 loop nrot := nrot + 1; return next; end loop; -- rotate step 2 if nrot = 0 then select count(1) from pgq.queue where queue_rotation_period is not null and queue_switch_step2 is null into nrot; end if; if nrot > 0 then func_name := 'pgq.maint_rotate_tables_step2'; func_arg := NULL; return next; end if; -- check if extra field exists perform 1 from pg_attribute where attrelid = 'pgq.queue'::regclass and attname = 'queue_extra_maint'; if found then -- add extra ops for func_arg, ops in select q.queue_name, queue_extra_maint from pgq.queue q where queue_extra_maint is not null order by 1 loop for i in array_lower(ops, 1) .. array_upper(ops, 1) loop func_name = ops[i]; return next; end loop; end loop; end if; -- vacuum tables func_name := 'vacuum'; for func_arg in select * from pgq.maint_tables_to_vacuum() loop return next; end loop; -- -- pgq_node & londiste -- -- although they belong to queue_extra_maint, they are -- common enough so its more effective to handle them here. -- perform 1 from pg_proc p, pg_namespace n where p.pronamespace = n.oid and n.nspname = 'pgq_node' and p.proname = 'maint_watermark'; if found then func_name := 'pgq_node.maint_watermark'; for func_arg in select n.queue_name from pgq_node.node_info n where n.node_type = 'root' loop return next; end loop; end if; perform 1 from pg_proc p, pg_namespace n where p.pronamespace = n.oid and n.nspname = 'londiste' and p.proname = 'root_check_seqs'; if found then func_name := 'londiste.root_check_seqs'; for func_arg in select distinct s.queue_name from londiste.seq_info s, pgq_node.node_info n where s.local and n.node_type = 'root' and n.queue_name = s.queue_name loop return next; end loop; end if; perform 1 from pg_proc p, pg_namespace n where p.pronamespace = n.oid and n.nspname = 'londiste' and p.proname = 'periodic_maintenance'; if found then func_name := 'londiste.periodic_maintenance'; func_arg := NULL; return next; end if; return; end; $$ language plpgsql; pgq-3.4.1/functions/pgq.maint_retry_events.sql000066400000000000000000000027101373233714300215540ustar00rootroot00000000000000create or replace function pgq.maint_retry_events() returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.maint_retry_events(0) -- -- Moves retry events back to main queue. -- -- It moves small amount at a time. It should be called -- until it returns 0 -- -- Returns: -- Number of events processed. -- ---------------------------------------------------------------------- declare cnt integer; rec record; begin cnt := 0; -- allow only single event mover at a time, without affecting inserts lock table pgq.retry_queue in share update exclusive mode; for rec in select queue_name, ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 from pgq.retry_queue, pgq.queue where ev_retry_after <= current_timestamp and queue_id = ev_queue order by ev_retry_after limit 10 loop cnt := cnt + 1; perform pgq.insert_event_raw(rec.queue_name, rec.ev_id, rec.ev_time, rec.ev_owner, rec.ev_retry, rec.ev_type, rec.ev_data, rec.ev_extra1, rec.ev_extra2, rec.ev_extra3, rec.ev_extra4); delete from pgq.retry_queue where ev_owner = rec.ev_owner and ev_id = rec.ev_id; end loop; return cnt; end; $$ language plpgsql; -- need admin access pgq-3.4.1/functions/pgq.maint_rotate_tables.sql000066400000000000000000000077401373233714300216630ustar00rootroot00000000000000create or replace function pgq.maint_rotate_tables_step1(i_queue_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.maint_rotate_tables_step1(1) -- -- Rotate tables for one queue. -- -- Parameters: -- i_queue_name - Name of the queue -- -- Returns: -- 0 -- ---------------------------------------------------------------------- declare badcnt integer; cf record; nr integer; tbl text; lowest_tick_id int8; lowest_xmin int8; begin -- check if needed and load record select * from pgq.queue into cf where queue_name = i_queue_name and queue_rotation_period is not null and queue_switch_step2 is not null and queue_switch_time + queue_rotation_period < current_timestamp for update; if not found then return 0; end if; -- if DB is in invalid state, stop if txid_current() < cf.queue_switch_step1 then raise exception 'queue % maint failure: step1=%, current=%', i_queue_name, cf.queue_switch_step1, txid_current(); end if; -- find lowest tick for that queue select min(sub_last_tick) into lowest_tick_id from pgq.subscription where sub_queue = cf.queue_id; -- if some consumer exists if lowest_tick_id is not null then -- is the slowest one still on previous table? select txid_snapshot_xmin(tick_snapshot) into lowest_xmin from pgq.tick where tick_queue = cf.queue_id and tick_id = lowest_tick_id; if not found then raise exception 'queue % maint failure: tick % not found', i_queue_name, lowest_tick_id; end if; if lowest_xmin <= cf.queue_switch_step2 then return 0; -- skip rotation then end if; end if; -- nobody on previous table, we can rotate -- calc next table number and name nr := cf.queue_cur_table + 1; if nr = cf.queue_ntables then nr := 0; end if; tbl := cf.queue_data_pfx || '_' || nr::text; -- there may be long lock on the table from pg_dump, -- detect it and skip rotate then begin execute 'lock table ' || pgq.quote_fqname(tbl) || ' nowait'; execute 'truncate ' || pgq.quote_fqname(tbl); exception when lock_not_available then -- cannot truncate, skipping rotate return 0; end; -- remember the moment update pgq.queue set queue_cur_table = nr, queue_switch_time = current_timestamp, queue_switch_step1 = txid_current(), queue_switch_step2 = NULL where queue_id = cf.queue_id; -- Clean ticks by using step2 txid from previous rotation. -- That should keep all ticks for all batches that are completely -- in old table. This keeps them for longer than needed, but: -- 1. we want the pgq.tick table to be big, to avoid Postgres -- accitentally switching to seqscans on that. -- 2. that way we guarantee to consumers that they an be moved -- back on the queue at least for one rotation_period. -- (may help in disaster recovery) delete from pgq.tick where tick_queue = cf.queue_id and txid_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2; return 0; end; $$ language plpgsql; -- need admin access create or replace function pgq.maint_rotate_tables_step2() returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.maint_rotate_tables_step2(0) -- -- Stores the txid when the rotation was visible. It should be -- called in separate transaction than pgq.maint_rotate_tables_step1() -- ---------------------------------------------------------------------- begin update pgq.queue set queue_switch_step2 = txid_current() where queue_switch_step2 is null; return 0; end; $$ language plpgsql; -- need admin access pgq-3.4.1/functions/pgq.maint_tables_to_vacuum.sql000066400000000000000000000032321373233714300223570ustar00rootroot00000000000000create or replace function pgq.maint_tables_to_vacuum() returns setof text as $$ -- ---------------------------------------------------------------------- -- Function: pgq.maint_tables_to_vacuum(0) -- -- Returns list of tablenames that need frequent vacuuming. -- -- The goal is to avoid hardcoding them into maintenance process. -- -- Returns: -- List of table names. -- ---------------------------------------------------------------------- declare scm text; tbl text; fqname text; begin -- assume autovacuum handles them fine if current_setting('autovacuum') = 'on' then return; end if; for scm, tbl in values ('pgq', 'subscription'), ('pgq', 'consumer'), ('pgq', 'queue'), ('pgq', 'tick'), ('pgq', 'retry_queue'), ('pgq_ext', 'completed_tick'), ('pgq_ext', 'completed_batch'), ('pgq_ext', 'completed_event'), ('pgq_ext', 'partial_batch'), --('pgq_node', 'node_location'), --('pgq_node', 'node_info'), ('pgq_node', 'local_state'), --('pgq_node', 'subscriber_info'), --('londiste', 'table_info'), ('londiste', 'seq_info'), --('londiste', 'applied_execute'), --('londiste', 'pending_fkeys'), ('txid', 'epoch'), ('londiste', 'completed') loop select n.nspname || '.' || t.relname into fqname from pg_class t, pg_namespace n where n.oid = t.relnamespace and n.nspname = scm and t.relname = tbl; if found then return next fqname; end if; end loop; return; end; $$ language plpgsql; pgq-3.4.1/functions/pgq.next_batch.sql000066400000000000000000000174001373233714300177540ustar00rootroot00000000000000create or replace function pgq.next_batch_info( in i_queue_name text, in i_consumer_name text, out batch_id int8, out cur_tick_id int8, out prev_tick_id int8, out cur_tick_time timestamptz, out prev_tick_time timestamptz, out cur_tick_event_seq int8, out prev_tick_event_seq int8) as $$ -- ---------------------------------------------------------------------- -- Function: pgq.next_batch_info(2) -- -- Makes next block of events active. -- -- If it returns NULL, there is no events available in queue. -- Consumer should sleep then. -- -- The values from event_id sequence may give hint how big the -- batch may be. But they are inexact, they do not give exact size. -- Client *MUST NOT* use them to detect whether the batch contains any -- events at all - the values are unfit for that purpose. -- -- Parameters: -- i_queue_name - Name of the queue -- i_consumer_name - Name of the consumer -- -- Returns: -- batch_id - Batch ID or NULL if there are no more events available. -- cur_tick_id - End tick id. -- cur_tick_time - End tick time. -- cur_tick_event_seq - Value from event id sequence at the time tick was issued. -- prev_tick_id - Start tick id. -- prev_tick_time - Start tick time. -- prev_tick_event_seq - value from event id sequence at the time tick was issued. -- Calls: -- pgq.next_batch_custom(5) -- Tables directly manipulated: -- None -- ---------------------------------------------------------------------- begin select f.batch_id, f.cur_tick_id, f.prev_tick_id, f.cur_tick_time, f.prev_tick_time, f.cur_tick_event_seq, f.prev_tick_event_seq into batch_id, cur_tick_id, prev_tick_id, cur_tick_time, prev_tick_time, cur_tick_event_seq, prev_tick_event_seq from pgq.next_batch_custom(i_queue_name, i_consumer_name, NULL, NULL, NULL) f; return; end; $$ language plpgsql; create or replace function pgq.next_batch( in i_queue_name text, in i_consumer_name text) returns int8 as $$ -- ---------------------------------------------------------------------- -- Function: pgq.next_batch(2) -- -- Old function that returns just batch_id. -- -- Parameters: -- i_queue_name - Name of the queue -- i_consumer_name - Name of the consumer -- -- Returns: -- Batch ID or NULL if there are no more events available. -- ---------------------------------------------------------------------- declare res int8; begin select batch_id into res from pgq.next_batch_info(i_queue_name, i_consumer_name); return res; end; $$ language plpgsql; create or replace function pgq.next_batch_custom( in i_queue_name text, in i_consumer_name text, in i_min_lag interval, in i_min_count int4, in i_min_interval interval, out batch_id int8, out cur_tick_id int8, out prev_tick_id int8, out cur_tick_time timestamptz, out prev_tick_time timestamptz, out cur_tick_event_seq int8, out prev_tick_event_seq int8) as $$ -- ---------------------------------------------------------------------- -- Function: pgq.next_batch_custom(5) -- -- Makes next block of events active. Block size can be tuned -- with i_min_count, i_min_interval parameters. Events age can -- be tuned with i_min_lag. -- -- If it returns NULL, there is no events available in queue. -- Consumer should sleep then. -- -- The values from event_id sequence may give hint how big the -- batch may be. But they are inexact, they do not give exact size. -- Client *MUST NOT* use them to detect whether the batch contains any -- events at all - the values are unfit for that purpose. -- -- Note: -- i_min_lag together with i_min_interval/i_min_count is inefficient. -- -- Parameters: -- i_queue_name - Name of the queue -- i_consumer_name - Name of the consumer -- i_min_lag - Consumer wants events older than that -- i_min_count - Consumer wants batch to contain at least this many events -- i_min_interval - Consumer wants batch to cover at least this much time -- -- Returns: -- batch_id - Batch ID or NULL if there are no more events available. -- cur_tick_id - End tick id. -- cur_tick_time - End tick time. -- cur_tick_event_seq - Value from event id sequence at the time tick was issued. -- prev_tick_id - Start tick id. -- prev_tick_time - Start tick time. -- prev_tick_event_seq - value from event id sequence at the time tick was issued. -- Calls: -- pgq.insert_event_raw(11) -- Tables directly manipulated: -- update - pgq.subscription -- ---------------------------------------------------------------------- declare errmsg text; queue_id integer; sub_id integer; cons_id integer; begin select s.sub_queue, s.sub_consumer, s.sub_id, s.sub_batch, t1.tick_id, t1.tick_time, t1.tick_event_seq, t2.tick_id, t2.tick_time, t2.tick_event_seq into queue_id, cons_id, sub_id, batch_id, prev_tick_id, prev_tick_time, prev_tick_event_seq, cur_tick_id, cur_tick_time, cur_tick_event_seq from pgq.consumer c, pgq.queue q, pgq.subscription s left join pgq.tick t1 on (t1.tick_queue = s.sub_queue and t1.tick_id = s.sub_last_tick) left join pgq.tick t2 on (t2.tick_queue = s.sub_queue and t2.tick_id = s.sub_next_tick) where q.queue_name = i_queue_name and c.co_name = i_consumer_name and s.sub_queue = q.queue_id and s.sub_consumer = c.co_id; if not found then errmsg := 'Not subscriber to queue: ' || coalesce(i_queue_name, 'NULL') || '/' || coalesce(i_consumer_name, 'NULL'); raise exception '%', errmsg; end if; -- sanity check if prev_tick_id is null then raise exception 'PgQ corruption: Consumer % on queue % does not see tick %', i_consumer_name, i_queue_name, prev_tick_id; end if; -- has already active batch if batch_id is not null then return; end if; if i_min_interval is null and i_min_count is null then -- find next tick select tick_id, tick_time, tick_event_seq into cur_tick_id, cur_tick_time, cur_tick_event_seq from pgq.tick where tick_id > prev_tick_id and tick_queue = queue_id order by tick_queue asc, tick_id asc limit 1; else -- find custom tick select next_tick_id, next_tick_time, next_tick_seq into cur_tick_id, cur_tick_time, cur_tick_event_seq from pgq.find_tick_helper(queue_id, prev_tick_id, prev_tick_time, prev_tick_event_seq, i_min_count, i_min_interval); end if; if i_min_lag is not null then -- enforce min lag if now() - cur_tick_time < i_min_lag then cur_tick_id := NULL; cur_tick_time := NULL; cur_tick_event_seq := NULL; end if; end if; if cur_tick_id is null then -- nothing to do prev_tick_id := null; prev_tick_time := null; prev_tick_event_seq := null; return; end if; -- get next batch batch_id := nextval('pgq.batch_id_seq'); update pgq.subscription set sub_batch = batch_id, sub_next_tick = cur_tick_id, sub_active = now() where sub_queue = queue_id and sub_consumer = cons_id; return; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.quote_fqname.sql000066400000000000000000000016211373233714300203170ustar00rootroot00000000000000 create or replace function pgq.quote_fqname(i_name text) returns text as $$ -- ---------------------------------------------------------------------- -- Function: pgq.quote_fqname(1) -- -- Quete fully-qualified object name for SQL. -- -- First dot is taken as schema separator. -- -- If schema is missing, 'public' is assumed. -- -- Parameters: -- i_name - fully qualified object name. -- -- Returns: -- Quoted name. -- ---------------------------------------------------------------------- declare res text; pos integer; s text; n text; begin pos := position('.' in i_name); if pos > 0 then s := substring(i_name for pos - 1); n := substring(i_name from pos + 1); else s := 'public'; n := i_name; end if; return quote_ident(s) || '.' || quote_ident(n); end; $$ language plpgsql strict immutable; pgq-3.4.1/functions/pgq.register_consumer.sql000066400000000000000000000075251373233714300214030ustar00rootroot00000000000000create or replace function pgq.register_consumer( x_queue_name text, x_consumer_id text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.register_consumer(2) -- -- Subscribe consumer on a queue. -- -- From this moment forward, consumer will see all events in the queue. -- -- Parameters: -- x_queue_name - Name of queue -- x_consumer_name - Name of consumer -- -- Returns: -- 0 - if already registered -- 1 - if new registration -- Calls: -- pgq.register_consumer_at(3) -- Tables directly manipulated: -- None -- ---------------------------------------------------------------------- begin return pgq.register_consumer_at(x_queue_name, x_consumer_id, NULL); end; $$ language plpgsql security definer; create or replace function pgq.register_consumer_at( x_queue_name text, x_consumer_name text, x_tick_pos bigint) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.register_consumer_at(3) -- -- Extended registration, allows to specify tick_id. -- -- Note: -- For usage in special situations. -- -- Parameters: -- x_queue_name - Name of a queue -- x_consumer_name - Name of consumer -- x_tick_pos - Tick ID -- -- Returns: -- 0/1 whether consumer has already registered. -- Calls: -- None -- Tables directly manipulated: -- update/insert - pgq.subscription -- ---------------------------------------------------------------------- declare tmp text; last_tick bigint; x_queue_id integer; x_consumer_id integer; queue integer; sub record; begin select queue_id into x_queue_id from pgq.queue where queue_name = x_queue_name; if not found then raise exception 'Event queue not created yet'; end if; -- get consumer and create if new select co_id into x_consumer_id from pgq.consumer where co_name = x_consumer_name for update; if not found then insert into pgq.consumer (co_name) values (x_consumer_name); x_consumer_id := currval('pgq.consumer_co_id_seq'); end if; -- if particular tick was requested, check if it exists if x_tick_pos is not null then perform 1 from pgq.tick where tick_queue = x_queue_id and tick_id = x_tick_pos; if not found then raise exception 'cannot reposition, tick not found: %', x_tick_pos; end if; end if; -- check if already registered select sub_last_tick, sub_batch into sub from pgq.subscription where sub_consumer = x_consumer_id and sub_queue = x_queue_id; if found then if x_tick_pos is not null then -- if requested, update tick pos and drop partial batch update pgq.subscription set sub_last_tick = x_tick_pos, sub_batch = null, sub_next_tick = null, sub_active = now() where sub_consumer = x_consumer_id and sub_queue = x_queue_id; end if; -- already registered return 0; end if; -- new registration if x_tick_pos is null then -- start from current tick select tick_id into last_tick from pgq.tick where tick_queue = x_queue_id order by tick_queue desc, tick_id desc limit 1; if not found then raise exception 'No ticks for this queue. Please run ticker on database.'; end if; else last_tick := x_tick_pos; end if; -- register insert into pgq.subscription (sub_queue, sub_consumer, sub_last_tick) values (x_queue_id, x_consumer_id, last_tick); return 1; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.seq_funcs.sql000066400000000000000000000032411373233714300176210ustar00rootroot00000000000000 create or replace function pgq.seq_getval(i_seq_name text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.seq_getval(1) -- -- Read current last_val from seq, without affecting it. -- -- Parameters: -- i_seq_name - Name of the sequence -- -- Returns: -- last value. -- ---------------------------------------------------------------------- declare res int8; fqname text; pos integer; s text; n text; begin pos := position('.' in i_seq_name); if pos > 0 then s := substring(i_seq_name for pos - 1); n := substring(i_seq_name from pos + 1); else s := 'public'; n := i_seq_name; end if; fqname := quote_ident(s) || '.' || quote_ident(n); execute 'select last_value from ' || fqname into res; return res; end; $$ language plpgsql strict; create or replace function pgq.seq_setval(i_seq_name text, i_new_value int8) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.seq_setval(2) -- -- Like setval() but does not allow going back. -- -- Parameters: -- i_seq_name - Name of the sequence -- i_new_value - new value -- -- Returns: -- current last value. -- ---------------------------------------------------------------------- declare res int8; fqname text; begin fqname := pgq.quote_fqname(i_seq_name); res := pgq.seq_getval(i_seq_name); if res < i_new_value then perform setval(fqname, i_new_value); return i_new_value; end if; return res; end; $$ language plpgsql strict; pgq-3.4.1/functions/pgq.set_queue_config.sql000066400000000000000000000032611373233714300211610ustar00rootroot00000000000000 create or replace function pgq.set_queue_config( x_queue_name text, x_param_name text, x_param_value text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.set_queue_config(3) -- -- -- Set configuration for specified queue. -- -- Parameters: -- x_queue_name - Name of the queue to configure. -- x_param_name - Configuration parameter name. -- x_param_value - Configuration parameter value. -- -- Returns: -- 0 if event was already in queue, 1 otherwise. -- Calls: -- None -- Tables directly manipulated: -- update - pgq.queue -- ---------------------------------------------------------------------- declare v_param_name text; begin -- discard NULL input if x_queue_name is null or x_param_name is null then raise exception 'Invalid NULL value'; end if; -- check if queue exists perform 1 from pgq.queue where queue_name = x_queue_name; if not found then raise exception 'No such event queue'; end if; -- check if valid parameter name v_param_name := 'queue_' || x_param_name; if v_param_name not in ( 'queue_ticker_max_count', 'queue_ticker_max_lag', 'queue_ticker_idle_period', 'queue_ticker_paused', 'queue_rotation_period', 'queue_external_ticker') then raise exception 'cannot change parameter "%s"', x_param_name; end if; execute 'update pgq.queue set ' || v_param_name || ' = ' || quote_literal(x_param_value) || ' where queue_name = ' || quote_literal(x_queue_name); return 1; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.ticker.sql000066400000000000000000000125721373233714300171230ustar00rootroot00000000000000create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint, i_orig_timestamp timestamptz, i_event_seq bigint) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.ticker(3) -- -- External ticker: Insert a tick with a particular tick_id and timestamp. -- -- Parameters: -- i_queue_name - Name of the queue -- i_tick_id - Id of new tick. -- -- Returns: -- Tick id. -- ---------------------------------------------------------------------- begin insert into pgq.tick (tick_queue, tick_id, tick_time, tick_event_seq) select queue_id, i_tick_id, i_orig_timestamp, i_event_seq from pgq.queue where queue_name = i_queue_name and queue_external_ticker and not queue_ticker_paused; if not found then raise exception 'queue not found or ticker disabled: %', i_queue_name; end if; -- make sure seqs stay current perform pgq.seq_setval(queue_tick_seq, i_tick_id), pgq.seq_setval(queue_event_seq, i_event_seq) from pgq.queue where queue_name = i_queue_name; return i_tick_id; end; $$ language plpgsql security definer; -- unsure about access create or replace function pgq.ticker(i_queue_name text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.ticker(1) -- -- Check if tick is needed for the queue and insert it. -- -- For pgqadm usage. -- -- Parameters: -- i_queue_name - Name of the queue -- -- Returns: -- Tick id or NULL if no tick was done. -- ---------------------------------------------------------------------- declare res bigint; q record; state record; last2 record; begin select queue_id, queue_tick_seq, queue_external_ticker, queue_ticker_max_count, queue_ticker_max_lag, queue_ticker_idle_period, queue_event_seq, pgq.seq_getval(queue_event_seq) as event_seq, queue_ticker_paused into q from pgq.queue where queue_name = i_queue_name; if not found then raise exception 'no such queue'; end if; if q.queue_external_ticker then raise exception 'This queue has external tick source.'; end if; if q.queue_ticker_paused then raise exception 'Ticker has been paused for this queue'; end if; -- load state from last tick select now() - tick_time as lag, q.event_seq - tick_event_seq as new_events, tick_id, tick_time, tick_event_seq, txid_snapshot_xmax(tick_snapshot) as sxmax, txid_snapshot_xmin(tick_snapshot) as sxmin into state from pgq.tick where tick_queue = q.queue_id order by tick_queue desc, tick_id desc limit 1; if found then if state.sxmin > txid_current() then raise exception 'Invalid PgQ state: old xmin=%, old xmax=%, cur txid=%', state.sxmin, state.sxmax, txid_current(); end if; if state.new_events < 0 then raise warning 'Negative new_events? old=% cur=%', state.tick_event_seq, q.event_seq; end if; if state.sxmax > txid_current() then raise warning 'Dubious PgQ state: old xmax=%, cur txid=%', state.sxmax, txid_current(); end if; if state.new_events > 0 then -- there are new events, should we wait a bit? if state.new_events < q.queue_ticker_max_count and state.lag < q.queue_ticker_max_lag then return NULL; end if; else -- no new events, should we apply idle period? -- check previous event from the last one. select state.tick_time - tick_time as lag into last2 from pgq.tick where tick_queue = q.queue_id and tick_id < state.tick_id order by tick_queue desc, tick_id desc limit 1; if found then -- gradually decrease the tick frequency if (state.lag < q.queue_ticker_max_lag / 2) or (state.lag < last2.lag * 2 and state.lag < q.queue_ticker_idle_period) then return NULL; end if; end if; end if; end if; insert into pgq.tick (tick_queue, tick_id, tick_event_seq) values (q.queue_id, nextval(q.queue_tick_seq), q.event_seq); return currval(q.queue_tick_seq); end; $$ language plpgsql security definer; -- unsure about access create or replace function pgq.ticker() returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.ticker(0) -- -- Creates ticks for all unpaused queues which dont have external ticker. -- -- Returns: -- Number of queues that were processed. -- ---------------------------------------------------------------------- declare res bigint; q record; begin res := 0; for q in select queue_name from pgq.queue where not queue_external_ticker and not queue_ticker_paused order by queue_name loop if pgq.ticker(q.queue_name) > 0 then res := res + 1; end if; end loop; return res; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.tune_storage.sql000066400000000000000000000025161373233714300203360ustar00rootroot00000000000000create or replace function pgq.tune_storage(i_queue_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.tune_storage(1) -- -- Tunes storage settings for queue data tables -- ---------------------------------------------------------------------- declare tbl text; tbloid oid; q record; i int4; sql text; pgver int4; begin pgver := current_setting('server_version_num'); select * into q from pgq.queue where queue_name = i_queue_name; if not found then return 0; end if; for i in 0 .. (q.queue_ntables - 1) loop tbl := q.queue_data_pfx || '_' || i::text; -- set fillfactor sql := 'alter table ' || tbl || ' set (fillfactor = 100'; -- autovacuum for 8.4+ if pgver >= 80400 then sql := sql || ', autovacuum_enabled=off, toast.autovacuum_enabled =off'; end if; sql := sql || ')'; execute sql; -- autovacuum for 8.3 if pgver < 80400 then tbloid := tbl::regclass::oid; delete from pg_catalog.pg_autovacuum where vacrelid = tbloid; insert into pg_catalog.pg_autovacuum values (tbloid, false, -1,-1,-1,-1,-1,-1,-1,-1); end if; end loop; return 1; end; $$ language plpgsql strict; pgq-3.4.1/functions/pgq.unregister_consumer.sql000066400000000000000000000045401373233714300217400ustar00rootroot00000000000000 create or replace function pgq.unregister_consumer( x_queue_name text, x_consumer_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.unregister_consumer(2) -- -- Unsubscribe consumer from the queue. -- Also consumer's retry events are deleted. -- -- Parameters: -- x_queue_name - Name of the queue -- x_consumer_name - Name of the consumer -- -- Returns: -- number of (sub)consumers unregistered -- Calls: -- None -- Tables directly manipulated: -- delete - pgq.retry_queue -- delete - pgq.subscription -- ---------------------------------------------------------------------- declare x_sub_id integer; _sub_id_cnt integer; _consumer_id integer; _is_subconsumer boolean; begin select s.sub_id, c.co_id, -- subconsumers can only have both null or both not null - main consumer for subconsumers has only one not null (s.sub_last_tick IS NULL AND s.sub_next_tick IS NULL) OR (s.sub_last_tick IS NOT NULL AND s.sub_next_tick IS NOT NULL) into x_sub_id, _consumer_id, _is_subconsumer from pgq.subscription s, pgq.consumer c, pgq.queue q where s.sub_queue = q.queue_id and s.sub_consumer = c.co_id and q.queue_name = x_queue_name and c.co_name = x_consumer_name for update of s, c; if not found then return 0; end if; -- consumer + subconsumer count select count(*) into _sub_id_cnt from pgq.subscription where sub_id = x_sub_id; -- delete only one subconsumer if _sub_id_cnt > 1 and _is_subconsumer then delete from pgq.subscription where sub_id = x_sub_id and sub_consumer = _consumer_id; return 1; else -- delete main consumer (including possible subconsumers) -- retry events delete from pgq.retry_queue where ev_owner = x_sub_id; -- this will drop subconsumers too delete from pgq.subscription where sub_id = x_sub_id; perform 1 from pgq.subscription where sub_consumer = _consumer_id; if not found then delete from pgq.consumer where co_id = _consumer_id; end if; return _sub_id_cnt; end if; end; $$ language plpgsql security definer; pgq-3.4.1/functions/pgq.upgrade_schema.sql000066400000000000000000000024751373233714300206120ustar00rootroot00000000000000 create or replace function pgq.upgrade_schema() returns int4 as $$ -- updates table structure if necessary declare cnt int4 = 0; begin -- pgq.subscription.sub_last_tick: NOT NULL -> NULL perform 1 from information_schema.columns where table_schema = 'pgq' and table_name = 'subscription' and column_name ='sub_last_tick' and is_nullable = 'NO'; if found then alter table pgq.subscription alter column sub_last_tick drop not null; cnt := cnt + 1; end if; -- create roles perform 1 from pg_catalog.pg_roles where rolname = 'pgq_reader'; if not found then create role pgq_reader; cnt := cnt + 1; end if; perform 1 from pg_catalog.pg_roles where rolname = 'pgq_writer'; if not found then create role pgq_writer; cnt := cnt + 1; end if; perform 1 from pg_catalog.pg_roles where rolname = 'pgq_admin'; if not found then create role pgq_admin in role pgq_reader, pgq_writer; cnt := cnt + 1; end if; perform 1 from pg_attribute where attrelid = 'pgq.queue'::regclass and attname = 'queue_extra_maint'; if not found then alter table pgq.queue add column queue_extra_maint text[]; end if; return 0; end; $$ language plpgsql; pgq-3.4.1/functions/pgq.version.sql000066400000000000000000000006771373233714300173320ustar00rootroot00000000000000create or replace function pgq.version() returns text as $$ -- ---------------------------------------------------------------------- -- Function: pgq.version(0) -- -- Returns version string for pgq. -- ---------------------------------------------------------------------- declare _vers text; begin select extversion from pg_catalog.pg_extension where extname = 'pgq' into _vers; return _vers; end; $$ language plpgsql; pgq-3.4.1/lowlevel/000077500000000000000000000000001373233714300141455ustar00rootroot00000000000000pgq-3.4.1/lowlevel/Makefile000066400000000000000000000002531373233714300156050ustar00rootroot00000000000000 MODULE_big = pgq_lowlevel DATA = pgq_lowlevel.sql SRCS = insert_event.c OBJS = $(SRCS:.c=.o) PG_CONFIG = pg_config PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) pgq-3.4.1/lowlevel/insert_event.c000066400000000000000000000212461373233714300170230ustar00rootroot00000000000000/* * insert_event.c - C implementation of pgq.insert_event_raw(). * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include "postgres.h" #include "funcapi.h" #include "access/hash.h" #include "catalog/pg_type.h" #include "commands/trigger.h" #include "executor/spi.h" #include "lib/stringinfo.h" #include "utils/builtins.h" #include "utils/datetime.h" #include "utils/hsearch.h" #include "access/xact.h" /* * Module tag */ #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif #ifndef TextDatumGetCString #define TextDatumGetCString(d) DatumGetCString(DirectFunctionCall1(textout, d)) #endif /* * Function tag */ Datum pgq_insert_event_raw(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(pgq_insert_event_raw); /* * Queue info fetching. * * Always touch ev_id sequence, even if ev_id is given as arg, * to notify ticker about new event. */ #define QUEUE_SQL \ "select queue_id::int4, queue_data_pfx::text," \ " queue_cur_table::int4, nextval(queue_event_seq)::int8," \ " queue_disable_insert::bool," \ " queue_per_tx_limit::int4" \ " from pgq.queue where queue_name = $1" #define COL_QUEUE_ID 1 #define COL_PREFIX 2 #define COL_TBLNO 3 #define COL_EVENT_ID 4 #define COL_DISABLED 5 #define COL_LIMIT 6 /* * Support inserting into pgq 2 queues. */ #define QUEUE_SQL_OLD \ "select queue_id::int4, queue_data_pfx::text," \ " queue_cur_table::int4, nextval(queue_event_seq)::int8," \ " false::bool as queue_disable_insert," \ " null::int4 as queue_per_tx_limit" \ " from pgq.queue where queue_name = $1" #define QUEUE_CHECK_NEW \ "select 1 from pg_catalog.pg_attribute" \ " where attname = 'queue_per_tx_limit'" \ " and attrelid = 'pgq.queue'::regclass" /* * Plan cache entry in HTAB. */ struct InsertCacheEntry { Oid queue_id; /* actually int32, but we want to use oid_hash */ int cur_table; TransactionId last_xid; int last_count; void *plan; }; /* * helper structure to pass values. */ struct QueueState { int queue_id; int cur_table; char *table_prefix; Datum next_event_id; bool disabled; int per_tx_limit; }; /* * Cached plans. */ static void *queue_plan; static HTAB *insert_cache; /* * Prepare utility plans and plan cache. */ static void init_cache(void) { static int init_done = 0; Oid types[1] = { TEXTOID }; HASHCTL ctl; int flags; int res; int max_queues = 128; const char *sql; if (init_done) return; /* * Check if old (v2.x) or new (v3.x) queue table. * * Needed for upgrades. */ res = SPI_execute(QUEUE_CHECK_NEW, 1, 0); if (res < 0) elog(ERROR, "pgq.insert_event: QUEUE_CHECK_NEW failed"); if (SPI_processed > 0) { sql = QUEUE_SQL; } else { sql = QUEUE_SQL_OLD; } /* * Init plans. */ queue_plan = SPI_saveplan(SPI_prepare(sql, 1, types)); if (queue_plan == NULL) elog(ERROR, "pgq_insert: SPI_prepare() failed"); /* * init insert plan cache. */ MemSet(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(struct InsertCacheEntry); ctl.hash = oid_hash; flags = HASH_ELEM | HASH_FUNCTION; insert_cache = hash_create("pgq_insert_raw plans cache", max_queues, &ctl, flags); init_done = 1; } /* * Create new plan for insertion into current queue table. */ static void *make_plan(struct QueueState *state) { void *plan; StringInfo sql; static Oid types[10] = { INT8OID, TIMESTAMPTZOID, INT4OID, INT4OID, TEXTOID, TEXTOID, TEXTOID, TEXTOID, TEXTOID, TEXTOID }; /* * create sql */ sql = makeStringInfo(); appendStringInfo(sql, "insert into %s_%d (ev_id, ev_time, ev_owner, ev_retry," " ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4)" " values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", state->table_prefix, state->cur_table); /* * create plan */ plan = SPI_prepare(sql->data, 10, types); return SPI_saveplan(plan); } /* * fetch insert plan from cache. */ static void *load_insert_plan(Datum qname, struct QueueState *state) { struct InsertCacheEntry *entry; Oid queue_id = state->queue_id; bool did_exist = false; entry = hash_search(insert_cache, &queue_id, HASH_ENTER, &did_exist); if (did_exist) { if (entry->plan && state->cur_table == entry->cur_table) goto valid_table; if (entry->plan) SPI_freeplan(entry->plan); } entry->cur_table = state->cur_table; entry->last_xid = 0; entry->plan = NULL; /* this can fail, struct must be valid before */ entry->plan = make_plan(state); valid_table: if (state->per_tx_limit >= 0) { TransactionId xid = GetTopTransactionId(); if (entry->last_xid != xid) { entry->last_xid = xid; entry->last_count = 0; } entry->last_count++; if (entry->last_count > state->per_tx_limit) elog(ERROR, "Queue '%s' allows max %d events from one TX", TextDatumGetCString(qname), state->per_tx_limit); } return entry->plan; } /* * load queue info from pgq.queue table. */ static void load_queue_info(Datum queue_name, struct QueueState *state) { Datum values[1]; int res; TupleDesc desc; HeapTuple row; bool isnull; values[0] = queue_name; res = SPI_execute_plan(queue_plan, values, NULL, false, 0); if (res != SPI_OK_SELECT) elog(ERROR, "Queue fetch failed"); if (SPI_processed == 0) elog(ERROR, "No such queue"); row = SPI_tuptable->vals[0]; desc = SPI_tuptable->tupdesc; state->queue_id = DatumGetInt32(SPI_getbinval(row, desc, COL_QUEUE_ID, &isnull)); if (isnull) elog(ERROR, "queue id NULL"); state->cur_table = DatumGetInt32(SPI_getbinval(row, desc, COL_TBLNO, &isnull)); if (isnull) elog(ERROR, "table nr NULL"); state->table_prefix = SPI_getvalue(row, desc, COL_PREFIX); if (state->table_prefix == NULL) elog(ERROR, "table prefix NULL"); state->next_event_id = SPI_getbinval(row, desc, COL_EVENT_ID, &isnull); if (isnull) elog(ERROR, "Seq name NULL"); state->disabled = SPI_getbinval(row, desc, COL_DISABLED, &isnull); if (isnull) elog(ERROR, "insert_disabled NULL"); state->per_tx_limit = SPI_getbinval(row, desc, COL_LIMIT, &isnull); if (isnull) state->per_tx_limit = -1; } /* * Arguments: * 0: queue_name text NOT NULL * 1: ev_id int8 if NULL take from SEQ * 2: ev_time timestamptz if NULL use now() * 3: ev_owner int4 * 4: ev_retry int4 * 5: ev_type text * 6: ev_data text * 7: ev_extra1 text * 8: ev_extra2 text * 9: ev_extra3 text * 10:ev_extra4 text */ Datum pgq_insert_event_raw(PG_FUNCTION_ARGS) { Datum values[11]; char nulls[11]; struct QueueState state; int64 ret_id; void *ins_plan; Datum ev_id, ev_time; int i, res; Datum qname; if (PG_NARGS() < 6) elog(ERROR, "Need at least 6 arguments"); if (PG_ARGISNULL(0)) elog(ERROR, "Queue name must not be NULL"); qname = PG_GETARG_DATUM(0); if (SPI_connect() < 0) elog(ERROR, "SPI_connect() failed"); init_cache(); load_queue_info(qname, &state); /* * Check if queue has disable_insert flag set. */ #if defined(PG_VERSION_NUM) && PG_VERSION_NUM >= 80300 /* 8.3+: allow insert_event() even if connection is in 'replica' role */ if (state.disabled) { if (SessionReplicationRole != SESSION_REPLICATION_ROLE_REPLICA) elog(ERROR, "Insert into queue disallowed"); } #else /* pre-8.3 */ if (state.disabled) elog(ERROR, "Insert into queue disallowed"); #endif if (PG_ARGISNULL(1)) ev_id = state.next_event_id; else ev_id = PG_GETARG_DATUM(1); if (PG_ARGISNULL(2)) ev_time = DirectFunctionCall1(now, 0); else ev_time = PG_GETARG_DATUM(2); /* * Prepare arguments for INSERT */ values[0] = ev_id; nulls[0] = ' '; values[1] = ev_time; nulls[1] = ' '; for (i = 3; i < 11; i++) { int dst = i - 1; if (i >= PG_NARGS() || PG_ARGISNULL(i)) { values[dst] = (Datum)NULL; nulls[dst] = 'n'; } else { values[dst] = PG_GETARG_DATUM(i); nulls[dst] = ' '; } } /* * Perform INSERT into queue table. */ ins_plan = load_insert_plan(qname, &state); res = SPI_execute_plan(ins_plan, values, nulls, false, 0); if (res != SPI_OK_INSERT) elog(ERROR, "Queue insert failed"); /* * ev_id cannot pass SPI_finish() */ ret_id = DatumGetInt64(ev_id); if (SPI_finish() < 0) elog(ERROR, "SPI_finish failed"); PG_RETURN_INT64(ret_id); } pgq-3.4.1/lowlevel/pgq_lowlevel.sql000066400000000000000000000022221373233714300173640ustar00rootroot00000000000000 -- ---------------------------------------------------------------------- -- Function: pgq.insert_event_raw(11) -- -- Actual event insertion. Used also by retry queue maintenance. -- -- Parameters: -- queue_name - Name of the queue -- ev_id - Event ID. If NULL, will be taken from seq. -- ev_time - Event creation time. -- ev_owner - Subscription ID when retry event. If NULL, the event is for everybody. -- ev_retry - Retry count. NULL for first-time events. -- ev_type - user data -- ev_data - user data -- ev_extra1 - user data -- ev_extra2 - user data -- ev_extra3 - user data -- ev_extra4 - user data -- -- Returns: -- Event ID. -- ---------------------------------------------------------------------- CREATE OR REPLACE FUNCTION pgq.insert_event_raw( queue_name text, ev_id bigint, ev_time timestamptz, ev_owner integer, ev_retry integer, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) RETURNS int8 AS '$libdir/pgq_lowlevel', 'pgq_insert_event_raw' LANGUAGE C; pgq-3.4.1/lowlevel_pl/000077500000000000000000000000001373233714300146405ustar00rootroot00000000000000pgq-3.4.1/lowlevel_pl/Makefile000066400000000000000000000000561373233714300163010ustar00rootroot00000000000000 all: clean: install: all distclean: clean pgq-3.4.1/lowlevel_pl/insert_event.sql000066400000000000000000000041741373233714300200740ustar00rootroot00000000000000 -- ---------------------------------------------------------------------- -- Function: pgq.insert_event_raw(11) -- -- Actual event insertion. Used also by retry queue maintenance. -- -- Parameters: -- queue_name - Name of the queue -- ev_id - Event ID. If NULL, will be taken from seq. -- ev_time - Event creation time. -- ev_owner - Subscription ID when retry event. If NULL, the event is for everybody. -- ev_retry - Retry count. NULL for first-time events. -- ev_type - user data -- ev_data - user data -- ev_extra1 - user data -- ev_extra2 - user data -- ev_extra3 - user data -- ev_extra4 - user data -- -- Returns: -- Event ID. -- ---------------------------------------------------------------------- create or replace function pgq.insert_event_raw( queue_name text, ev_id bigint, ev_time timestamptz, ev_owner integer, ev_retry integer, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns int8 as $$ declare qstate record; _qname text; begin _qname := queue_name; select q.queue_id, pgq.quote_fqname(q.queue_data_pfx || '_' || q.queue_cur_table::text) as cur_table_name, nextval(q.queue_event_seq) as next_ev_id, q.queue_disable_insert, q.queue_per_tx_limit from pgq.queue q where q.queue_name = _qname into qstate; if ev_id is null then ev_id := qstate.next_ev_id; end if; if qstate.queue_disable_insert then if current_setting('session_replication_role') <> 'replica' then raise exception 'Insert into queue disallowed'; end if; end if; execute 'insert into ' || qstate.cur_table_name || ' (ev_id, ev_time, ev_owner, ev_retry,' || ' ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4)' || 'values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)' using ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return ev_id; end; $$ language plpgsql; pgq-3.4.1/lowlevel_pl/jsontriga.sql000066400000000000000000000276561373233714300174010ustar00rootroot00000000000000create or replace function pgq.jsontriga() returns trigger as $$ -- ---------------------------------------------------------------------- -- Function: pgq.jsontriga() -- -- Trigger function that puts row data in JSON-encoded form into queue. -- -- Purpose: -- Convert row data into easily parseable form. -- -- Trigger parameters: -- arg1 - queue name -- argX - any number of optional arg, in any order -- -- Optional arguments: -- SKIP - The actual operation should be skipped (BEFORE trigger) -- ignore=col1[,col2] - don't look at the specified arguments -- pkey=col1[,col2] - Set pkey fields for the table, autodetection will be skipped -- backup - Put urlencoded contents of old row to ev_extra2 -- colname=EXPR - Override field value with SQL expression. Can reference table -- columns. colname can be: ev_type, ev_data, ev_extra1 .. ev_extra4 -- when=EXPR - If EXPR returns false, don't insert event. -- -- Queue event fields: -- ev_type - I/U/D ':' pkey_column_list -- ev_data - column values urlencoded -- ev_extra1 - table name -- ev_extra2 - optional urlencoded backup -- -- Regular listen trigger example: -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.jsontriga('qname'); -- -- Redirect trigger example: -- > CREATE TRIGGER triga_nimi BEFORE INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.jsontriga('qname', 'SKIP'); -- ---------------------------------------------------------------------- declare qname text; ev_type text; ev_data text; ev_extra1 text; ev_extra2 text; ev_extra3 text; ev_extra4 text; do_skip boolean := false; do_backup boolean := false; do_insert boolean := true; do_deny boolean := false; extra_ignore_list text[]; full_ignore_list text[]; ignore_list text[] := '{}'; pkey_list text[]; pkey_str text; field_sql_sfx text; field_sql text[] := '{}'; data_sql text; ignore_col_changes int4 := 0; begin if TG_NARGS < 1 then raise exception 'Trigger needs queue name'; end if; qname := TG_ARGV[0]; -- standard output ev_extra1 := TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME; -- prepare to handle magic fields field_sql_sfx := ')::text as val from (select $1.*) r'; extra_ignore_list := array['_pgq_ev_type', '_pgq_ev_extra1', '_pgq_ev_extra2', '_pgq_ev_extra3', '_pgq_ev_extra4']::text[]; -- parse trigger args declare got boolean; argpair text[]; i integer; begin for i in 1 .. TG_NARGS-1 loop if TG_ARGV[i] in ('skip', 'SKIP') then do_skip := true; elsif TG_ARGV[i] = 'backup' then do_backup := true; elsif TG_ARGV[i] = 'deny' then do_deny := true; else got := false; for argpair in select regexp_matches(TG_ARGV[i], '^([^=]+)=(.*)') loop got := true; if argpair[1] = 'pkey' then pkey_str := argpair[2]; pkey_list := string_to_array(pkey_str, ','); elsif argpair[1] = 'ignore' then ignore_list := string_to_array(argpair[2], ','); elsif argpair[1] ~ '^ev_(type|extra[1-4])$' then field_sql := array_append(field_sql, 'select ' || quote_literal(argpair[1]) || '::text as key, (' || argpair[2] || field_sql_sfx); elsif argpair[1] = 'when' then field_sql := array_append(field_sql, 'select ' || quote_literal(argpair[1]) || '::text as key, (case when (' || argpair[2] || ')::boolean then ''proceed'' else null end' || field_sql_sfx); else got := false; end if; end loop; if not got then raise exception 'bad argument: %', TG_ARGV[i]; end if; end if; end loop; end; full_ignore_list := ignore_list || extra_ignore_list; if pkey_str is null then select array_agg(pk.attname) from (select k.attname from pg_index i, pg_attribute k where i.indrelid = TG_RELID and k.attrelid = i.indexrelid and i.indisprimary and k.attnum > 0 and not k.attisdropped order by k.attnum) pk into pkey_list; if pkey_list is null then pkey_list := '{}'; pkey_str := ''; else pkey_str := array_to_string(pkey_list, ','); end if; end if; if pkey_str = '' and TG_OP in ('UPDATE', 'DELETE') then raise exception 'Update/Delete on table without pkey'; end if; if TG_OP not in ('INSERT', 'UPDATE', 'DELETE', 'TRUNCATE') then raise exception 'TG_OP not supported: %', TG_OP; end if; -- fill ev_type select to_json(t.*)::text from (select TG_OP as op, array[TG_TABLE_SCHEMA,TG_TABLE_NAME] as "table", pkey_list as "pkey") t into ev_type; -- early exit? if current_setting('session_replication_role') = 'local' then if TG_WHEN = 'AFTER' or TG_OP = 'TRUNCATE' then return null; elsif TG_OP = 'DELETE' then return OLD; else return NEW; end if; elsif do_deny then raise exception 'Table ''%.%'' to queue ''%'': change not allowed (%)', TG_TABLE_SCHEMA, TG_TABLE_NAME, qname, TG_OP; elsif TG_OP = 'TRUNCATE' then perform pgq.insert_event(qname, ev_type, '{}', ev_extra1, ev_extra2, ev_extra3, ev_extra4); return null; end if; -- process table columns declare attr record; pkey_sql_buf text[]; qcol text; data_sql_buf text[]; ignore_sql text; ignore_sql_buf text[]; pkey_change_sql text; pkey_col_changes int4 := 0; valexp text; begin for attr in select k.attnum, k.attname, k.atttypid from pg_attribute k where k.attrelid = TG_RELID and k.attnum > 0 and not k.attisdropped order by k.attnum loop qcol := quote_ident(attr.attname); if attr.attname = any (ignore_list) then ignore_sql_buf := array_append(ignore_sql_buf, 'select case when rold.' || qcol || ' is null and rnew.' || qcol || ' is null then false' || ' when rold.' || qcol || ' is null or rnew.' || qcol || ' is null then true' || ' else rold.' || qcol || ' <> rnew.' || qcol || ' end as is_changed ' || 'from (select $1.*) rold, (select $2.*) rnew'); continue; elsif attr.attname = any (extra_ignore_list) then field_sql := array_prepend('select ' || quote_literal(substring(attr.attname from 6)) || '::text as key, (r.' || qcol || field_sql_sfx, field_sql); continue; end if; -- force cast to text or not if attr.atttypid in ('timestamptz'::regtype::oid, 'timestamp'::regtype::oid, 'int8'::regtype::oid, 'int4'::regtype::oid, 'int2'::regtype::oid, 'date'::regtype::oid, 'boolean'::regtype::oid) then valexp := 'to_json(r.' || qcol || ')::text'; else valexp := 'to_json(r.' || qcol || '::text)::text'; end if; if attr.attname = any (pkey_list) then pkey_sql_buf := array_append(pkey_sql_buf, 'select case when rold.' || qcol || ' is null and rnew.' || qcol || ' is null then false' || ' when rold.' || qcol || ' is null or rnew.' || qcol || ' is null then true' || ' else rold.' || qcol || ' <> rnew.' || qcol || ' end as is_changed ' || 'from (select $1.*) rold, (select $2.*) rnew'); end if; data_sql_buf := array_append(data_sql_buf, 'select ' || quote_literal(to_json(attr.attname) || ':') || ' || coalesce(' || valexp || ', ''null'') as jpair from (select $1.*) r'); end loop; -- SQL to see if pkey columns have changed if TG_OP = 'UPDATE' then pkey_change_sql := 'select count(1) from (' || array_to_string(pkey_sql_buf, ' union all ') || ') cols where cols.is_changed'; execute pkey_change_sql using OLD, NEW into pkey_col_changes; if pkey_col_changes > 0 then raise exception 'primary key update not allowed'; end if; end if; -- SQL to see if ignored columns have changed if TG_OP = 'UPDATE' and array_length(ignore_list, 1) is not null then ignore_sql := 'select count(1) from (' || array_to_string(ignore_sql_buf, ' union all ') || ') cols where cols.is_changed'; execute ignore_sql using OLD, NEW into ignore_col_changes; end if; -- SQL to load data data_sql := 'select ''{'' || array_to_string(array_agg(cols.jpair), '','') || ''}'' from (' || array_to_string(data_sql_buf, ' union all ') || ') cols'; end; -- render data declare old_data text; begin if TG_OP = 'INSERT' then execute data_sql using NEW into ev_data; elsif TG_OP = 'UPDATE' then -- render NEW execute data_sql using NEW into ev_data; -- render OLD when needed if do_backup or array_length(ignore_list, 1) is not null then execute data_sql using OLD into old_data; end if; -- only change was to ignored columns? if old_data = ev_data and ignore_col_changes > 0 then do_insert := false; end if; -- is backup needed? if do_backup then ev_extra2 := old_data; end if; elsif TG_OP = 'DELETE' then execute data_sql using OLD into ev_data; end if; end; -- apply magic args and columns declare col text; val text; rmain record; sql text; begin if do_insert and array_length(field_sql, 1) is not null then if TG_OP = 'DELETE' then rmain := OLD; else rmain := NEW; end if; sql := array_to_string(field_sql, ' union all '); for col, val in execute sql using rmain loop if col = 'ev_type' then ev_type := val; elsif col = 'ev_extra1' then ev_extra1 := val; elsif col = 'ev_extra2' then ev_extra2 := val; elsif col = 'ev_extra3' then ev_extra3 := val; elsif col = 'ev_extra4' then ev_extra4 := val; elsif col = 'when' then if val is null then do_insert := false; end if; end if; end loop; end if; end; -- insert final values if do_insert then perform pgq.insert_event(qname, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); end if; if do_skip or TG_WHEN = 'AFTER' or TG_OP = 'TRUNCATE' then return null; elsif TG_OP = 'DELETE' then return OLD; else return NEW; end if; end; $$ language plpgsql; pgq-3.4.1/lowlevel_pl/logutriga.sql000066400000000000000000000303631373233714300173630ustar00rootroot00000000000000create or replace function pgq.logutriga() returns trigger as $$ -- ---------------------------------------------------------------------- -- Function: pgq.logutriga() -- -- Trigger function that puts row data in urlencoded form into queue. -- -- Purpose: -- Used as producer for several PgQ standard consumers (cube_dispatcher, -- queue_mover, table_dispatcher). Basically for cases where the -- consumer wants to parse the event and look at the actual column values. -- -- Trigger parameters: -- arg1 - queue name -- argX - any number of optional arg, in any order -- -- Optional arguments: -- SKIP - The actual operation should be skipped (BEFORE trigger) -- ignore=col1[,col2] - don't look at the specified arguments -- pkey=col1[,col2] - Set pkey fields for the table, autodetection will be skipped -- backup - Put urlencoded contents of old row to ev_extra2 -- colname=EXPR - Override field value with SQL expression. Can reference table -- columns. colname can be: ev_type, ev_data, ev_extra1 .. ev_extra4 -- when=EXPR - If EXPR returns false, don't insert event. -- -- Queue event fields: -- ev_type - I/U/D ':' pkey_column_list -- ev_data - column values urlencoded -- ev_extra1 - table name -- ev_extra2 - optional urlencoded backup -- -- Regular listen trigger example: -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname'); -- -- Redirect trigger example: -- > CREATE TRIGGER triga_nimi BEFORE INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname', 'SKIP'); -- ---------------------------------------------------------------------- declare qname text; ev_type text; ev_data text; ev_extra1 text; ev_extra2 text; ev_extra3 text; ev_extra4 text; do_skip boolean := false; do_backup boolean := false; do_insert boolean := true; do_deny boolean := false; extra_ignore_list text[]; full_ignore_list text[]; ignore_list text[] := '{}'; pkey_list text[]; pkey_str text; field_sql_sfx text; field_sql text[] := '{}'; data_sql text; ignore_col_changes int4 := 0; begin if TG_NARGS < 1 then raise exception 'Trigger needs queue name'; end if; qname := TG_ARGV[0]; -- standard output ev_extra1 := TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME; -- prepare to handle magic fields field_sql_sfx := ')::text as val from (select $1.*) r'; extra_ignore_list := array['_pgq_ev_type', '_pgq_ev_extra1', '_pgq_ev_extra2', '_pgq_ev_extra3', '_pgq_ev_extra4']::text[]; -- parse trigger args declare got boolean; argpair text[]; i integer; begin for i in 1 .. TG_NARGS-1 loop if TG_ARGV[i] in ('skip', 'SKIP') then do_skip := true; elsif TG_ARGV[i] = 'backup' then do_backup := true; elsif TG_ARGV[i] = 'deny' then do_deny := true; else got := false; for argpair in select regexp_matches(TG_ARGV[i], '^([^=]+)=(.*)') loop got := true; if argpair[1] = 'pkey' then pkey_str := argpair[2]; pkey_list := string_to_array(pkey_str, ','); elsif argpair[1] = 'ignore' then ignore_list := string_to_array(argpair[2], ','); elsif argpair[1] ~ '^ev_(type|extra[1-4])$' then field_sql := array_append(field_sql, 'select ' || quote_literal(argpair[1]) || '::text as key, (' || argpair[2] || field_sql_sfx); elsif argpair[1] = 'when' then field_sql := array_append(field_sql, 'select ' || quote_literal(argpair[1]) || '::text as key, (case when (' || argpair[2] || ')::boolean then ''proceed'' else null end' || field_sql_sfx); else got := false; end if; end loop; if not got then raise exception 'bad argument: %', TG_ARGV[i]; end if; end if; end loop; end; full_ignore_list := ignore_list || extra_ignore_list; if pkey_str is null then select array_agg(pk.attname) from (select k.attname from pg_index i, pg_attribute k where i.indrelid = TG_RELID and k.attrelid = i.indexrelid and i.indisprimary and k.attnum > 0 and not k.attisdropped order by k.attnum) pk into pkey_list; if pkey_list is null then pkey_list := '{}'; pkey_str := ''; else pkey_str := array_to_string(pkey_list, ','); end if; end if; if pkey_str = '' and TG_OP in ('UPDATE', 'DELETE') then raise exception 'Update/Delete on table without pkey'; end if; if TG_OP = 'INSERT' then ev_type := 'I:' || pkey_str; elsif TG_OP = 'UPDATE' then ev_type := 'U:' || pkey_str; elsif TG_OP = 'DELETE' then ev_type := 'D:' || pkey_str; elsif TG_OP = 'TRUNCATE' then ev_type := 'R'; else raise exception 'TG_OP not supported: %', TG_OP; end if; if current_setting('session_replication_role') = 'local' then if TG_WHEN = 'AFTER' or TG_OP = 'TRUNCATE' then return null; elsif TG_OP = 'DELETE' then return OLD; else return NEW; end if; elsif do_deny then raise exception 'Table ''%.%'' to queue ''%'': change not allowed (%)', TG_TABLE_SCHEMA, TG_TABLE_NAME, qname, TG_OP; elsif TG_OP = 'TRUNCATE' then perform pgq.insert_event(qname, ev_type, '', ev_extra1, ev_extra2, ev_extra3, ev_extra4); return null; end if; -- process table columns declare attr record; pkey_sql_buf text[]; qcol text; data_sql_buf text[]; ignore_sql text; ignore_sql_buf text[]; pkey_change_sql text; pkey_col_changes int4 := 0; valexp text; begin for attr in select k.attnum, k.attname, k.atttypid from pg_attribute k where k.attrelid = TG_RELID and k.attnum > 0 and not k.attisdropped order by k.attnum loop qcol := quote_ident(attr.attname); if attr.attname = any (ignore_list) then ignore_sql_buf := array_append(ignore_sql_buf, 'select case when rold.' || qcol || ' is null and rnew.' || qcol || ' is null then false' || ' when rold.' || qcol || ' is null or rnew.' || qcol || ' is null then true' || ' else rold.' || qcol || ' <> rnew.' || qcol || ' end as is_changed ' || 'from (select $1.*) rold, (select $2.*) rnew'); continue; elsif attr.attname = any (extra_ignore_list) then field_sql := array_prepend('select ' || quote_literal(substring(attr.attname from 6)) || '::text as key, (r.' || qcol || field_sql_sfx, field_sql); continue; end if; if attr.atttypid = 'boolean'::regtype::oid then valexp := 'case r.' || qcol || ' when true then ''t'' when false then ''f'' else null end'; else valexp := 'r.' || qcol || '::text'; end if; if attr.attname = any (pkey_list) then pkey_sql_buf := array_append(pkey_sql_buf, 'select case when rold.' || qcol || ' is null and rnew.' || qcol || ' is null then false' || ' when rold.' || qcol || ' is null or rnew.' || qcol || ' is null then true' || ' else rold.' || qcol || ' <> rnew.' || qcol || ' end as is_changed ' || 'from (select $1.*) rold, (select $2.*) rnew'); end if; data_sql_buf := array_append(data_sql_buf, 'select pgq._urlencode(' || quote_literal(attr.attname) || ') || coalesce(''='' || pgq._urlencode(' || valexp || '), '''') as upair from (select $1.*) r'); end loop; -- SQL to see if pkey columns have changed if TG_OP = 'UPDATE' then pkey_change_sql := 'select count(1) from (' || array_to_string(pkey_sql_buf, ' union all ') || ') cols where cols.is_changed'; execute pkey_change_sql using OLD, NEW into pkey_col_changes; if pkey_col_changes > 0 then raise exception 'primary key update not allowed'; end if; end if; -- SQL to see if ignored columns have changed if TG_OP = 'UPDATE' and array_length(ignore_list, 1) is not null then ignore_sql := 'select count(1) from (' || array_to_string(ignore_sql_buf, ' union all ') || ') cols where cols.is_changed'; execute ignore_sql using OLD, NEW into ignore_col_changes; end if; -- SQL to load data data_sql := 'select array_to_string(array_agg(cols.upair), ''&'') from (' || array_to_string(data_sql_buf, ' union all ') || ') cols'; end; -- render data declare old_data text; begin if TG_OP = 'INSERT' then execute data_sql using NEW into ev_data; elsif TG_OP = 'UPDATE' then -- render NEW execute data_sql using NEW into ev_data; -- render OLD when needed if do_backup or array_length(ignore_list, 1) is not null then execute data_sql using OLD into old_data; end if; -- only change was to ignored columns? if old_data = ev_data and ignore_col_changes > 0 then do_insert := false; end if; -- is backup needed? if do_backup then ev_extra2 := old_data; end if; elsif TG_OP = 'DELETE' then execute data_sql using OLD into ev_data; end if; end; -- apply magic args and columns declare col text; val text; rmain record; sql text; begin if do_insert and array_length(field_sql, 1) is not null then if TG_OP = 'DELETE' then rmain := OLD; else rmain := NEW; end if; sql := array_to_string(field_sql, ' union all '); for col, val in execute sql using rmain loop if col = 'ev_type' then ev_type := val; elsif col = 'ev_extra1' then ev_extra1 := val; elsif col = 'ev_extra2' then ev_extra2 := val; elsif col = 'ev_extra3' then ev_extra3 := val; elsif col = 'ev_extra4' then ev_extra4 := val; elsif col = 'when' then if val is null then do_insert := false; end if; end if; end loop; end if; end; -- insert final values if do_insert then perform pgq.insert_event(qname, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); end if; if do_skip or TG_WHEN = 'AFTER' or TG_OP = 'TRUNCATE' then return null; elsif TG_OP = 'DELETE' then return OLD; else return NEW; end if; end; $$ language plpgsql; create or replace function pgq._urlencode(val text) returns text as $$ select replace(string_agg(pair[1] || regexp_replace(encode(convert_to(pair[2], 'utf8'), 'hex'), '..', E'%\\&', 'g'), ''), '%20', '+') from regexp_matches($1, '([-_.a-zA-Z0-9]*)([^-_.a-zA-Z0-9]*)', 'g') pair $$ language sql strict immutable; pgq-3.4.1/lowlevel_pl/sqltriga.sql000066400000000000000000000342511373233714300172140ustar00rootroot00000000000000create or replace function pgq.sqltriga() returns trigger as $$ -- ---------------------------------------------------------------------- -- Function: pgq.logutriga() -- -- Trigger function that puts row data in SQL-fragment form into queue. -- -- Purpose: -- Anciant way to implement replication. -- -- Trigger parameters: -- arg1 - queue name -- argX - any number of optional arg, in any order -- -- Optional arguments: -- SKIP - The actual operation should be skipped (BEFORE trigger) -- ignore=col1[,col2] - don't look at the specified arguments -- pkey=col1[,col2] - Set pkey fields for the table, autodetection will be skipped -- backup - Put urlencoded contents of old row to ev_extra2 -- colname=EXPR - Override field value with SQL expression. Can reference table -- columns. colname can be: ev_type, ev_data, ev_extra1 .. ev_extra4 -- when=EXPR - If EXPR returns false, don't insert event. -- -- Queue event fields: -- ev_type - I/U/D ':' pkey_column_list -- ev_data - column values urlencoded -- ev_extra1 - table name -- ev_extra2 - optional urlencoded backup -- -- Regular listen trigger example: -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname'); -- -- Redirect trigger example: -- > CREATE TRIGGER triga_nimi BEFORE INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname', 'SKIP'); -- ---------------------------------------------------------------------- declare qname text; ev_type text; ev_data text; ev_extra1 text; ev_extra2 text; ev_extra3 text; ev_extra4 text; do_skip boolean := false; do_backup boolean := false; do_insert boolean := true; do_deny boolean := false; extra_ignore_list text[]; full_ignore_list text[]; ignore_list text[] := '{}'; pkey_list text[]; pkey_str text; field_sql_sfx text; field_sql text[] := '{}'; data_sql text; ignore_col_changes int4 := 0; begin if TG_NARGS < 1 then raise exception 'Trigger needs queue name'; end if; qname := TG_ARGV[0]; -- standard output ev_extra1 := TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME; -- prepare to handle magic fields field_sql_sfx := ')::text as val from (select $1.*) r'; extra_ignore_list := array['_pgq_ev_type', '_pgq_ev_extra1', '_pgq_ev_extra2', '_pgq_ev_extra3', '_pgq_ev_extra4']::text[]; -- parse trigger args declare got boolean; argpair text[]; i integer; begin for i in 1 .. TG_NARGS-1 loop if TG_ARGV[i] in ('skip', 'SKIP') then do_skip := true; elsif TG_ARGV[i] = 'backup' then do_backup := true; elsif TG_ARGV[i] = 'deny' then do_deny := true; else got := false; for argpair in select regexp_matches(TG_ARGV[i], '^([^=]+)=(.*)') loop got := true; if argpair[1] = 'pkey' then pkey_str := argpair[2]; pkey_list := string_to_array(pkey_str, ','); elsif argpair[1] = 'ignore' then ignore_list := string_to_array(argpair[2], ','); elsif argpair[1] ~ '^ev_(type|extra[1-4])$' then field_sql := array_append(field_sql, 'select ' || quote_literal(argpair[1]) || '::text as key, (' || argpair[2] || field_sql_sfx); elsif argpair[1] = 'when' then field_sql := array_append(field_sql, 'select ' || quote_literal(argpair[1]) || '::text as key, (case when (' || argpair[2] || ')::boolean then ''proceed'' else null end' || field_sql_sfx); else got := false; end if; end loop; if not got then raise exception 'bad argument: %', TG_ARGV[i]; end if; end if; end loop; end; full_ignore_list := ignore_list || extra_ignore_list; if pkey_str is null then select array_agg(pk.attname) from (select k.attname from pg_index i, pg_attribute k where i.indrelid = TG_RELID and k.attrelid = i.indexrelid and i.indisprimary and k.attnum > 0 and not k.attisdropped order by k.attnum) pk into pkey_list; if pkey_list is null then pkey_list := '{}'; pkey_str := ''; else pkey_str := array_to_string(pkey_list, ','); end if; end if; if pkey_str = '' and TG_OP in ('UPDATE', 'DELETE') then raise exception 'Update/Delete on table without pkey'; end if; if TG_OP = 'INSERT' then ev_type := 'I'; elsif TG_OP = 'UPDATE' then ev_type := 'U'; elsif TG_OP = 'DELETE' then ev_type := 'D'; elsif TG_OP = 'TRUNCATE' then ev_type := 'R'; else raise exception 'TG_OP not supported: %', TG_OP; end if; if current_setting('session_replication_role') = 'local' then if TG_WHEN = 'AFTER' or TG_OP = 'TRUNCATE' then return null; elsif TG_OP = 'DELETE' then return OLD; else return NEW; end if; elsif do_deny then raise exception 'Table ''%.%'' to queue ''%'': change not allowed (%)', TG_TABLE_SCHEMA, TG_TABLE_NAME, qname, TG_OP; elsif TG_OP = 'TRUNCATE' then perform pgq.insert_event(qname, ev_type, '', ev_extra1, ev_extra2, ev_extra3, ev_extra4); return null; end if; -- process table columns declare attr record; pkey_sql_buf text[]; qcol text; data_sql_buf text[]; ignore_sql text; ignore_sql_buf text[]; pkey_change_sql text; pkey_col_changes int4 := 0; valexp text; sql1_buf text[] := '{}'; -- I:cols, U:vals, D:- sql2_buf text[] := '{}'; -- I:vals, U:pks, D:pks sql1_buf_fallback text[] := '{}'; val_sql text; has_changed boolean; begin for attr in select k.attnum, k.attname, k.atttypid from pg_attribute k where k.attrelid = TG_RELID and k.attnum > 0 and not k.attisdropped order by k.attnum loop qcol := quote_ident(attr.attname); if attr.attname = any (ignore_list) then ignore_sql_buf := array_append(ignore_sql_buf, 'select case when rold.' || qcol || ' is null and rnew.' || qcol || ' is null then false' || ' when rold.' || qcol || ' is null or rnew.' || qcol || ' is null then true' || ' else rold.' || qcol || ' <> rnew.' || qcol || ' end as is_changed ' || 'from (select $1.*) rold, (select $2.*) rnew'); continue; elsif attr.attname = any (extra_ignore_list) then field_sql := array_prepend('select ' || quote_literal(substring(attr.attname from 6)) || '::text as key, (r.' || qcol || field_sql_sfx, field_sql); continue; end if; if attr.atttypid = 'boolean'::regtype::oid then valexp := 'case r.' || qcol || ' when true then ''t'' when false then ''f'' else null end'; else valexp := 'r.' || qcol || '::text'; end if; if attr.attname = any (pkey_list) then pkey_sql_buf := array_append(pkey_sql_buf, 'select case when rold.' || qcol || ' is null and rnew.' || qcol || ' is null then false' || ' when rold.' || qcol || ' is null or rnew.' || qcol || ' is null then true' || ' else rold.' || qcol || ' <> rnew.' || qcol || ' end as is_changed ' || 'from (select $1.*) rold, (select $2.*) rnew'); if TG_OP in ('UPDATE', 'DELETE') then sql2_buf := array_append(sql2_buf, 'select ' || quote_literal(qcol) || ' || coalesce(''='' || quote_literal(' || valexp|| '), '' is null'') as val' || ' from (select $1.*) r'); if array_length(sql1_buf_fallback, 1) is null then sql1_buf_fallback := array_append(sql1_buf_fallback, 'select ' || quote_literal(qcol || '=') || ' || quote_nullable(' || valexp || ') as val' || ' from (select $1.*) r'); end if; continue; end if; end if; if TG_OP = 'INSERT' then sql1_buf := array_append(sql1_buf, qcol); sql2_buf := array_append(sql2_buf, 'select coalesce(quote_literal(' || valexp || '), ''null'') as val' || ' from (select $1.*) r'); elsif TG_OP = 'UPDATE' then execute 'select quote_nullable(rold.' || qcol || ') <> quote_nullable(rnew.' || qcol || ') as has_changed' || ' from (select $1.*) rold, (select $2.*) rnew' using OLD, NEW into has_changed; if has_changed then sql1_buf := array_append(sql1_buf, 'select ' || quote_literal(qcol || '=') || ' || quote_nullable(' || valexp || ') as val' || ' from (select $1.*) r'); end if; end if; end loop; -- SQL to see if pkey columns have changed if TG_OP = 'UPDATE' then pkey_change_sql := 'select count(1) from (' || array_to_string(pkey_sql_buf, ' union all ') || ') cols where cols.is_changed'; execute pkey_change_sql using OLD, NEW into pkey_col_changes; if pkey_col_changes > 0 then raise exception 'primary key update not allowed'; end if; end if; -- SQL to see if ignored columns have changed if TG_OP = 'UPDATE' and array_length(ignore_list, 1) is not null then ignore_sql := 'select count(1) from (' || array_to_string(ignore_sql_buf, ' union all ') || ') cols where cols.is_changed'; execute ignore_sql using OLD, NEW into ignore_col_changes; end if; -- SQL to load data if TG_OP = 'INSERT' then data_sql := 'select array_to_string(array[''('', ' || quote_literal(array_to_string(sql1_buf, ',')) || ', '') values ('',' || '(select array_to_string(array_agg(s.val), '','') from (' || array_to_string(sql2_buf, ' union all ') || ') s)' || ', '')''' || '], '''')'; elsif TG_OP = 'UPDATE' then if array_length(sql1_buf, 1) is null then sql1_buf := sql1_buf_fallback; end if; data_sql := 'select array_to_string(array[' || '(select array_to_string(array_agg(s.val), '','') from (' || array_to_string(sql1_buf, ' union all ') || ') s)' || ', '' where '',' || '(select array_to_string(array_agg(s.val), '' and '') from (' || array_to_string(sql2_buf, ' union all ') || ') s)' || '], '''')'; else data_sql := 'select array_to_string(array[' || '(select array_to_string(array_agg(s.val), '' and '') from (' || array_to_string(sql2_buf, ' union all ') || ') s)' || '], '''')'; end if; end; -- render data declare old_data text; begin if TG_OP = 'INSERT' then execute data_sql using NEW into ev_data; elsif TG_OP = 'UPDATE' then -- render NEW execute data_sql using NEW into ev_data; -- render OLD when needed if do_backup or array_length(ignore_list, 1) is not null then execute data_sql using OLD into old_data; end if; -- only change was to ignored columns? if old_data = ev_data and ignore_col_changes > 0 then do_insert := false; end if; -- is backup needed? if do_backup then ev_extra2 := old_data; end if; elsif TG_OP = 'DELETE' then execute data_sql using OLD into ev_data; end if; end; -- apply magic args and columns declare col text; val text; rmain record; sql text; begin if do_insert and array_length(field_sql, 1) is not null then if TG_OP = 'DELETE' then rmain := OLD; else rmain := NEW; end if; sql := array_to_string(field_sql, ' union all '); for col, val in execute sql using rmain loop if col = 'ev_type' then ev_type := val; elsif col = 'ev_extra1' then ev_extra1 := val; elsif col = 'ev_extra2' then ev_extra2 := val; elsif col = 'ev_extra3' then ev_extra3 := val; elsif col = 'ev_extra4' then ev_extra4 := val; elsif col = 'when' then if val is null then do_insert := false; end if; end if; end loop; end if; end; -- insert final values if do_insert then perform pgq.insert_event(qname, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); end if; if do_skip or TG_WHEN = 'AFTER' or TG_OP = 'TRUNCATE' then return null; elsif TG_OP = 'DELETE' then return OLD; else return NEW; end if; end; $$ language plpgsql; pgq-3.4.1/mk/000077500000000000000000000000001373233714300127235ustar00rootroot00000000000000pgq-3.4.1/mk/catsql.py000077500000000000000000000070161373233714300145730ustar00rootroot00000000000000#! /usr/bin/env python3 """Prints out SQL files with psql command execution. Supported psql commands: \i, \cd, \q Others are skipped. Aditionally does some pre-processing for NDoc. NDoc is looks nice but needs some hand-holding. Bug: - function def end detection searches for 'as'/'is' but does not check word boundaries - finds them even in function name. That means in main conf, as/is must be disabled and $ ' added. This script can remove the unnecessary AS from output. Niceties: - Ndoc includes function def in output only if def is after comment. But for SQL functions its better to have it after def. This script can swap comment and def. - Optionally remove CREATE FUNCTION (OR REPLACE) from def to keep it shorter in doc. Note: - NDoc compares real function name and name in comment. if differ, it decides detection failed. """ import sys, os, re, getopt def usage(x): print("usage: catsql [--ndoc] FILE [FILE ...]") sys.exit(x) # NDoc specific changes cf_ndoc = 0 # compile regexes func_re = r"create\s+(or\s+replace\s+)?function\s+" func_rc = re.compile(func_re, re.I) comm_rc = re.compile(r"^\s*([#]\s*)?(?P--.*)", re.I) end_rc = re.compile(r"\b([;]|begin|declare|end)\b", re.I) as_rc = re.compile(r"\s+as\s+", re.I) cmd_rc = re.compile(r"^\\([a-z]*)(\s+.*)?", re.I) # conversion func def fix_func(ln): # if ndoc, replace AS with ' ' if cf_ndoc: return as_rc.sub(' ', ln) else: return ln # got function def def proc_func(f, ln): # remove CREATE OR REPLACE if cf_ndoc: ln = func_rc.sub('', ln) ln = fix_func(ln) pre_list = [ln] comm_list = [] while 1: ln = f.readline() if not ln: break com = None if cf_ndoc: com = comm_rc.search(ln) if cf_ndoc and com: pos = com.start('com') comm_list.append(ln[pos:]) elif end_rc.search(ln): break elif len(comm_list) > 0: break else: pre_list.append(fix_func(ln)) if len(comm_list) > 2: for el in comm_list: sys.stdout.write(el) for el in pre_list: sys.stdout.write(el) else: for el in pre_list: sys.stdout.write(el) for el in comm_list: sys.stdout.write(el) if ln: sys.stdout.write(fix_func(ln)) def cat_file(fn): sys.stdout.write("\n") f = open(fn) while 1: ln = f.readline() if not ln: break m = cmd_rc.search(ln) if m: cmd = m.group(1) if cmd == "i": # include a file fn2 = m.group(2).strip() cat_file(fn2) elif cmd == "q": # quit sys.exit(0) elif cmd == "cd": # chdir cd_dir = m.group(2).strip() os.chdir(cd_dir) else: # skip all others pass else: if func_rc.search(ln): # function header proc_func(f, ln) else: # normal sql sys.stdout.write(ln) sys.stdout.write("\n") def main(): global cf_ndoc try: opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['ndoc']) except getopt.error as d: print(str(d)) usage(1) for o, v in opts: if o == "-h": usage(0) elif o == "--ndoc": cf_ndoc = 1 for fn in args: cat_file(fn) if __name__ == '__main__': main() pgq-3.4.1/mk/common-pgxs.mk000066400000000000000000000105761373233714300155340ustar00rootroot00000000000000 # PGXS does not support modules that are supposed # to run on different Postgres versions very well. # Here are some workarounds for them. # Variables that are used when extensions are available Extension_data ?= Extension_data_built ?= $(EXTENSION)--$(EXT_VERSION).sql $(EXTENSION)--unpackaged--$(EXT_VERSION).sql Extension_regress ?= # Variables that are used when extensions are not available Contrib_data ?= Contrib_data_built += $(EXTENSION).sql $(EXTENSION).upgrade.sql \ structure/newgrants_$(EXTENSION).sql \ structure/oldgrants_$(EXTENSION).sql Contrib_regress ?= EXT_VERSION ?= EXT_OLD_VERSIONS ?= Extension_upgrade_files = $(if $(EXT_OLD_VERSIONS),$(foreach v,$(EXT_OLD_VERSIONS),$(EXTENSION)--$(v)--$(EXT_VERSION).sql)) Extension_data_built += $(Extension_upgrade_files) # Should the Contrib* files installed (under ../contrib/) # even when extensions are available? Contrib_install_always ?= yes # # switch variables # IfExt = $(if $(filter 8.% 9.0%,$(MAJORVERSION)8.3),$(2),$(1)) DATA = $(call IfExt,$(Extension_data),$(Contrib_data)) DATA_built = $(call IfExt,$(Extension_data_built),$(Contrib_data_built)) REGRESS = $(call IfExt,$(Extension_regress),$(Contrib_regress)) EXTRA_CLEAN += $(call IfExt,$(Contrib_data_built),$(Extension_data_built)) test.dump # have deterministic dbname for regtest database override CONTRIB_TESTDB = regression REGRESS_OPTS = --dbname=$(CONTRIB_TESTDB) # # Calculate actual sql files # GRANT_SQL = structure/newgrants_$(EXTENSION).sql SQLS = $(shell $(AWK) '/^\\i / { print $$2; }' structure/install.sql) FUNCS = $(shell $(AWK) '/^\\i / { print $$2; }' $(SQLS)) SRCS = $(SQLS) $(FUNCS) $(GRANT_SQL) # # load PGXS # PG_CONFIG ?= pg_config PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # when compiling locally and with postgres without python, # the variable may be empty PYTHON3 := $(if $(PYTHON3),$(PYTHON3),python3) # # common tools # NDOC = NaturalDocs NDOCARGS = -r -o html docs/html -p docs -i docs/sql CATSQL = $(PYTHON3) mk/catsql.py GRANTFU = $(PYTHON3) mk/grantfu.py # # build rules, in case Contrib data must be always installed # ifeq ($(call IfExt,$(Contrib_install_always),no),yes) all: $(Contrib_data) $(Contrib_data_built) installdirs: installdirs-old-contrib install: install-old-contrib installdirs-old-contrib: $(MKDIR_P) '$(DESTDIR)$(datadir)/contrib' install-old-contrib: $(Contrib_data) $(Contrib_data_built) installdirs-old-contrib $(INSTALL_DATA) $(addprefix $(srcdir)/, $(Contrib_data)) $(Contrib_data_built) '$(DESTDIR)$(datadir)/contrib/' endif # # regtest shortcuts # test: install $(MAKE) installcheck || { filterdiff --format=unified regression.diffs | less; exit 1; } pg_dump regression > test.dump citest: checkver $(MAKE) installcheck || { filterdiff --format=unified regression.diffs; exit 1; } ack: cp results/*.out expected/ cleandox: rm -rf docs/html docs/Data docs/sql clean: cleandox .PHONY: test ack installdirs-old-contrib install-old-contrib cleandox dox # # common files # $(EXTENSION)--$(EXT_VERSION).sql: $(EXTENSION).sql structure/ext_postproc.sql $(CATSQL) $^ > $@ $(EXTENSION)--unpackaged--$(EXT_VERSION).sql: $(EXTENSION).upgrade.sql structure/ext_unpackaged.sql structure/ext_postproc.sql $(CATSQL) $^ > $@ $(EXTENSION).sql: $(SRCS) $(CATSQL) structure/install.sql $(GRANT_SQL) > $@ $(EXTENSION).upgrade.sql: $(SRCS) $(CATSQL) structure/upgrade.sql $(GRANT_SQL) > $@ ifneq ($(Extension_upgrade_files),) $(Extension_upgrade_files): $(EXTENSION).upgrade.sql cp $< $@ endif structure/newgrants_$(EXTENSION).sql: structure/grants.ini $(GRANTFU) -r -d $< > $@ structure/oldgrants_$(EXTENSION).sql: structure/grants.ini structure/grants.sql echo "begin;" > $@ $(GRANTFU) -R -o $< >> $@ cat structure/grants.sql >> $@ echo "commit;" >> $@ checkver: @echo "Checking version numbers" @grep -q "^default_version *= *'$(EXT_VERSION)'" $(EXTENSION).control \ || { echo "ERROR: $(EXTENSION).control has wrong version"; exit 1; } @test -f "docs/notes/v$(EXT_VERSION).md" \ || { echo "ERROR: notes missing: docs/notes/v$(EXT_VERSION).md"; exit 1; } @head debian/changelog | grep -q "[(]$(EXT_VERSION)-" debian/changelog \ || { echo "ERROR: debian/changelog has wrong version"; exit 1; } all: checkver TARNAME = $(EXTENSION)-$(EXT_VERSION) dist: checkver git archive --format=tar.gz --prefix=$(TARNAME)/ -o $(TARNAME).tar.gz HEAD release: checkver git tag v$(EXT_VERSION) git push github git push github --tag pgq-3.4.1/mk/grantfu.py000077500000000000000000000261371373233714300147570ustar00rootroot00000000000000#! /usr/bin/env python3 # GrantFu - GRANT/REVOKE generator for Postgres # # Copyright (c) 2005 Marko Kreen # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """Generator for PostgreSQL permissions. Loads config where roles, objects and their mapping is described and generates grants based on them. ConfigParser docs: http://docs.python.org/lib/module-ConfigParser.html Example: -------------------------------------------------------------------- [DEFAULT] users = user1, user2 # users to handle groups = group1, group2 # groups to handle auto_seq = 0 # dont handle seqs (default) # '!' after a table negates this setting for a table seq_name = id # the name for serial field (default: id) seq_usage = 0 # should we grant "usage" or "select, update" # for automatically handled sequences # section names can be random, but if you want to see them # in same order as in config file, then order them alphabetically [1.section] on.tables = testtbl, testtbl_id_seq, # here we handle seq by hand table_with_seq! # handle seq automatically # (table_with_seq_id_seq) user1 = select group1 = select, insert, update # instead of 'tables', you may use 'functions', 'languages', # 'schemas', 'tablespaces' --------------------------------------------------------------------- """ import sys, os, getopt try: from configparser import ConfigParser except ImportError: from ConfigParser import SafeConfigParser as ConfigParser __version__ = "1.0" R_NEW = 0x01 R_DEFS = 0x02 G_DEFS = 0x04 R_ONLY = 0x80 def usage(err): sys.stderr.write("usage: %s [-r|-R] CONF_FILE\n" % sys.argv[0]) sys.stderr.write(" -r Generate also REVOKE commands\n") sys.stderr.write(" -R Generate only REVOKE commands\n") sys.stderr.write(" -d Also REVOKE default perms\n") sys.stderr.write(" -D Only REVOKE default perms\n") sys.stderr.write(" -o Generate default GRANTS\n") sys.stderr.write(" -v Print program version\n") sys.stderr.write(" -t Put everything in one big transaction\n") sys.exit(err) class PConf(ConfigParser): "List support for ConfigParser" def __init__(self, defaults = None): ConfigParser.__init__(self, defaults) def get_list(self, sect, key): str = self.get(sect, key).strip() res = [] if not str: return res for val in str.split(","): res.append(val.strip()) return res class GrantFu: def __init__(self, cf, revoke): self.cf = cf self.revoke = revoke # avoid putting grantfu vars into defaults, thus into every section self.group_list = [] self.user_list = [] self.auto_seq = 0 self.seq_name = "id" self.seq_usage = 0 if self.cf.has_option('GrantFu', 'groups'): self.group_list = self.cf.get_list('GrantFu', 'groups') if self.cf.has_option('GrantFu', 'users'): self.user_list += self.cf.get_list('GrantFu', 'users') if self.cf.has_option('GrantFu', 'roles'): self.user_list += self.cf.get_list('GrantFu', 'roles') if self.cf.has_option('GrantFu', 'auto_seq'): self.auto_seq = self.cf.getint('GrantFu', 'auto_seq') if self.cf.has_option('GrantFu', 'seq_name'): self.seq_name = self.cf.get('GrantFu', 'seq_name') if self.cf.has_option('GrantFu', 'seq_usage'): self.seq_usage = self.cf.getint('GrantFu', 'seq_usage') # make string of all subjects tmp = [] for g in self.group_list: tmp.append("group " + g) for u in self.user_list: tmp.append(u) self.all_subjs = ", ".join(tmp) # per-section vars self.sect = None self.seq_list = [] self.seq_allowed = [] def process(self): if len(self.user_list) == 0 and len(self.group_list) == 0: return sect_list = self.cf.sections() sect_list.sort() for self.sect in sect_list: if self.sect == "GrantFu": continue print("\n-- %s --" % self.sect) self.handle_tables() self.handle_other('on.databases', 'DATABASE') self.handle_other('on.functions', 'FUNCTION') self.handle_other('on.languages', 'LANGUAGE') self.handle_other('on.schemas', 'SCHEMA') self.handle_other('on.tablespaces', 'TABLESPACE') self.handle_other('on.sequences', 'SEQUENCE') self.handle_other('on.types', 'TYPE') self.handle_other('on.domains', 'DOMAIN') def handle_other(self, listname, obj_type): """Handle grants for all objects except tables.""" if not self.sect_hasvar(listname): return # don't parse list, as in case of functions it may be complicated obj_str = obj_type + " " + self.sect_var(listname) if self.revoke & R_NEW: self.gen_revoke(obj_str) if self.revoke & R_DEFS: self.gen_revoke_defs(obj_str, obj_type) if not self.revoke & R_ONLY: self.gen_one_type(obj_str) if self.revoke & G_DEFS: self.gen_defs(obj_str, obj_type) def handle_tables(self): """Handle grants for tables and sequences. The tricky part here is the automatic handling of sequences.""" if not self.sect_hasvar('on.tables'): return cleaned_list = [] table_list = self.sect_list('on.tables') for table in table_list: if table[-1] == '!': table = table[:-1] if not self.auto_seq: self.seq_list.append("%s_%s_seq" % (table, self.seq_name)) else: if self.auto_seq: self.seq_list.append("%s_%s_seq" % (table, self.seq_name)) cleaned_list.append(table) obj_str = "TABLE " + ", ".join(cleaned_list) if self.revoke & R_NEW: self.gen_revoke(obj_str) if self.revoke & R_DEFS: self.gen_revoke_defs(obj_str, "TABLE") if not self.revoke & R_ONLY: self.gen_one_type(obj_str) if self.revoke & G_DEFS: self.gen_defs(obj_str, "TABLE") # cleanup self.seq_list = [] self.seq_allowed = [] def gen_revoke(self, obj_str): "Generate revoke for one section / subject type (user or group)" if len(self.seq_list) > 0: obj_str += ", " + ", ".join(self.seq_list) obj_str = obj_str.strip().replace('\n', '\n ') print("REVOKE ALL ON %s\n FROM %s CASCADE;" % (obj_str, self.all_subjs)) def gen_revoke_defs(self, obj_str, obj_type): "Generate revoke defaults for one section" # process only things that have default grants to public if obj_type not in ('FUNCTION', 'DATABASE', 'LANGUAGE', 'TYPE', 'DOMAIN'): return defrole = 'public' # if the sections contains grants to 'public', dont drop if self.sect_hasvar(defrole): return obj_str = obj_str.strip().replace('\n', '\n ') print("REVOKE ALL ON %s\n FROM %s CASCADE;" % (obj_str, defrole)) def gen_defs(self, obj_str, obj_type): "Generate defaults grants for one section" if obj_type == "FUNCTION": defgrants = "execute" elif obj_type == "DATABASE": defgrants = "connect, temp" elif obj_type in ("LANGUAGE", "TYPE", "DOMAIN"): defgrants = "usage" else: return defrole = 'public' obj_str = obj_str.strip().replace('\n', '\n ') print("GRANT %s ON %s\n TO %s;" % (defgrants, obj_str, defrole)) def gen_one_subj(self, subj, fqsubj, obj_str): if not self.sect_hasvar(subj): return obj_str = obj_str.strip().replace('\n', '\n ') perm = self.sect_var(subj).strip() if perm: print("GRANT %s ON %s\n TO %s;" % (perm, obj_str, fqsubj)) # check for seq perms if len(self.seq_list) > 0: loperm = perm.lower() if loperm.find("insert") >= 0 or loperm.find("all") >= 0: self.seq_allowed.append(fqsubj) def gen_one_type(self, obj_str): "Generate GRANT for one section / one object type in section" for u in self.user_list: self.gen_one_subj(u, u, obj_str) for g in self.group_list: self.gen_one_subj(g, "group " + g, obj_str) # if there was any seq perms, generate grants if len(self.seq_allowed) > 0: seq_str = ", ".join(self.seq_list) subj_str = ", ".join(self.seq_allowed) if self.seq_usage: cmd = "GRANT usage ON SEQUENCE %s\n TO %s;" else: cmd = "GRANT select, update ON %s\n TO %s;" print(cmd % (seq_str, subj_str)) def sect_var(self, name): return self.cf.get(self.sect, name).strip() def sect_list(self, name): return self.cf.get_list(self.sect, name) def sect_hasvar(self, name): return self.cf.has_option(self.sect, name) def main(): revoke = 0 tx = False try: opts, args = getopt.getopt(sys.argv[1:], "vhrRdDot") except getopt.error as det: print("getopt error:", det) usage(1) for o, v in opts: if o == "-h": usage(0) elif o == "-r": revoke |= R_NEW elif o == "-R": revoke |= R_NEW | R_ONLY elif o == "-d": revoke |= R_DEFS elif o == "-D": revoke |= R_DEFS | R_ONLY elif o == "-o": revoke |= G_DEFS elif o == "-t": tx = True elif o == "-v": print("GrantFu version", __version__) sys.exit(0) if len(args) != 1: usage(1) # load config cf = PConf() cf.read(args[0]) if not cf.has_section("GrantFu"): print("Incorrect config file, GrantFu sction missing") sys.exit(1) if tx: print("begin;\n") # revokes and default grants if revoke & (R_NEW | R_DEFS): g = GrantFu(cf, revoke | R_ONLY) g.process() revoke = revoke & R_ONLY # grants if revoke & R_ONLY == 0: g = GrantFu(cf, revoke & G_DEFS) g.process() if tx: print("\ncommit;\n") if __name__ == '__main__': main() pgq-3.4.1/pgq.control000066400000000000000000000002171373233714300145050ustar00rootroot00000000000000# pgq extension comment = 'Generic queue for PostgreSQL' default_version = '3.4.1' relocatable = false superuser = true schema = 'pg_catalog' pgq-3.4.1/sql/000077500000000000000000000000001373233714300131135ustar00rootroot00000000000000pgq-3.4.1/sql/clean.sql000066400000000000000000000007021373233714300147150ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; drop schema pgq cascade; drop sequence tmptest_seq; drop table custom_expr; drop table custom_expr2; drop table custom_fields; drop table custom_fields2; drop table custom_pkey; drop table deny_test; drop table nopkey; drop table nopkey2; drop table rtest; drop table if exists trunctrg1; drop table if exists trunctrg2; drop table ucustom_pkey; drop table udata; drop table when_test; pgq-3.4.1/sql/clean_ext.sql000066400000000000000000000000261373233714300155740ustar00rootroot00000000000000 drop extension pgq; pgq-3.4.1/sql/pgq_core.sql000066400000000000000000000073071373233714300154420ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; select * from pgq.maint_tables_to_vacuum(); select * from pgq.maint_retry_events(); select pgq.create_queue('tmpqueue'); select pgq.register_consumer('tmpqueue', 'consumer'); select pgq.unregister_consumer('tmpqueue', 'consumer'); select pgq.drop_queue('tmpqueue'); select pgq.create_queue('myqueue'); select pgq.register_consumer('myqueue', 'consumer'); update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0'; select pgq.next_batch('myqueue', 'consumer'); select pgq.next_batch('myqueue', 'consumer'); select pgq.ticker(); select pgq.next_batch('myqueue', 'consumer'); select pgq.next_batch('myqueue', 'consumer'); select queue_name, consumer_name, prev_tick_id, tick_id, lag < '30 seconds' as lag_exists from pgq.get_batch_info(1); select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, queue_switch_time <= now() as switch_time_exists, queue_external_ticker, queue_ticker_max_count, queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag < '2 hours' as ticker_lag_exists, last_tick_id from pgq.get_queue_info() order by 1; select queue_name, consumer_name, lag < '30 seconds' as lag_exists, last_seen < '30 seconds' as last_seen_exists, last_tick, current_batch, next_tick from pgq.get_consumer_info() order by 1, 2; select pgq.finish_batch(1); select pgq.finish_batch(1); select pgq.ticker(); select pgq.next_batch('myqueue', 'consumer'); select * from pgq.batch_event_tables(2); select * from pgq.get_batch_events(2); select pgq.finish_batch(2); select pgq.insert_event('myqueue', 'r1', 'data'); select pgq.insert_event('myqueue', 'r2', 'data', 'extra1', 'extra2', 'extra3', 'extra4'); select pgq.insert_event('myqueue', 'r3', 'data'); select pgq.current_event_table('myqueue'); select pgq.ticker(); select * from pgq.next_batch_custom('myqueue', 'consumer', '1 hour', null, null); select * from pgq.next_batch_custom('myqueue', 'consumer', null, 10000, null); select * from pgq.next_batch_custom('myqueue', 'consumer', null, null, '10 minutes'); select pgq.next_batch('myqueue', 'consumer'); select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3); begin; select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_cursor(3, 'acurs', 10); close acurs; select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_cursor(3, 'acurs', 2); close acurs; select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_cursor(3, 'acurs', 2, 'ev_id = 1'); close acurs; end; select pgq.event_retry(3, 2, 0); select pgq.batch_retry(3, 0); select pgq.finish_batch(3); select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0, 'rawtest', 'data', null, null, null, null); select pgq.ticker(); -- test maint update pgq.queue set queue_rotation_period = '0 seconds'; select queue_name, pgq.maint_rotate_tables_step1(queue_name) from pgq.queue; select pgq.maint_rotate_tables_step2(); -- test extra select nextval(queue_event_seq) from pgq.queue where queue_name = 'myqueue'; select pgq.force_tick('myqueue'); select nextval(queue_event_seq) from pgq.queue where queue_name = 'myqueue'; create sequence tmptest_seq; select pgq.seq_getval('tmptest_seq'); select pgq.seq_setval('tmptest_seq', 10); select pgq.seq_setval('tmptest_seq', 5); select pgq.seq_setval('tmptest_seq', 15); select pgq.seq_getval('tmptest_seq'); drop sequence tmptest_seq; select * from pgq.maint_operations(); update pgq.queue set queue_extra_maint = array['baz', 'foo.bar']; select * from pgq.maint_operations(); select pgq.drop_queue('myqueue', true); pgq-3.4.1/sql/pgq_core_disabled.sql000066400000000000000000000010061373233714300172570ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; select pgq.create_queue('queue_disabled'); -- test disabled select pgq.insert_event('queue_disabled', 'test', 'event'); update pgq.queue set queue_disable_insert = true where queue_name = 'queue_disabled'; select pgq.insert_event('queue_disabled', 'test', 'event'); update pgq.queue set queue_disable_insert = false where queue_name = 'queue_disabled'; select pgq.insert_event('queue_disabled', 'test', 'event'); select pgq.drop_queue('queue_disabled'); pgq-3.4.1/sql/pgq_core_tx_limit.sql000066400000000000000000000015501373233714300173450ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; select pgq.create_queue('queue_tx_limit'); -- test limit update pgq.queue set queue_per_tx_limit = 2 where queue_name = 'queue_tx_limit'; begin; select pgq.insert_event('queue_tx_limit', 'test', 'event1'); select pgq.insert_event('queue_tx_limit', 'test', 'event2'); select pgq.insert_event('queue_tx_limit', 'test', 'event3'); end; update pgq.queue set queue_per_tx_limit = 0 where queue_name = 'queue_tx_limit'; begin; select pgq.insert_event('queue_tx_limit', 'test', 'event1'); end; update pgq.queue set queue_per_tx_limit = null where queue_name = 'queue_tx_limit'; begin; select pgq.insert_event('queue_tx_limit', 'test', 'event1'); select pgq.insert_event('queue_tx_limit', 'test', 'event2'); select pgq.insert_event('queue_tx_limit', 'test', 'event3'); end; select pgq.drop_queue('queue_tx_limit'); pgq-3.4.1/sql/pgq_init_ext.sql000066400000000000000000000011601373233714300163240ustar00rootroot00000000000000 \set ECHO none \set VERBOSITY 'terse' set client_min_messages = 'warning'; -- just create to extension (used to be "from unpackaged" test) create extension pgq; select pgq.create_queue('testqueue1'); \set ECHO all select array_length(extconfig, 1) from pg_catalog.pg_extension where extname = 'pgq'; select pgq.create_queue('testqueue2'); --drop extension pgq; -- will fail select pgq.drop_queue('testqueue2'); select pgq.drop_queue('testqueue1'); -- drop extension drop extension pgq; -- create clean schema create extension pgq; select array_length(extconfig, 1) from pg_catalog.pg_extension where extname = 'pgq'; pgq-3.4.1/sql/pgq_init_noext.sql000066400000000000000000000002321373233714300166600ustar00rootroot00000000000000 \set ECHO none \set VERBOSITY 'terse' set client_min_messages = 'warning'; -- \i ../txid/txid.sql -- \i pgq.sql \i structure/install.sql \set ECHO all pgq-3.4.1/sql/pgq_init_upgrade.sql000066400000000000000000000003121373233714300171510ustar00rootroot00000000000000\set ECHO none \set VERBOSITY 'terse' set client_min_messages = 'warning'; \i ../../upgrade/final/pgq_core_2.1.13.sql \i ../../upgrade/final/pgq.upgrade_2.1_to_3.0.sql \i pgq.upgrade.sql \set ECHO all pgq-3.4.1/sql/pgq_perms.sql000066400000000000000000000021721373233714300156330ustar00rootroot00000000000000\set ECHO none \set VERBOSITY 'terse' set client_min_messages = 'warning'; -- drop public perms \i structure/newgrants_pgq.sql -- select proname, proacl from pg_proc p, pg_namespace n where n.nspname = 'pgq' and p.pronamespace = n.oid; \set ECHO all drop role if exists pgq_test_producer; drop role if exists pgq_test_consumer; drop role if exists pgq_test_admin; create role pgq_test_consumer with login in role pgq_reader; create role pgq_test_producer with login in role pgq_writer; create role pgq_test_admin with login in role pgq_admin; \c - pgq_test_admin select * from pgq.create_queue('pqueue'); -- ok \c - pgq_test_producer select * from pgq.create_queue('pqueue'); -- fail select * from pgq.insert_event('pqueue', 'test', 'data'); -- ok select * from pgq.register_consumer('pqueue', 'prod'); -- fail \c - pgq_test_consumer select * from pgq.create_queue('pqueue'); -- fail select * from pgq.insert_event('pqueue', 'test', 'data'); -- fail select * from pgq.register_consumer('pqueue', 'cons'); -- ok select * from pgq.next_batch('pqueue', 'cons'); -- ok \c - pgq_test_admin select * from pgq.drop_queue('pqueue', true); pgq-3.4.1/sql/pgq_session_role.sql000066400000000000000000000022211373233714300172040ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; show session_replication_role; select pgq.create_queue('role_test_enabled'); select pgq.create_queue('role_test_disabled'); update pgq.queue set queue_disable_insert=true where queue_name = 'role_test_disabled'; select pgq.insert_event('role_test_enabled', 'enabled', 'role:origin'); select pgq.insert_event('role_test_disabled', 'disabled', 'role:origin'); set session_replication_role = 'replica'; show session_replication_role; select pgq.insert_event('role_test_enabled', 'enabled', 'role:replica'); select pgq.insert_event('role_test_disabled', 'disabled', 'role:replica'); set session_replication_role = 'local'; show session_replication_role; select pgq.insert_event('role_test_enabled', 'enabled', 'role:local'); select pgq.insert_event('role_test_disabled', 'disabled', 'role:local'); set session_replication_role = 'origin'; show session_replication_role; select pgq.insert_event('role_test_enabled', 'enabled', 'role:origin'); select pgq.insert_event('role_test_disabled', 'disabled', 'role:origin'); select pgq.drop_queue('role_test_enabled'); select pgq.drop_queue('role_test_disabled'); pgq-3.4.1/sql/switch_plonly.sql000066400000000000000000000001141373233714300165260ustar00rootroot00000000000000\set ECHO none \i lowlevel_pl/insert_event.sql \i structure/triggers_pl.sql pgq-3.4.1/sql/trigger_base.sql000066400000000000000000000064201373233714300162730ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; set bytea_output = 'hex'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_base ( id serial primary key, txt text, val float ); create trigger base_trig_0 after insert or update or delete on trigger_base for each row execute procedure pgq.jsontriga('jsontriga'); create trigger base_trig_1 after insert or update or delete on trigger_base for each row execute procedure pgq.logutriga('logutriga'); create trigger base_trig_2 after insert or update or delete on trigger_base for each row execute procedure pgq.sqltriga('sqltriga'); insert into trigger_base (txt) values ('text1'); insert into trigger_base (val) values (1.5); update trigger_base set txt='text2' where id=1; delete from trigger_base where id=2; -- test missing pkey create table trigger_nopkey_jsontriga (dat text); create table trigger_nopkey_logutriga (dat text); create table trigger_nopkey_sqltriga (dat text); create trigger nopkey after insert or update or delete on trigger_nopkey_jsontriga for each row execute procedure pgq.jsontriga('jsontriga'); create trigger nopkey after insert or update or delete on trigger_nopkey_logutriga for each row execute procedure pgq.logutriga('logutriga'); create trigger nopkey after insert or update or delete on trigger_nopkey_sqltriga for each row execute procedure pgq.sqltriga('sqltriga'); insert into trigger_nopkey_jsontriga values ('foo'); insert into trigger_nopkey_logutriga values ('foo'); insert into trigger_nopkey_sqltriga values ('foo'); update trigger_nopkey_jsontriga set dat = 'bat'; update trigger_nopkey_logutriga set dat = 'bat'; update trigger_nopkey_sqltriga set dat = 'bat'; delete from trigger_nopkey_jsontriga; delete from trigger_nopkey_logutriga; delete from trigger_nopkey_sqltriga; -- test invalid pk update create table trigger_pkey_jsontriga (id int4 primary key); create table trigger_pkey_logutriga (id int4 primary key); create table trigger_pkey_sqltriga (id int4 primary key); insert into trigger_pkey_jsontriga values (1); insert into trigger_pkey_logutriga values (1); insert into trigger_pkey_sqltriga values (1); create trigger nopkey after insert or update or delete on trigger_pkey_jsontriga for each row execute procedure pgq.jsontriga('jsontriga'); create trigger nopkey after insert or update or delete on trigger_pkey_logutriga for each row execute procedure pgq.logutriga('logutriga'); create trigger nopkey after insert or update or delete on trigger_pkey_sqltriga for each row execute procedure pgq.sqltriga('sqltriga'); update trigger_pkey_jsontriga set id = 6; update trigger_pkey_logutriga set id = 6; update trigger_pkey_sqltriga set id = 6; -- restore drop table trigger_base; drop table trigger_nopkey_jsontriga; drop table trigger_nopkey_logutriga; drop table trigger_nopkey_sqltriga; drop table trigger_pkey_jsontriga; drop table trigger_pkey_logutriga; drop table trigger_pkey_sqltriga; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_deny.sql000066400000000000000000000032211373233714300163140ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; -- create tables with data create table jsontriga_deny (dat1 text primary key); create table logutriga_deny (dat1 text primary key); create table sqltriga_deny (dat1 text primary key); insert into jsontriga_deny values ('a'); insert into logutriga_deny values ('a'); insert into sqltriga_deny values ('a'); -- create triggers create trigger deny_trig after insert or update or delete on jsontriga_deny for each row execute procedure pgq.jsontriga('jsontriga', 'deny'); create trigger deny_trig after insert or update or delete on logutriga_deny for each row execute procedure pgq.logutriga('logutriga', 'deny'); create trigger deny_trig after insert or update or delete on sqltriga_deny for each row execute procedure pgq.sqltriga('sqltriga', 'deny'); -- see what happens insert into jsontriga_deny values ('b'); insert into logutriga_deny values ('b'); insert into sqltriga_deny values ('b'); update jsontriga_deny set dat1 = 'c'; update logutriga_deny set dat1 = 'c'; update sqltriga_deny set dat1 = 'c'; delete from jsontriga_deny; delete from logutriga_deny; delete from sqltriga_deny; -- restore drop table jsontriga_deny; drop table logutriga_deny; drop table sqltriga_deny; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_extra_args.sql000066400000000000000000000031401373233714300175140ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_extra_args (nr int4 primary key, col1 text, col2 text); create trigger extra_trig_0 after insert or update or delete on trigger_extra_args for each row execute procedure pgq.jsontriga('jsontriga', 'ev_extra1=(nr+3)::text', 'ev_extra2=col1||col2', 'ev_extra3=$$333$$', 'ev_extra4=$$444$$', 'ev_type=$$badidea$$'); create trigger extra_trig_1 after insert or update or delete on trigger_extra_args for each row execute procedure pgq.logutriga('logutriga', 'ev_extra1=(nr+3)::text', 'ev_extra2=col1||col2', 'ev_extra3=$$333$$', 'ev_extra4=$$444$$', 'ev_type=$$badidea$$'); create trigger extra_trig_2 after insert or update or delete on trigger_extra_args for each row execute procedure pgq.sqltriga('sqltriga', 'ev_extra1=(nr+3)::text', 'ev_extra2=col1||col2', 'ev_extra3=$$333$$', 'ev_extra4=$$444$$', 'ev_type=$$badidea$$'); -- test insert insert into trigger_extra_args values (1, 'col1', 'col2'); -- test update update trigger_extra_args set col1 = 'col1x', col2='col2x' where nr=1; -- test delete delete from trigger_extra_args where nr=1; -- restore drop table trigger_extra_args; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_extra_cols.sql000066400000000000000000000030231373233714300175200ustar00rootroot00000000000000-- this functionality is deprecated \set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_extra_columns (nr int4 primary key, col1 text, col2 text, _pgq_ev_type text, _pgq_ev_extra1 text, _pgq_ev_extra2 text default 'e2', _pgq_ev_extra3 text default 'e3', _pgq_ev_extra4 text default 'e4'); create trigger extra_trig_0 after insert or update or delete on trigger_extra_columns for each row execute procedure pgq.jsontriga('jsontriga'); create trigger extra_trig_1 after insert or update or delete on trigger_extra_columns for each row execute procedure pgq.logutriga('logutriga'); create trigger extra_trig_2 after insert or update or delete on trigger_extra_columns for each row execute procedure pgq.sqltriga('sqltriga'); -- test insert insert into trigger_extra_columns (nr, col1, col2, _pgq_ev_type, _pgq_ev_extra1) values (1, 'col1', 'col2', 'xt', 'E1'); -- test update update trigger_extra_columns set col1 = 'col1x', col2='col2x', _pgq_ev_extra1='X1' where nr=1; -- test delete delete from trigger_extra_columns where nr=1; -- restore drop table trigger_extra_columns; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_ignore.sql000066400000000000000000000027151373233714300166470ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_ignore (dat1 text primary key, col1 text, col2 text); create trigger ignore_trig_0 after insert or update or delete on trigger_ignore for each row execute procedure pgq.jsontriga('jsontriga', 'ignore=col2'); create trigger ignore_trig_1 after insert or update or delete on trigger_ignore for each row execute procedure pgq.logutriga('logutriga', 'ignore=col2'); create trigger ignore_trig_2 after insert or update or delete on trigger_ignore for each row execute procedure pgq.sqltriga('sqltriga', 'ignore=col2'); -- test insert insert into trigger_ignore values ('a', 'col1', 'col2'); -- test update of non-ignored column update trigger_ignore set col1 = 'col1x' where dat1 = 'a'; update trigger_ignore set col1 = 'col1y', col2='col2y' where dat1 = 'a'; -- test update of ignored column update trigger_ignore set col2 = 'col2z' where dat1 = 'a'; -- test null update update trigger_ignore set col2 = col2 where dat1 = 'a'; -- restore drop table trigger_ignore; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_pkey.sql000066400000000000000000000022411373233714300163260ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_pkey (nr int4, col1 text, col2 text); create trigger pkey_trig_0 after insert or update or delete on trigger_pkey for each row execute procedure pgq.jsontriga('jsontriga', 'pkey=nr,col1'); create trigger pkey_trig_1 after insert or update or delete on trigger_pkey for each row execute procedure pgq.logutriga('logutriga', 'pkey=nr,col1'); create trigger pkey_trig_2 after insert or update or delete on trigger_pkey for each row execute procedure pgq.sqltriga('sqltriga', 'pkey=nr,col1'); insert into trigger_pkey values (1, 'col1', 'col2'); update trigger_pkey set col2='col2x' where nr=1; delete from trigger_pkey where nr=1; -- restore drop table trigger_pkey; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_sess_role.sql000066400000000000000000000052441373233714300173620ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ declare ev_id int8; begin raise warning 'calling insert_event_raw'; ev_id := pgq.insert_event_raw(queue_name, null, now(), null, null, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%]) = %', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4, ev_id; return ev_id; end; $$ language plpgsql; select pgq.create_queue('jsontriga_role'); select pgq.create_queue('logutriga_role'); select pgq.create_queue('sqltriga_role'); update pgq.queue set queue_disable_insert = true where queue_name = 'jsontriga_role'; update pgq.queue set queue_disable_insert = true where queue_name = 'logutriga_role'; update pgq.queue set queue_disable_insert = true where queue_name = 'sqltriga_role'; -- create tables create table jsontriga_role (dat1 text primary key); create table logutriga_role (dat1 text primary key); create table sqltriga_role (dat1 text primary key); create trigger trig after insert or update or delete on jsontriga_role for each row execute procedure pgq.jsontriga('jsontriga_role'); create trigger trig after insert or update or delete on logutriga_role for each row execute procedure pgq.logutriga('logutriga_role'); create trigger trig after insert or update or delete on sqltriga_role for each row execute procedure pgq.sqltriga('sqltriga_role'); -- origin: expect insert_event error show session_replication_role; insert into jsontriga_role values ('a'); insert into logutriga_role values ('a'); insert into sqltriga_role values ('a'); -- local: silence, trigger does not call insert_event set session_replication_role = 'local'; show session_replication_role; insert into jsontriga_role values ('b'); insert into logutriga_role values ('b'); insert into sqltriga_role values ('b'); -- replica: silence, trigger does not call insert_event set session_replication_role = 'replica'; show session_replication_role; insert into jsontriga_role values ('c'); insert into logutriga_role values ('c'); insert into sqltriga_role values ('c'); select * from jsontriga_role; select * from logutriga_role; select * from sqltriga_role; -- restore set session_replication_role = 'origin'; drop table jsontriga_role; drop table logutriga_role; drop table sqltriga_role; select pgq.drop_queue('jsontriga_role'); select pgq.drop_queue('logutriga_role'); select pgq.drop_queue('sqltriga_role'); \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_trunc.sql000066400000000000000000000037221373233714300165160ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table jsontriga_trunc (dat1 text primary key); create table logutriga_trunc (dat1 text primary key); create table sqltriga_trunc (dat1 text primary key); -- test successful truncate create trigger trunc1_trig after truncate on jsontriga_trunc for each statement execute procedure pgq.jsontriga('jsontriga'); create trigger trunc1_trig after truncate on logutriga_trunc for each statement execute procedure pgq.logutriga('logutriga'); create trigger trunc1_trig after truncate on sqltriga_trunc for each statement execute procedure pgq.sqltriga('sqltriga'); truncate jsontriga_trunc; truncate logutriga_trunc; truncate sqltriga_trunc; -- test deny create table jsontriga_trunc2 (dat1 text primary key); create table logutriga_trunc2 (dat1 text primary key); create table sqltriga_trunc2 (dat1 text primary key); create trigger trunc_trig after truncate on jsontriga_trunc2 for each statement execute procedure pgq.jsontriga('jsontriga_trunc2', 'deny'); create trigger trunc_trig after truncate on logutriga_trunc2 for each statement execute procedure pgq.sqltriga('logutriga_trunc2', 'deny'); create trigger trunc_trig after truncate on sqltriga_trunc2 for each statement execute procedure pgq.sqltriga('sqltriga_trunc2', 'deny'); truncate jsontriga_trunc2; truncate logutriga_trunc2; truncate sqltriga_trunc2; -- restore drop table jsontriga_trunc; drop table logutriga_trunc; drop table sqltriga_trunc; drop table jsontriga_trunc2; drop table logutriga_trunc2; drop table sqltriga_trunc2; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_types.sql000066400000000000000000000060571373233714300165330ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; set DateStyle = 'ISO, YMD'; set timezone = 'UTC'; set bytea_output = 'hex'; set extra_float_digits = 0; \set ECHO none create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%])', queue_name, ev_type, ev_data; return 1; end; $$ language plpgsql; create function typetest(typ text, val text) returns void as $$ declare rec text; begin execute 'create table ttest (nr int4, val '||typ||' default '||quote_nullable(val) ||', arr '||typ||'[] default array['||quote_nullable(val)||','||quote_nullable(val) ||']::'||typ||'[])'; execute 'create trigger types_trig_0 after insert or update or delete on ttest' || ' for each row execute procedure pgq.jsontriga(''jsontriga'');'; execute 'create trigger types_trig_1 after insert or update or delete on ttest' || ' for each row execute procedure pgq.logutriga(''logutriga'')'; execute 'create trigger types_trig_2 after insert or update or delete on ttest' || ' for each row execute procedure pgq.sqltriga(''sqltriga'');'; execute 'insert into ttest (nr) values (1)'; /* perform 1 from pg_catalog.pg_proc p, pg_catalog.pg_namespace n where n.nspname = 'pg_catalog' and p.proname = 'to_json' and p.pronamespace = n.oid; if found then for rec in execute 'select row_to_json(t.*) from ttest t where nr=1' loop raise warning 'type: % row_to_json: %', typ, rec; end loop; end if; */ execute 'drop table ttest'; exception when invalid_text_representation then raise exception 'invalid input syntax for type %: "%"', typ, val; end; $$ language plpgsql; \set ECHO all select typetest('text', null); select typetest('text', $$'"quoted\string$%,@"'$$); select typetest('bytea', $$\x00FF01$$); select typetest('bool', 'true'); select typetest('bool', 'false'); select typetest('timestamptz', '2009-09-19 11:59:48.599'); select typetest('timestamp', '2009-09-19 11:59:48.599'); select typetest('date', '2009-09-19'); select typetest('time', '11:59:48.599'); select typetest('interval', '2 minutes'); select typetest('int2', '10010'); select typetest('int4', '100100100'); select typetest('int8', '100200300400500600'); select typetest('int8', '9223372036854775807'); select typetest('int8', '-9223372036854775808'); select typetest('oid', '100200300'); select typetest('xid', '100200300'); select typetest('tid', '100200300'); select typetest('real', '100100.666'); select typetest('float', '100100.6005005665'); select typetest('numeric(40,15)', '100100.600500566501811'); select typetest('box', '((1.1, 2.1), (5.6, 5.7))'); select typetest('uuid', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); select typetest('json', '{"a": [false, null, true]}'); select typetest('json', '[1,2,3]'); -- restore drop function typetest(text,text); \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/sql/trigger_when.sql000066400000000000000000000022631373233714300163230ustar00rootroot00000000000000\set VERBOSITY 'terse' set client_min_messages = 'warning'; create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) returns bigint as $$ begin raise warning 'insert_event(q=[%], t=[%], d=[%], 1=[%], 2=[%], 3=[%], 4=[%])', queue_name, ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4; return 1; end; $$ language plpgsql; create table trigger_when (nr int4 primary key, col1 text, col2 text); create trigger when_trig_0 after insert or update or delete on trigger_when for each row execute procedure pgq.jsontriga('jsontriga', 'when=col2=''foo'''); create trigger when_trig_1 after insert or update or delete on trigger_when for each row execute procedure pgq.logutriga('logutriga', 'when=col2=''foo'''); create trigger when_trig_2 after insert or update or delete on trigger_when for each row execute procedure pgq.sqltriga('sqltriga', 'when=col2=''foo'''); -- test insert with when insert into trigger_when values (1, 'col1', 'col2'); insert into trigger_when values (2, 'col1', 'foo'); -- restore drop table trigger_when; \set ECHO none \i functions/pgq.insert_event.sql pgq-3.4.1/structure/000077500000000000000000000000001373233714300143545ustar00rootroot00000000000000pgq-3.4.1/structure/ext_postproc.sql000066400000000000000000000010131373233714300176210ustar00rootroot00000000000000 -- tag data objects as dumpable SELECT pg_catalog.pg_extension_config_dump('pgq.queue', ''); SELECT pg_catalog.pg_extension_config_dump('pgq.consumer', ''); SELECT pg_catalog.pg_extension_config_dump('pgq.tick', ''); SELECT pg_catalog.pg_extension_config_dump('pgq.subscription', ''); SELECT pg_catalog.pg_extension_config_dump('pgq.event_template', ''); SELECT pg_catalog.pg_extension_config_dump('pgq.retry_queue', ''); -- This needs pg_dump 9.1.7+ SELECT pg_catalog.pg_extension_config_dump('pgq.batch_id_seq', ''); pgq-3.4.1/structure/ext_unpackaged.sql000066400000000000000000000005511373233714300200600ustar00rootroot00000000000000 ALTER EXTENSION pgq ADD SCHEMA pgq; ALTER EXTENSION pgq ADD TABLE pgq.queue; ALTER EXTENSION pgq ADD TABLE pgq.consumer; ALTER EXTENSION pgq ADD TABLE pgq.tick; ALTER EXTENSION pgq ADD TABLE pgq.subscription; ALTER EXTENSION pgq ADD TABLE pgq.event_template; ALTER EXTENSION pgq ADD TABLE pgq.retry_queue; ALTER EXTENSION pgq ADD SEQUENCE pgq.batch_id_seq; pgq-3.4.1/structure/func_internal.sql000066400000000000000000000013761373233714300177330ustar00rootroot00000000000000-- Section: Internal Functions -- install & launch schema upgrade \i functions/pgq.upgrade_schema.sql select pgq.upgrade_schema(); -- Group: Low-level event handling \i functions/pgq.batch_event_sql.sql \i functions/pgq.batch_event_tables.sql \i functions/pgq.event_retry_raw.sql \i functions/pgq.find_tick_helper.sql -- Group: Ticker \i functions/pgq.ticker.sql -- Group: Periodic maintenence \i functions/pgq.maint_retry_events.sql \i functions/pgq.maint_rotate_tables.sql \i functions/pgq.maint_tables_to_vacuum.sql \i functions/pgq.maint_operations.sql -- Group: Random utility functions \i functions/pgq.grant_perms.sql \i functions/pgq.tune_storage.sql \i functions/pgq.force_tick.sql \i functions/pgq.seq_funcs.sql \i functions/pgq.quote_fqname.sql pgq-3.4.1/structure/func_public.sql000066400000000000000000000042631373233714300173730ustar00rootroot00000000000000-- ---------------------------------------------------------------------- -- Section: Public Functions -- -- The queue is used by a client in the following steps -- -- 1. Register the client (a queue consumer) -- -- pgq.register_consumer(queue_name, consumer_id) -- -- 2. run a loop createing, consuming and closing batches -- -- 2a. pgq.get_batch_events(batch_id int8) - returns an int8 batch handle -- -- 2b. pgq.get_batch_events(batch_id int8) - returns a set of events for current batch -- -- the event structure is :(ev_id int8, ev_time timestamptz, ev_txid int8, ev_retry -- int4, ev_type text, ev_data text, ev_extra1, ev_extra2, ev_extra3, ev_extra4) -- -- 2c. if any of the events need to be tagged as failed, use a the function -- -- pgq.event_failed(batch_id int8, event_id int8, reason text) -- -- 2d. if you want the event to be re-inserted in the main queue afrer N seconds, use -- -- pgq.event_retry(batch_id int8, event_id int8, retry_seconds int4) -- -- 2e. To finish processing and release the batch, use -- -- pgq.finish_batch(batch_id int8) -- -- Until this is not done, the consumer will get same batch again. -- -- After calling finish_batch consumer cannot do any operations with events -- of that batch. All operations must be done before. -- -- -- ---------------------------------------------------------------------- -- Group: Queue creation \i functions/pgq.create_queue.sql \i functions/pgq.drop_queue.sql \i functions/pgq.set_queue_config.sql -- Group: Event publishing \i functions/pgq.insert_event.sql \i functions/pgq.current_event_table.sql -- Group: Subscribing to queue \i functions/pgq.register_consumer.sql \i functions/pgq.unregister_consumer.sql -- Group: Batch processing \i functions/pgq.next_batch.sql \i functions/pgq.get_batch_events.sql \i functions/pgq.get_batch_cursor.sql \i functions/pgq.event_retry.sql \i functions/pgq.batch_retry.sql \i functions/pgq.finish_batch.sql -- Group: General info functions \i functions/pgq.get_queue_info.sql \i functions/pgq.get_consumer_info.sql \i functions/pgq.version.sql \i functions/pgq.get_batch_info.sql pgq-3.4.1/structure/grants.ini000066400000000000000000000051431373233714300163560ustar00rootroot00000000000000 [GrantFu] roles = pgq_reader, pgq_writer, pgq_admin, public [1.public] on.functions = %(pgq_generic_fns)s public = execute [2.consumer] on.functions = %(pgq_read_fns)s pgq_reader = execute [3.producer] on.functions = %(pgq_write_fns)s pgq_writer = execute [4.admin] on.functions = %(pgq_system_fns)s pgq_admin = execute [5.meta.tables] on.tables = pgq.consumer, pgq.queue, pgq.tick, pgq.subscription pgq_admin = select, insert, update, delete pgq_reader = select public = select [5.event.tables] on.tables = pgq.event_template pgq_reader = select pgq_admin = select, insert, truncate # drop public access to events public = [6.retry.event] on.tables = pgq.retry_queue pgq_admin = select, insert, update, delete # # define various groups of functions # [DEFAULT] pgq_generic_fns = pgq.seq_getval(text), pgq.get_queue_info(), pgq.get_queue_info(text), pgq.get_consumer_info(), pgq.get_consumer_info(text), pgq.get_consumer_info(text, text), pgq.quote_fqname(text), pgq.version() pgq_read_fns = pgq.batch_event_sql(bigint), pgq.batch_event_tables(bigint), pgq.find_tick_helper(int4, int8, timestamptz, int8, int8, interval), pgq.register_consumer(text, text), pgq.register_consumer_at(text, text, bigint), pgq.unregister_consumer(text, text), pgq.next_batch_info(text, text), pgq.next_batch(text, text), pgq.next_batch_custom(text, text, interval, int4, interval), pgq.get_batch_events(bigint), pgq.get_batch_info(bigint), pgq.get_batch_cursor(bigint, text, int4, text), pgq.get_batch_cursor(bigint, text, int4), pgq.event_retry(bigint, bigint, timestamptz), pgq.event_retry(bigint, bigint, integer), pgq.batch_retry(bigint, integer), pgq.force_tick(text), pgq.finish_batch(bigint) pgq_write_fns = pgq.insert_event(text, text, text), pgq.insert_event(text, text, text, text, text, text, text), pgq.current_event_table(text), pgq.jsontriga(), pgq.sqltriga(), pgq.logutriga() pgq_system_fns = pgq.ticker(text, bigint, timestamptz, bigint), pgq.ticker(text), pgq.ticker(), pgq.maint_retry_events(), pgq.maint_rotate_tables_step1(text), pgq.maint_rotate_tables_step2(), pgq.maint_tables_to_vacuum(), pgq.maint_operations(), pgq.upgrade_schema(), pgq.grant_perms(text), pgq._grant_perms_from(text,text,text,text), pgq.tune_storage(text), pgq.seq_setval(text, int8), pgq.create_queue(text), pgq.drop_queue(text, bool), pgq.drop_queue(text), pgq.set_queue_config(text, text, text), pgq.insert_event_raw(text, bigint, timestamptz, integer, integer, text, text, text, text, text, text), pgq.event_retry_raw(text, text, timestamptz, bigint, timestamptz, integer, text, text, text, text, text, text) pgq-3.4.1/structure/grants.sql000066400000000000000000000006041373233714300163730ustar00rootroot00000000000000 grant usage on schema pgq to public; -- old default grants grant select on table pgq.consumer to public; grant select on table pgq.queue to public; grant select on table pgq.tick to public; grant select on table pgq.queue to public; grant select on table pgq.subscription to public; grant select on table pgq.event_template to public; grant select on table pgq.retry_queue to public; pgq-3.4.1/structure/install.sql000066400000000000000000000002451373233714300165440ustar00rootroot00000000000000 \i structure/tables.sql \i structure/func_internal.sql \i lowlevel/pgq_lowlevel.sql \i structure/func_public.sql \i structure/triggers.sql \i structure/grants.sql pgq-3.4.1/structure/install_pl.sql000066400000000000000000000002531373233714300172360ustar00rootroot00000000000000 \i structure/tables.sql \i structure/func_internal.sql \i lowlevel_pl/insert_event.sql \i structure/func_public.sql \i structure/triggers_pl.sql \i structure/grants.sql pgq-3.4.1/structure/tables.sql000066400000000000000000000230671373233714300163570ustar00rootroot00000000000000-- ---------------------------------------------------------------------- -- Section: Internal Tables -- -- Overview: -- pgq.queue - Queue configuration -- pgq.consumer - Consumer names -- pgq.subscription - Consumer registrations -- pgq.tick - Per-queue snapshots (ticks) -- pgq.event_* - Data tables -- pgq.retry_queue - Events to be retried later -- -- -- Standard triggers store events in the pgq.event_* data tables -- There is one top event table pgq.event_ for each queue -- inherited from pgq.event_template wuith three tables for actual data -- pgq.event__0 to pgq.event__2. -- -- The active table is rotated at interval, so that if all the consubers -- have passed some poin the oldes one can be emptied using TRUNCATE command -- for efficiency -- -- -- ---------------------------------------------------------------------- set client_min_messages = 'warning'; set default_with_oids = 'off'; -- drop schema if exists pgq cascade; create schema pgq; -- ---------------------------------------------------------------------- -- Table: pgq.consumer -- -- Name to id lookup for consumers -- -- Columns: -- co_id - consumer's id for internal usage -- co_name - consumer's id for external usage -- ---------------------------------------------------------------------- create table pgq.consumer ( co_id serial, co_name text not null, constraint consumer_pkey primary key (co_id), constraint consumer_name_uq UNIQUE (co_name) ); -- ---------------------------------------------------------------------- -- Table: pgq.queue -- -- Information about available queues -- -- Columns: -- queue_id - queue id for internal usage -- queue_name - queue name visible outside -- queue_ntables - how many data tables the queue has -- queue_cur_table - which data table is currently active -- queue_rotation_period - period for data table rotation -- queue_switch_step1 - tx when rotation happened -- queue_switch_step2 - tx after rotation was committed -- queue_switch_time - time when switch happened -- queue_external_ticker - ticks come from some external sources -- queue_ticker_paused - ticker is paused -- queue_disable_insert - disallow pgq.insert_event() -- queue_ticker_max_count - batch should not contain more events -- queue_ticker_max_lag - events should not age more -- queue_ticker_idle_period - how often to tick when no events happen -- queue_per_tx_limit - Max number of events single TX can insert -- queue_data_pfx - prefix for data table names -- queue_event_seq - sequence for event id's -- queue_tick_seq - sequence for tick id's -- queue_extra_maint - array of functon names to call during maintenance -- ---------------------------------------------------------------------- create table pgq.queue ( queue_id serial, queue_name text not null, queue_ntables integer not null default 3, queue_cur_table integer not null default 0, queue_rotation_period interval not null default '2 hours', queue_switch_step1 bigint not null default txid_current(), queue_switch_step2 bigint default txid_current(), queue_switch_time timestamptz not null default now(), queue_external_ticker boolean not null default false, queue_disable_insert boolean not null default false, queue_ticker_paused boolean not null default false, queue_ticker_max_count integer not null default 500, queue_ticker_max_lag interval not null default '3 seconds', queue_ticker_idle_period interval not null default '1 minute', queue_per_tx_limit integer, queue_data_pfx text not null, queue_event_seq text not null, queue_tick_seq text not null, queue_extra_maint text[], constraint queue_pkey primary key (queue_id), constraint queue_name_uq unique (queue_name) ); -- ---------------------------------------------------------------------- -- Table: pgq.tick -- -- Snapshots for event batching -- -- Columns: -- tick_queue - queue id whose tick it is -- tick_id - ticks id (per-queue) -- tick_time - time when tick happened -- tick_snapshot - transaction state -- tick_event_seq - last value for event seq -- ---------------------------------------------------------------------- create table pgq.tick ( tick_queue int4 not null, tick_id bigint not null, tick_time timestamptz not null default now(), tick_snapshot txid_snapshot not null default txid_current_snapshot(), tick_event_seq bigint not null, -- may be NULL on upgraded dbs constraint tick_pkey primary key (tick_queue, tick_id), constraint tick_queue_fkey foreign key (tick_queue) references pgq.queue (queue_id) ); -- ---------------------------------------------------------------------- -- Sequence: pgq.batch_id_seq -- -- Sequence for batch id's. -- ---------------------------------------------------------------------- create sequence pgq.batch_id_seq; -- ---------------------------------------------------------------------- -- Table: pgq.subscription -- -- Consumer registration on a queue. -- -- Columns: -- -- sub_id - subscription id for internal usage -- sub_queue - queue id -- sub_consumer - consumer's id -- sub_last_tick - last tick the consumer processed -- sub_batch - shortcut for queue_id/consumer_id/tick_id -- sub_next_tick - batch end pos -- ---------------------------------------------------------------------- create table pgq.subscription ( sub_id serial not null, sub_queue int4 not null, sub_consumer int4 not null, sub_last_tick bigint, sub_active timestamptz not null default now(), sub_batch bigint, sub_next_tick bigint, constraint subscription_pkey primary key (sub_queue, sub_consumer), constraint subscription_batch_idx unique (sub_batch), constraint sub_queue_fkey foreign key (sub_queue) references pgq.queue (queue_id), constraint sub_consumer_fkey foreign key (sub_consumer) references pgq.consumer (co_id) ); -- ---------------------------------------------------------------------- -- Table: pgq.event_template -- -- Parent table for all event tables -- -- Columns: -- ev_id - event's id, supposed to be unique per queue -- ev_time - when the event was inserted -- ev_txid - transaction id which inserted the event -- ev_owner - subscription id that wanted to retry this -- ev_retry - how many times the event has been retried, NULL for new events -- ev_type - consumer/producer can specify what the data fields contain -- ev_data - data field -- ev_extra1 - extra data field -- ev_extra2 - extra data field -- ev_extra3 - extra data field -- ev_extra4 - extra data field -- ---------------------------------------------------------------------- create table pgq.event_template ( ev_id bigint not null, ev_time timestamptz not null, ev_txid bigint not null default txid_current(), ev_owner int4, ev_retry int4, ev_type text, ev_data text, ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text ); -- ---------------------------------------------------------------------- -- Table: pgq.retry_queue -- -- Events to be retried. When retry time reaches, they will -- be put back into main queue. -- -- Columns: -- ev_retry_after - time when it should be re-inserted to main queue -- ev_queue - queue id, used to speed up event copy into queue -- * - same as pgq.event_template -- ---------------------------------------------------------------------- create table pgq.retry_queue ( ev_retry_after timestamptz not null, ev_queue int4 not null, like pgq.event_template, constraint rq_pkey primary key (ev_owner, ev_id), constraint rq_queue_id_fkey foreign key (ev_queue) references pgq.queue (queue_id) ); alter table pgq.retry_queue alter column ev_owner set not null; alter table pgq.retry_queue alter column ev_txid drop not null; create index rq_retry_idx on pgq.retry_queue (ev_retry_after); pgq-3.4.1/structure/triggers.sql000066400000000000000000000001721373233714300167230ustar00rootroot00000000000000 -- Section: Public Triggers -- Group: Trigger Functions -- \i triggers/pgq.logutriga.sql \i triggers/pgq_triggers.sql pgq-3.4.1/structure/triggers_pl.sql000066400000000000000000000002221373233714300174120ustar00rootroot00000000000000 -- Section: Public Triggers -- Group: Trigger Functions \i lowlevel_pl/jsontriga.sql \i lowlevel_pl/logutriga.sql \i lowlevel_pl/sqltriga.sql pgq-3.4.1/structure/uninstall_pgq.sql000066400000000000000000000000641373233714300177550ustar00rootroot00000000000000 -- brute-force uninstall drop schema pgq cascade; pgq-3.4.1/structure/upgrade.sql000066400000000000000000000001631373233714300165240ustar00rootroot00000000000000\i structure/func_internal.sql \i lowlevel/pgq_lowlevel.sql \i structure/func_public.sql \i structure/triggers.sql pgq-3.4.1/structure/upgrade_pl.sql000066400000000000000000000001711373233714300172160ustar00rootroot00000000000000\i structure/func_internal.sql \i lowlevel_pl/insert_event.sql \i structure/func_public.sql \i structure/triggers_pl.sql pgq-3.4.1/triggers/000077500000000000000000000000001373233714300141425ustar00rootroot00000000000000pgq-3.4.1/triggers/Makefile000066400000000000000000000004651373233714300156070ustar00rootroot00000000000000 MODULE_big = pgq_triggers SRCS = logtriga.c logutriga.c sqltriga.c jsontriga.c \ common.c makesql.c stringutil.c \ parsesql.c qbuilder.c OBJS = $(SRCS:.c=.o) DATA = pgq_triggers.sql PG_CONFIG = pg_config PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) cs: cscope -b -f .cscope.out *.c pgq-3.4.1/triggers/common.c000066400000000000000000000564161373233714300156120ustar00rootroot00000000000000/* * common.c - functions used by all trigger variants. * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if PG_VERSION_NUM >= 90300 #include #endif #include "common.h" #include "stringutil.h" #include "qbuilder.h" /* * Module tag */ #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif /* memcmp is ok on NameData fields */ #define is_magic_field(s) (memcmp(s, "_pgq_ev_", 8) == 0) static void make_query(struct PgqTriggerEvent *ev, int fld, const char *arg); static void override_fields(struct PgqTriggerEvent *ev); /* * primary key info */ static bool tbl_cache_invalid; static MemoryContext tbl_cache_ctx; static HTAB *tbl_cache_map; static const char pkey_sql[] = "SELECT k.attnum, k.attname FROM pg_catalog.pg_index i, pg_catalog.pg_attribute k" " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid" " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped" " ORDER BY k.attnum"; static void *pkey_plan; static void relcache_reset_cb(Datum arg, Oid relid); /* * helper for queue insertion. * * does not support NULL arguments. */ void pgq_simple_insert(const char *queue_name, Datum ev_type, Datum ev_data, Datum ev_extra1, Datum ev_extra2, Datum ev_extra3, Datum ev_extra4) { Datum values[7]; char nulls[7]; static void *plan = NULL; int res; if (!plan) { const char *sql; Oid types[7] = { TEXTOID, TEXTOID, TEXTOID, TEXTOID, TEXTOID, TEXTOID, TEXTOID }; sql = "select pgq.insert_event($1, $2, $3, $4, $5, $6, $7)"; plan = SPI_saveplan(SPI_prepare(sql, 7, types)); if (plan == NULL) elog(ERROR, "logtriga: SPI_prepare() failed"); } values[0] = DirectFunctionCall1(textin, (Datum)queue_name); values[1] = ev_type; values[2] = ev_data; values[3] = ev_extra1; values[4] = ev_extra2; values[5] = ev_extra3; values[6] = ev_extra4; nulls[0] = ' '; nulls[1] = ev_type ? ' ' : 'n'; nulls[2] = ev_data ? ' ' : 'n'; nulls[3] = ev_extra1 ? ' ' : 'n'; nulls[4] = ev_extra2 ? ' ' : 'n'; nulls[5] = ev_extra3 ? ' ' : 'n'; nulls[6] = ev_extra4 ? ' ' : 'n'; res = SPI_execute_plan(plan, values, nulls, false, 0); if (res != SPI_OK_SELECT) elog(ERROR, "call of pgq.insert_event failed"); } static void fill_magic_columns(PgqTriggerEvent *ev) { TriggerData *tg = ev->tgdata; int i; char *col_name, *col_value; StringInfo *dst = NULL; TupleDesc tupdesc = tg->tg_relation->rd_att; HeapTuple row; if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) row = tg->tg_newtuple; else row = tg->tg_trigtuple; for (i = 0; i < tupdesc->natts; i++) { /* Skip dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; col_name = NameStr(TupleDescAttr(tupdesc, i)->attname); if (!is_magic_field(col_name)) continue; if (strcmp(col_name, "_pgq_ev_type") == 0) dst = &ev->field[EV_TYPE]; else if (strcmp(col_name, "_pgq_ev_data") == 0) dst = &ev->field[EV_DATA]; else if (strcmp(col_name, "_pgq_ev_extra1") == 0) dst = &ev->field[EV_EXTRA1]; else if (strcmp(col_name, "_pgq_ev_extra2") == 0) dst = &ev->field[EV_EXTRA2]; else if (strcmp(col_name, "_pgq_ev_extra3") == 0) dst = &ev->field[EV_EXTRA3]; else if (strcmp(col_name, "_pgq_ev_extra4") == 0) dst = &ev->field[EV_EXTRA4]; else elog(ERROR, "Unknown magic column: %s", col_name); col_value = SPI_getvalue(row, tupdesc, i + 1); if (col_value != NULL) { *dst = pgq_init_varbuf(); appendStringInfoString(*dst, col_value); } else { *dst = NULL; } } } void pgq_insert_tg_event(PgqTriggerEvent *ev) { if (ev->tgargs->custom_fields) fill_magic_columns(ev); override_fields(ev); if (ev->skip_event) return; pgq_simple_insert(ev->queue_name, pgq_finish_varbuf(ev->field[EV_TYPE]), pgq_finish_varbuf(ev->field[EV_DATA]), pgq_finish_varbuf(ev->field[EV_EXTRA1]), pgq_finish_varbuf(ev->field[EV_EXTRA2]), pgq_finish_varbuf(ev->field[EV_EXTRA3]), pgq_finish_varbuf(ev->field[EV_EXTRA4])); } static char *find_table_name(Relation rel, StringInfo jsbuf) { Oid nsoid = rel->rd_rel->relnamespace; char namebuf[NAMEDATALEN * 2 + 3]; HeapTuple ns_tup; Form_pg_namespace ns_struct; const char *tname = NameStr(rel->rd_rel->relname); const char *nspname; /* find namespace info */ ns_tup = SearchSysCache(NAMESPACEOID, ObjectIdGetDatum(nsoid), 0, 0, 0); if (!HeapTupleIsValid(ns_tup)) elog(ERROR, "Cannot find namespace %u", nsoid); ns_struct = (Form_pg_namespace) GETSTRUCT(ns_tup); nspname = NameStr(ns_struct->nspname); /* fill name */ snprintf(namebuf, sizeof(namebuf), "%s.%s", nspname, tname); appendStringInfoString(jsbuf, ",\"table\":["); pgq_encode_cstring(jsbuf, nspname, TBUF_QUOTE_JSON); appendStringInfoChar(jsbuf, ','); pgq_encode_cstring(jsbuf, tname, TBUF_QUOTE_JSON); appendStringInfoChar(jsbuf, ']'); ReleaseSysCache(ns_tup); return pstrdup(namebuf); } static void init_pkey_plan(void) { Oid types[1] = { OIDOID }; pkey_plan = SPI_saveplan(SPI_prepare(pkey_sql, 1, types)); if (pkey_plan == NULL) elog(ERROR, "pgq_triggers: SPI_prepare() failed"); } static void init_cache(void) { HASHCTL ctl; int flags; int max_tables = 128; /* * create own context */ tbl_cache_ctx = AllocSetContextCreate(TopMemoryContext, "pgq_triggers table info", #if (PG_VERSION_NUM >= 110000) ALLOCSET_SMALL_SIZES #else ALLOCSET_SMALL_MINSIZE, ALLOCSET_SMALL_INITSIZE, ALLOCSET_SMALL_MAXSIZE #endif ); /* * init pkey cache. */ MemSet(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(struct PgqTableInfo); ctl.hash = oid_hash; flags = HASH_ELEM | HASH_FUNCTION; tbl_cache_map = hash_create("pgq_triggers pkey cache", max_tables, &ctl, flags); } /* * Prepare utility plans and plan cache. */ static void init_module(void) { static int callback_init = 0; /* do full reset if requested */ if (tbl_cache_invalid) { if (tbl_cache_map) hash_destroy(tbl_cache_map); if (tbl_cache_ctx) MemoryContextDelete(tbl_cache_ctx); tbl_cache_map = NULL; tbl_cache_ctx = NULL; tbl_cache_invalid = false; } /* re-initialize cache */ if (tbl_cache_ctx) return; init_cache(); /* * Rest is done only once. */ if (!pkey_plan) init_pkey_plan(); if (!callback_init) { CacheRegisterRelcacheCallback(relcache_reset_cb, (Datum)0); callback_init = 1; } } /* * Fill table information in hash table. */ static void fill_tbl_info(Relation rel, struct PgqTableInfo *info) { StringInfo pkeys; Datum values[1]; const char *name; TupleDesc desc; HeapTuple row; bool isnull; int res, i, attno; StringInfo jsbuf; jsbuf = makeStringInfo(); name = find_table_name(rel, jsbuf); appendStringInfoString(jsbuf, ",\"pkey\":["); /* load pkeys */ values[0] = ObjectIdGetDatum(rel->rd_id); res = SPI_execute_plan(pkey_plan, values, NULL, false, 0); if (res != SPI_OK_SELECT) elog(ERROR, "pkey_plan exec failed: %d", res); /* * Fill info */ desc = SPI_tuptable->tupdesc; pkeys = makeStringInfo(); info->n_pkeys = SPI_processed; info->table_name = MemoryContextStrdup(tbl_cache_ctx, name); info->pkey_attno = MemoryContextAlloc(tbl_cache_ctx, info->n_pkeys * sizeof(int)); for (i = 0; i < SPI_processed; i++) { row = SPI_tuptable->vals[i]; attno = DatumGetInt16(SPI_getbinval(row, desc, 1, &isnull)); name = SPI_getvalue(row, desc, 2); info->pkey_attno[i] = attno; if (i > 0) { appendStringInfoChar(pkeys, ','); appendStringInfoChar(jsbuf, ','); } appendStringInfoString(pkeys, name); pgq_encode_cstring(jsbuf, name, TBUF_QUOTE_JSON); } appendStringInfoChar(jsbuf, ']'); info->pkey_list = MemoryContextStrdup(tbl_cache_ctx, pkeys->data); info->json_info = MemoryContextStrdup(tbl_cache_ctx, jsbuf->data); info->tg_cache = NULL; } static void clean_info(struct PgqTableInfo *info, bool found) { struct PgqTriggerInfo *tg, *tmp = info->tg_cache; int i; if (!found) goto uninitialized; for (tg = info->tg_cache; tg; ) { tmp = tg->next; if (tg->ignore_list) pfree((void *)tg->ignore_list); if (tg->pkey_list) pfree((void *)tg->pkey_list); for (i = 0; i < EV_NFIELDS; i++) { if (tg->query[i]) qb_free(tg->query[i]); } pfree(tg); tg = tmp; } if (info->table_name) pfree(info->table_name); if (info->pkey_attno) pfree(info->pkey_attno); if (info->pkey_list) pfree((void *)info->pkey_list); if (info->json_info) pfree((void *)info->json_info); uninitialized: info->tg_cache = NULL; info->table_name = NULL; info->pkey_attno = NULL; info->pkey_list = NULL; info->n_pkeys = 0; info->invalid = true; info->json_info = NULL; } /* * the callback can be launched any time from signal callback, * only minimal tagging can be done here. */ static void relcache_reset_cb(Datum arg, Oid relid) { if (relid == InvalidOid) { tbl_cache_invalid = true; } else if (tbl_cache_map && !tbl_cache_invalid) { struct PgqTableInfo *entry; entry = hash_search(tbl_cache_map, &relid, HASH_FIND, NULL); if (entry) entry->invalid = true; } } /* * fetch table struct from cache. */ static struct PgqTableInfo *find_table_info(Relation rel) { struct PgqTableInfo *entry; bool found = false; init_module(); entry = hash_search(tbl_cache_map, &rel->rd_id, HASH_ENTER, &found); if (!found || entry->invalid) { clean_info(entry, found); /* * During fill_tbl_info() 2 events can happen: * - table info reset * - exception * To survive both, always clean struct and tag * as invalid but differently from reset. */ entry->invalid = 2; /* find info */ fill_tbl_info(rel, entry); /* * If no reset happened, it's valid. Actual reset * is postponed to next call. */ if (entry->invalid == 2) entry->invalid = false; } return entry; } static struct PgqTriggerInfo *find_trigger_info(struct PgqTableInfo *info, Oid tgoid, bool create) { struct PgqTriggerInfo *tgargs = info->tg_cache; for (tgargs = info->tg_cache; tgargs; tgargs = tgargs->next) { if (tgargs->tgoid == tgoid) return tgargs; } if (!create) return NULL; tgargs = MemoryContextAllocZero(tbl_cache_ctx, sizeof(*tgargs)); tgargs->tgoid = tgoid; tgargs->next = info->tg_cache; info->tg_cache = tgargs; return tgargs; } static void parse_newstyle_args(PgqTriggerEvent *ev, TriggerData *tg) { int i; /* * parse args */ for (i = 1; i < tg->tg_trigger->tgnargs; i++) { const char *arg = tg->tg_trigger->tgargs[i]; if (strcmp(arg, "SKIP") == 0) ev->tgargs->skip = true; else if (strncmp(arg, "ignore=", 7) == 0) ev->tgargs->ignore_list = MemoryContextStrdup(tbl_cache_ctx, arg + 7); else if (strncmp(arg, "pkey=", 5) == 0) ev->tgargs->pkey_list = MemoryContextStrdup(tbl_cache_ctx, arg + 5); else if (strcmp(arg, "backup") == 0) ev->tgargs->backup = true; else if (strcmp(arg, "deny") == 0) ev->tgargs->deny = true; else if (strncmp(arg, "ev_extra4=", 10) == 0) make_query(ev, EV_EXTRA4, arg + 10); else if (strncmp(arg, "ev_extra3=", 10) == 0) make_query(ev, EV_EXTRA3, arg + 10); else if (strncmp(arg, "ev_extra2=", 10) == 0) make_query(ev, EV_EXTRA2, arg + 10); else if (strncmp(arg, "ev_extra1=", 10) == 0) make_query(ev, EV_EXTRA1, arg + 10); else if (strncmp(arg, "ev_data=", 8) == 0) make_query(ev, EV_DATA, arg + 8); else if (strncmp(arg, "ev_type=", 8) == 0) make_query(ev, EV_TYPE, arg + 8); else if (strncmp(arg, "when=", 5) == 0) make_query(ev, EV_WHEN, arg + 5); else elog(ERROR, "bad param to pgq trigger"); } if (ev->op_type == 'R') { if (ev->tgargs->ignore_list) elog(ERROR, "Column ignore does not make sense for truncate trigger"); if (ev->tgargs->pkey_list) elog(ERROR, "Custom pkey_list does not make sense for truncate trigger"); if (ev->tgargs->backup) elog(ERROR, "Backup does not make sense for truncate trigger"); } } static void parse_oldstyle_args(PgqTriggerEvent *ev, TriggerData *tg) { const char *kpos; int attcnt, i; TupleDesc tupdesc = tg->tg_relation->rd_att; if (tg->tg_trigger->tgnargs < 2 || tg->tg_trigger->tgnargs > 3) elog(ERROR, "pgq.logtriga must be used with 2 or 3 args"); ev->attkind = tg->tg_trigger->tgargs[1]; ev->attkind_len = strlen(ev->attkind); if (tg->tg_trigger->tgnargs > 2) ev->table_name = tg->tg_trigger->tgargs[2]; /* * Count number of active columns */ tupdesc = tg->tg_relation->rd_att; for (i = 0, attcnt = 0; i < tupdesc->natts; i++) { if (!TupleDescAttr(tupdesc, i)->attisdropped) attcnt++; } /* * look if last pkey column exists */ kpos = strrchr(ev->attkind, 'k'); if (kpos == NULL) elog(ERROR, "need at least one key column"); if (kpos - ev->attkind >= attcnt) elog(ERROR, "key column does not exist"); } /* * parse trigger arguments. */ void pgq_prepare_event(struct PgqTriggerEvent *ev, TriggerData *tg, bool newstyle) { memset(ev, 0, sizeof(*ev)); /* * Check trigger calling conventions */ if (TRIGGER_FIRED_BY_TRUNCATE(tg->tg_event)) { if (!TRIGGER_FIRED_FOR_STATEMENT(tg->tg_event)) elog(ERROR, "pgq tRuncate trigger must be fired FOR EACH STATEMENT"); } else if (!TRIGGER_FIRED_FOR_ROW(tg->tg_event)) { elog(ERROR, "pgq Ins/Upd/Del trigger must be fired FOR EACH ROW"); } if (tg->tg_trigger->tgnargs < 1) elog(ERROR, "pgq trigger must have destination queue as argument"); /* * check operation type */ if (TRIGGER_FIRED_BY_INSERT(tg->tg_event)) { ev->op_type = 'I'; ev->op_type_str = "INSERT"; } else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) { ev->op_type = 'U'; ev->op_type_str = "UPDATE"; } else if (TRIGGER_FIRED_BY_DELETE(tg->tg_event)) { ev->op_type = 'D'; ev->op_type_str = "DELETE"; } else if (TRIGGER_FIRED_BY_TRUNCATE(tg->tg_event)) { ev->op_type = 'R'; ev->op_type_str = "TRUNCATE"; } else { elog(ERROR, "unknown event for pgq trigger"); } /* * load table info */ ev->tgdata = tg; ev->info = find_table_info(tg->tg_relation); ev->table_name = ev->info->table_name; ev->pkey_list = ev->info->pkey_list; ev->queue_name = tg->tg_trigger->tgargs[0]; /* * parse args, newstyle args are cached */ ev->tgargs = find_trigger_info(ev->info, tg->tg_trigger->tgoid, true); if (newstyle) { if (!ev->tgargs->finalized) parse_newstyle_args(ev, tg); if (ev->tgargs->pkey_list) ev->pkey_list = ev->tgargs->pkey_list; /* Check if we have pkey */ if (ev->op_type == 'U' || ev->op_type == 'D') { if (ev->pkey_list[0] == 0) elog(ERROR, "Update/Delete on table without pkey"); } } else { parse_oldstyle_args(ev, tg); } ev->tgargs->finalized = true; /* * Check if BEFORE/AFTER makes sense. */ if (ev->tgargs->skip) { if (TRIGGER_FIRED_AFTER(tg->tg_event)) elog(ERROR, "SKIP does not work in AFTER trigger."); } else { if (!TRIGGER_FIRED_AFTER(tg->tg_event)) /* dont care ??? */ ; } if (ev->tgargs->deny) { elog(ERROR, "Table '%s' to queue '%s': change not allowed (%s)", ev->table_name, ev->queue_name, ev->op_type_str); } /* * init data */ ev->field[EV_TYPE] = pgq_init_varbuf(); ev->field[EV_DATA] = pgq_init_varbuf(); ev->field[EV_EXTRA1] = pgq_init_varbuf(); /* * Do the backup, if requested. */ if (ev->tgargs->backup) { ev->field[EV_EXTRA2] = pgq_init_varbuf(); pgq_urlenc_row(ev, tg->tg_trigtuple, ev->field[EV_EXTRA2]); } } /* * Check if column should be skipped */ bool pgqtriga_skip_col(PgqTriggerEvent *ev, int i, int attkind_idx) { TriggerData *tg = ev->tgdata; TupleDesc tupdesc; const char *name; tupdesc = tg->tg_relation->rd_att; if (TupleDescAttr(tupdesc, i)->attisdropped) return true; name = NameStr(TupleDescAttr(tupdesc, i)->attname); if (is_magic_field(name)) { ev->tgargs->custom_fields = 1; return true; } if (ev->attkind) { if (attkind_idx >= ev->attkind_len) return true; return ev->attkind[attkind_idx] == 'i'; } else if (ev->tgargs->ignore_list) { return pgq_strlist_contains(ev->tgargs->ignore_list, name); } return false; } /* * Check if column is pkey. */ bool pgqtriga_is_pkey(PgqTriggerEvent *ev, int i, int attkind_idx) { TriggerData *tg = ev->tgdata; TupleDesc tupdesc; const char *name; if (ev->attkind) { if (attkind_idx >= ev->attkind_len) return false; return ev->attkind[attkind_idx] == 'k'; } else if (ev->pkey_list) { tupdesc = tg->tg_relation->rd_att; if (TupleDescAttr(tupdesc, i)->attisdropped) return false; name = NameStr(TupleDescAttr(tupdesc, i)->attname); if (is_magic_field(name)) { ev->tgargs->custom_fields = 1; return false; } return pgq_strlist_contains(ev->pkey_list, name); } return false; } /* * Check if trigger action should be skipped. */ bool pgq_is_logging_disabled(void) { #if defined(PG_VERSION_NUM) && PG_VERSION_NUM >= 80300 /* * Force-disable the trigger in local replication role. In other * roles rely on the enabled/disabled status of the trigger. */ if (SessionReplicationRole == SESSION_REPLICATION_ROLE_LOCAL) return true; #endif return false; } /* * Callbacks for queryfilter */ static int tg_name_lookup(void *arg, const char *name, int len) { TriggerData *tg = arg; TupleDesc desc = tg->tg_relation->rd_att; char namebuf[NAMEDATALEN + 1]; int nr; if (len >= sizeof(namebuf)) return -1; memcpy(namebuf, name, len); namebuf[len] = 0; nr = SPI_fnumber(desc, namebuf); if (nr > 0) return nr; return -1; } static Oid tg_type_lookup(void *arg, int spi_nr) { TriggerData *tg = arg; TupleDesc desc = tg->tg_relation->rd_att; return SPI_gettypeid(desc, spi_nr); } static Datum tg_value_lookup(void *arg, int spi_nr, bool *isnull) { TriggerData *tg = arg; TupleDesc desc = tg->tg_relation->rd_att; HeapTuple row; if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) row = tg->tg_newtuple; else row = tg->tg_trigtuple; return SPI_getbinval(row, desc, spi_nr, isnull); } static const struct QueryBuilderOps tg_ops = { tg_name_lookup, tg_type_lookup, tg_value_lookup, }; /* * Custom override queries for field values. */ static void make_query(struct PgqTriggerEvent *ev, int fld, const char *arg) { struct TriggerData *tg = ev->tgdata; struct PgqTriggerInfo *tgargs; struct QueryBuilder *q; Oid tgoid = tg->tg_trigger->tgoid; const char *pfx = "select "; if (ev->op_type == 'R') elog(ERROR, "Custom expressions do not make sense for truncater trigger"); /* make sure tgargs exists */ if (!ev->tgargs) ev->tgargs = find_trigger_info(ev->info, tgoid, true); tgargs = ev->tgargs; if (tgargs->query[fld]) { /* seems we already have prepared query */ if (tgargs->query[fld]->plan) return; /* query is broken, last prepare failed? */ qb_free(tgargs->query[fld]); tgargs->query[fld] = NULL; } /* allocate query in right context */ q = qb_create(&tg_ops, tbl_cache_ctx); /* attach immediately */ tgargs->query[fld] = q; /* prepare the query */ qb_add_raw(q, pfx, strlen(pfx)); qb_add_parse(q, arg, tg); qb_prepare(q, tg); } static void override_fields(struct PgqTriggerEvent *ev) { TriggerData *tg = ev->tgdata; int res, i; char *val; /* no overrides */ if (!ev->tgargs) return; for (i = 0; i < EV_NFIELDS; i++) { if (!ev->tgargs->query[i]) continue; res = qb_execute(ev->tgargs->query[i], tg); if (res != SPI_OK_SELECT) elog(ERROR, "Override query failed"); if (SPI_processed != 1) elog(ERROR, "Expect 1 row from override query, got %d", (int)SPI_processed); /* special handling for EV_WHEN */ if (i == EV_WHEN) { bool isnull; Oid oid = SPI_gettypeid(SPI_tuptable->tupdesc, 1); Datum res; if (oid != BOOLOID) elog(ERROR, "when= query result must be boolean, got=%u", oid); res = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); if (isnull) elog(ERROR, "when= should not be NULL"); if (DatumGetBool(res) == 0) ev->skip_event = true; continue; } /* normal field */ val = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); if (ev->field[i]) { pfree(ev->field[i]->data); pfree(ev->field[i]); ev->field[i] = NULL; } if (val) { ev->field[i] = pgq_init_varbuf(); appendStringInfoString(ev->field[i], val); } } } /* * need to ignore UPDATE where only ignored columns change */ int pgq_is_interesting_change(PgqTriggerEvent *ev, TriggerData *tg) { HeapTuple old_row = tg->tg_trigtuple; HeapTuple new_row = tg->tg_newtuple; TupleDesc tupdesc = tg->tg_relation->rd_att; Datum old_value; Datum new_value; bool old_isnull; bool new_isnull; bool is_pk; int i, attkind_idx = -1; int ignore_count = 0; /* only UPDATE may need to be ignored */ if (!TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) return 1; for (i = 0; i < tupdesc->natts; i++) { /* * Ignore dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; is_pk = pgqtriga_is_pkey(ev, i, attkind_idx); if (!is_pk && ev->tgargs->ignore_list == NULL) continue; old_value = SPI_getbinval(old_row, tupdesc, i + 1, &old_isnull); new_value = SPI_getbinval(new_row, tupdesc, i + 1, &new_isnull); /* * If old and new value are NULL, the column is unchanged */ if (old_isnull && new_isnull) continue; /* * If both are NOT NULL, we need to compare the values and skip * setting the column if equal */ if (!old_isnull && !new_isnull) { Oid opr_oid; FmgrInfo *opr_finfo_p; /* * Lookup the equal operators function call info using the * typecache if available */ TypeCacheEntry *type_cache; type_cache = lookup_type_cache(SPI_gettypeid(tupdesc, i + 1), TYPECACHE_EQ_OPR | TYPECACHE_EQ_OPR_FINFO); opr_oid = type_cache->eq_opr; if (opr_oid == ARRAY_EQ_OP) opr_oid = InvalidOid; else opr_finfo_p = &(type_cache->eq_opr_finfo); /* * If we have an equal operator, use that to do binary * comparison. Else get the string representation of both * attributes and do string comparison. */ if (OidIsValid(opr_oid)) { if (DatumGetBool(FunctionCall2Coll(opr_finfo_p, TupleDescAttr(tupdesc, i)->attcollation, old_value, new_value))) continue; } else { char *old_strval = SPI_getvalue(old_row, tupdesc, i + 1); char *new_strval = SPI_getvalue(new_row, tupdesc, i + 1); if (strcmp(old_strval, new_strval) == 0) continue; } } if (is_pk) elog(ERROR, "primary key update not allowed"); if (pgqtriga_skip_col(ev, i, attkind_idx)) { /* this change should be ignored */ ignore_count++; continue; } /* a non-ignored column has changed */ return 1; } /* skip if only ignored column had changed */ if (ignore_count) return 0; /* do show NOP updates */ return 1; } pgq-3.4.1/triggers/common.h000066400000000000000000000047041373233714300156100ustar00rootroot00000000000000#if PG_VERSION_NUM < 100000 /* from src/include/access/tupdesc.h, introduced in 2cd708452 */ #define TupleDescAttr(tupdesc, i) ((tupdesc)->attrs[(i)]) #endif enum PgqFields { EV_TYPE = 0, EV_DATA, EV_EXTRA1, EV_EXTRA2, EV_EXTRA3, EV_EXTRA4, EV_WHEN, EV_NFIELDS }; /* * Per-event temporary data. */ struct PgqTriggerEvent { char op_type; const char *op_type_str; /* overridable fields */ // fixme: check proper usage const char *table_name; const char *queue_name; const char *pkey_list; /* no cache for old-style args */ const char *attkind; int attkind_len; /* cached per-table info */ struct PgqTableInfo *info; /* cached per-trigger args */ struct PgqTriggerInfo *tgargs; /* current event data */ TriggerData *tgdata; /* result fields */ StringInfo field[EV_NFIELDS]; /* if 'when=' query fails */ bool skip_event; }; typedef struct PgqTriggerEvent PgqTriggerEvent; /* * Per trigger cached info, stored under table cache, * so that invalidate can drop it. */ struct PgqTriggerInfo { struct PgqTriggerInfo *next; Oid tgoid; bool finalized; bool skip; bool backup; bool custom_fields; bool deny; const char *ignore_list; const char *pkey_list; struct QueryBuilder *query[EV_NFIELDS]; }; /* * Per-table cached info. * * Per-trigger info should be cached under tg_cache. */ struct PgqTableInfo { Oid reloid; /* must be first, used by htab */ int n_pkeys; /* number of pkeys */ const char *pkey_list; /* pk column name list */ int *pkey_attno; /* pk column positions */ char *table_name; /* schema-quelified table name */ int invalid; /* set if the info was invalidated */ const char *json_info; struct PgqTriggerInfo *tg_cache; }; /* common.c */ void pgq_prepare_event(struct PgqTriggerEvent *ev, TriggerData *tg, bool newstyle); void pgq_simple_insert(const char *queue_name, Datum ev_type, Datum ev_data, Datum ev_extra1, Datum ev_extra2, Datum ev_extra3, Datum ev_extra4); bool pgqtriga_skip_col(PgqTriggerEvent *ev, int i, int attkind_idx); bool pgqtriga_is_pkey(PgqTriggerEvent *ev, int i, int attkind_idx); void pgq_insert_tg_event(PgqTriggerEvent *ev); bool pgq_is_logging_disabled(void); /* makesql.c */ int pgqtriga_make_sql(PgqTriggerEvent *ev, StringInfo sql); /* logutriga.c */ void pgq_urlenc_row(PgqTriggerEvent *ev, HeapTuple row, StringInfo buf); int pgq_is_interesting_change(PgqTriggerEvent *ev, TriggerData *tg); #ifndef TRIGGER_FIRED_BY_TRUNCATE #define TRIGGER_FIRED_BY_TRUNCATE(tg) 0 #endif pgq-3.4.1/triggers/jsontriga.c000066400000000000000000000206711373233714300163140ustar00rootroot00000000000000/* * jsontriga.c - Smart trigger that logs JSON-encoded changes. * * Copyright (c) 2016 Marko Kreen * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "common.h" #include "stringutil.h" PG_FUNCTION_INFO_V1(pgq_jsontriga); Datum pgq_jsontriga(PG_FUNCTION_ARGS); /* * Force DateStyle like to_json() does. Unfortunately * there is no sane way to call to_json() or it's underlying * helpers from C. Need to reimplement, with some compat goo. * * But from plus side, timestamps work same way in 9.[012] now too. */ #if PG_VERSION_NUM < 90200 static void CompatEncodeDateTime(struct pg_tm * tm, fsec_t fsec, bool print_tz, int tz, const char *tzn, int style, char *str) { if (print_tz) { EncodeDateTime(tm, fsec, &tz, NULL, style, str); } else { EncodeDateTime(tm, fsec, NULL, NULL, style, str); } } #define EncodeDateTime CompatEncodeDateTime static int compat_timestamp2tm(Timestamp dt, int *tzp, struct pg_tm * tm, fsec_t *fsec, const char **tzn, pg_tz *attimezone) { return timestamp2tm(dt, tzp, tm, fsec, (char **)tzn, attimezone); } #define timestamp2tm compat_timestamp2tm #endif #if PG_VERSION_NUM < 90400 static void EncodeSpecialTimestamp(Timestamp dt, char *str) { if (TIMESTAMP_IS_NOBEGIN(dt)) strcpy(str, EARLY); else if (TIMESTAMP_IS_NOEND(dt)) strcpy(str, LATE); else /* shouldn't happen */ elog(ERROR, "invalid argument for EncodeSpecialTimestamp"); } static void EncodeSpecialDate(DateADT dt, char *str) { if (DATE_IS_NOBEGIN(dt)) strcpy(str, EARLY); else if (DATE_IS_NOEND(dt)) strcpy(str, LATE); else /* shouldn't happen */ elog(ERROR, "invalid argument for EncodeSpecialDate"); } #endif static void timestamp_to_json(Datum val, StringInfo dst) { char buf[MAXDATELEN + 1]; struct pg_tm tm; fsec_t fsec; Timestamp timestamp = DatumGetTimestamp(val); if (TIMESTAMP_NOT_FINITE(timestamp)) EncodeSpecialTimestamp(timestamp, buf); else if (timestamp2tm(timestamp, NULL, &tm, &fsec, NULL, NULL) == 0) EncodeDateTime(&tm, fsec, false, 0, NULL, USE_XSD_DATES, buf); else ereport(ERROR, (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), errmsg("timestamp out of range"))); appendStringInfo(dst, "\"%s\"", buf); } static void timestamptz_to_json(Datum val, StringInfo dst) { char buf[MAXDATELEN + 1]; int tz; struct pg_tm tm; fsec_t fsec; const char *tzn = NULL; TimestampTz timestamp = DatumGetTimestampTz(val); if (TIMESTAMP_NOT_FINITE(timestamp)) EncodeSpecialTimestamp(timestamp, buf); else if (timestamp2tm(timestamp, &tz, &tm, &fsec, &tzn, NULL) == 0) EncodeDateTime(&tm, fsec, true, tz, tzn, USE_XSD_DATES, buf); else ereport(ERROR, (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), errmsg("timestamp out of range"))); appendStringInfo(dst, "\"%s\"", buf); } static void date_to_json(Datum val, StringInfo dst) { struct pg_tm tm; char buf[MAXDATELEN + 1]; DateADT date = DatumGetDateADT(val); if (DATE_NOT_FINITE(date)) { EncodeSpecialDate(date, buf); } else { j2date(date + POSTGRES_EPOCH_JDATE, &(tm.tm_year), &(tm.tm_mon), &(tm.tm_mday)); EncodeDateOnly(&tm, USE_XSD_DATES, buf); } appendStringInfo(dst, "\"%s\"", buf); } /* * Convert row to JSON */ static void pgq_jsonenc_row(PgqTriggerEvent *ev, HeapTuple row, StringInfo buf) { Oid col_type; Datum col_datum; bool isnull; TriggerData *tg = ev->tgdata; TupleDesc tupdesc = tg->tg_relation->rd_att; bool first = true; int i; const char *col_ident, *col_value; int attkind_idx = -1; if (ev->op_type == 'R') { appendStringInfoString(buf, "{}"); return; } appendStringInfoChar(buf, '{'); for (i = 0; i < tg->tg_relation->rd_att->natts; i++) { /* Skip dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; if (pgqtriga_skip_col(ev, i, attkind_idx)) continue; if (first) first = false; else appendStringInfoChar(buf, ','); /* quote column name */ col_ident = SPI_fname(tupdesc, i + 1); pgq_encode_cstring(buf, col_ident, TBUF_QUOTE_JSON); appendStringInfoChar(buf, ':'); /* quote column value */ col_type = TupleDescAttr(tupdesc, i)->atttypid; col_datum = SPI_getbinval(row, tupdesc, i + 1, &isnull); col_value = NULL; if (isnull) { appendStringInfoString(buf, "null"); continue; } switch (col_type) { case BOOLOID: if (DatumGetBool(col_datum)) { appendStringInfoString(buf, "true"); } else { appendStringInfoString(buf, "false"); } break; case TIMESTAMPOID: timestamp_to_json(col_datum, buf); break; case TIMESTAMPTZOID: timestamptz_to_json(col_datum, buf); break; case DATEOID: date_to_json(col_datum, buf); break; case INT2OID: appendStringInfo(buf, "%d", (int)DatumGetInt16(col_datum)); break; case INT4OID: appendStringInfo(buf, "%d", (int)DatumGetInt32(col_datum)); break; case INT8OID: col_value = SPI_getvalue(row, tupdesc, i + 1); appendStringInfoString(buf, col_value); break; default: col_value = SPI_getvalue(row, tupdesc, i + 1); pgq_encode_cstring(buf, col_value, TBUF_QUOTE_JSON); break; } if (col_value) pfree((void*)col_value); } appendStringInfoChar(buf, '}'); } static void fill_json_type(PgqTriggerEvent *ev, HeapTuple row, StringInfo ev_type) { appendStringInfo(ev_type, "{\"op\":\"%s\"", ev->op_type_str); if (ev->tgargs->pkey_list) { static const char pkey_tag[] = "\"pkey\":"; const char *cstart, *cpos; char *start, *pos, *tmp; char sep = '['; cstart = ev->info->json_info; cpos = strstr(cstart, "\"pkey\":"); appendBinaryStringInfo(ev_type, cstart, cpos - cstart + strlen(pkey_tag)); start = tmp = pstrdup(ev->tgargs->pkey_list); pos = strchr(start, ','); while (pos) { appendStringInfoChar(ev_type, sep); sep = ','; *pos = 0; pgq_encode_cstring(ev_type, start, TBUF_QUOTE_JSON); start = pos + 1; pos = strchr(start, ','); } appendStringInfoChar(ev_type, sep); pgq_encode_cstring(ev_type, start, TBUF_QUOTE_JSON); appendStringInfoChar(ev_type, ']'); pfree(tmp); } else { appendStringInfoString(ev_type, ev->info->json_info); } appendStringInfoChar(ev_type, '}'); } /* * PgQ log trigger, takes 2 arguments: * 1. queue name to be inserted to. * * Queue events will be in format: * ev_type - operation type, I/U/D * ev_data - urlencoded column values * ev_extra1 - table name * ev_extra2 - optional urlencoded backup */ Datum pgq_jsontriga(PG_FUNCTION_ARGS) { TriggerData *tg; struct PgqTriggerEvent ev; HeapTuple row; /* * Get the trigger call context */ if (!CALLED_AS_TRIGGER(fcinfo)) elog(ERROR, "pgq.logutriga not called as trigger"); tg = (TriggerData *)(fcinfo->context); if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) row = tg->tg_newtuple; else row = tg->tg_trigtuple; if (pgq_is_logging_disabled()) goto skip_it; /* * Connect to the SPI manager */ if (SPI_connect() < 0) elog(ERROR, "logutriga: SPI_connect() failed"); pgq_prepare_event(&ev, tg, true); appendStringInfoString(ev.field[EV_EXTRA1], ev.info->table_name); fill_json_type(&ev, row, ev.field[EV_TYPE]); if (pgq_is_interesting_change(&ev, tg)) { /* * create type, data */ pgq_jsonenc_row(&ev, row, ev.field[EV_DATA]); /* * Construct the parameter array and insert the log row. */ pgq_insert_tg_event(&ev); } if (SPI_finish() < 0) elog(ERROR, "SPI_finish failed"); /* * After trigger ignores result, * before trigger skips event if NULL. */ skip_it: if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.tgargs->skip) return PointerGetDatum(NULL); else return PointerGetDatum(row); } pgq-3.4.1/triggers/logtriga.c000066400000000000000000000042651373233714300161250ustar00rootroot00000000000000/* * logtriga.c - Dumb SQL logging trigger. * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include "common.h" #include "stringutil.h" PG_FUNCTION_INFO_V1(pgq_logtriga); Datum pgq_logtriga(PG_FUNCTION_ARGS); /* * PGQ log trigger, takes 2 arguments: * 1. queue name to be inserted to. * 2. column type string * * Queue events will be in format: * ev_type - operation type, I/U/D * ev_data - partial SQL describing operation * ev_extra1 - table name */ Datum pgq_logtriga(PG_FUNCTION_ARGS) { TriggerData *tg; PgqTriggerEvent ev; /* * Get the trigger call context */ if (!CALLED_AS_TRIGGER(fcinfo)) elog(ERROR, "pgq.logtriga not called as trigger"); tg = (TriggerData *)(fcinfo->context); if (!TRIGGER_FIRED_AFTER(tg->tg_event)) elog(ERROR, "pgq.logtriga must be fired AFTER"); if (pgq_is_logging_disabled()) goto skip_it; /* * Connect to the SPI manager */ if (SPI_connect() < 0) elog(ERROR, "logtriga: SPI_connect() failed"); pgq_prepare_event(&ev, tg, false); appendStringInfoChar(ev.field[EV_TYPE], ev.op_type); appendStringInfoString(ev.field[EV_EXTRA1], ev.info->table_name); /* * create sql and insert if interesting */ if (pgqtriga_make_sql(&ev, ev.field[EV_DATA])) pgq_insert_tg_event(&ev); if (SPI_finish() < 0) elog(ERROR, "SPI_finish failed"); skip_it: return PointerGetDatum(NULL); } pgq-3.4.1/triggers/logutriga.c000066400000000000000000000070641373233714300163120ustar00rootroot00000000000000/* * logutriga.c - Smart trigger that logs urlencoded changes. * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include "common.h" #include "stringutil.h" PG_FUNCTION_INFO_V1(pgq_logutriga); Datum pgq_logutriga(PG_FUNCTION_ARGS); void pgq_urlenc_row(PgqTriggerEvent *ev, HeapTuple row, StringInfo buf) { TriggerData *tg = ev->tgdata; TupleDesc tupdesc = tg->tg_relation->rd_att; bool first = true; int i; const char *col_ident, *col_value; int attkind_idx = -1; if (ev->op_type == 'R') return; for (i = 0; i < tg->tg_relation->rd_att->natts; i++) { /* Skip dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; if (pgqtriga_skip_col(ev, i, attkind_idx)) continue; if (first) first = false; else appendStringInfoChar(buf, '&'); /* quote column name */ col_ident = SPI_fname(tupdesc, i + 1); pgq_encode_cstring(buf, col_ident, TBUF_QUOTE_URLENC); /* quote column value */ col_value = SPI_getvalue(row, tupdesc, i + 1); if (col_value != NULL) { appendStringInfoChar(buf, '='); pgq_encode_cstring(buf, col_value, TBUF_QUOTE_URLENC); } } } /* * PgQ log trigger, takes 2 arguments: * 1. queue name to be inserted to. * * Queue events will be in format: * ev_type - operation type, I/U/D * ev_data - urlencoded column values * ev_extra1 - table name * ev_extra2 - optional urlencoded backup */ Datum pgq_logutriga(PG_FUNCTION_ARGS) { TriggerData *tg; struct PgqTriggerEvent ev; HeapTuple row; /* * Get the trigger call context */ if (!CALLED_AS_TRIGGER(fcinfo)) elog(ERROR, "pgq.logutriga not called as trigger"); tg = (TriggerData *)(fcinfo->context); if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) row = tg->tg_newtuple; else row = tg->tg_trigtuple; if (pgq_is_logging_disabled()) goto skip_it; /* * Connect to the SPI manager */ if (SPI_connect() < 0) elog(ERROR, "logutriga: SPI_connect() failed"); pgq_prepare_event(&ev, tg, true); appendStringInfoString(ev.field[EV_EXTRA1], ev.info->table_name); appendStringInfoChar(ev.field[EV_TYPE], ev.op_type); if (ev.op_type != 'R') { appendStringInfoChar(ev.field[EV_TYPE], ':'); appendStringInfoString(ev.field[EV_TYPE], ev.pkey_list); } if (pgq_is_interesting_change(&ev, tg)) { /* * create type, data */ pgq_urlenc_row(&ev, row, ev.field[EV_DATA]); /* * Construct the parameter array and insert the log row. */ pgq_insert_tg_event(&ev); } if (SPI_finish() < 0) elog(ERROR, "SPI_finish failed"); /* * After trigger ignores result, * before trigger skips event if NULL. */ skip_it: if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.tgargs->skip) return PointerGetDatum(NULL); else return PointerGetDatum(row); } pgq-3.4.1/triggers/makesql.c000066400000000000000000000210561373233714300157470ustar00rootroot00000000000000/* * makesql.c - generate partial SQL statement for row change. * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Based on Slony-I log trigger: * * Copyright (c) 2003-2006, PostgreSQL Global Development Group * Author: Jan Wieck, Afilias USA INC. * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include "common.h" #include "stringutil.h" static void append_key_eq(StringInfo buf, const char *col_ident, const char *col_value) { if (col_value == NULL) elog(ERROR, "logtriga: Unexpected NULL key value"); pgq_encode_cstring(buf, col_ident, TBUF_QUOTE_IDENT); appendStringInfoChar(buf, '='); pgq_encode_cstring(buf, col_value, TBUF_QUOTE_LITERAL); } static void append_normal_eq(StringInfo buf, const char *col_ident, const char *col_value) { pgq_encode_cstring(buf, col_ident, TBUF_QUOTE_IDENT); appendStringInfoChar(buf, '='); if (col_value != NULL) pgq_encode_cstring(buf, col_value, TBUF_QUOTE_LITERAL); else appendStringInfoString(buf, "NULL"); } static void process_insert(PgqTriggerEvent *ev, StringInfo sql) { TriggerData *tg = ev->tgdata; HeapTuple new_row = tg->tg_trigtuple; TupleDesc tupdesc = tg->tg_relation->rd_att; int i; int need_comma = false; int attkind_idx; /* * Specify all the columns */ appendStringInfoChar(sql, '('); attkind_idx = -1; for (i = 0; i < tupdesc->natts; i++) { char *col_ident; /* Skip dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; /* Check if allowed by colstring */ attkind_idx++; if (pgqtriga_skip_col(ev, i, attkind_idx)) continue; if (need_comma) appendStringInfoChar(sql, ','); else need_comma = true; /* quote column name */ col_ident = SPI_fname(tupdesc, i + 1); pgq_encode_cstring(sql, col_ident, TBUF_QUOTE_IDENT); } /* * Append the string ") values (" */ appendStringInfoString(sql, ") values ("); /* * Append the values */ need_comma = false; attkind_idx = -1; for (i = 0; i < tupdesc->natts; i++) { char *col_value; /* Skip dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; /* Check if allowed by colstring */ attkind_idx++; if (pgqtriga_skip_col(ev, i, attkind_idx)) continue; if (need_comma) appendStringInfoChar(sql, ','); else need_comma = true; /* quote column value */ col_value = SPI_getvalue(new_row, tupdesc, i + 1); if (col_value == NULL) appendStringInfoString(sql, "null"); else pgq_encode_cstring(sql, col_value, TBUF_QUOTE_LITERAL); } /* * Terminate and done */ appendStringInfoChar(sql, ')'); } static int process_update(PgqTriggerEvent *ev, StringInfo sql) { TriggerData *tg = ev->tgdata; HeapTuple old_row = tg->tg_trigtuple; HeapTuple new_row = tg->tg_newtuple; TupleDesc tupdesc = tg->tg_relation->rd_att; Datum old_value; Datum new_value; bool old_isnull; bool new_isnull; char *col_ident; char *col_value; int i; int need_comma = false; int need_and = false; int attkind_idx; int ignore_count = 0; attkind_idx = -1; for (i = 0; i < tupdesc->natts; i++) { /* * Ignore dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; old_value = SPI_getbinval(old_row, tupdesc, i + 1, &old_isnull); new_value = SPI_getbinval(new_row, tupdesc, i + 1, &new_isnull); /* * If old and new value are NULL, the column is unchanged */ if (old_isnull && new_isnull) continue; /* * If both are NOT NULL, we need to compare the values and skip * setting the column if equal */ if (!old_isnull && !new_isnull) { Oid opr_oid; FmgrInfo *opr_finfo_p; /* * Lookup the equal operators function call info using the * typecache if available */ TypeCacheEntry *type_cache; type_cache = lookup_type_cache(SPI_gettypeid(tupdesc, i + 1), TYPECACHE_EQ_OPR | TYPECACHE_EQ_OPR_FINFO); opr_oid = type_cache->eq_opr; if (opr_oid == ARRAY_EQ_OP) opr_oid = InvalidOid; else opr_finfo_p = &(type_cache->eq_opr_finfo); /* * If we have an equal operator, use that to do binary * comparison. Else get the string representation of both * attributes and do string comparison. */ if (OidIsValid(opr_oid)) { if (DatumGetBool(FunctionCall2Coll(opr_finfo_p, TupleDescAttr(tupdesc, i)->attcollation, old_value, new_value))) continue; } else { char *old_strval = SPI_getvalue(old_row, tupdesc, i + 1); char *new_strval = SPI_getvalue(new_row, tupdesc, i + 1); if (strcmp(old_strval, new_strval) == 0) continue; } } if (pgqtriga_is_pkey(ev, i, attkind_idx)) elog(ERROR, "primary key update not allowed"); if (pgqtriga_skip_col(ev, i, attkind_idx)) { /* this change should be ignored */ ignore_count++; continue; } if (need_comma) appendStringInfoChar(sql, ','); else need_comma = true; col_ident = SPI_fname(tupdesc, i + 1); col_value = SPI_getvalue(new_row, tupdesc, i + 1); append_normal_eq(sql, col_ident, col_value); } /* * It can happen that the only UPDATE an application does is to set a * column to the same value again. In that case, we'd end up here with * no columns in the SET clause yet. We add the first key column here * with it's old value to simulate the same for the replication * engine. */ if (!need_comma) { /* there was change in ignored columns, skip whole event */ if (ignore_count > 0) return 0; for (i = 0, attkind_idx = -1; i < tupdesc->natts; i++) { if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; if (pgqtriga_is_pkey(ev, i, attkind_idx)) break; } col_ident = SPI_fname(tupdesc, i + 1); col_value = SPI_getvalue(old_row, tupdesc, i + 1); append_key_eq(sql, col_ident, col_value); } appendStringInfoString(sql, " where "); for (i = 0, attkind_idx = -1; i < tupdesc->natts; i++) { /* * Ignore dropped columns */ if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; if (!pgqtriga_is_pkey(ev, i, attkind_idx)) continue; col_ident = SPI_fname(tupdesc, i + 1); col_value = SPI_getvalue(old_row, tupdesc, i + 1); if (need_and) appendStringInfoString(sql, " and "); else need_and = true; append_key_eq(sql, col_ident, col_value); } return 1; } static void process_delete(PgqTriggerEvent *ev, StringInfo sql) { TriggerData *tg = ev->tgdata; HeapTuple old_row = tg->tg_trigtuple; TupleDesc tupdesc = tg->tg_relation->rd_att; char *col_ident; char *col_value; int i; int need_and = false; int attkind_idx; for (i = 0, attkind_idx = -1; i < tupdesc->natts; i++) { if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attkind_idx++; if (!pgqtriga_is_pkey(ev, i, attkind_idx)) continue; col_ident = SPI_fname(tupdesc, i + 1); col_value = SPI_getvalue(old_row, tupdesc, i + 1); if (need_and) appendStringInfoString(sql, " and "); else need_and = true; append_key_eq(sql, col_ident, col_value); } } int pgqtriga_make_sql(PgqTriggerEvent *ev, StringInfo sql) { TriggerData *tg = ev->tgdata; TupleDesc tupdesc; int i; int attcnt; int need_event = 1; tupdesc = tg->tg_relation->rd_att; /* * Count number of active columns */ for (i = 0, attcnt = 0; i < tupdesc->natts; i++) { if (TupleDescAttr(tupdesc, i)->attisdropped) continue; attcnt++; } /* * Determine cmdtype and op_data depending on the command type */ if (TRIGGER_FIRED_BY_INSERT(tg->tg_event)) { process_insert(ev, sql); } else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) { need_event = process_update(ev, sql); } else if (TRIGGER_FIRED_BY_DELETE(tg->tg_event)) { process_delete(ev, sql); } else if (TRIGGER_FIRED_BY_TRUNCATE(tg->tg_event)) { /* nothing to do for truncate */ } else elog(ERROR, "logtriga fired for unhandled event"); return need_event; } pgq-3.4.1/triggers/parsesql.c000066400000000000000000000101471373233714300161430ustar00rootroot00000000000000 #ifndef TEST #include #else #include #include #endif #include "parsesql.h" /* * Small SQL tokenizer. For cases where flex/bison is overkill. * * To simplify further processing, it merges words separated * with dots together. That also means it does not support * whitespace/comments before and after dot. * * Otherwise it's relatively compatible with main parser. * * Return value: * -1 - error * 0 - end of string * 1..255 - single char * >255 - token code */ int sql_tokenizer(const char *sql, int *len_p, bool stdstr) { const char *p = sql; int tok; *len_p = 0; if (!*p) { /* end */ return 0; } else if (isspace(*p) || (p[0] == '-' && p[1] == '-') || (p[0] == '/' && p[1] == '*')) { /* whitespace */ tok = T_SPACE; while (1) { if (p[0] == '-' && p[1] == '-') { /* line comment */ while (*p && *p != '\n') p++; } else if (p[0] == '/' && p[1] == '*') { /* c-comment, potentially nested */ int level = 1; p += 2; while (level) { if (p[0] == '*' && p[1] == '/') { level--; p += 2; } else if (p[0] == '/' && p[1] == '*') { level++; p += 2; } else if (!*p) { return -1; } else p++; } } else if (isspace(p[0])) { /* plain whitespace */ while (isspace(p[0])) p++; } else break; } } else if ((p[0] == '\'' && !stdstr) || ((p[0] == 'E' || p[0] == 'e') && p[1] == '\'')) { /* extended string */ tok = T_STRING; if (p[0] == '\'') p++; else p += 2; for (; *p; p++) { if (p[0] == '\'') { if (p[1] == '\'') p++; else break; } else if (p[0] == '\\') { if (!p[1]) return -1; p++; } } if (*p++ != '\'') return -1; } else if (p[0] == '\'' && stdstr) { /* standard string */ tok = T_STRING; for (p++; *p; p++) { if (p[0] == '\'') { if (p[1] == '\'') p++; else break; } } if (*p++ != '\'') return -1; } else if (isalpha(*p) || (*p == '_')) { /* plain/quoted words separated with '.' */ tok = T_WORD; while (1) { /* plain ident */ while (*p && (isalnum(*p) || *p == '_' || *p == '.')) p++; if (p[0] == '"') { /* quoted ident */ for (p++; *p; p++) { if (p[0] == '"') { if (p[1] == '"') p++; else break; } } if (*p++ != '"') return -1; } else if (p[0] == '.') { tok = T_FQIDENT; p++; } else { break; } } } else if (isdigit(p[0]) || (p[0] == '.' && isdigit(p[1]))) { /* number */ tok = T_NUMBER; while (*p) { if (isdigit(*p) || *p == '.') { p++; } else if ((*p == 'e' || *p == 'E')) { if (p[1] == '.' || p[1] == '+' || p[1] == '-') { p += 2; } else if (isdigit(p[1])) { p += 2; } else break; } else break; } } else if (p[0] == '$') { if (isdigit(p[1])) { /* dollar ident */ tok = T_WORD; for (p += 2; *p; p++) { if (!isdigit(*p)) break; } } else if (isalpha(p[1]) || p[1] == '_' || p[1] == '$') { /* dollar quote */ const char *p2, *p3; tok = T_STRING; p2 = strchr(p+1, '$'); if (!p2) return -1; p3 = ++p2; while (1) { p3 = strchr(p3, '$'); if (!p3) return -1; if (strncmp(p3, p, p2 - p) == 0) break; p3++; } p = p3 + (p2 - p); } else return -1; } else if (*p == '.') { /* disallow standalone dot - seems ident parsing missed it */ return -1; } else { /* return other symbols as-is */ tok = *p++; } *len_p = p - sql; return tok; } #ifdef TEST /* * test code */ const char test_sql[] = "\r\n\t " "-- foo\n" "/*/**//* nested *//**/*/\n" "select 1, .600, $1, $150, 1.44e+.1," " bzo.\"fo'\"\".o\".zoo.fa, " "E'a\\\\ \\000 \\' baz '''," "'foo''baz' from \"quoted\"\"id\";" "$$$$ $_$ $x$ $ $_ $_$" ; int main(void) { const char *sql = test_sql; int tlen; int tok; bool stdstr = false; while (1) { tok = sql_tokenizer(sql, &tlen, stdstr); if (tok == 0) { printf("EOF\n"); break; } else if (tok < 0) { printf("ERR\n"); return 1; } printf("tok=%d len=%d str=<%.*s>\n", tok, tlen, tlen, sql); sql += tlen; } return 0; } #endif pgq-3.4.1/triggers/parsesql.h000066400000000000000000000002461373233714300161470ustar00rootroot00000000000000 /* multi-char tokens */ enum SqlToken { T_SPACE = 257, T_STRING, T_NUMBER, T_WORD, T_FQIDENT, }; int sql_tokenizer(const char *sql, int *len_p, bool stdstr); pgq-3.4.1/triggers/pgq_triggers.sql000066400000000000000000000115311373233714300173610ustar00rootroot00000000000000-- ---------------------------------------------------------------------- -- Function: pgq.jsontriga() -- -- Trigger function that puts row data in JSON-encoded form into queue. -- -- Purpose: -- Convert row data into easily parseable form. -- -- Trigger parameters: -- arg1 - queue name -- argX - any number of optional arg, in any order -- -- Optional arguments: -- SKIP - The actual operation should be skipped (BEFORE trigger) -- ignore=col1[,col2] - don't look at the specified arguments -- pkey=col1[,col2] - Set pkey fields for the table, autodetection will be skipped -- backup - Put urlencoded contents of old row to ev_extra2 -- colname=EXPR - Override field value with SQL expression. Can reference table -- columns. colname can be: ev_type, ev_data, ev_extra1 .. ev_extra4 -- when=EXPR - If EXPR returns false, don't insert event. -- -- Queue event fields: -- ev_type - I/U/D ':' pkey_column_list -- ev_data - column values urlencoded -- ev_extra1 - table name -- ev_extra2 - optional urlencoded backup -- -- Regular listen trigger example: -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.jsontriga('qname'); -- -- Redirect trigger example: -- > CREATE TRIGGER triga_nimi BEFORE INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.jsontriga('qname', 'SKIP'); -- ---------------------------------------------------------------------- CREATE OR REPLACE FUNCTION pgq.jsontriga() RETURNS TRIGGER AS '$libdir/pgq_triggers', 'pgq_jsontriga' LANGUAGE C; -- ---------------------------------------------------------------------- -- Function: pgq.logutriga() -- -- Trigger function that puts row data in urlencoded form into queue. -- -- Purpose: -- Used as producer for several PgQ standard consumers (cube_dispatcher, -- queue_mover, table_dispatcher). Basically for cases where the -- consumer wants to parse the event and look at the actual column values. -- -- Trigger parameters: -- arg1 - queue name -- argX - any number of optional arg, in any order -- -- Optinal arguments: -- SKIP - The actual operation should be skipped (BEFORE trigger) -- ignore=col1[,col2] - don't look at the specified arguments -- pkey=col1[,col2] - Set pkey fields for the table, autodetection will be skipped -- backup - Put urlencoded contents of old row to ev_extra2 -- colname=EXPR - Override field value with SQL expression. Can reference table -- columns. colname can be: ev_type, ev_data, ev_extra1 .. ev_extra4 -- when=EXPR - If EXPR returns false, don't insert event. -- -- Queue event fields: -- ev_type - I/U/D ':' pkey_column_list -- ev_data - column values urlencoded -- ev_extra1 - table name -- ev_extra2 - optional urlencoded backup -- -- Regular listen trigger example: -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname'); -- -- Redirect trigger example: -- > CREATE TRIGGER triga_nimi BEFORE INSERT OR UPDATE ON customer -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname', 'SKIP'); -- ---------------------------------------------------------------------- CREATE OR REPLACE FUNCTION pgq.logutriga() RETURNS TRIGGER AS '$libdir/pgq_triggers', 'pgq_logutriga' LANGUAGE C; -- ---------------------------------------------------------------------- -- Function: pgq.sqltriga() -- -- Trigger that generates queue events containing partial SQL. -- It autodetects table structure. -- -- Purpose: -- Replication events, that only need changed column values. -- -- Parameters: -- arg1 - queue name -- argX - any number of optional arg, in any order -- -- Optinal arguments: -- SKIP - The actual operation should be skipped (BEFORE trigger) -- ignore=col1[,col2] - don't look at the specified arguments -- pkey=col1[,col2] - Set pkey fields for the table, PK autodetection will be skipped -- backup - Put urlencoded contents of old row to ev_extra2 -- colname=EXPR - Override field value with SQL expression. Can reference table -- columns. colname can be: ev_type, ev_data, ev_extra1 .. ev_extra4 -- when=EXPR - If EXPR returns false, don't insert event. -- -- Queue event fields: -- ev_type - I/U/D -- ev_data - partial SQL statement -- ev_extra1 - table name -- ev_extra2 - optional urlencoded backup -- -- ---------------------------------------------------------------------- CREATE OR REPLACE FUNCTION pgq.sqltriga() RETURNS trigger AS '$libdir/pgq_triggers', 'pgq_sqltriga' LANGUAGE C; pgq-3.4.1/triggers/qbuilder.c000066400000000000000000000062731373233714300161250ustar00rootroot00000000000000 #include #include #include "qbuilder.h" #include "parsesql.h" /* import standard_conforming_strings */ #if PG_VERSION_NUM >= 80500 #include #else #ifndef PGDLLIMPORT #define PGDLLIMPORT DLLIMPORT #endif extern PGDLLIMPORT bool standard_conforming_strings; #endif /* create QB in right context */ struct QueryBuilder *qb_create(const struct QueryBuilderOps *ops, MemoryContext ctx) { struct QueryBuilder *q; q = MemoryContextAllocZero(ctx, sizeof(*q)); q->op = ops; q->stdstr = standard_conforming_strings; q->maxargs = 8; q->arg_map = MemoryContextAlloc(ctx, q->maxargs * sizeof(int)); /* default size too large? */ q->sql.maxlen = 64; q->sql.data = MemoryContextAlloc(ctx, q->sql.maxlen); q->sql.data[0] = 0; return q; } /* add fragment without parsing */ void qb_add_raw(struct QueryBuilder *q, const char *str, int len) { if (len < 0) len = strlen(str); appendBinaryStringInfo(&q->sql, str, len); } /* the ident may or may not be argument reference */ static void qb_handle_ident(struct QueryBuilder *q, const char *ident, int len, void *arg) { int real_idx; int local_idx = -1, i; char abuf[32]; /* is argument reference? */ real_idx = q->op->name_lookup(arg, ident, len); if (real_idx < 0) { qb_add_raw(q, ident, len); return; } /* already referenced? */ for (i = 0; i < q->nargs; i++) { if (q->arg_map[i] == real_idx) { local_idx = i; break; } } /* new reference? */ if (local_idx < 0) { if (q->nargs >= FUNC_MAX_ARGS) elog(ERROR, "Too many args"); if (q->nargs >= q->maxargs) { q->arg_map = repalloc(q->arg_map, q->maxargs * 2 * sizeof(int)); q->maxargs *= 2; } local_idx = q->nargs++; q->arg_map[local_idx] = real_idx; } /* add $n to query */ snprintf(abuf, sizeof(abuf), "$%d", local_idx + 1); return qb_add_raw(q, abuf, strlen(abuf)); } /* add fragment with parsing - argument references are replaced with $n */ void qb_add_parse(struct QueryBuilder *q, const char *sql, void *arg) { int tlen, tok; /* tokenize sql, pick out argument references */ while (1) { tok = sql_tokenizer(sql, &tlen, q->stdstr); if (!tok) break; if (tok < 0) elog(ERROR, "QB: syntax error"); if (tok == T_WORD) { qb_handle_ident(q, sql, tlen, arg); } else { qb_add_raw(q, sql, tlen); } sql += tlen; } } /* prepare */ void qb_prepare(struct QueryBuilder *q, void *arg) { Oid types[FUNC_MAX_ARGS]; void *plan; int i; for (i = 0; i < q->nargs; i++) types[i] = q->op->type_lookup(arg, q->arg_map[i]); plan = SPI_prepare(q->sql.data, q->nargs, types); q->plan = SPI_saveplan(plan); } /* lookup values and run plan. returns result from SPI_execute_plan() */ int qb_execute(struct QueryBuilder *q, void *arg) { Datum values[FUNC_MAX_ARGS]; char nulls[FUNC_MAX_ARGS]; int i; if (!q->plan) elog(ERROR, "QB: query not prepared yet"); for (i = 0; i < q->nargs; i++) { bool isnull = false; values[i] = q->op->value_lookup(arg, q->arg_map[i], &isnull); nulls[i] = isnull ? 'n' : ' '; } return SPI_execute_plan(q->plan, values, nulls, true, 0); } void qb_free(struct QueryBuilder *q) { if (!q) return; if (q->plan) SPI_freeplan(q->plan); if (q->sql.data) pfree(q->sql.data); pfree(q); } pgq-3.4.1/triggers/qbuilder.h000066400000000000000000000017551373233714300161320ustar00rootroot00000000000000 #include /* * Callbacks that to argument name/type/value lookups. */ struct QueryBuilderOps { /* returns name index or < 0 if unknown. str is not null-terminated */ int (*name_lookup)(void *arg, const char *str, int len); /* returns type oid for nr that .name_lookup returned */ Oid (*type_lookup)(void *arg, int nr); /* returns value for nr that .name_lookup returned */ Datum (*value_lookup)(void *arg, int nr, bool *isnull); }; /* * Parsed query */ struct QueryBuilder { StringInfoData sql; bool stdstr; const struct QueryBuilderOps *op; void *plan; int nargs; int maxargs; int *arg_map; }; struct QueryBuilder *qb_create(const struct QueryBuilderOps *ops, MemoryContext ctx); void qb_add_raw(struct QueryBuilder *q, const char *str, int len); void qb_add_parse(struct QueryBuilder *q, const char *str, void *arg); void qb_free(struct QueryBuilder *q); void qb_prepare(struct QueryBuilder *q, void *arg); int qb_execute(struct QueryBuilder *q, void *arg); pgq-3.4.1/triggers/sqltriga.c000066400000000000000000000046011373233714300161350ustar00rootroot00000000000000/* * sqltriga.c - Smart SQL-logging trigger. * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include "common.h" #include "stringutil.h" PG_FUNCTION_INFO_V1(pgq_sqltriga); Datum pgq_sqltriga(PG_FUNCTION_ARGS); /* * PgQ log trigger, takes 2 arguments: * 1. queue name to be inserted to. * * Queue events will be in format: * ev_type - operation type, I/U/D/R * ev_data - urlencoded column values * ev_extra1 - table name * ev_extra2 - optional urlencoded backup */ Datum pgq_sqltriga(PG_FUNCTION_ARGS) { TriggerData *tg; PgqTriggerEvent ev; /* * Get the trigger call context */ if (!CALLED_AS_TRIGGER(fcinfo)) elog(ERROR, "pgq.sqltriga not called as trigger"); tg = (TriggerData *)(fcinfo->context); if (pgq_is_logging_disabled()) goto skip_it; /* * Connect to the SPI manager */ if (SPI_connect() < 0) elog(ERROR, "sqltriga: SPI_connect() failed"); pgq_prepare_event(&ev, tg, true); appendStringInfoChar(ev.field[EV_TYPE], ev.op_type); appendStringInfoString(ev.field[EV_EXTRA1], ev.info->table_name); /* * create sql and insert if interesting */ if (pgqtriga_make_sql(&ev, ev.field[EV_DATA])) pgq_insert_tg_event(&ev); if (SPI_finish() < 0) elog(ERROR, "SPI_finish failed"); /* * After trigger ignores result, * before trigger skips event if NULL. */ skip_it: if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.tgargs->skip) return PointerGetDatum(NULL); else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) return PointerGetDatum(tg->tg_newtuple); else return PointerGetDatum(tg->tg_trigtuple); } pgq-3.4.1/triggers/stringutil.c000066400000000000000000000113071373233714300165140ustar00rootroot00000000000000/* * stringutil.c - some tools for string handling * * Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #if PG_VERSION_NUM >= 90200 #include #endif #include "stringutil.h" #ifndef SET_VARSIZE #define SET_VARSIZE(x, len) VARATT_SIZEP(x) = len #endif #if PG_VERSION_NUM < 90100 static char *quote_literal_cstr(const char *str) { StringInfoData buf; initStringInfo(&buf); if (strchr(str, '\\')) appendStringInfoCharMacro(&buf, 'E'); appendStringInfoCharMacro(&buf, '\''); for (; *str; str++) { if (*str == '\'' || *str == '\\') appendStringInfoCharMacro(&buf, *str); appendStringInfoCharMacro(&buf, *str); } appendStringInfoCharMacro(&buf, '\''); return buf.data; } #endif StringInfo pgq_init_varbuf(void) { StringInfo buf; buf = makeStringInfo(); appendStringInfoString(buf, "XXXX"); return buf; } Datum pgq_finish_varbuf(StringInfo buf) { if (!buf) return (Datum)0; SET_VARSIZE(buf->data, buf->len); return PointerGetDatum(buf->data); } /* * Find a string in comma-separated list. * * It does not support space inside tokens. */ bool pgq_strlist_contains(const char *liststr, const char *str) { int c, len = strlen(str); const char *p, *listpos = liststr; loop: /* find string fragment, later check if actual token */ p = strstr(listpos, str); if (p == NULL) return false; /* move listpos further */ listpos = p + len; /* survive len=0 and avoid unnecessary compare */ if (*listpos) listpos++; /* check previous symbol */ if (p > liststr) { c = *(p - 1); if (!isspace(c) && c != ',') goto loop; } /* check following symbol */ c = p[len]; if (c != 0 && !isspace(c) && c != ',') goto loop; return true; } /* * quoting */ static void pgq_urlencode(StringInfo buf, const char *src) { static const char hextbl[] = "0123456789abcdef"; while (*src) { unsigned c = (unsigned char)*src++; if (c == ' ') { appendStringInfoCharMacro(buf, '+'); } else if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || c == '.' || c == '-') { appendStringInfoCharMacro(buf, c); } else { appendStringInfoCharMacro(buf, '%'); appendStringInfoCharMacro(buf, hextbl[c >> 4]); appendStringInfoCharMacro(buf, hextbl[c & 15]); } } } static void pgq_quote_literal(StringInfo buf, const char *src) { const char *quoted = quote_literal_cstr(src); appendStringInfoString(buf, quoted); pfree((char*)quoted); } /* * pgq_quote_ident - Quote an identifier only if needed */ static void pgq_quote_ident(StringInfo buf, const char *src) { const char *quoted = quote_identifier(src); appendStringInfoString(buf, quoted); if (quoted != src) pfree((char *)quoted); } #if PG_VERSION_NUM < 90200 static void escape_json(StringInfo buf, const char *p) { appendStringInfoCharMacro(buf, '\"'); for (; *p; p++) { switch (*p) { case '\b': appendStringInfoString(buf, "\\b"); break; case '\f': appendStringInfoString(buf, "\\f"); break; case '\n': appendStringInfoString(buf, "\\n"); break; case '\r': appendStringInfoString(buf, "\\r"); break; case '\t': appendStringInfoString(buf, "\\t"); break; case '"': appendStringInfoString(buf, "\\\""); break; case '\\': appendStringInfoString(buf, "\\\\"); break; default: if ((unsigned char) *p < ' ') appendStringInfo(buf, "\\u%04x", (int) *p); else appendStringInfoCharMacro(buf, *p); break; } } appendStringInfoCharMacro(buf, '\"'); } #endif void pgq_encode_cstring(StringInfo tbuf, const char *str, enum PgqEncode encoding) { if (str == NULL) elog(ERROR, "tbuf_encode_cstring: NULL"); switch (encoding) { case TBUF_QUOTE_LITERAL: pgq_quote_literal(tbuf, str); break; case TBUF_QUOTE_IDENT: pgq_quote_ident(tbuf, str); break; case TBUF_QUOTE_URLENC: pgq_urlencode(tbuf, str); break; case TBUF_QUOTE_JSON: escape_json(tbuf, str); break; default: elog(ERROR, "bad encoding"); } } pgq-3.4.1/triggers/stringutil.h000066400000000000000000000005041373233714300165160ustar00rootroot00000000000000 enum PgqEncode { TBUF_QUOTE_IDENT, TBUF_QUOTE_LITERAL, TBUF_QUOTE_URLENC, TBUF_QUOTE_JSON, }; StringInfo pgq_init_varbuf(void); Datum pgq_finish_varbuf(StringInfo buf); bool pgq_strlist_contains(const char *liststr, const char *str); void pgq_encode_cstring(StringInfo tbuf, const char *str, enum PgqEncode encoding);