pax_global_header00006660000000000000000000000064130212702700014504gustar00rootroot0000000000000052 comment=54da8eace1946dd049e1a01ae717a10c8d7423d2 pgq-node/000077500000000000000000000000001302127027000125625ustar00rootroot00000000000000pgq-node/.gitignore000066400000000000000000000004001302127027000145440ustar00rootroot00000000000000*.o *.so .deps *.swp *.dump *--* sql/*/*.sql sql/*/results regression.* newgrants_*.sql oldgrants_*.sql debian/control debian/tmp debian/files debian/postgresql-* debian/*-stamp *.substvars *.log *.debhelper pgq_node.sql pgq_node.upgrade.sql results/ pgq-node/Makefile000066400000000000000000000010371302127027000142230ustar00rootroot00000000000000 EXTENSION = pgq_node EXT_VERSION = 3.2.5 EXT_OLD_VERSIONS = 3.1 3.1.3 3.1.6 3.2 Extension_regress = pgq_node_init_ext pgq_node_test Contrib_regress = pgq_node_init_noext pgq_node_test include mk/common-pgxs.mk # # docs # dox: cleandox $(SRCS) mkdir -p docs/html mkdir -p docs/sql $(CATSQL) --ndoc structure/tables.sql > docs/sql/pgq_node.sql $(CATSQL) --ndoc structure/functions.sql > docs/sql/functions.sql $(NDOC) $(NDOCARGS) deb: make -f debian/rules genfiles debuild -us -uc -b debclean: make -f debian/rules debclean pgq-node/debian/000077500000000000000000000000001302127027000140045ustar00rootroot00000000000000pgq-node/debian/changelog000066400000000000000000000001741302127027000156600ustar00rootroot00000000000000pgq-node (3.2.5-1) unstable; urgency=low * v3.2.5 -- Marko Kreen Sat, 11 Jun 2016 16:21:25 +0300 pgq-node/debian/compat000066400000000000000000000000021302127027000152020ustar00rootroot000000000000009 pgq-node/debian/control.in000066400000000000000000000010261302127027000160130ustar00rootroot00000000000000Source: pgq-node Section: database Priority: extra Maintainer: Marko Kreen Build-Depends: debhelper (>= 9), python, postgresql-server-dev-all Standards-Version: 3.9.1 Homepage: https://github.io/pgq Package: postgresql-PGVERSION-pgq-node Architecture: all Depends: ${misc:Depends}, postgresql-PGVERSION, postgresql-PGVERSION-pgq Conflicts: postgresql-PGVERSION-pgq3 Description: Cascaded queueing on top of PgQ PostgreSQL extension that provider queue node registration and coordination for cascaded queueing. pgq-node/debian/copyright000066400000000000000000000016631302127027000157450ustar00rootroot00000000000000Based on Skytools 3 packaging: Copyright (C) 2011 Dimitri Fontaine Upstream Author: Marko Kreen Copyright: Copyright (C) 2007-2016 Marko Kreen License: Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. pgq-node/debian/pgversions000066400000000000000000000000041302127027000161200ustar00rootroot00000000000000all pgq-node/debian/rules000077500000000000000000000020511302127027000150620ustar00rootroot00000000000000#!/usr/bin/make -f # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 DEB_BUILD_OPTIONS := nostrip nocheck export DEB_BUILD_OPTIONS PG_BUILDEXT = pg_buildext export PG_UPDATECONTROL=1 PG_VERSIONS = $$($(PG_BUILDEXT) supported-versions . ) include /usr/share/postgresql-common/pgxs_debian_control.mk %: dh $@ override_dh_auto_clean: dh_auto_clean -- clean || exit 0 override_dh_auto_install: mkdir -p $(CURDIR)/debian/tmp for v in $(PG_VERSIONS); do \ echo "### Building for PostgreSQL $${v}" && \ $(MAKE) clean install \ PG_CONFIG=/usr/lib/postgresql/$${v}/bin/pg_config \ DESTDIR=$(CURDIR)/debian/tmp \ || exit 1 ; \ done genfiles: rm -f debian/control rm -rf debian/postgresql-* set -e; for ver in $(PG_VERSIONS); do \ xmod=pgq-node; \ dst="debian/postgresql-$${ver}-$${xmod}.install"; \ echo "usr/share/postgresql/$${ver}/extension" > "$${dst}"; \ echo "usr/share/postgresql/$${ver}/contrib" >> "$${dst}"; \ done $(PG_BUILDEXT) updatecontrol debclean: clean rm -f debian/control rm -rf debian/postgresql-* pgq-node/debian/source/000077500000000000000000000000001302127027000153045ustar00rootroot00000000000000pgq-node/debian/source/format000066400000000000000000000000141302127027000165120ustar00rootroot000000000000003.0 (quilt) pgq-node/docs/000077500000000000000000000000001302127027000135125ustar00rootroot00000000000000pgq-node/docs/Languages.txt000066400000000000000000000120211302127027000161550ustar00rootroot00000000000000Format: 1.52 # This is the Natural Docs languages file for this project. If you change # anything here, it will apply to THIS PROJECT ONLY. If you'd like to change # something for all your projects, edit the Languages.txt in Natural Docs' # Config directory instead. Ignore Extension: sql #------------------------------------------------------------------------------- # SYNTAX: # # Unlike other Natural Docs configuration files, in this file all comments # MUST be alone on a line. Some languages deal with the # character, so you # cannot put comments on the same line as content. # # Also, all lists are separated with spaces, not commas, again because some # languages may need to use them. # # Language: [name] # Alter Language: [name] # Defines a new language or alters an existing one. Its name can use any # characters. If any of the properties below have an add/replace form, you # must use that when using Alter Language. # # The language Shebang Script is special. It's entry is only used for # extensions, and files with those extensions have their shebang (#!) lines # read to determine the real language of the file. Extensionless files are # always treated this way. # # The language Text File is also special. It's treated as one big comment # so you can put Natural Docs content in them without special symbols. Also, # if you don't specify a package separator, ignored prefixes, or enum value # behavior, it will copy those settings from the language that is used most # in the source tree. # # Extensions: [extension] [extension] ... # [Add/Replace] Extensions: [extension] [extension] ... # Defines the file extensions of the language's source files. You can # redefine extensions found in the main languages file. You can use * to # mean any undefined extension. # # Shebang Strings: [string] [string] ... # [Add/Replace] Shebang Strings: [string] [string] ... # Defines a list of strings that can appear in the shebang (#!) line to # designate that it's part of the language. You can redefine strings found # in the main languages file. # # Ignore Prefixes in Index: [prefix] [prefix] ... # [Add/Replace] Ignored Prefixes in Index: [prefix] [prefix] ... # # Ignore [Topic Type] Prefixes in Index: [prefix] [prefix] ... # [Add/Replace] Ignored [Topic Type] Prefixes in Index: [prefix] [prefix] ... # Specifies prefixes that should be ignored when sorting symbols in an # index. Can be specified in general or for a specific topic type. # #------------------------------------------------------------------------------ # For basic language support only: # # Line Comments: [symbol] [symbol] ... # Defines a space-separated list of symbols that are used for line comments, # if any. # # Block Comments: [opening sym] [closing sym] [opening sym] [closing sym] ... # Defines a space-separated list of symbol pairs that are used for block # comments, if any. # # Package Separator: [symbol] # Defines the default package separator symbol. The default is a dot. # # [Topic Type] Prototype Enders: [symbol] [symbol] ... # When defined, Natural Docs will attempt to get a prototype from the code # immediately following the topic type. It stops when it reaches one of # these symbols. Use \n for line breaks. # # Line Extender: [symbol] # Defines the symbol that allows a prototype to span multiple lines if # normally a line break would end it. # # Enum Values: [global|under type|under parent] # Defines how enum values are referenced. The default is global. # global - Values are always global, referenced as 'value'. # under type - Values are under the enum type, referenced as # 'package.enum.value'. # under parent - Values are under the enum's parent, referenced as # 'package.value'. # # Perl Package: [perl package] # Specifies the Perl package used to fine-tune the language behavior in ways # too complex to do in this file. # #------------------------------------------------------------------------------ # For full language support only: # # Full Language Support: [perl package] # Specifies the Perl package that has the parsing routines necessary for full # language support. # #------------------------------------------------------------------------------- # The following languages are defined in the main file, if you'd like to alter # them: # # Text File, Shebang Script, C/C++, C#, Java, JavaScript, Perl, Python, # PHP, SQL, Visual Basic, Pascal, Assembly, Ada, Tcl, Ruby, Makefile, # ActionScript, ColdFusion, R, Fortran # If you add a language that you think would be useful to other developers # and should be included in Natural Docs by default, please e-mail it to # languages [at] naturaldocs [dot] org. Language: PLPGSQL Extension: sql Line Comment: -- Block Comment: /* */ Enum Values: Global Function Prototype Enders: , ; ) $ ' Variable Prototype Enders: , ; ) := default Default DEFAULT Database Index Prototype Enders: , ; ) Database Trigger Prototype Enders: begin Begin BEGIN pgq-node/docs/Menu.txt000066400000000000000000000037061302127027000151650ustar00rootroot00000000000000Format: 1.52 # You can add a title and sub-title to your menu like this: # Title: [project name] # SubTitle: [subtitle] # You can add a footer to your documentation like this: # Footer: [text] # If you want to add a copyright notice, this would be the place to do it. # You can add a timestamp to your documentation like one of these: # Timestamp: Generated on month day, year # Timestamp: Updated mm/dd/yyyy # Timestamp: Last updated mon day # # m - One or two digit month. January is "1" # mm - Always two digit month. January is "01" # mon - Short month word. January is "Jan" # month - Long month word. January is "January" # d - One or two digit day. 1 is "1" # dd - Always two digit day. 1 is "01" # day - Day with letter extension. 1 is "1st" # yy - Two digit year. 2006 is "06" # yyyy - Four digit year. 2006 is "2006" # year - Four digit year. 2006 is "2006" # -------------------------------------------------------------------------- # # Cut and paste the lines below to change the order in which your files # appear on the menu. Don't worry about adding or removing files, Natural # Docs will take care of that. # # You can further organize the menu by grouping the entries. Add a # "Group: [name] {" line to start a group, and add a "}" to end it. # # You can add text and web links to the menu by adding "Text: [text]" and # "Link: [name] ([URL])" lines, respectively. # # The formatting and comments are auto-generated, so don't worry about # neatness when editing the file. Natural Docs will clean it up the next # time it is run. When working with groups, just deal with the braces and # forget about the indentation and comments. # # -------------------------------------------------------------------------- File: Functions (functions.sql) File: Tables (pgq_node.sql) Group: Index { Index: Everything Database Table Index: Database Tables Function Index: Functions } # Group: Index pgq-node/docs/Topics.txt000066400000000000000000000065051302127027000155220ustar00rootroot00000000000000Format: 1.52 # This is the Natural Docs topics file for this project. If you change anything # here, it will apply to THIS PROJECT ONLY. If you'd like to change something # for all your projects, edit the Topics.txt in Natural Docs' Config directory # instead. # If you'd like to prevent keywords from being recognized by Natural Docs, you # can do it like this: # Ignore Keywords: [keyword], [keyword], ... # # Or you can use the list syntax like how they are defined: # Ignore Keywords: # [keyword] # [keyword], [plural keyword] # ... #------------------------------------------------------------------------------- # SYNTAX: # # Topic Type: [name] # Alter Topic Type: [name] # Creates a new topic type or alters one from the main file. Each type gets # its own index and behavior settings. Its name can have letters, numbers, # spaces, and these charaters: - / . ' # # Plural: [name] # Sets the plural name of the topic type, if different. # # Keywords: # [keyword] # [keyword], [plural keyword] # ... # Defines or adds to the list of keywords for the topic type. They may only # contain letters, numbers, and spaces and are not case sensitive. Plural # keywords are used for list topics. You can redefine keywords found in the # main topics file. # # Index: [yes|no] # Whether the topics get their own index. Defaults to yes. Everything is # included in the general index regardless of this setting. # # Scope: [normal|start|end|always global] # How the topics affects scope. Defaults to normal. # normal - Topics stay within the current scope. # start - Topics start a new scope for all the topics beneath it, # like class topics. # end - Topics reset the scope back to global for all the topics # beneath it. # always global - Topics are defined as global, but do not change the scope # for any other topics. # # Class Hierarchy: [yes|no] # Whether the topics are part of the class hierarchy. Defaults to no. # # Page Title If First: [yes|no] # Whether the topic's title becomes the page title if it's the first one in # a file. Defaults to no. # # Break Lists: [yes|no] # Whether list topics should be broken into individual topics in the output. # Defaults to no. # # Can Group With: [type], [type], ... # Defines a list of topic types that this one can possibly be grouped with. # Defaults to none. #------------------------------------------------------------------------------- # The following topics are defined in the main file, if you'd like to alter # their behavior or add keywords: # # Generic, Class, Interface, Section, File, Group, Function, Variable, # Property, Type, Constant, Enumeration, Event, Delegate, Macro, # Database, Database Table, Database View, Database Index, Database # Cursor, Database Trigger, Cookie, Build Target # If you add something that you think would be useful to other developers # and should be included in Natural Docs by default, please e-mail it to # topics [at] naturaldocs [dot] org. Topic Type: Schema Plural: Schemas Index: No Scope: Start Class Hierarchy: Yes Keywords: schema, schemas Alter Topic Type: Function Add Keywords: public function internal function Alter Topic Type: File Index: No pgq-node/expected/000077500000000000000000000000001302127027000143635ustar00rootroot00000000000000pgq-node/expected/pgq_node_init_ext.out000066400000000000000000000007371302127027000206220ustar00rootroot00000000000000create extension pgq; \set ECHO none upgrade_schema ---------------- 0 (1 row) create extension pgq_node from unpackaged; select array_length(extconfig, 1) as dumpable from pg_catalog.pg_extension where extname = 'pgq_node'; dumpable ---------- 4 (1 row) drop extension pgq_node; create extension pgq_node; select array_length(extconfig, 1) as dumpable from pg_catalog.pg_extension where extname = 'pgq_node'; dumpable ---------- 4 (1 row) pgq-node/expected/pgq_node_init_noext.out000066400000000000000000000002051302127027000211450ustar00rootroot00000000000000\set ECHO none upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 0 (1 row) pgq-node/expected/pgq_node_test.out000066400000000000000000000557241302127027000177640ustar00rootroot00000000000000select * from pgq_node.register_location('aqueue', 'node1', 'dbname=node1', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('aqueue', 'node2', 'dbname=node2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('aqueue', 'node3', 'dbname=node3', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('aqueue', 'node4', 'dbname=node44', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('aqueue', 'node4', 'dbname=node4', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('aqueue', 'node5', 'dbname=node4', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.get_queue_locations('aqueue'); node_name | node_location | dead -----------+---------------+------ node1 | dbname=node1 | f node2 | dbname=node2 | f node3 | dbname=node3 | f node4 | dbname=node4 | f node5 | dbname=node4 | f (5 rows) select * from pgq_node.unregister_location('aqueue', 'node5'); ret_code | ret_note ----------+---------- 200 | Ok (1 row) select * from pgq_node.unregister_location('aqueue', 'node5'); ret_code | ret_note ----------+---------------------------------- 301 | Location not found: aqueue/node5 (1 row) select * from pgq_node.get_queue_locations('aqueue'); node_name | node_location | dead -----------+---------------+------ node1 | dbname=node1 | f node2 | dbname=node2 | f node3 | dbname=node3 | f node4 | dbname=node4 | f (4 rows) select * from pgq_node.create_node('aqueue', 'root', 'node1', 'node1_worker', null, null, null); ret_code | ret_note ----------+-------------------------------------------------------------- 200 | Node "node1" initialized for queue "aqueue" with type "root" (1 row) select * from pgq_node.register_subscriber('aqueue', 'node2', 'node2_worker', null); ret_code | ret_note | global_watermark ----------+------------------------------+------------------ 200 | Subscriber registered: node2 | 1 (1 row) select * from pgq_node.register_subscriber('aqueue', 'node3', 'node3_worker', null); ret_code | ret_note | global_watermark ----------+------------------------------+------------------ 200 | Subscriber registered: node3 | 1 (1 row) select * from pgq_node.maint_watermark('aqueue'); maint_watermark ----------------- 0 (1 row) select * from pgq_node.maint_watermark('aqueue-x'); maint_watermark ----------------- 0 (1 row) select * from pgq_node.get_consumer_info('aqueue'); consumer_name | provider_node | last_tick_id | paused | uptodate | cur_error ---------------+---------------+--------------+--------+----------+----------- node1_worker | node1 | 1 | f | f | (1 row) select * from pgq_node.unregister_subscriber('aqueue', 'node3'); ret_code | ret_note ----------+-------------------------------- 200 | Subscriber unregistered: node3 (1 row) select queue_name, consumer_name, last_tick from pgq.get_consumer_info(); queue_name | consumer_name | last_tick ------------+-------------------+----------- aqueue | .global_watermark | 1 aqueue | .node2.watermark | 1 aqueue | node2_worker | 1 (3 rows) select * from pgq_node.get_worker_state('aqueue'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error | worker_name | global_watermark | local_watermark | local_queue_top | combined_queue | combined_type ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+-----------+--------------+------------------+-----------------+-----------------+----------------+--------------- 100 | Ok | root | node1 | 1 | node1 | dbname=node1 | f | f | | node1_worker | 1 | 1 | 1 | | (1 row) update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0'; select * from pgq.ticker('aqueue'); ticker -------- 2 (1 row) select * from pgq.ticker('aqueue'); ticker -------- 3 (1 row) select * from pgq_node.set_subscriber_watermark('aqueue', 'node2', 3); ret_code | ret_note ----------+--------------------------- 200 | .node2.watermark set to 3 (1 row) select queue_name, consumer_name, last_tick from pgq.get_consumer_info(); queue_name | consumer_name | last_tick ------------+-------------------+----------- aqueue | .global_watermark | 1 aqueue | .node2.watermark | 3 aqueue | node2_worker | 1 (3 rows) select * from pgq_node.set_node_attrs('aqueue', 'test=1'); ret_code | ret_note ----------+------------------------- 200 | Node attributes updated (1 row) select * from pgq_node.get_node_info('aqueue'); ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs ----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ 100 | Ok | root | node1 | 1 | 1 | node1 | dbname=node1 | | | node1_worker | f | f | 3 | test=1 (1 row) select * from pgq_node.get_subscriber_info('aqueue'); node_name | worker_name | node_watermark -----------+--------------+---------------- node2 | node2_worker | 3 (1 row) -- branch node select * from pgq_node.register_location('bqueue', 'node1', 'dbname=node1', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('bqueue', 'node2', 'dbname=node2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('bqueue', 'node3', 'dbname=node3', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('bqueue', 'branch', 'node2', 'node2_worker', 'node1', 1, null); ret_code | ret_note ----------+---------------------------------------------------------------- 200 | Node "node2" initialized for queue "bqueue" with type "branch" (1 row) select * from pgq_node.register_consumer('bqueue', 'random_consumer', 'node1', 1); ret_code | ret_note ----------+----------------------------------------------------- 200 | Consumer random_consumer registered on queue bqueue (1 row) select * from pgq_node.register_consumer('bqueue', 'random_consumer2', 'node1', 1); ret_code | ret_note ----------+------------------------------------------------------ 200 | Consumer random_consumer2 registered on queue bqueue (1 row) select * from pgq_node.local_state; queue_name | consumer_name | provider_node | last_tick_id | cur_error | paused | uptodate ------------+------------------+---------------+--------------+-----------+--------+---------- aqueue | node1_worker | node1 | 1 | | f | f bqueue | node2_worker | node1 | 1 | | f | f bqueue | random_consumer | node1 | 1 | | f | f bqueue | random_consumer2 | node1 | 1 | | f | f (4 rows) select * from pgq_node.node_info; queue_name | node_type | node_name | worker_name | combined_queue | node_attrs ------------+-----------+-----------+--------------+----------------+------------ aqueue | root | node1 | node1_worker | | test=1 bqueue | branch | node2 | node2_worker | | (2 rows) select * from pgq_node.get_node_info('aqueue'); ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs ----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ 100 | Ok | root | node1 | 1 | 1 | node1 | dbname=node1 | | | node1_worker | f | f | 3 | test=1 (1 row) select * from pgq_node.get_node_info('bqueue'); ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs ----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ 100 | Ok | branch | node2 | 1 | 1 | node1 | dbname=node1 | | | node2_worker | f | f | 1 | (1 row) select * from pgq_node.get_node_info('cqueue'); ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs ----------+-----------------------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+-------------+---------------+-----------------+------------------+------------ 404 | Unknown queue: cqueue | | | | | | | | | | | | | (1 row) select * from pgq_node.get_worker_state('aqueue'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error | worker_name | global_watermark | local_watermark | local_queue_top | combined_queue | combined_type ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+-----------+--------------+------------------+-----------------+-----------------+----------------+--------------- 100 | Ok | root | node1 | 1 | node1 | dbname=node1 | f | f | | node1_worker | 1 | 1 | 3 | | (1 row) select * from pgq_node.get_worker_state('bqueue'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error | worker_name | global_watermark | local_watermark | local_queue_top | combined_queue | combined_type ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+-----------+--------------+------------------+-----------------+-----------------+----------------+--------------- 100 | Ok | branch | node2 | 1 | node1 | dbname=node1 | f | f | | node2_worker | 1 | 1 | 1 | | (1 row) select * from pgq_node.get_worker_state('cqueue'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error | worker_name | global_watermark | local_watermark | local_queue_top | combined_queue | combined_type ----------+-----------------------+-----------+-----------+----------------+---------------+-------------------+--------+----------+-----------+-------------+------------------+-----------------+-----------------+----------------+--------------- 404 | Unknown queue: cqueue | | | | | | | | | | | | | | (1 row) select * from pgq_node.is_root_node('aqueue'); is_root_node -------------- t (1 row) select * from pgq_node.is_root_node('bqueue'); is_root_node -------------- f (1 row) select * from pgq_node.is_root_node('cqueue'); ERROR: queue does not exist: cqueue select * from pgq_node.get_consumer_state('bqueue', 'random_consumer'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+----------- 100 | Ok | branch | node2 | 1 | node1 | dbname=node1 | f | f | (1 row) select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+----------- 100 | Ok | branch | node2 | 1 | node1 | dbname=node1 | f | f | (1 row) select * from pgq_node.set_consumer_error('bqueue', 'random_consumer2', 'failure'); ret_code | ret_note ----------+------------------------------------------- 100 | Consumer random_consumer2 error = failure (1 row) select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+----------- 100 | Ok | branch | node2 | 1 | node1 | dbname=node1 | f | f | failure (1 row) select * from pgq_node.set_consumer_completed('bqueue', 'random_consumer2', 2); ret_code | ret_note ----------+---------------------------------------------- 100 | Consumer random_consumer2 compleded tick = 2 (1 row) select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+----------- 100 | Ok | branch | node2 | 2 | node1 | dbname=node1 | f | f | (1 row) select * from pgq_node.set_consumer_paused('bqueue', 'random_consumer2', true); ret_code | ret_note ----------+-------------------------------------------- 200 | Consumer random_consumer2 tagged as paused (1 row) select * from pgq_node.set_consumer_uptodate('bqueue', 'random_consumer2', true); ret_code | ret_note ----------+----------------------- 200 | Consumer uptodate = 1 (1 row) select * from pgq_node.change_consumer_provider('bqueue', 'random_consumer2', 'node3'); ret_code | ret_note ----------+--------------------------------------- 200 | Consumer provider node set to : node3 (1 row) select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+----------- 100 | Ok | branch | node2 | 2 | node3 | dbname=node3 | t | f | (1 row) select * from pgq_node.unregister_consumer('bqueue', 'random_consumer2'); ret_code | ret_note ----------+---------------------------------------------------- 200 | Consumer random_consumer2 unregistered from bqueue (1 row) select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error ----------+-------------------------------------------+-----------+-----------+----------------+---------------+-------------------+--------+----------+----------- 404 | Unknown consumer: bqueue/random_consumer2 | branch | node2 | | | | | | (1 row) select * from pgq_node.get_node_info('bqueue'); ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs ----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ 100 | Ok | branch | node2 | 1 | 1 | node1 | dbname=node1 | | | node2_worker | f | f | 1 | (1 row) set session_replication_role = 'replica'; select * from pgq_node.demote_root('aqueue', 1, 'node3'); ret_code | ret_note | last_tick ----------+--------------------------------------+----------- 200 | Step 1: Writing disabled for: aqueue | (1 row) select * from pgq_node.demote_root('aqueue', 1, 'node3'); ret_code | ret_note | last_tick ----------+--------------------------------------+----------- 200 | Step 1: Writing disabled for: aqueue | (1 row) select * from pgq_node.demote_root('aqueue', 2, 'node3'); ret_code | ret_note | last_tick ----------+------------------------------------+----------- 200 | Step 2: Inserted last tick: aqueue | 4 (1 row) select * from pgq_node.demote_root('aqueue', 2, 'node3'); ret_code | ret_note | last_tick ----------+------------------------------------+----------- 200 | Step 2: Inserted last tick: aqueue | 5 (1 row) select * from pgq_node.demote_root('aqueue', 3, 'node3'); ret_code | ret_note | last_tick ----------+----------------------------------------+----------- 200 | Step 3: Demoted root to branch: aqueue | 5 (1 row) select * from pgq_node.demote_root('aqueue', 3, 'node3'); ret_code | ret_note | last_tick ----------+---------------+----------- 301 | Node not root | (1 row) -- leaf node select * from pgq_node.register_location('mqueue', 'node1', 'dbname=node1', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('mqueue', 'node2', 'dbname=node2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('mqueue', 'node3', 'dbname=node3', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('mqueue', 'leaf', 'node2', 'node2_worker', 'node1', 13, 'aqueue'); ret_code | ret_note ----------+-------------------------------------------------------------- 200 | Node "node2" initialized for queue "mqueue" with type "leaf" (1 row) select * from pgq_node.get_worker_state('mqueue'); ret_code | ret_note | node_type | node_name | completed_tick | provider_node | provider_location | paused | uptodate | cur_error | worker_name | global_watermark | local_watermark | local_queue_top | combined_queue | combined_type ----------+----------+-----------+-----------+----------------+---------------+-------------------+--------+----------+-----------+--------------+------------------+-----------------+-----------------+----------------+--------------- 100 | Ok | leaf | node2 | 13 | node1 | dbname=node1 | f | f | | node2_worker | | 13 | | aqueue | branch (1 row) select * from pgq_node.drop_node('asd', 'asd'); ret_code | ret_note ----------+------------------- 200 | Node dropped: asd (1 row) select * from pgq_node.drop_node('mqueue', 'node3'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node3 (1 row) select * from pgq_node.drop_node('mqueue', 'node2'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node2 (1 row) select * from pgq_node.drop_node('mqueue', 'node1'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node1 (1 row) select * from pgq_node.drop_node('aqueue', 'node5'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node5 (1 row) select * from pgq_node.drop_node('aqueue', 'node4'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node4 (1 row) select * from pgq_node.drop_node('aqueue', 'node1'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node1 (1 row) select * from pgq_node.drop_node('aqueue', 'node2'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node2 (1 row) select * from pgq_node.drop_node('aqueue', 'node3'); ret_code | ret_note ----------+--------------------- 200 | Node dropped: node3 (1 row) \q pgq-node/functions/000077500000000000000000000000001302127027000145725ustar00rootroot00000000000000pgq-node/functions/pgq_node.change_consumer_provider.sql000066400000000000000000000027361302127027000241700ustar00rootroot00000000000000 create or replace function pgq_node.change_consumer_provider( in i_queue_name text, in i_consumer_name text, in i_new_provider text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.change_consumer_provider(3) -- -- Change provider for this consumer. -- -- Parameters: -- i_queue_name - queue name -- i_consumer_name - consumer name -- i_new_provider - node name for new provider -- Returns: -- ret_code - error code -- 200 - ok -- 404 - no such consumer or new node -- ret_note - description -- ---------------------------------------------------------------------- begin perform 1 from pgq_node.node_location where queue_name = i_queue_name and node_name = i_new_provider; if not found then select 404, 'New node not found: ' || i_new_provider into ret_code, ret_note; return; end if; update pgq_node.local_state set provider_node = i_new_provider, uptodate = false where queue_name = i_queue_name and consumer_name = i_consumer_name; if not found then select 404, 'Unknown consumer: ' || i_queue_name || '/' || i_consumer_name into ret_code, ret_note; return; end if; select 200, 'Consumer provider node set to : ' || i_new_provider into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.create_node.sql000066400000000000000000000102721302127027000213600ustar00rootroot00000000000000 create or replace function pgq_node.create_node( in i_queue_name text, in i_node_type text, in i_node_name text, in i_worker_name text, in i_provider_name text, in i_global_watermark bigint, in i_combined_queue text, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.create_node(7) -- -- Initialize node. -- -- Parameters: -- i_node_name - cascaded queue name -- i_node_type - node type -- i_node_name - node name -- i_worker_name - worker consumer name -- i_provider_name - provider node name for non-root nodes -- i_global_watermark - global lowest tick_id -- i_combined_queue - merge-leaf: target queue -- -- Returns: -- 200 - Ok -- 401 - node already initialized -- ???? - maybe we coud use more error codes ? -- -- Node Types: -- root - master node -- branch - subscriber node that can be provider to others -- leaf - subscriber node that cannot be provider to others -- Calls: -- None -- Tables directly manipulated: -- None -- ---------------------------------------------------------------------- declare _wm_consumer text; _global_wm bigint; begin perform 1 from pgq_node.node_info where queue_name = i_queue_name; if found then select 401, 'Node already initialized' into ret_code, ret_note; return; end if; _wm_consumer := '.global_watermark'; if i_node_type = 'root' then if coalesce(i_provider_name, i_global_watermark::text, i_combined_queue) is not null then select 401, 'unexpected args for '||i_node_type into ret_code, ret_note; return; end if; perform pgq.create_queue(i_queue_name); perform pgq.register_consumer(i_queue_name, _wm_consumer); _global_wm := (select last_tick from pgq.get_consumer_info(i_queue_name, _wm_consumer)); elsif i_node_type = 'branch' then if i_provider_name is null then select 401, 'provider not set for '||i_node_type into ret_code, ret_note; return; end if; if i_global_watermark is null then select 401, 'global watermark not set for '||i_node_type into ret_code, ret_note; return; end if; perform pgq.create_queue(i_queue_name); update pgq.queue set queue_external_ticker = true, queue_disable_insert = true where queue_name = i_queue_name; if i_global_watermark > 1 then perform pgq.ticker(i_queue_name, i_global_watermark, now(), 1); end if; perform pgq.register_consumer_at(i_queue_name, _wm_consumer, i_global_watermark); _global_wm := i_global_watermark; elsif i_node_type = 'leaf' then _global_wm := i_global_watermark; if i_combined_queue is not null then perform 1 from pgq.get_queue_info(i_combined_queue); if not found then select 401, 'non-existing queue on leaf side: '||i_combined_queue into ret_code, ret_note; return; end if; end if; else select 401, 'bad node type: '||i_node_type into ret_code, ret_note; end if; insert into pgq_node.node_info (queue_name, node_type, node_name, worker_name, combined_queue) values (i_queue_name, i_node_type, i_node_name, i_worker_name, i_combined_queue); if i_node_type <> 'root' then select f.ret_code, f.ret_note into ret_code, ret_note from pgq_node.register_consumer(i_queue_name, i_worker_name, i_provider_name, _global_wm) f; else select f.ret_code, f.ret_note into ret_code, ret_note from pgq_node.register_consumer(i_queue_name, i_worker_name, i_node_name, _global_wm) f; end if; if ret_code <> 200 then return; end if; select 200, 'Node "' || i_node_name || '" initialized for queue "' || i_queue_name || '" with type "' || i_node_type || '"' into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.demote_root.sql000066400000000000000000000072121302127027000214300ustar00rootroot00000000000000 create or replace function pgq_node.demote_root( in i_queue_name text, in i_step int4, in i_new_provider text, out ret_code int4, out ret_note text, out last_tick int8) as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.demote_root(3) -- -- Multi-step root demotion to branch. -- -- Must be be called for each step in sequence: -- -- Step 1 - disable writing to queue. -- Step 2 - wait until writers go away, do tick. -- Step 3 - change type, register. -- -- Parameters: -- i_queue_name - queue name -- i_step - step number -- i_new_provider - new provider node -- Returns: -- 200 - success -- 404 - node not initialized for queue -- 301 - node is not root -- ---------------------------------------------------------------------- declare n_type text; w_name text; sql text; ev_id int8; ev_tbl text; begin select node_type, worker_name into n_type, w_name from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Node not initialized for queue: ' || i_queue_name into ret_code, ret_note; return; end if; if n_type != 'root' then select 301, 'Node not root' into ret_code, ret_note; return; end if; if i_step > 1 then select queue_data_pfx into ev_tbl from pgq.queue where queue_name = i_queue_name and queue_disable_insert and queue_external_ticker; if not found then raise exception 'steps in wrong order'; end if; end if; if i_step = 1 then update pgq.queue set queue_disable_insert = true, queue_external_ticker = true where queue_name = i_queue_name; if not found then select 404, 'Huh, no queue?: ' || i_queue_name into ret_code, ret_note; return; end if; select 200, 'Step 1: Writing disabled for: ' || i_queue_name into ret_code, ret_note; elsif i_step = 2 then set local session_replication_role = 'replica'; -- lock parent table to stop updates, allow reading sql := 'lock table ' || ev_tbl || ' in exclusive mode'; execute sql; select nextval(queue_tick_seq), nextval(queue_event_seq) into last_tick, ev_id from pgq.queue where queue_name = i_queue_name; perform pgq.ticker(i_queue_name, last_tick, now(), ev_id); select 200, 'Step 2: Inserted last tick: ' || i_queue_name into ret_code, ret_note; elsif i_step = 3 then -- change type, point worker to new provider select t.tick_id into last_tick from pgq.tick t, pgq.queue q where q.queue_name = i_queue_name and t.tick_queue = q.queue_id order by t.tick_queue desc, t.tick_id desc limit 1; update pgq_node.node_info set node_type = 'branch' where queue_name = i_queue_name; update pgq_node.local_state set provider_node = i_new_provider, last_tick_id = last_tick, uptodate = false where queue_name = i_queue_name and consumer_name = w_name; select 200, 'Step 3: Demoted root to branch: ' || i_queue_name into ret_code, ret_note; else raise exception 'incorrect step number'; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.drop_node.sql000066400000000000000000000047341302127027000210670ustar00rootroot00000000000000 create or replace function pgq_node.drop_node( in i_queue_name text, in i_node_name text, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.drop_node(2) -- -- Drop node. This needs to be run on all the members of a set -- to properly get rid of the node. -- -- Parameters: -- i_queue_name - queue name -- i_node_name - node_name -- -- Returns: -- ret_code - error code -- ret_note - error description -- -- Return Codes: -- 200 - Ok -- 304 - No such queue -- 406 - That is a provider -- Calls: -- None -- Tables directly manipulated: -- None ------------------------------------------------------------------------ declare _is_local boolean; _is_prov boolean; begin select (n.node_name = i_node_name), (select s.provider_node = i_node_name from pgq_node.local_state s where s.queue_name = i_queue_name and s.consumer_name = n.worker_name) into _is_local, _is_prov from pgq_node.node_info n where n.queue_name = i_queue_name; if not found then -- proceed with cleaning anyway, as there schenarios -- where some data is left around _is_prov := false; _is_local := true; end if; -- drop local state if _is_local then delete from pgq_node.subscriber_info where queue_name = i_queue_name; delete from pgq_node.local_state where queue_name = i_queue_name; delete from pgq_node.node_info where queue_name = i_queue_name and node_name = i_node_name; perform pgq.drop_queue(queue_name, true) from pgq.queue where queue_name = i_queue_name; delete from pgq_node.node_location where queue_name = i_queue_name and node_name <> i_node_name; elsif _is_prov then select 405, 'Cannot drop provider node: ' || i_node_name into ret_code, ret_note; return; else perform pgq_node.unregister_subscriber(i_queue_name, i_node_name); end if; -- let the unregister_location send event if needed select f.ret_code, f.ret_note from pgq_node.unregister_location(i_queue_name, i_node_name) f into ret_code, ret_note; select 200, 'Node dropped: ' || i_node_name into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.get_consumer_info.sql000066400000000000000000000024311302127027000226130ustar00rootroot00000000000000 create or replace function pgq_node.get_consumer_info( in i_queue_name text, out consumer_name text, out provider_node text, out last_tick_id int8, out paused boolean, out uptodate boolean, out cur_error text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_consumer_info(1) -- -- Get consumer list that work on the local node. -- -- Parameters: -- i_queue_name - cascaded queue name -- -- Returns: -- consumer_name - cascaded consumer name -- provider_node - node from where the consumer reads from -- last_tick_id - last committed tick -- paused - if consumer is paused -- uptodate - if consumer is uptodate -- cur_error - failure reason -- ---------------------------------------------------------------------- begin for consumer_name, provider_node, last_tick_id, paused, uptodate, cur_error in select s.consumer_name, s.provider_node, s.last_tick_id, s.paused, s.uptodate, s.cur_error from pgq_node.local_state s where s.queue_name = i_queue_name order by 1 loop return next; end loop; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.get_consumer_state.sql000066400000000000000000000043561302127027000230100ustar00rootroot00000000000000 create or replace function pgq_node.get_consumer_state( in i_queue_name text, in i_consumer_name text, out ret_code int4, out ret_note text, out node_type text, out node_name text, out completed_tick bigint, out provider_node text, out provider_location text, out paused boolean, out uptodate boolean, out cur_error text ) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_consumer_state(2) -- -- Get info for cascaded consumer that targets local node. -- -- Parameters: -- i_node_name - cascaded queue name -- i_consumer_name - cascaded consumer name -- -- Returns: -- node_type - local node type -- node_name - local node name -- completed_tick - last committed tick -- provider_node - provider node name -- provider_location - connect string to provider node -- paused - this node should not do any work -- uptodate - if consumer has loaded last changes -- cur_error - failure reason -- ---------------------------------------------------------------------- begin select n.node_type, n.node_name into node_type, node_name from pgq_node.node_info n where n.queue_name = i_queue_name; if not found then select 404, 'Unknown queue: ' || i_queue_name into ret_code, ret_note; return; end if; select s.last_tick_id, s.provider_node, s.paused, s.uptodate, s.cur_error into completed_tick, provider_node, paused, uptodate, cur_error from pgq_node.local_state s where s.queue_name = i_queue_name and s.consumer_name = i_consumer_name; if not found then select 404, 'Unknown consumer: ' || i_queue_name || '/' || i_consumer_name into ret_code, ret_note; return; end if; select 100, 'Ok', p.node_location into ret_code, ret_note, provider_location from pgq_node.node_location p where p.queue_name = i_queue_name and p.node_name = provider_node; if not found then select 404, 'Unknown provider node: ' || i_queue_name || '/' || provider_node into ret_code, ret_note; return; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.get_node_info.sql000066400000000000000000000070021302127027000217040ustar00rootroot00000000000000 drop function if exists pgq_node.get_node_info(text); create or replace function pgq_node.get_node_info( in i_queue_name text, out ret_code int4, out ret_note text, out node_type text, out node_name text, out global_watermark bigint, out local_watermark bigint, out provider_node text, out provider_location text, out combined_queue text, out combined_type text, out worker_name text, out worker_paused bool, out worker_uptodate bool, out worker_last_tick bigint, out node_attrs text ) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_node_info(1) -- -- Get local node info for cascaded queue. -- -- Parameters: -- i_queue_name - cascaded queue name -- -- Returns: -- node_type - local node type -- node_name - local node name -- global_watermark - queue's global watermark -- local_watermark - queue's local watermark, for this and below nodes -- provider_node - provider node name -- provider_location - provider connect string -- combined_queue - queue name for target set -- combined_type - node type of target set -- worker_name - consumer name that maintains this node -- worker_paused - is worker paused -- worker_uptodate - is worker seen the changes -- worker_last_tick - last committed tick_id by worker -- node_attrs - urlencoded dict of random attrs for worker (eg. sync_watermark) -- ---------------------------------------------------------------------- declare sql text; begin select 100, 'Ok', n.node_type, n.node_name, c.node_type, c.queue_name, w.provider_node, l.node_location, n.worker_name, w.paused, w.uptodate, w.last_tick_id, n.node_attrs into ret_code, ret_note, node_type, node_name, combined_type, combined_queue, provider_node, provider_location, worker_name, worker_paused, worker_uptodate, worker_last_tick, node_attrs from pgq_node.node_info n left join pgq_node.node_info c on (c.queue_name = n.combined_queue) left join pgq_node.local_state w on (w.queue_name = n.queue_name and w.consumer_name = n.worker_name) left join pgq_node.node_location l on (l.queue_name = w.queue_name and l.node_name = w.provider_node) where n.queue_name = i_queue_name; if not found then select 404, 'Unknown queue: ' || i_queue_name into ret_code, ret_note; return; end if; if node_type in ('root', 'branch') then select min(case when consumer_name = '.global_watermark' then null else last_tick end), min(case when consumer_name = '.global_watermark' then last_tick else null end) into local_watermark, global_watermark from pgq.get_consumer_info(i_queue_name); if local_watermark is null then select t.tick_id into local_watermark from pgq.tick t, pgq.queue q where t.tick_queue = q.queue_id and q.queue_name = i_queue_name order by 1 desc limit 1; end if; else local_watermark := worker_last_tick; end if; if node_type = 'root' then select tick_id from pgq.tick t, pgq.queue q where q.queue_name = i_queue_name and t.tick_queue = q.queue_id order by t.tick_queue desc, t.tick_id desc limit 1 into worker_last_tick; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.get_queue_locations.sql000066400000000000000000000016521302127027000231500ustar00rootroot00000000000000 create or replace function pgq_node.get_queue_locations( in i_queue_name text, out node_name text, out node_location text, out dead boolean ) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_queue_locations(1) -- -- Get node list for the queue. -- -- Parameters: -- i_queue_name - queue name -- -- Returns: -- node_name - node name -- node_location - libpq connect string for the node -- dead - whether the node should be considered dead -- ---------------------------------------------------------------------- begin for node_name, node_location, dead in select l.node_name, l.node_location, l.dead from pgq_node.node_location l where l.queue_name = i_queue_name loop return next; end loop; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.get_subscriber_info.sql000066400000000000000000000025631302127027000231310ustar00rootroot00000000000000 create or replace function pgq_node.get_subscriber_info( in i_queue_name text, out node_name text, out worker_name text, out node_watermark int8) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_subscriber_info(1) -- -- Get subscriber list for the local node. -- -- It may be out-of-date, due to in-progress -- administrative change. -- Node's local provider info ( pgq_node.get_node_info() or pgq_node.get_worker_state(1) ) -- is the authoritative source. -- -- Parameters: -- i_queue_name - cascaded queue name -- -- Returns: -- node_name - node name that uses current node as provider -- worker_name - consumer that maintains remote node -- local_watermark - lowest tick_id on subscriber -- ---------------------------------------------------------------------- declare _watermark_name text; begin for node_name, worker_name, _watermark_name in select s.subscriber_node, s.worker_name, s.watermark_name from pgq_node.subscriber_info s where s.queue_name = i_queue_name order by 1 loop select last_tick into node_watermark from pgq.get_consumer_info(i_queue_name, _watermark_name); return next; end loop; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.get_worker_state.sql000066400000000000000000000077471302127027000224750ustar00rootroot00000000000000 create or replace function pgq_node.get_worker_state( in i_queue_name text, out ret_code int4, out ret_note text, out node_type text, out node_name text, out completed_tick bigint, out provider_node text, out provider_location text, out paused boolean, out uptodate boolean, out cur_error text, out worker_name text, out global_watermark bigint, out local_watermark bigint, out local_queue_top bigint, out combined_queue text, out combined_type text ) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_worker_state(1) -- -- Get info for consumer that maintains local node. -- -- Parameters: -- i_queue_name - cascaded queue name -- -- Returns: -- node_type - local node type -- node_name - local node name -- completed_tick - last committed tick -- provider_node - provider node name -- provider_location - connect string to provider node -- paused - this node should not do any work -- uptodate - if consumer has loaded last changes -- cur_error - failure reason -- worker_name - consumer name that maintains this node -- global_watermark - queue's global watermark -- local_watermark - queue's local watermark, for this and below nodes -- local_queue_top - last tick in local queue -- combined_queue - queue name for target set -- combined_type - node type of target setA -- ---------------------------------------------------------------------- begin select n.node_type, n.node_name, n.worker_name, n.combined_queue into node_type, node_name, worker_name, combined_queue from pgq_node.node_info n where n.queue_name = i_queue_name; if not found then select 404, 'Unknown queue: ' || i_queue_name into ret_code, ret_note; return; end if; select s.last_tick_id, s.provider_node, s.paused, s.uptodate, s.cur_error into completed_tick, provider_node, paused, uptodate, cur_error from pgq_node.local_state s where s.queue_name = i_queue_name and s.consumer_name = worker_name; if not found then select 404, 'Unknown consumer: ' || i_queue_name || '/' || worker_name into ret_code, ret_note; return; end if; select 100, 'Ok', p.node_location into ret_code, ret_note, provider_location from pgq_node.node_location p where p.queue_name = i_queue_name and p.node_name = provider_node; if not found then select 404, 'Unknown provider node: ' || i_queue_name || '/' || provider_node into ret_code, ret_note; return; end if; if combined_queue is not null then select n.node_type into combined_type from pgq_node.node_info n where n.queue_name = get_worker_state.combined_queue; if not found then select 404, 'Combinde queue node not found: ' || combined_queue into ret_code, ret_note; return; end if; end if; if node_type in ('root', 'branch') then select min(case when consumer_name = '.global_watermark' then null else last_tick end), min(case when consumer_name = '.global_watermark' then last_tick else null end) into local_watermark, global_watermark from pgq.get_consumer_info(i_queue_name); if local_watermark is null then select t.tick_id into local_watermark from pgq.tick t, pgq.queue q where t.tick_queue = q.queue_id and q.queue_name = i_queue_name order by 1 desc limit 1; end if; select tick_id from pgq.tick t, pgq.queue q where q.queue_name = i_queue_name and t.tick_queue = q.queue_id order by t.tick_queue desc, t.tick_id desc limit 1 into local_queue_top; else local_watermark := completed_tick; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.is_leaf_node.sql000066400000000000000000000013051302127027000215140ustar00rootroot00000000000000create or replace function pgq_node.is_leaf_node(i_queue_name text) returns bool as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.is_leaf_node(1) -- -- Checs if node is leaf. -- -- Parameters: -- i_queue_name - queue name -- Returns: -- true - if this this the leaf node for queue -- ---------------------------------------------------------------------- declare res bool; begin select n.node_type = 'leaf' into res from pgq_node.node_info n where n.queue_name = i_queue_name; if not found then raise exception 'queue does not exist: %', i_queue_name; end if; return res; end; $$ language plpgsql; pgq-node/functions/pgq_node.is_root_node.sql000066400000000000000000000013051302127027000215700ustar00rootroot00000000000000create or replace function pgq_node.is_root_node(i_queue_name text) returns bool as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.is_root_node(1) -- -- Checs if node is root. -- -- Parameters: -- i_queue_name - queue name -- Returns: -- true - if this this the root node for queue -- ---------------------------------------------------------------------- declare res bool; begin select n.node_type = 'root' into res from pgq_node.node_info n where n.queue_name = i_queue_name; if not found then raise exception 'queue does not exist: %', i_queue_name; end if; return res; end; $$ language plpgsql; pgq-node/functions/pgq_node.maint_watermark.sql000066400000000000000000000015151302127027000222750ustar00rootroot00000000000000 create or replace function pgq_node.maint_watermark(i_queue_name text) returns int4 as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.maint_watermark(1) -- -- Move global watermark on root node. -- -- Returns: -- 0 - tells pgqd to call just once -- ---------------------------------------------------------------------- declare _lag interval; begin perform 1 from pgq_node.node_info where queue_name = i_queue_name and node_type = 'root' for update; if not found then return 0; end if; select lag into _lag from pgq.get_consumer_info(i_queue_name, '.global_watermark'); if _lag >= '5 minutes'::interval then perform pgq_node.set_global_watermark(i_queue_name, NULL); end if; return 0; end; $$ language plpgsql; pgq-node/functions/pgq_node.promote_branch.sql000066400000000000000000000041251302127027000221120ustar00rootroot00000000000000 create or replace function pgq_node.promote_branch( in i_queue_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.promote_branch(1) -- -- Promote branch node to root. -- -- Parameters: -- i_queue_name - queue name -- -- Returns: -- 200 - success -- 404 - node not initialized for queue -- 301 - node is not branch -- ---------------------------------------------------------------------- declare n_name text; n_type text; w_name text; last_tick bigint; sql text; begin select node_name, node_type, worker_name into n_name, n_type, w_name from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Node not initialized for queue: ' || i_queue_name into ret_code, ret_note; return; end if; if n_type != 'branch' then select 301, 'Node not branch' into ret_code, ret_note; return; end if; update pgq.queue set queue_disable_insert = false, queue_external_ticker = false where queue_name = i_queue_name; -- change type, point worker to itself select t.tick_id into last_tick from pgq.tick t, pgq.queue q where q.queue_name = i_queue_name and t.tick_queue = q.queue_id order by t.tick_queue desc, t.tick_id desc limit 1; -- make tick seq larger than last tick perform pgq.seq_setval(queue_tick_seq, last_tick) from pgq.queue where queue_name = i_queue_name; update pgq_node.node_info set node_type = 'root' where queue_name = i_queue_name; update pgq_node.local_state set provider_node = n_name, last_tick_id = last_tick, uptodate = false where queue_name = i_queue_name and consumer_name = w_name; select 200, 'Branch node promoted to root' into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.register_consumer.sql000066400000000000000000000040571302127027000226530ustar00rootroot00000000000000 create or replace function pgq_node.register_consumer( in i_queue_name text, in i_consumer_name text, in i_provider_node text, in i_custom_tick_id int8, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.register_consumer(4) -- -- Subscribe plain cascaded consumer to a target node. -- That means it's planning to read from remote node -- and write to local node. -- -- Parameters: -- i_queue_name - set name -- i_consumer_name - cascaded consumer name -- i_provider_node - node name -- i_custom_tick_id - tick id -- -- Returns: -- ret_code - error code -- 200 - ok -- 201 - already registered -- 401 - no such queue -- ret_note - description -- ---------------------------------------------------------------------- declare n record; node_wm_name text; node_pos bigint; begin select node_type into n from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Unknown queue: ' || i_queue_name into ret_code, ret_note; return; end if; perform 1 from pgq_node.local_state where queue_name = i_queue_name and consumer_name = i_consumer_name; if found then update pgq_node.local_state set provider_node = i_provider_node, last_tick_id = i_custom_tick_id where queue_name = i_queue_name and consumer_name = i_consumer_name; select 201, 'Consumer already registered: ' || i_queue_name || '/' || i_consumer_name into ret_code, ret_note; return; end if; insert into pgq_node.local_state (queue_name, consumer_name, provider_node, last_tick_id) values (i_queue_name, i_consumer_name, i_provider_node, i_custom_tick_id); select 200, 'Consumer '||i_consumer_name||' registered on queue '||i_queue_name into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.register_location.sql000066400000000000000000000035471302127027000226330ustar00rootroot00000000000000 create or replace function pgq_node.register_location( in i_queue_name text, in i_node_name text, in i_node_location text, in i_dead boolean, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.register_location(4) -- -- Add new node location. -- -- Parameters: -- i_queue_name - queue name -- i_node_name - node name -- i_node_location - node connect string -- i_dead - dead flag for node -- -- Returns: -- ret_code - error code -- ret_note - error description -- -- Return Codes: -- 200 - Ok -- ---------------------------------------------------------------------- declare node record; begin select node_type = 'root' as is_root into node from pgq_node.node_info where queue_name = i_queue_name for update; -- may return 0 rows perform 1 from pgq_node.node_location where queue_name = i_queue_name and node_name = i_node_name; if found then update pgq_node.node_location set node_location = coalesce(i_node_location, node_location), dead = i_dead where queue_name = i_queue_name and node_name = i_node_name; elsif i_node_location is not null then insert into pgq_node.node_location (queue_name, node_name, node_location, dead) values (i_queue_name, i_node_name, i_node_location, i_dead); end if; if node.is_root then perform pgq.insert_event(i_queue_name, 'pgq.location-info', i_node_name, i_queue_name, i_node_location, i_dead::text, null) from pgq_node.node_info n where n.queue_name = i_queue_name; end if; select 200, 'Location registered' into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.register_subscriber.sql000066400000000000000000000041321302127027000231550ustar00rootroot00000000000000 create or replace function pgq_node.register_subscriber( in i_queue_name text, in i_remote_node_name text, in i_remote_worker_name text, in i_custom_tick_id int8, out ret_code int4, out ret_note text, out global_watermark bigint) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.register_subscriber(4) -- -- Subscribe remote node to local node at custom position. -- Should be used when changing provider for existing node. -- -- Parameters: -- i_node_name - set name -- i_remote_node_name - node name -- i_remote_worker_name - consumer name -- i_custom_tick_id - tick id [optional] -- -- Returns: -- ret_code - error code -- ret_note - description -- global_watermark - minimal watermark -- ---------------------------------------------------------------------- declare n record; node_wm_name text; node_pos bigint; begin select node_type into n from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Unknown queue: ' || i_queue_name into ret_code, ret_note; return; end if; select last_tick into global_watermark from pgq.get_consumer_info(i_queue_name, '.global_watermark'); if n.node_type not in ('root', 'branch') then select 401, 'Cannot subscribe to ' || n.node_type || ' node' into ret_code, ret_note; return; end if; node_wm_name := '.' || i_remote_node_name || '.watermark'; node_pos := coalesce(i_custom_tick_id, global_watermark); perform pgq.register_consumer_at(i_queue_name, node_wm_name, global_watermark); perform pgq.register_consumer_at(i_queue_name, i_remote_worker_name, node_pos); insert into pgq_node.subscriber_info (queue_name, subscriber_node, worker_name, watermark_name) values (i_queue_name, i_remote_node_name, i_remote_worker_name, node_wm_name); select 200, 'Subscriber registered: '||i_remote_node_name into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_consumer_completed.sql000066400000000000000000000023341302127027000236520ustar00rootroot00000000000000 create or replace function pgq_node.set_consumer_completed( in i_queue_name text, in i_consumer_name text, in i_tick_id int8, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_consumer_completed(3) -- -- Set last completed tick id for the cascaded consumer -- that it has committed to local node. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_consumer_name - cascaded consumer name -- i_tick_id - tick id -- Returns: -- 200 - ok -- 404 - consumer not known -- ---------------------------------------------------------------------- begin update pgq_node.local_state set last_tick_id = i_tick_id, cur_error = NULL where queue_name = i_queue_name and consumer_name = i_consumer_name; if found then select 100, 'Consumer ' || i_consumer_name || ' compleded tick = ' || i_tick_id::text into ret_code, ret_note; else select 404, 'Consumer not known: ' || i_queue_name || '/' || i_consumer_name into ret_code, ret_note; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_consumer_error.sql000066400000000000000000000020141302127027000230220ustar00rootroot00000000000000 create or replace function pgq_node.set_consumer_error( in i_queue_name text, in i_consumer_name text, in i_error_msg text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_consumer_error(3) -- -- If batch processing fails, consumer can store it's last error in db. -- Returns: -- 100 - ok -- 101 - consumer not known -- ---------------------------------------------------------------------- begin update pgq_node.local_state set cur_error = i_error_msg where queue_name = i_queue_name and consumer_name = i_consumer_name; if found then select 100, 'Consumer ' || i_consumer_name || ' error = ' || i_error_msg into ret_code, ret_note; else select 101, 'Consumer not known, ignoring: ' || i_queue_name || '/' || i_consumer_name into ret_code, ret_note; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_consumer_paused.sql000066400000000000000000000031271302127027000231600ustar00rootroot00000000000000 create or replace function pgq_node.set_consumer_paused( in i_queue_name text, in i_consumer_name text, in i_paused boolean, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_consumer_paused(3) -- -- Set consumer paused flag. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_consumer_name - cascaded consumer name -- i_paused - new flag state -- Returns: -- 200 - ok -- 201 - already paused -- 404 - consumer not found -- ---------------------------------------------------------------------- declare old_flag boolean; word text; begin if i_paused then word := 'paused'; else word := 'resumed'; end if; select paused into old_flag from pgq_node.local_state where queue_name = i_queue_name and consumer_name = i_consumer_name for update; if not found then select 404, 'Unknown consumer: ' || i_consumer_name into ret_code, ret_note; elsif old_flag = i_paused then select 201, 'Consumer ' || i_consumer_name || ' already ' || word into ret_code, ret_note; else update pgq_node.local_state set paused = i_paused, uptodate = false where queue_name = i_queue_name and consumer_name = i_consumer_name; select 200, 'Consumer '||i_consumer_name||' tagged as '||word into ret_code, ret_note; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_consumer_uptodate.sql000066400000000000000000000021541302127027000235230ustar00rootroot00000000000000 create or replace function pgq_node.set_consumer_uptodate( in i_queue_name text, in i_consumer_name text, in i_uptodate boolean, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_consumer_uptodate(3) -- -- Set consumer uptodate flag..... -- -- Parameters: -- i_queue_name - queue name -- i_consumer_name - consumer name -- i_uptodate - new flag state -- -- Returns: -- 200 - ok -- 404 - consumer not known -- ---------------------------------------------------------------------- begin update pgq_node.local_state set uptodate = i_uptodate where queue_name = i_queue_name and consumer_name = i_consumer_name; if found then select 200, 'Consumer uptodate = ' || i_uptodate::int4::text into ret_code, ret_note; else select 404, 'Consumer not known: ' || i_queue_name || '/' || i_consumer_name into ret_code, ret_note; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_global_watermark.sql000066400000000000000000000060551302127027000233040ustar00rootroot00000000000000 create or replace function pgq_node.set_global_watermark( in i_queue_name text, in i_watermark bigint, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_global_watermark(2) -- -- Move global watermark on branch/leaf, publish on root. -- -- Parameters: -- i_queue_name - queue name -- i_watermark - global tick_id that is processed everywhere. -- NULL on root, then local wm is published. -- ---------------------------------------------------------------------- declare this record; _wm bigint; wm_consumer text; begin wm_consumer = '.global_watermark'; select node_type, queue_name, worker_name into this from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Queue' || i_queue_name || ' not found' into ret_code, ret_note; return; end if; _wm = i_watermark; if this.node_type = 'root' then if i_watermark is null then select f.ret_code, f.ret_note, f.local_watermark into ret_code, ret_note, _wm from pgq_node.get_node_info(i_queue_name) f; if ret_code >= 300 then return; end if; if _wm is null then raise exception 'local_watermark=NULL from get_node_info()?'; end if; end if; -- move watermark perform pgq.register_consumer_at(i_queue_name, wm_consumer, _wm); -- send event downstream perform pgq.insert_event(i_queue_name, 'pgq.global-watermark', _wm::text, i_queue_name, null, null, null); -- update root workers pos to avoid it getting stale update pgq_node.local_state set last_tick_id = _wm where queue_name = i_queue_name and consumer_name = this.worker_name; elsif this.node_type = 'branch' then if i_watermark is null then select 500, 'bad usage: wm=null on branch node' into ret_code, ret_note; return; end if; -- tick can be missing if we are processing -- old batches that set watermark outside -- current range perform 1 from pgq.tick t, pgq.queue q where q.queue_name = i_queue_name and t.tick_queue = q.queue_id and t.tick_id = _wm; if not found then select 200, 'Skipping global watermark update to ' || _wm::text into ret_code, ret_note; return; end if; -- move watermark perform pgq.register_consumer_at(i_queue_name, wm_consumer, _wm); else select 100, 'Ignoring global watermark in leaf' into ret_code, ret_note; return; end if; select 200, 'Global watermark set to ' || _wm::text into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_node_attrs.sql000066400000000000000000000016241302127027000221260ustar00rootroot00000000000000 create or replace function pgq_node.set_node_attrs( in i_queue_name text, in i_node_attrs text, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.create_attrs(2) -- -- Set node attributes. -- -- Parameters: -- i_node_name - cascaded queue name -- i_node_attrs - urlencoded node attrs -- -- Returns: -- 200 - ok -- 404 - node not found -- ---------------------------------------------------------------------- begin update pgq_node.node_info set node_attrs = i_node_attrs where queue_name = i_queue_name; if not found then select 404, 'Node not found' into ret_code, ret_note; return; end if; select 200, 'Node attributes updated' into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_partition_watermark.sql000066400000000000000000000032571302127027000240560ustar00rootroot00000000000000 create or replace function pgq_node.set_partition_watermark( in i_combined_queue_name text, in i_part_queue_name text, in i_watermark bigint, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_partition_watermark(3) -- -- Move merge-leaf position on combined-branch. -- -- Parameters: -- i_combined_queue_name - local combined queue name -- i_part_queue_name - local part queue name (merge-leaf) -- i_watermark - partition tick_id that came inside combined-root batch -- -- Returns: -- 200 - success -- 201 - no partition queue -- 401 - worker registration not found -- ---------------------------------------------------------------------- declare n record; begin -- check if combined-branch exists select c.node_type, p.worker_name into n from pgq_node.node_info c, pgq_node.node_info p where p.queue_name = i_part_queue_name and c.queue_name = i_combined_queue_name and p.combined_queue = c.queue_name and p.node_type = 'leaf' and c.node_type = 'branch'; if not found then select 201, 'Part-queue does not exist' into ret_code, ret_note; return; end if; update pgq_node.local_state set last_tick_id = i_watermark where queue_name = i_part_queue_name and consumer_name = n.worker_name; if not found then select 401, 'Worker registration not found' into ret_code, ret_note; return; end if; select 200, 'Ok' into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.set_subscriber_watermark.sql000066400000000000000000000031571302127027000242070ustar00rootroot00000000000000 create or replace function pgq_node.set_subscriber_watermark( in i_queue_name text, in i_node_name text, in i_watermark bigint, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.set_subscriber_watermark(3) -- -- Notify provider about subscribers lowest watermark. -- -- Called on provider at interval by each worker -- -- Parameters: -- i_queue_name - cascaded queue name -- i_node_name - subscriber node name -- i_watermark - tick_id -- -- Returns: -- ret_code - error code -- ret_note - description -- ---------------------------------------------------------------------- declare n record; wm_name text; begin wm_name := '.' || i_node_name || '.watermark'; select * into n from pgq.get_consumer_info(i_queue_name, wm_name); if not found then select 404, 'node '||i_node_name||' not subscribed to queue ', i_queue_name into ret_code, ret_note; return; end if; -- todo: check if wm sane? if i_watermark < n.last_tick then select 405, 'watermark must not be moved backwards' into ret_code, ret_note; return; elsif i_watermark = n.last_tick then select 100, 'watermark already set' into ret_code, ret_note; return; end if; perform pgq.register_consumer_at(i_queue_name, wm_name, i_watermark); select 200, wm_name || ' set to ' || i_watermark::text into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.unregister_consumer.sql000066400000000000000000000022321302127027000232070ustar00rootroot00000000000000 create or replace function pgq_node.unregister_consumer( in i_queue_name text, in i_consumer_name text, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.unregister_consumer(2) -- -- Unregister cascaded consumer from local node. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_consumer_name - cascaded consumer name -- -- Returns: -- ret_code - error code -- 200 - ok -- 404 - no such queue -- ret_note - description -- ---------------------------------------------------------------------- begin perform 1 from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Unknown queue: ' || i_queue_name into ret_code, ret_note; return; end if; delete from pgq_node.local_state where queue_name = i_queue_name and consumer_name = i_consumer_name; select 200, 'Consumer '||i_consumer_name||' unregistered from '||i_queue_name into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.unregister_location.sql000066400000000000000000000044361302127027000231740ustar00rootroot00000000000000 create or replace function pgq_node.unregister_location( in i_queue_name text, in i_node_name text, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.unregister_location(2) -- -- Drop unreferenced node. -- -- Parameters: -- i_queue_name - queue name -- i_node_name - node to drop -- -- Returns: -- ret_code - error code -- ret_note - error description -- -- Return Codes: -- 200 - Ok -- 301 - Location not found -- 403 - Cannot drop nodes own or parent location -- ---------------------------------------------------------------------- declare _queue_name text; _wm_consumer text; _global_wm bigint; sub record; node record; begin select n.node_name, n.node_type, s.provider_node into node from pgq_node.node_info n left join pgq_node.local_state s on (s.consumer_name = n.worker_name and s.queue_name = n.queue_name) where n.queue_name = i_queue_name; if found then if node.node_name = i_node_name then select 403, 'Cannot drop nodes own location' into ret_code, ret_note; return; end if; if node.provider_node = i_node_name then select 403, 'Cannot drop location of nodes parent' into ret_code, ret_note; return; end if; end if; -- -- There may be obsolete subscriptions around -- drop them silently. -- perform pgq_node.unregister_subscriber(i_queue_name, i_node_name); -- -- Actual removal -- delete from pgq_node.node_location where queue_name = i_queue_name and node_name = i_node_name; if found then select 200, 'Ok' into ret_code, ret_note; else select 301, 'Location not found: ' || i_queue_name || '/' || i_node_name into ret_code, ret_note; end if; if node.node_type = 'root' then perform pgq.insert_event(i_queue_name, 'pgq.unregister-location', i_node_name, i_queue_name, null, null, null) from pgq_node.node_info n where n.queue_name = i_queue_name; end if; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.unregister_subscriber.sql000066400000000000000000000025741302127027000235300ustar00rootroot00000000000000 create or replace function pgq_node.unregister_subscriber( in i_queue_name text, in i_remote_node_name text, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.unregister_subscriber(2) -- -- Unsubscribe remote node from local node. -- -- Parameters: -- i_queue_name - set name -- i_remote_node_name - node name -- -- Returns: -- ret_code - error code -- ret_note - description -- ---------------------------------------------------------------------- declare n_wm_name text; worker_name text; begin n_wm_name := '.' || i_remote_node_name || '.watermark'; select s.worker_name into worker_name from pgq_node.subscriber_info s where queue_name = i_queue_name and subscriber_node = i_remote_node_name; if not found then select 304, 'Subscriber not found' into ret_code, ret_note; return; end if; delete from pgq_node.subscriber_info where queue_name = i_queue_name and subscriber_node = i_remote_node_name; perform pgq.unregister_consumer(i_queue_name, n_wm_name); perform pgq.unregister_consumer(i_queue_name, worker_name); select 200, 'Subscriber unregistered: '||i_remote_node_name into ret_code, ret_note; return; end; $$ language plpgsql security definer; pgq-node/functions/pgq_node.upgrade_schema.sql000066400000000000000000000007671302127027000220670ustar00rootroot00000000000000 create or replace function pgq_node.upgrade_schema() returns int4 as $$ -- updates table structure if necessary declare cnt int4 = 0; begin -- node_info.node_attrs perform 1 from information_schema.columns where table_schema = 'pgq_node' and table_name = 'node_info' and column_name = 'node_attrs'; if not found then alter table pgq_node.node_info add column node_attrs text; cnt := cnt + 1; end if; return cnt; end; $$ language plpgsql; pgq-node/functions/pgq_node.version.sql000066400000000000000000000006651302127027000206020ustar00rootroot00000000000000 create or replace function pgq_node.version() returns text as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.version(0) -- -- Returns version string for pgq_node. ATM it is based on SkyTools -- version and only bumped when database code changes. -- ---------------------------------------------------------------------- begin return '3.2.5'; end; $$ language plpgsql; pgq-node/mk/000077500000000000000000000000001302127027000131715ustar00rootroot00000000000000pgq-node/mk/catsql.py000077500000000000000000000066671302127027000150540ustar00rootroot00000000000000#! /usr/bin/env python """Prints out SQL files with psql command execution. Supported psql commands: \i, \cd, \q Others are skipped. Aditionally does some pre-processing for NDoc. NDoc is looks nice but needs some hand-holding. Bug: - function def end detection searches for 'as'/'is' but does not check word boundaries - finds them even in function name. That means in main conf, as/is must be disabled and $ ' added. This script can remove the unnecessary AS from output. Niceties: - Ndoc includes function def in output only if def is after comment. But for SQL functions its better to have it after def. This script can swap comment and def. - Optionally remove CREATE FUNCTION (OR REPLACE) from def to keep it shorter in doc. Note: - NDoc compares real function name and name in comment. if differ, it decides detection failed. """ import sys, os, re, getopt def usage(x): print("usage: catsql [--ndoc] FILE [FILE ...]") sys.exit(x) # NDoc specific changes cf_ndoc = 0 # compile regexes func_re = r"create\s+(or\s+replace\s+)?function\s+" func_rc = re.compile(func_re, re.I) comm_rc = re.compile(r"^\s*([#]\s*)?(?P--.*)", re.I) end_rc = re.compile(r"\b([;]|begin|declare|end)\b", re.I) as_rc = re.compile(r"\s+as\s+", re.I) cmd_rc = re.compile(r"^\\([a-z]*)(\s+.*)?", re.I) # conversion func def fix_func(ln): # if ndoc, replace AS with ' ' if cf_ndoc: return as_rc.sub(' ', ln) else: return ln # got function def def proc_func(f, ln): # remove CREATE OR REPLACE if cf_ndoc: ln = func_rc.sub('', ln) ln = fix_func(ln) pre_list = [ln] comm_list = [] while 1: ln = f.readline() if not ln: break com = None if cf_ndoc: com = comm_rc.search(ln) if cf_ndoc and com: pos = com.start('com') comm_list.append(ln[pos:]) elif end_rc.search(ln): break elif len(comm_list) > 0: break else: pre_list.append(fix_func(ln)) if len(comm_list) > 2: map(sys.stdout.write, comm_list) map(sys.stdout.write, pre_list) else: map(sys.stdout.write, pre_list) map(sys.stdout.write, comm_list) if ln: sys.stdout.write(fix_func(ln)) def cat_file(fn): sys.stdout.write("\n") f = open(fn) while 1: ln = f.readline() if not ln: break m = cmd_rc.search(ln) if m: cmd = m.group(1) if cmd == "i": # include a file fn2 = m.group(2).strip() cat_file(fn2) elif cmd == "q": # quit sys.exit(0) elif cmd == "cd": # chdir cd_dir = m.group(2).strip() os.chdir(cd_dir) else: # skip all others pass else: if func_rc.search(ln): # function header proc_func(f, ln) else: # normal sql sys.stdout.write(ln) sys.stdout.write("\n") def main(): global cf_ndoc try: opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['ndoc']) except getopt.error, d: print(str(d)) usage(1) for o, v in opts: if o == "-h": usage(0) elif o == "--ndoc": cf_ndoc = 1 for fn in args: cat_file(fn) if __name__ == '__main__': main() pgq-node/mk/common-pgxs.mk000066400000000000000000000072031302127027000157730ustar00rootroot00000000000000 # PGXS does not support modules that are supposed # to run on different Postgres versions very well. # Here are some workarounds for them. # Variables that are used when extensions are available Extension_data ?= Extension_data_built ?= $(EXTENSION)--$(EXT_VERSION).sql $(EXTENSION)--unpackaged--$(EXT_VERSION).sql Extension_regress ?= # Variables that are used when extensions are not available Contrib_data ?= Contrib_data_built += $(EXTENSION).sql $(EXTENSION).upgrade.sql \ structure/newgrants_$(EXTENSION).sql \ structure/oldgrants_$(EXTENSION).sql Contrib_regress ?= EXT_VERSION ?= EXT_OLD_VERSIONS ?= Extension_upgrade_files = $(if $(EXT_OLD_VERSIONS),$(foreach v,$(EXT_OLD_VERSIONS),$(EXTENSION)--$(v)--$(EXT_VERSION).sql)) Extension_data_built += $(Extension_upgrade_files) # Should the Contrib* files installed (under ../contrib/) # even when extensions are available? Contrib_install_always ?= yes # # switch variables # IfExt = $(if $(filter 8.% 9.0%,$(MAJORVERSION)8.3),$(2),$(1)) DATA = $(call IfExt,$(Extension_data),$(Contrib_data)) DATA_built = $(call IfExt,$(Extension_data_built),$(Contrib_data_built)) REGRESS = $(call IfExt,$(Extension_regress),$(Contrib_regress)) EXTRA_CLEAN += $(call IfExt,$(Contrib_data_built),$(Extension_data_built)) test.dump # have deterministic dbname for regtest database override CONTRIB_TESTDB = regression REGRESS_OPTS = --load-language=plpgsql --dbname=$(CONTRIB_TESTDB) # # Calculate actual sql files # GRANT_SQL = structure/newgrants_$(EXTENSION).sql SQLS = $(shell $(AWK) '/^\\i / { print $$2; }' structure/install.sql) FUNCS = $(shell $(AWK) '/^\\i / { print $$2; }' $(SQLS)) SRCS = $(SQLS) $(FUNCS) $(GRANT_SQL) # # load PGXS # PG_CONFIG ?= pg_config PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # when compiling locally and with postgres without python, # the variable may be empty PYTHON := $(if $(PYTHON),$(PYTHON),python) # # common tools # NDOC = NaturalDocs NDOCARGS = -r -o html docs/html -p docs -i docs/sql CATSQL = $(PYTHON) mk/catsql.py GRANTFU = $(PYTHON) mk/grantfu.py # # build rules, in case Contrib data must be always installed # ifeq ($(call IfExt,$(Contrib_install_always),no),yes) all: $(Contrib_data) $(Contrib_data_built) installdirs: installdirs-old-contrib install: install-old-contrib installdirs-old-contrib: $(MKDIR_P) '$(DESTDIR)$(datadir)/contrib' install-old-contrib: $(Contrib_data) $(Contrib_data_built) installdirs-old-contrib $(INSTALL_DATA) $(addprefix $(srcdir)/, $(Contrib_data)) $(Contrib_data_built) '$(DESTDIR)$(datadir)/contrib/' endif # # regtest shortcuts # test: install $(MAKE) installcheck || { filterdiff --format=unified regression.diffs | less; exit 1; } pg_dump regression > test.dump ack: cp results/*.out expected/ cleandox: rm -rf docs/html docs/Data docs/sql clean: cleandox .PHONY: test ack installdirs-old-contrib install-old-contrib cleandox dox # # common files # $(EXTENSION)--$(EXT_VERSION).sql: $(EXTENSION).sql structure/ext_postproc.sql $(CATSQL) $^ > $@ $(EXTENSION)--unpackaged--$(EXT_VERSION).sql: $(EXTENSION).upgrade.sql structure/ext_unpackaged.sql structure/ext_postproc.sql $(CATSQL) $^ > $@ $(EXTENSION).sql: $(SRCS) $(CATSQL) structure/install.sql $(GRANT_SQL) > $@ $(EXTENSION).upgrade.sql: $(SRCS) $(CATSQL) structure/upgrade.sql $(GRANT_SQL) > $@ ifneq ($(Extension_upgrade_files),) $(Extension_upgrade_files): $(EXTENSION).upgrade.sql cp $< $@ endif structure/newgrants_$(EXTENSION).sql: structure/grants.ini $(GRANTFU) -r -d $< > $@ structure/oldgrants_$(EXTENSION).sql: structure/grants.ini structure/grants.sql echo "begin;" > $@ $(GRANTFU) -R -o $< >> $@ cat structure/grants.sql >> $@ echo "commit;" >> $@ pgq-node/mk/grantfu.py000077500000000000000000000260021302127027000152140ustar00rootroot00000000000000#! /usr/bin/env python # GrantFu - GRANT/REVOKE generator for Postgres # # Copyright (c) 2005 Marko Kreen # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """Generator for PostgreSQL permissions. Loads config where roles, objects and their mapping is described and generates grants based on them. ConfigParser docs: http://docs.python.org/lib/module-ConfigParser.html Example: -------------------------------------------------------------------- [DEFAULT] users = user1, user2 # users to handle groups = group1, group2 # groups to handle auto_seq = 0 # dont handle seqs (default) # '!' after a table negates this setting for a table seq_name = id # the name for serial field (default: id) seq_usage = 0 # should we grant "usage" or "select, update" # for automatically handled sequences # section names can be random, but if you want to see them # in same order as in config file, then order them alphabetically [1.section] on.tables = testtbl, testtbl_id_seq, # here we handle seq by hand table_with_seq! # handle seq automatically # (table_with_seq_id_seq) user1 = select group1 = select, insert, update # instead of 'tables', you may use 'functions', 'languages', # 'schemas', 'tablespaces' --------------------------------------------------------------------- """ import sys, os, getopt from ConfigParser import SafeConfigParser __version__ = "1.0" R_NEW = 0x01 R_DEFS = 0x02 G_DEFS = 0x04 R_ONLY = 0x80 def usage(err): sys.stderr.write("usage: %s [-r|-R] CONF_FILE\n" % sys.argv[0]) sys.stderr.write(" -r Generate also REVOKE commands\n") sys.stderr.write(" -R Generate only REVOKE commands\n") sys.stderr.write(" -d Also REVOKE default perms\n") sys.stderr.write(" -D Only REVOKE default perms\n") sys.stderr.write(" -o Generate default GRANTS\n") sys.stderr.write(" -v Print program version\n") sys.stderr.write(" -t Put everything in one big transaction\n") sys.exit(err) class PConf(SafeConfigParser): "List support for ConfigParser" def __init__(self, defaults = None): SafeConfigParser.__init__(self, defaults) def get_list(self, sect, key): str = self.get(sect, key).strip() res = [] if not str: return res for val in str.split(","): res.append(val.strip()) return res class GrantFu: def __init__(self, cf, revoke): self.cf = cf self.revoke = revoke # avoid putting grantfu vars into defaults, thus into every section self.group_list = [] self.user_list = [] self.auto_seq = 0 self.seq_name = "id" self.seq_usage = 0 if self.cf.has_option('GrantFu', 'groups'): self.group_list = self.cf.get_list('GrantFu', 'groups') if self.cf.has_option('GrantFu', 'users'): self.user_list += self.cf.get_list('GrantFu', 'users') if self.cf.has_option('GrantFu', 'roles'): self.user_list += self.cf.get_list('GrantFu', 'roles') if self.cf.has_option('GrantFu', 'auto_seq'): self.auto_seq = self.cf.getint('GrantFu', 'auto_seq') if self.cf.has_option('GrantFu', 'seq_name'): self.seq_name = self.cf.get('GrantFu', 'seq_name') if self.cf.has_option('GrantFu', 'seq_usage'): self.seq_usage = self.cf.getint('GrantFu', 'seq_usage') # make string of all subjects tmp = [] for g in self.group_list: tmp.append("group " + g) for u in self.user_list: tmp.append(u) self.all_subjs = ", ".join(tmp) # per-section vars self.sect = None self.seq_list = [] self.seq_allowed = [] def process(self): if len(self.user_list) == 0 and len(self.group_list) == 0: return sect_list = self.cf.sections() sect_list.sort() for self.sect in sect_list: if self.sect == "GrantFu": continue print "\n-- %s --" % self.sect self.handle_tables() self.handle_other('on.databases', 'DATABASE') self.handle_other('on.functions', 'FUNCTION') self.handle_other('on.languages', 'LANGUAGE') self.handle_other('on.schemas', 'SCHEMA') self.handle_other('on.tablespaces', 'TABLESPACE') self.handle_other('on.sequences', 'SEQUENCE') self.handle_other('on.types', 'TYPE') self.handle_other('on.domains', 'DOMAIN') def handle_other(self, listname, obj_type): """Handle grants for all objects except tables.""" if not self.sect_hasvar(listname): return # don't parse list, as in case of functions it may be complicated obj_str = obj_type + " " + self.sect_var(listname) if self.revoke & R_NEW: self.gen_revoke(obj_str) if self.revoke & R_DEFS: self.gen_revoke_defs(obj_str, obj_type) if not self.revoke & R_ONLY: self.gen_one_type(obj_str) if self.revoke & G_DEFS: self.gen_defs(obj_str, obj_type) def handle_tables(self): """Handle grants for tables and sequences. The tricky part here is the automatic handling of sequences.""" if not self.sect_hasvar('on.tables'): return cleaned_list = [] table_list = self.sect_list('on.tables') for table in table_list: if table[-1] == '!': table = table[:-1] if not self.auto_seq: self.seq_list.append("%s_%s_seq" % (table, self.seq_name)) else: if self.auto_seq: self.seq_list.append("%s_%s_seq" % (table, self.seq_name)) cleaned_list.append(table) obj_str = "TABLE " + ", ".join(cleaned_list) if self.revoke & R_NEW: self.gen_revoke(obj_str) if self.revoke & R_DEFS: self.gen_revoke_defs(obj_str, "TABLE") if not self.revoke & R_ONLY: self.gen_one_type(obj_str) if self.revoke & G_DEFS: self.gen_defs(obj_str, "TABLE") # cleanup self.seq_list = [] self.seq_allowed = [] def gen_revoke(self, obj_str): "Generate revoke for one section / subject type (user or group)" if len(self.seq_list) > 0: obj_str += ", " + ", ".join(self.seq_list) obj_str = obj_str.strip().replace('\n', '\n ') print "REVOKE ALL ON %s\n FROM %s CASCADE;" % (obj_str, self.all_subjs) def gen_revoke_defs(self, obj_str, obj_type): "Generate revoke defaults for one section" # process only things that have default grants to public if obj_type not in ('FUNCTION', 'DATABASE', 'LANGUAGE', 'TYPE', 'DOMAIN'): return defrole = 'public' # if the sections contains grants to 'public', dont drop if self.sect_hasvar(defrole): return obj_str = obj_str.strip().replace('\n', '\n ') print "REVOKE ALL ON %s\n FROM %s CASCADE;" % (obj_str, defrole) def gen_defs(self, obj_str, obj_type): "Generate defaults grants for one section" if obj_type == "FUNCTION": defgrants = "execute" elif obj_type == "DATABASE": defgrants = "connect, temp" elif obj_type in ("LANGUAGE", "TYPE", "DOMAIN"): defgrants = "usage" else: return defrole = 'public' obj_str = obj_str.strip().replace('\n', '\n ') print "GRANT %s ON %s\n TO %s;" % (defgrants, obj_str, defrole) def gen_one_subj(self, subj, fqsubj, obj_str): if not self.sect_hasvar(subj): return obj_str = obj_str.strip().replace('\n', '\n ') perm = self.sect_var(subj).strip() if perm: print "GRANT %s ON %s\n TO %s;" % (perm, obj_str, fqsubj) # check for seq perms if len(self.seq_list) > 0: loperm = perm.lower() if loperm.find("insert") >= 0 or loperm.find("all") >= 0: self.seq_allowed.append(fqsubj) def gen_one_type(self, obj_str): "Generate GRANT for one section / one object type in section" for u in self.user_list: self.gen_one_subj(u, u, obj_str) for g in self.group_list: self.gen_one_subj(g, "group " + g, obj_str) # if there was any seq perms, generate grants if len(self.seq_allowed) > 0: seq_str = ", ".join(self.seq_list) subj_str = ", ".join(self.seq_allowed) if self.seq_usage: cmd = "GRANT usage ON SEQUENCE %s\n TO %s;" else: cmd = "GRANT select, update ON %s\n TO %s;" print cmd % (seq_str, subj_str) def sect_var(self, name): return self.cf.get(self.sect, name).strip() def sect_list(self, name): return self.cf.get_list(self.sect, name) def sect_hasvar(self, name): return self.cf.has_option(self.sect, name) def main(): revoke = 0 tx = False try: opts, args = getopt.getopt(sys.argv[1:], "vhrRdDot") except getopt.error, det: print "getopt error:", det usage(1) for o, v in opts: if o == "-h": usage(0) elif o == "-r": revoke |= R_NEW elif o == "-R": revoke |= R_NEW | R_ONLY elif o == "-d": revoke |= R_DEFS elif o == "-D": revoke |= R_DEFS | R_ONLY elif o == "-o": revoke |= G_DEFS elif o == "-t": tx = True elif o == "-v": print "GrantFu version", __version__ sys.exit(0) if len(args) != 1: usage(1) # load config cf = PConf() cf.read(args[0]) if not cf.has_section("GrantFu"): print "Incorrect config file, GrantFu sction missing" sys.exit(1) if tx: print "begin;\n" # revokes and default grants if revoke & (R_NEW | R_DEFS): g = GrantFu(cf, revoke | R_ONLY) g.process() revoke = revoke & R_ONLY # grants if revoke & R_ONLY == 0: g = GrantFu(cf, revoke & G_DEFS) g.process() if tx: print "\ncommit;\n" if __name__ == '__main__': main() pgq-node/pgq_node.control000066400000000000000000000002331302127027000157560ustar00rootroot00000000000000# pgq_node comment = 'Cascaded queue infrastructure' default_version = '3.2.5' relocatable = false superuser = true schema = 'pg_catalog' requires = 'pgq' pgq-node/sql/000077500000000000000000000000001302127027000133615ustar00rootroot00000000000000pgq-node/sql/pgq_node_init_ext.sql000066400000000000000000000005751302127027000176100ustar00rootroot00000000000000 create extension pgq; \set ECHO none \i structure/install.sql \set ECHO all create extension pgq_node from unpackaged; select array_length(extconfig, 1) as dumpable from pg_catalog.pg_extension where extname = 'pgq_node'; drop extension pgq_node; create extension pgq_node; select array_length(extconfig, 1) as dumpable from pg_catalog.pg_extension where extname = 'pgq_node'; pgq-node/sql/pgq_node_init_noext.sql000066400000000000000000000001261302127027000201350ustar00rootroot00000000000000 \set ECHO none \i ../pgq/pgq.sql \i structure/tables.sql \i structure/functions.sql pgq-node/sql/pgq_node_test.sql000066400000000000000000000130571302127027000167430ustar00rootroot00000000000000 select * from pgq_node.register_location('aqueue', 'node1', 'dbname=node1', false); select * from pgq_node.register_location('aqueue', 'node2', 'dbname=node2', false); select * from pgq_node.register_location('aqueue', 'node3', 'dbname=node3', false); select * from pgq_node.register_location('aqueue', 'node4', 'dbname=node44', false); select * from pgq_node.register_location('aqueue', 'node4', 'dbname=node4', false); select * from pgq_node.register_location('aqueue', 'node5', 'dbname=node4', false); select * from pgq_node.get_queue_locations('aqueue'); select * from pgq_node.unregister_location('aqueue', 'node5'); select * from pgq_node.unregister_location('aqueue', 'node5'); select * from pgq_node.get_queue_locations('aqueue'); select * from pgq_node.create_node('aqueue', 'root', 'node1', 'node1_worker', null, null, null); select * from pgq_node.register_subscriber('aqueue', 'node2', 'node2_worker', null); select * from pgq_node.register_subscriber('aqueue', 'node3', 'node3_worker', null); select * from pgq_node.maint_watermark('aqueue'); select * from pgq_node.maint_watermark('aqueue-x'); select * from pgq_node.get_consumer_info('aqueue'); select * from pgq_node.unregister_subscriber('aqueue', 'node3'); select queue_name, consumer_name, last_tick from pgq.get_consumer_info(); select * from pgq_node.get_worker_state('aqueue'); update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0'; select * from pgq.ticker('aqueue'); select * from pgq.ticker('aqueue'); select * from pgq_node.set_subscriber_watermark('aqueue', 'node2', 3); select queue_name, consumer_name, last_tick from pgq.get_consumer_info(); select * from pgq_node.set_node_attrs('aqueue', 'test=1'); select * from pgq_node.get_node_info('aqueue'); select * from pgq_node.get_subscriber_info('aqueue'); -- branch node select * from pgq_node.register_location('bqueue', 'node1', 'dbname=node1', false); select * from pgq_node.register_location('bqueue', 'node2', 'dbname=node2', false); select * from pgq_node.register_location('bqueue', 'node3', 'dbname=node3', false); select * from pgq_node.create_node('bqueue', 'branch', 'node2', 'node2_worker', 'node1', 1, null); select * from pgq_node.register_consumer('bqueue', 'random_consumer', 'node1', 1); select * from pgq_node.register_consumer('bqueue', 'random_consumer2', 'node1', 1); select * from pgq_node.local_state; select * from pgq_node.node_info; select * from pgq_node.get_node_info('aqueue'); select * from pgq_node.get_node_info('bqueue'); select * from pgq_node.get_node_info('cqueue'); select * from pgq_node.get_worker_state('aqueue'); select * from pgq_node.get_worker_state('bqueue'); select * from pgq_node.get_worker_state('cqueue'); select * from pgq_node.is_root_node('aqueue'); select * from pgq_node.is_root_node('bqueue'); select * from pgq_node.is_root_node('cqueue'); select * from pgq_node.get_consumer_state('bqueue', 'random_consumer'); select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); select * from pgq_node.set_consumer_error('bqueue', 'random_consumer2', 'failure'); select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); select * from pgq_node.set_consumer_completed('bqueue', 'random_consumer2', 2); select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); select * from pgq_node.set_consumer_paused('bqueue', 'random_consumer2', true); select * from pgq_node.set_consumer_uptodate('bqueue', 'random_consumer2', true); select * from pgq_node.change_consumer_provider('bqueue', 'random_consumer2', 'node3'); select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); select * from pgq_node.unregister_consumer('bqueue', 'random_consumer2'); select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); select * from pgq_node.get_node_info('bqueue'); set session_replication_role = 'replica'; select * from pgq_node.demote_root('aqueue', 1, 'node3'); select * from pgq_node.demote_root('aqueue', 1, 'node3'); select * from pgq_node.demote_root('aqueue', 2, 'node3'); select * from pgq_node.demote_root('aqueue', 2, 'node3'); select * from pgq_node.demote_root('aqueue', 3, 'node3'); select * from pgq_node.demote_root('aqueue', 3, 'node3'); -- leaf node select * from pgq_node.register_location('mqueue', 'node1', 'dbname=node1', false); select * from pgq_node.register_location('mqueue', 'node2', 'dbname=node2', false); select * from pgq_node.register_location('mqueue', 'node3', 'dbname=node3', false); select * from pgq_node.create_node('mqueue', 'leaf', 'node2', 'node2_worker', 'node1', 13, 'aqueue'); select * from pgq_node.get_worker_state('mqueue'); select * from pgq_node.drop_node('asd', 'asd'); select * from pgq_node.drop_node('mqueue', 'node3'); select * from pgq_node.drop_node('mqueue', 'node2'); select * from pgq_node.drop_node('mqueue', 'node1'); select * from pgq_node.drop_node('aqueue', 'node5'); select * from pgq_node.drop_node('aqueue', 'node4'); select * from pgq_node.drop_node('aqueue', 'node1'); select * from pgq_node.drop_node('aqueue', 'node2'); select * from pgq_node.drop_node('aqueue', 'node3'); \q select * from pgq_node.subscribe_node('aqueue', 'node2'); select * from pgq_node.subscribe_node('aqueue', 'node3', 1); select * from pgq_node.unsubscribe_node('aqueue', 'node3'); select * from pgq_node.get_node_info('aqueue'); select * from pgq_node.is_root('q'); select * from pgq_node.is_root('aqueue'); select * from pgq_node.is_root(null); select * from pgq_node.rename_node_step1('aqueue', 'node2', 'node2x'); select * from pgq_node.rename_node_step2('aqueue', 'node2', 'node2x'); select * from pgq_node.get_subscriber_info('aqueue'); pgq-node/structure/000077500000000000000000000000001302127027000146225ustar00rootroot00000000000000pgq-node/structure/ext_postproc.sql000066400000000000000000000005101302127027000200700ustar00rootroot00000000000000 -- tag data objects as dumpable SELECT pg_catalog.pg_extension_config_dump('pgq_node.node_location', ''); SELECT pg_catalog.pg_extension_config_dump('pgq_node.node_info', ''); SELECT pg_catalog.pg_extension_config_dump('pgq_node.local_state', ''); SELECT pg_catalog.pg_extension_config_dump('pgq_node.subscriber_info', ''); pgq-node/structure/ext_unpackaged.sql000066400000000000000000000004311302127027000203230ustar00rootroot00000000000000 ALTER EXTENSION pgq_node ADD SCHEMA pgq_node; ALTER EXTENSION pgq_node ADD TABLE pgq_node.node_location; ALTER EXTENSION pgq_node ADD TABLE pgq_node.node_info; ALTER EXTENSION pgq_node ADD TABLE pgq_node.local_state; ALTER EXTENSION pgq_node ADD TABLE pgq_node.subscriber_info; pgq-node/structure/functions.sql000066400000000000000000000063671302127027000173670ustar00rootroot00000000000000-- File: Functions -- -- Database functions for cascaded pgq. -- -- Cascaded consumer flow: -- -- - (1) [target] call pgq_node.get_consumer_state() -- - (2) If .paused is true, sleep, go to (1). -- This is allows to control consumer remotely. -- - (3) If .uptodate is false, call pgq_node.set_consumer_uptodate(true). -- This allows remote controller to know that consumer has seen the changes. -- - (4) [source] call pgq.next_batch(). If returns NULL, sleep, goto (1) -- - (5) [source] if batch already done, call pgq.finish_batch(), go to (1) -- - (6) [source] read events -- - (7) [target] process events, call pgq_node.set_consumer_completed() in same tx. -- - (8) [source] call pgq.finish_batch() -- -- Cascaded worker flow: -- -- Worker is consumer that also copies to queue contents to local node (branch), -- so it can act as provider to other nodes. There can be only one worker per -- node. Or zero if node is leaf. In addition to cascaded consumer logic above, it must - -- - [branch] copy all events to local queue and create ticks -- - [merge-leaf] copy all events to combined-queue -- - [branch] publish local watermark upwards to provider so it reaches root. -- - [branch] apply global watermark event to local node -- - [merge-leaf] wait-behind on combined-branch (failover combined-root). -- It's last_tick_id is set by combined-branch worker, it must call -- pgq.next_batch()+pgq.finish_batch() without processing events -- when behind, but not move further. When the combined-branch -- becomes root, it will be in right position to continue updating. -- \i functions/pgq_node.upgrade_schema.sql select pgq_node.upgrade_schema(); -- Group: Global Node Map \i functions/pgq_node.register_location.sql \i functions/pgq_node.unregister_location.sql \i functions/pgq_node.get_queue_locations.sql -- Group: Node operations \i functions/pgq_node.create_node.sql \i functions/pgq_node.drop_node.sql -- \i functions/pgq_node.rename_node.sql \i functions/pgq_node.get_node_info.sql \i functions/pgq_node.is_root_node.sql \i functions/pgq_node.is_leaf_node.sql \i functions/pgq_node.get_subscriber_info.sql \i functions/pgq_node.get_consumer_info.sql \i functions/pgq_node.demote_root.sql \i functions/pgq_node.promote_branch.sql \i functions/pgq_node.set_node_attrs.sql -- Group: Provider side operations - worker \i functions/pgq_node.register_subscriber.sql \i functions/pgq_node.unregister_subscriber.sql \i functions/pgq_node.set_subscriber_watermark.sql -- Group: Subscriber side operations - worker \i functions/pgq_node.get_worker_state.sql \i functions/pgq_node.set_global_watermark.sql \i functions/pgq_node.set_partition_watermark.sql -- Group: Subscriber side operations - any consumer \i functions/pgq_node.register_consumer.sql \i functions/pgq_node.unregister_consumer.sql \i functions/pgq_node.get_consumer_state.sql \i functions/pgq_node.change_consumer_provider.sql \i functions/pgq_node.set_consumer_uptodate.sql \i functions/pgq_node.set_consumer_paused.sql \i functions/pgq_node.set_consumer_completed.sql \i functions/pgq_node.set_consumer_error.sql -- Group: Maintenance operations \i functions/pgq_node.maint_watermark.sql \i functions/pgq_node.version.sql pgq-node/structure/grants.ini000066400000000000000000000043471302127027000166310ustar00rootroot00000000000000[GrantFu] roles = pgq_writer, pgq_admin, pgq_reader, public [1.public.fns] on.functions = %(pgq_node_public_fns)s public = execute # cascaded consumer, target side [2.consumer.fns] on.functions = %(pgq_node_consumer_fns)s pgq_writer = execute pgq_admin = execute # cascaded worker, target side [3.worker.fns] on.functions = %(pgq_node_worker_fns)s pgq_admin = execute # cascaded consumer/worker, source side [4.remote.fns] on.functions = %(pgq_node_remote_fns)s pgq_reader = execute pgq_writer = execute pgq_admin = execute # called by ticker, upgrade script [4.admin.fns] on.functions = %(pgq_node_admin_fns)s pgq_admin = execute [5.tables] pgq_reader = select pgq_writer = select pgq_admin = select, insert, update, delete on.tables = pgq_node.node_location, pgq_node.node_info, pgq_node.local_state, pgq_node.subscriber_info # define various groups of functions [DEFAULT] pgq_node_remote_fns = pgq_node.get_consumer_info(text), pgq_node.get_consumer_state(text, text), pgq_node.get_queue_locations(text), pgq_node.get_node_info(text), pgq_node.get_subscriber_info(text), pgq_node.register_subscriber(text, text, text, int8), pgq_node.unregister_subscriber(text, text), pgq_node.set_subscriber_watermark(text, text, bigint) pgq_node_public_fns = pgq_node.is_root_node(text), pgq_node.is_leaf_node(text), pgq_node.version() pgq_node_admin_fns = pgq_node.register_location(text, text, text, boolean), pgq_node.unregister_location(text, text), pgq_node.upgrade_schema(), pgq_node.maint_watermark(text) pgq_node_consumer_fns = pgq_node.register_consumer(text, text, text, int8), pgq_node.unregister_consumer(text, text), pgq_node.change_consumer_provider(text, text, text), pgq_node.set_consumer_uptodate(text, text, boolean), pgq_node.set_consumer_paused(text, text, boolean), pgq_node.set_consumer_completed(text, text, int8), pgq_node.set_consumer_error(text, text, text) pgq_node_worker_fns = pgq_node.create_node(text, text, text, text, text, bigint, text), pgq_node.drop_node(text, text), pgq_node.demote_root(text, int4, text), pgq_node.promote_branch(text), pgq_node.set_node_attrs(text, text), pgq_node.get_worker_state(text), pgq_node.set_global_watermark(text, bigint), pgq_node.set_partition_watermark(text, text, bigint) pgq-node/structure/grants.sql000066400000000000000000000000541302127027000166400ustar00rootroot00000000000000 grant usage on schema pgq_node to public; pgq-node/structure/install.sql000066400000000000000000000001151302127027000170060ustar00rootroot00000000000000 \i structure/tables.sql \i structure/functions.sql \i structure/grants.sql pgq-node/structure/tables.sql000066400000000000000000000117641302127027000166260ustar00rootroot00000000000000-- ---------------------------------------------------------------------- -- File: Tables -- -- Schema 'pgq_node', contains tables for cascaded pgq. -- -- Event types for cascaded queue: -- pgq.location-info - ev_data: node_name, extra1: queue_name, extra2: location, extra3: dead -- It contains updated node connect string. -- -- pgq.global-watermark - ev_data: tick_id, extra1: queue_name -- Root node sends minimal tick_id that must be kept. -- -- pgq.tick-id - ev_data: tick_id, extra1: queue_name -- Partition node inserts its tick-id into combined queue. -- -- ---------------------------------------------------------------------- create schema pgq_node; -- ---------------------------------------------------------------------- -- Table: pgq_node.node_location -- -- Static table that just lists all members in set. -- -- Columns: -- queue_name - cascaded queue name -- node_name - node name -- node_location - libpq connect string for connecting to node -- dead - whether the node is offline -- ---------------------------------------------------------------------- create table pgq_node.node_location ( queue_name text not null, node_name text not null, node_location text not null, dead boolean not null default false, primary key (queue_name, node_name) ); -- ---------------------------------------------------------------------- -- Table: pgq_node.node_info -- -- Local node info. -- -- Columns: -- queue_name - cascaded queue name -- node_type - local node type -- node_name - local node name -- worker_name - consumer name that maintains this node -- combined_queue - on 'leaf' the target combined set name -- node_attrs - urlencoded fields for worker -- -- Node types: -- root - data + batches is generated here -- branch - replicates full queue contents and maybe contains some tables -- leaf - does not replicate queue / or uses combined queue for that -- ---------------------------------------------------------------------- create table pgq_node.node_info ( queue_name text not null primary key, node_type text not null, node_name text not null, worker_name text, combined_queue text, node_attrs text, foreign key (queue_name, node_name) references pgq_node.node_location, check (node_type in ('root', 'branch', 'leaf')), check (case when node_type = 'root' then (worker_name is not null and combined_queue is null) when node_type = 'branch' then (worker_name is not null and combined_queue is null) when node_type = 'leaf' then (worker_name is not null) else false end) ); -- ---------------------------------------------------------------------- -- Table: pgq_node.local_state -- -- All cascaded consumers (both worker and non-worker) -- keep their state here. -- -- Columns: -- queue_name - cascaded queue name -- consumer_name - cascaded consumer name -- provider_node - node name the consumer reads from -- last_tick_id - last committed tick id on this node -- cur_error - reason why current batch failed -- paused - whether consumer should wait -- uptodate - if consumer has seen new state -- ---------------------------------------------------------------------- create table pgq_node.local_state ( queue_name text not null, consumer_name text not null, provider_node text not null, last_tick_id bigint not null, cur_error text, paused boolean not null default false, uptodate boolean not null default false, primary key (queue_name, consumer_name), foreign key (queue_name) references pgq_node.node_info, foreign key (queue_name, provider_node) references pgq_node.node_location ); -- ---------------------------------------------------------------------- -- Table: pgq_node.subscriber_info -- -- List of nodes that subscribe to local node. -- -- Columns: -- queue_name - cascaded queue name -- subscriber_node - node name that uses this node as provider. -- worker_name - consumer name that maintains remote node -- ---------------------------------------------------------------------- create table pgq_node.subscriber_info ( queue_name text not null, subscriber_node text not null, worker_name text not null, watermark_name text not null, primary key (queue_name, subscriber_node), foreign key (queue_name) references pgq_node.node_info, foreign key (queue_name, subscriber_node) references pgq_node.node_location, foreign key (worker_name) references pgq.consumer (co_name), foreign key (watermark_name) references pgq.consumer (co_name) ); pgq-node/structure/upgrade.sql000066400000000000000000000000331302127027000167660ustar00rootroot00000000000000\i structure/functions.sql