pax_global_header00006660000000000000000000000064143215323530014513gustar00rootroot0000000000000052 comment=55085fc4e8e6cb00e2b6255427fd86c50cd6fef3 londiste-sql-3.8/000077500000000000000000000000001432153235300140035ustar00rootroot00000000000000londiste-sql-3.8/.github/000077500000000000000000000000001432153235300153435ustar00rootroot00000000000000londiste-sql-3.8/.github/workflows/000077500000000000000000000000001432153235300174005ustar00rootroot00000000000000londiste-sql-3.8/.github/workflows/ci.yml000066400000000000000000000044601432153235300205220ustar00rootroot00000000000000# # https://docs.github.com/en/actions # https://github.com/actions # name: CI on: pull_request: {} push: {} jobs: linux: name: "Ubuntu 20.04 + PostgreSQL ${{matrix.PGVER}}" runs-on: ubuntu-20.04 strategy: matrix: PGVER: [10, 11, 12, 13, 14] steps: - name: "Checkout" uses: actions/checkout@v2 - name: "InstallDB" run: | echo "::group::apt-get-update" echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main ${{matrix.PGVER}}" \ | sudo tee /etc/apt/sources.list.d/pgdg.list sudo -nH apt-get -q update echo "::endgroup::" echo "::group::apt-get-install" # disable new cluster creation sudo -nH mkdir -p /etc/postgresql-common/createcluster.d echo "create_main_cluster = false" | sudo -nH tee /etc/postgresql-common/createcluster.d/no-main.conf sudo -nH apt-get -qyu install \ postgresql-${{matrix.PGVER}} \ postgresql-server-dev-${{matrix.PGVER}} \ libpq-dev patchutils echo "::endgroup::" # tune environment echo "/usr/lib/postgresql/${{matrix.PGVER}}/bin" >> $GITHUB_PATH echo "PGHOST=/tmp" >> $GITHUB_ENV dpkg -l postgres\* libpq\* bison\* flex\* gcc\* clang\* libllvm\* - name: "Build" run: make - name: "Install" run: | echo "::group::install-pgq" git clone -q https://github.com/pgq/pgq; make -C pgq sudo -nH bash -c "PATH='${PATH}' make install -C pgq" echo "::endgroup::" echo "::group::install-pgq-node" git clone -q https://github.com/pgq/pgq-node; make -C pgq-node sudo -nH bash -c "PATH='${PATH}' make install -C pgq-node" echo "::endgroup::" sudo -nH bash -c "PATH='${PATH}' make install" - name: "StartDB" run: | rm -rf data log mkdir -p log LANG=C initdb data sed -ri -e "s,^[# ]*(unix_socket_directories).*,\\1='/tmp'," data/postgresql.conf pg_ctl -D data -l log/pg.log start || { cat log/pg.log ; exit 1; } - name: "Test" run: make citest - name: "StopDB" run: | pg_ctl -D data stop rm -rf data log /tmp/.s.PGSQL* londiste-sql-3.8/.github/workflows/release.yml000066400000000000000000000027051432153235300215470ustar00rootroot00000000000000 name: REL on: push: tags: ["v[0-9]*"] jobs: release: name: Release runs-on: ubuntu-latest steps: - name: Checkout code id: checkout uses: actions/checkout@v2 - name: Build tarball id: build run: | make checkver make dist EXTENSION=$(grep ^EXTENSION Makefile | sed 's/.*= *//') EXT_VERSION=$(grep ^EXT_VERSION Makefile | sed 's/.*= *//') test "${{github.ref}}" = "refs/tags/v${EXT_VERSION}" || { echo "ERR: tag mismatch"; exit 1; } echo "EXTENSION=${EXTENSION}" >> $GITHUB_ENV echo "EXT_VERSION=${EXT_VERSION}" >> $GITHUB_ENV echo "TGZ=${EXTENSION}-${EXT_VERSION}.tar.gz" >> $GITHUB_ENV - name: Create release id: release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} with: tag_name: ${{github.ref}} release_name: ${{github.event.repository.name}} v${{env.EXT_VERSION}} body_path: docs/notes/v${{env.EXT_VERSION}}.md draft: false prerelease: false - name: Upload source id: upload uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} with: upload_url: ${{steps.release.outputs.upload_url}} asset_path: ${{env.TGZ}} asset_name: ${{env.TGZ}} asset_content_type: application/x-gzip londiste-sql-3.8/.gitignore000066400000000000000000000004441432153235300157750ustar00rootroot00000000000000*.o *.so .deps *.swp *.dump *--* *.gz sql/*/*.sql sql/*/results regression.* newgrants_*.sql oldgrants_*.sql debian/control debian/tmp debian/files debian/postgresql-* debian/*-stamp *.substvars *.log *.debhelper londiste.sql londiste.upgrade.sql results/ docs/Data docs/html docs/sql londiste-sql-3.8/Makefile000066400000000000000000000011551432153235300154450ustar00rootroot00000000000000 EXTENSION = londiste EXT_VERSION = 3.8 EXT_OLD_VERSIONS = 3.2 3.2.3 3.2.4 3.4 3.4.1 3.5 3.6 3.7 base_regress = londiste_provider londiste_subscriber \ londiste_fkeys londiste_execute londiste_seqs londiste_merge \ londiste_leaf londiste_create_part \ londiste_merge_fkeys Contrib_regress = init_noext $(base_regress) Extension_regress = init_ext $(base_regress) include mk/common-pgxs.mk dox: cleandox mkdir -p docs/html mkdir -p docs/sql $(CATSQL) --ndoc structure/tables.sql > docs/sql/schema.sql $(CATSQL) --ndoc structure/functions.sql > docs/sql/functions.sql $(NDOC) $(NDOCARGS) londiste-sql-3.8/README.rst000066400000000000000000000001301432153235300154640ustar00rootroot00000000000000londiste-sql ============ Database extension that contains support code for Londiste. londiste-sql-3.8/docs/000077500000000000000000000000001432153235300147335ustar00rootroot00000000000000londiste-sql-3.8/docs/Languages.txt000066400000000000000000000120211432153235300173760ustar00rootroot00000000000000Format: 1.52 # This is the Natural Docs languages file for this project. If you change # anything here, it will apply to THIS PROJECT ONLY. If you'd like to change # something for all your projects, edit the Languages.txt in Natural Docs' # Config directory instead. Ignore Extension: sql #------------------------------------------------------------------------------- # SYNTAX: # # Unlike other Natural Docs configuration files, in this file all comments # MUST be alone on a line. Some languages deal with the # character, so you # cannot put comments on the same line as content. # # Also, all lists are separated with spaces, not commas, again because some # languages may need to use them. # # Language: [name] # Alter Language: [name] # Defines a new language or alters an existing one. Its name can use any # characters. If any of the properties below have an add/replace form, you # must use that when using Alter Language. # # The language Shebang Script is special. It's entry is only used for # extensions, and files with those extensions have their shebang (#!) lines # read to determine the real language of the file. Extensionless files are # always treated this way. # # The language Text File is also special. It's treated as one big comment # so you can put Natural Docs content in them without special symbols. Also, # if you don't specify a package separator, ignored prefixes, or enum value # behavior, it will copy those settings from the language that is used most # in the source tree. # # Extensions: [extension] [extension] ... # [Add/Replace] Extensions: [extension] [extension] ... # Defines the file extensions of the language's source files. You can # redefine extensions found in the main languages file. You can use * to # mean any undefined extension. # # Shebang Strings: [string] [string] ... # [Add/Replace] Shebang Strings: [string] [string] ... # Defines a list of strings that can appear in the shebang (#!) line to # designate that it's part of the language. You can redefine strings found # in the main languages file. # # Ignore Prefixes in Index: [prefix] [prefix] ... # [Add/Replace] Ignored Prefixes in Index: [prefix] [prefix] ... # # Ignore [Topic Type] Prefixes in Index: [prefix] [prefix] ... # [Add/Replace] Ignored [Topic Type] Prefixes in Index: [prefix] [prefix] ... # Specifies prefixes that should be ignored when sorting symbols in an # index. Can be specified in general or for a specific topic type. # #------------------------------------------------------------------------------ # For basic language support only: # # Line Comments: [symbol] [symbol] ... # Defines a space-separated list of symbols that are used for line comments, # if any. # # Block Comments: [opening sym] [closing sym] [opening sym] [closing sym] ... # Defines a space-separated list of symbol pairs that are used for block # comments, if any. # # Package Separator: [symbol] # Defines the default package separator symbol. The default is a dot. # # [Topic Type] Prototype Enders: [symbol] [symbol] ... # When defined, Natural Docs will attempt to get a prototype from the code # immediately following the topic type. It stops when it reaches one of # these symbols. Use \n for line breaks. # # Line Extender: [symbol] # Defines the symbol that allows a prototype to span multiple lines if # normally a line break would end it. # # Enum Values: [global|under type|under parent] # Defines how enum values are referenced. The default is global. # global - Values are always global, referenced as 'value'. # under type - Values are under the enum type, referenced as # 'package.enum.value'. # under parent - Values are under the enum's parent, referenced as # 'package.value'. # # Perl Package: [perl package] # Specifies the Perl package used to fine-tune the language behavior in ways # too complex to do in this file. # #------------------------------------------------------------------------------ # For full language support only: # # Full Language Support: [perl package] # Specifies the Perl package that has the parsing routines necessary for full # language support. # #------------------------------------------------------------------------------- # The following languages are defined in the main file, if you'd like to alter # them: # # Text File, Shebang Script, C/C++, C#, Java, JavaScript, Perl, Python, # PHP, SQL, Visual Basic, Pascal, Assembly, Ada, Tcl, Ruby, Makefile, # ActionScript, ColdFusion, R, Fortran # If you add a language that you think would be useful to other developers # and should be included in Natural Docs by default, please e-mail it to # languages [at] naturaldocs [dot] org. Language: PLPGSQL Extension: sql Line Comment: -- Block Comment: /* */ Enum Values: Global Function Prototype Enders: , ; ) $ ' Variable Prototype Enders: , ; ) := default Default DEFAULT Database Index Prototype Enders: , ; ) Database Trigger Prototype Enders: begin Begin BEGIN londiste-sql-3.8/docs/Menu.txt000066400000000000000000000037311432153235300164040ustar00rootroot00000000000000Format: 1.52 # You can add a title and sub-title to your menu like this: # Title: [project name] # SubTitle: [subtitle] # You can add a footer to your documentation like this: # Footer: [text] # If you want to add a copyright notice, this would be the place to do it. # You can add a timestamp to your documentation like one of these: # Timestamp: Generated on month day, year # Timestamp: Updated mm/dd/yyyy # Timestamp: Last updated mon day # # m - One or two digit month. January is "1" # mm - Always two digit month. January is "01" # mon - Short month word. January is "Jan" # month - Long month word. January is "January" # d - One or two digit day. 1 is "1" # dd - Always two digit day. 1 is "01" # day - Day with letter extension. 1 is "1st" # yy - Two digit year. 2006 is "06" # yyyy - Four digit year. 2006 is "2006" # year - Four digit year. 2006 is "2006" # -------------------------------------------------------------------------- # # Cut and paste the lines below to change the order in which your files # appear on the menu. Don't worry about adding or removing files, Natural # Docs will take care of that. # # You can further organize the menu by grouping the entries. Add a # "Group: [name] {" line to start a group, and add a "}" to end it. # # You can add text and web links to the menu by adding "Text: [text]" and # "Link: [name] ([URL])" lines, respectively. # # The formatting and comments are auto-generated, so don't worry about # neatness when editing the file. Natural Docs will clean it up the next # time it is run. When working with groups, just deal with the braces and # forget about the indentation and comments. # # -------------------------------------------------------------------------- File: Londiste functions (functions.sql) File: Londiste internals (schema.sql) Group: Index { Index: Everything Database Table Index: Database Tables Function Index: Functions } # Group: Index londiste-sql-3.8/docs/Topics.txt000066400000000000000000000065051432153235300167430ustar00rootroot00000000000000Format: 1.52 # This is the Natural Docs topics file for this project. If you change anything # here, it will apply to THIS PROJECT ONLY. If you'd like to change something # for all your projects, edit the Topics.txt in Natural Docs' Config directory # instead. # If you'd like to prevent keywords from being recognized by Natural Docs, you # can do it like this: # Ignore Keywords: [keyword], [keyword], ... # # Or you can use the list syntax like how they are defined: # Ignore Keywords: # [keyword] # [keyword], [plural keyword] # ... #------------------------------------------------------------------------------- # SYNTAX: # # Topic Type: [name] # Alter Topic Type: [name] # Creates a new topic type or alters one from the main file. Each type gets # its own index and behavior settings. Its name can have letters, numbers, # spaces, and these charaters: - / . ' # # Plural: [name] # Sets the plural name of the topic type, if different. # # Keywords: # [keyword] # [keyword], [plural keyword] # ... # Defines or adds to the list of keywords for the topic type. They may only # contain letters, numbers, and spaces and are not case sensitive. Plural # keywords are used for list topics. You can redefine keywords found in the # main topics file. # # Index: [yes|no] # Whether the topics get their own index. Defaults to yes. Everything is # included in the general index regardless of this setting. # # Scope: [normal|start|end|always global] # How the topics affects scope. Defaults to normal. # normal - Topics stay within the current scope. # start - Topics start a new scope for all the topics beneath it, # like class topics. # end - Topics reset the scope back to global for all the topics # beneath it. # always global - Topics are defined as global, but do not change the scope # for any other topics. # # Class Hierarchy: [yes|no] # Whether the topics are part of the class hierarchy. Defaults to no. # # Page Title If First: [yes|no] # Whether the topic's title becomes the page title if it's the first one in # a file. Defaults to no. # # Break Lists: [yes|no] # Whether list topics should be broken into individual topics in the output. # Defaults to no. # # Can Group With: [type], [type], ... # Defines a list of topic types that this one can possibly be grouped with. # Defaults to none. #------------------------------------------------------------------------------- # The following topics are defined in the main file, if you'd like to alter # their behavior or add keywords: # # Generic, Class, Interface, Section, File, Group, Function, Variable, # Property, Type, Constant, Enumeration, Event, Delegate, Macro, # Database, Database Table, Database View, Database Index, Database # Cursor, Database Trigger, Cookie, Build Target # If you add something that you think would be useful to other developers # and should be included in Natural Docs by default, please e-mail it to # topics [at] naturaldocs [dot] org. Topic Type: Schema Plural: Schemas Index: No Scope: Start Class Hierarchy: Yes Keywords: schema, schemas Alter Topic Type: Function Add Keywords: public function internal function Alter Topic Type: File Index: No londiste-sql-3.8/docs/notes/000077500000000000000000000000001432153235300160635ustar00rootroot00000000000000londiste-sql-3.8/docs/notes/v3.5.md000066400000000000000000000001051432153235300170740ustar00rootroot00000000000000* Set up Github actions * Support Postgres 13 * Fix version function londiste-sql-3.8/docs/notes/v3.6.md000066400000000000000000000000501432153235300170740ustar00rootroot00000000000000 * Support foreign keys on merge nodes londiste-sql-3.8/docs/notes/v3.7.md000066400000000000000000000000721432153235300171010ustar00rootroot00000000000000* Support multi-step fkey restore * Drop debian packaging londiste-sql-3.8/docs/notes/v3.8.md000066400000000000000000000001221432153235300170760ustar00rootroot00000000000000* `execute_start`: fix insert race * `create_trigger`: enforce no-triggers better londiste-sql-3.8/docs/release.txt000066400000000000000000000002441432153235300171140ustar00rootroot00000000000000 * Set new version number in .control, Makefile, debian/changelog * Add release notes to docs/notes * git commit -m "vX.Y" * "make release" to tag and push it out londiste-sql-3.8/expected/000077500000000000000000000000001432153235300156045ustar00rootroot00000000000000londiste-sql-3.8/expected/init_ext.out000066400000000000000000000002721432153235300201610ustar00rootroot00000000000000\set ECHO none create extension londiste; select array_length(extconfig, 1) as dumpable from pg_catalog.pg_extension where extname = 'londiste'; dumpable ---------- 4 (1 row) londiste-sql-3.8/expected/init_noext.out000066400000000000000000000003001432153235300205060ustar00rootroot00000000000000\set ECHO none upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 0 (1 row) londiste-sql-3.8/expected/londiste_create_part.out000066400000000000000000000070111432153235300225260ustar00rootroot00000000000000\set ECHO none drop role if exists londiste_test_part1; drop role if exists londiste_test_part2; create group londiste_test_part1; create group londiste_test_part2; create table events ( id int4 primary key, txt text not null, ctime timestamptz not null default now(), someval int4 check (someval > 0) ); create index ctime_idx on events (ctime); create rule ignore_dups AS on insert to events where (exists (select 1 from events where (events.id = new.id))) do instead nothing; create or replace function "NullTrigger"() returns trigger as $$ begin return null; end; $$ language plpgsql; create trigger "Fooza" after delete on events for each row execute procedure "NullTrigger"(); alter table events enable always trigger "Fooza"; grant select,delete on events to londiste_test_part1; grant select,update,delete on events to londiste_test_part2 with grant option; grant select,insert on events to public; select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01', 'month'); create_partition ------------------ 1 (1 row) select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01'::timestamptz, 'month'); create_partition ------------------ 0 (1 row) select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01'::timestamp, 'month'); create_partition ------------------ 0 (1 row) select count(*) from pg_indexes where schemaname='public' and tablename = 'events_2011_01'; count ------- 2 (1 row) select count(*) from pg_constraint where conrelid = 'public.events_2011_01'::regclass; count ------- 3 (1 row) select count(*) from pg_rules where schemaname = 'public' and tablename = 'events_2011_01'; count ------- 1 (1 row) select trigger_name, event_manipulation, action_statement from information_schema.triggers where event_object_schema = 'public' and event_object_table = 'events_2011_01'; trigger_name | event_manipulation | action_statement --------------+--------------------+----------------------------------- Fooza | DELETE | EXECUTE PROCEDURE "NullTrigger"() (1 row) select tgenabled, pg_get_triggerdef(oid) from pg_trigger where tgrelid = 'events_2011_01'::regclass::oid; tgenabled | pg_get_triggerdef -----------+------------------------------------------------------------------------------------------------------------- A | CREATE TRIGGER "Fooza" AFTER DELETE ON public.events_2011_01 FOR EACH ROW EXECUTE PROCEDURE "NullTrigger"() (1 row) -- test weird quoting create table "Bad "" table '.' name!" ( id int4 primary key, txt text not null, ctime timestamptz not null default now(), someval int4 check (someval > 0) ); create rule "Ignore Dups" AS on insert to "Bad "" table '.' name!" where (exists (select 1 from "Bad "" table '.' name!" where ("Bad "" table '.' name!".id = new.id))) do instead nothing; alter table "Bad "" table '.' name!" ENABLE ALWAYS RULE "Ignore Dups"; select londiste.create_partition('public.Bad " table ''.'' name!', 'public.Bad " table ''.'' part!', 'id', 'ctime', '2011-01-01', 'month'); create_partition ------------------ 1 (1 row) select count(*) from pg_rules where schemaname = 'public' and tablename ilike 'bad%'; count ------- 2 (1 row) -- \d events_2011_01 -- \dp events -- \dp events_2011_01 londiste-sql-3.8/expected/londiste_create_part_1.out000066400000000000000000000070031432153235300227470ustar00rootroot00000000000000\set ECHO none drop role if exists londiste_test_part1; drop role if exists londiste_test_part2; create group londiste_test_part1; create group londiste_test_part2; create table events ( id int4 primary key, txt text not null, ctime timestamptz not null default now(), someval int4 check (someval > 0) ); create index ctime_idx on events (ctime); create rule ignore_dups AS on insert to events where (exists (select 1 from events where (events.id = new.id))) do instead nothing; create or replace function "NullTrigger"() returns trigger as $$ begin return null; end; $$ language plpgsql; create trigger "Fooza" after delete on events for each row execute procedure "NullTrigger"(); alter table events enable always trigger "Fooza"; grant select,delete on events to londiste_test_part1; grant select,update,delete on events to londiste_test_part2 with grant option; grant select,insert on events to public; select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01', 'month'); create_partition ------------------ 1 (1 row) select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01'::timestamptz, 'month'); create_partition ------------------ 0 (1 row) select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01'::timestamp, 'month'); create_partition ------------------ 0 (1 row) select count(*) from pg_indexes where schemaname='public' and tablename = 'events_2011_01'; count ------- 2 (1 row) select count(*) from pg_constraint where conrelid = 'public.events_2011_01'::regclass; count ------- 3 (1 row) select count(*) from pg_rules where schemaname = 'public' and tablename = 'events_2011_01'; count ------- 1 (1 row) select trigger_name, event_manipulation, action_statement from information_schema.triggers where event_object_schema = 'public' and event_object_table = 'events_2011_01'; trigger_name | event_manipulation | action_statement --------------+--------------------+---------------------------------- Fooza | DELETE | EXECUTE FUNCTION "NullTrigger"() (1 row) select tgenabled, pg_get_triggerdef(oid) from pg_trigger where tgrelid = 'events_2011_01'::regclass::oid; tgenabled | pg_get_triggerdef -----------+------------------------------------------------------------------------------------------------------------ A | CREATE TRIGGER "Fooza" AFTER DELETE ON public.events_2011_01 FOR EACH ROW EXECUTE FUNCTION "NullTrigger"() (1 row) -- test weird quoting create table "Bad "" table '.' name!" ( id int4 primary key, txt text not null, ctime timestamptz not null default now(), someval int4 check (someval > 0) ); create rule "Ignore Dups" AS on insert to "Bad "" table '.' name!" where (exists (select 1 from "Bad "" table '.' name!" where ("Bad "" table '.' name!".id = new.id))) do instead nothing; alter table "Bad "" table '.' name!" ENABLE ALWAYS RULE "Ignore Dups"; select londiste.create_partition('public.Bad " table ''.'' name!', 'public.Bad " table ''.'' part!', 'id', 'ctime', '2011-01-01', 'month'); create_partition ------------------ 1 (1 row) select count(*) from pg_rules where schemaname = 'public' and tablename ilike 'bad%'; count ------- 2 (1 row) -- \d events_2011_01 -- \dp events -- \dp events_2011_01 londiste-sql-3.8/expected/londiste_execute.out000066400000000000000000000046621432153235300217100ustar00rootroot00000000000000set log_error_verbosity = 'terse'; select * from londiste.execute_start('branch_set', 'DDL-A.sql', 'drop all', false); ret_code | ret_note ----------+---------------------- 200 | Executing: DDL-A.sql (1 row) select * from londiste.execute_start('branch_set', 'DDL-A.sql', 'drop all', false); ret_code | ret_note ----------+------------------------------------------------ 201 | EXECUTE: "DDL-A.sql" already applied, skipping (1 row) select * from londiste.execute_finish('branch_set', 'DDL-A.sql'); ret_code | ret_note ----------+----------------------------- 200 | Execute finished: DDL-A.sql (1 row) select * from londiste.execute_finish('branch_set', 'DDL-A.sql'); ret_code | ret_note ----------+----------------------------- 200 | Execute finished: DDL-A.sql (1 row) select * from londiste.execute_finish('branch_set', 'DDL-XXX.sql'); ret_code | ret_note ----------+------------------------------------------- 404 | execute_file called without execute_start (1 row) select * from londiste.execute_start('branch_set', 'DDL-B.sql', 'drop all', true); ret_code | ret_note ----------+----------------------------------- 401 | Node is not root node: branch_set (1 row) select * from londiste.execute_start('branch_set', 'DDL-B.sql', 'drop all', true); ret_code | ret_note ----------+----------------------------------- 401 | Node is not root node: branch_set (1 row) select * from londiste.execute_start('aset', 'DDL-root.sql', 'drop all', true); ret_code | ret_note ----------+------------------------- 200 | Executing: DDL-root.sql (1 row) select * from londiste.execute_start('aset', 'DDL-root.sql', 'drop all', true); ret_code | ret_note ----------+--------------------------------------------------- 201 | EXECUTE: "DDL-root.sql" already applied, skipping (1 row) select * from londiste.execute_finish('aset', 'DDL-root.sql'); ret_code | ret_note ----------+-------------------------------- 200 | Execute finished: DDL-root.sql (1 row) select * from londiste.execute_finish('aset', 'DDL-root.sql'); ret_code | ret_note ----------+-------------------------------- 200 | Execute finished: DDL-root.sql (1 row) londiste-sql-3.8/expected/londiste_fkeys.out000066400000000000000000000322201432153235300213560ustar00rootroot00000000000000set log_error_verbosity = 'terse'; set client_min_messages = 'warning'; create table ref_1 ( id int4 primary key, val text ); create table ref_2 ( id int4 primary key, ref int4 not null references ref_1, val text ); create table ref_3 ( id int4 primary key, ref2 int4 not null references ref_2, val text ); select * from londiste.global_add_table('branch_set', 'public.ref_1'); ret_code | ret_note ----------+--------------------------- 200 | Table added: public.ref_1 (1 row) select * from londiste.global_add_table('branch_set', 'public.ref_2'); ret_code | ret_note ----------+--------------------------- 200 | Table added: public.ref_2 (1 row) select * from londiste.global_add_table('branch_set', 'public.ref_3'); ret_code | ret_note ----------+--------------------------- 200 | Table added: public.ref_3 (1 row) select * from londiste.local_add_table('branch_set', 'public.ref_1'); ret_code | ret_note ----------+--------------------------- 200 | Table added: public.ref_1 (1 row) select * from londiste.local_add_table('branch_set', 'public.ref_2'); ret_code | ret_note ----------+--------------------------- 200 | Table added: public.ref_2 (1 row) select * from londiste.local_add_table('branch_set', 'public.ref_3'); ret_code | ret_note ----------+--------------------------- 200 | Table added: public.ref_3 (1 row) select * from londiste.find_table_fkeys('public.ref_1'); from_table | to_table | fkey_name | fkey_def --------------+--------------+----------------+---------------------------------------------------------------------------------------------------- public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) (1 row) select * from londiste.find_table_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (2 rows) select * from londiste.find_table_fkeys('public.ref_3'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (1 row) select * from londiste.get_table_pending_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys('branch_set'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) -- drop fkeys select * from londiste.drop_table_fkey('public.ref_2', 'ref_2_ref_fkey'); drop_table_fkey ----------------- 1 (1 row) select * from londiste.find_table_fkeys('public.ref_1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.find_table_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (1 row) select * from londiste.find_table_fkeys('public.ref_3'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (1 row) select * from londiste.drop_table_fkey('public.ref_3', 'ref_3_ref2_fkey'); drop_table_fkey ----------------- 1 (1 row) -- check if dropped select * from londiste.find_table_fkeys('public.ref_1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.find_table_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.find_table_fkeys('public.ref_3'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) -- look state select * from londiste.get_table_pending_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (2 rows) select * from londiste.get_valid_pending_fkeys('branch_set'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) -- toggle sync select * from londiste.local_set_table_state('branch_set', 'public.ref_1', null, 'ok'); ret_code | ret_note ----------+-------------------------------------- 200 | Table public.ref_1 state set to 'ok' (1 row) select * from londiste.get_valid_pending_fkeys('branch_set'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.local_set_table_state('branch_set', 'public.ref_2', null, 'ok'); ret_code | ret_note ----------+-------------------------------------- 200 | Table public.ref_2 state set to 'ok' (1 row) select * from londiste.get_valid_pending_fkeys('branch_set'); from_table | to_table | fkey_name | fkey_def --------------+--------------+----------------+---------------------------------------------------------------------------------------------------- public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) (1 row) select * from londiste.local_set_table_state('branch_set', 'public.ref_3', null, 'ok'); ret_code | ret_note ----------+-------------------------------------- 200 | Table public.ref_3 state set to 'ok' (1 row) select * from londiste.get_valid_pending_fkeys('branch_set'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (2 rows) -- restore: old select * from londiste.get_table_pending_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (2 rows) select * from londiste.restore_table_fkey('public.ref_2', 'ref_2_ref_fkey'); restore_table_fkey -------------------- (1 row) select * from londiste.get_table_pending_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (1 row) -- restore: new select * from londiste.get_table_pending_fkeys('public.ref_3'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (1 row) select londiste.restore_table_fkey('public.ref_3', 'ref_3_ref2_fkey', true) as step1; step1 ---------------------------------------------------------------------------------------------------------------- alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) not valid (1 row) \gset :step1 ; select londiste.restore_table_fkey('public.ref_3', 'ref_3_ref2_fkey', true) as step2; step2 ------------------------------------------------------------------- alter table only public.ref_3 validate constraint ref_3_ref2_fkey (1 row) \gset :step2 ; select londiste.restore_table_fkey('public.ref_3', 'ref_3_ref2_fkey', true) as step3; step3 ------- (1 row) select * from londiste.get_table_pending_fkeys('public.ref_3'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) -- look state select * from londiste.get_table_pending_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys('branch_set'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.find_table_fkeys('public.ref_1'); from_table | to_table | fkey_name | fkey_def --------------+--------------+----------------+---------------------------------------------------------------------------------------------------- public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) (1 row) select * from londiste.find_table_fkeys('public.ref_2'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_2 | public.ref_1 | ref_2_ref_fkey | alter table only public.ref_2 add constraint ref_2_ref_fkey FOREIGN KEY (ref) REFERENCES ref_1(id) public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (2 rows) select * from londiste.find_table_fkeys('public.ref_3'); from_table | to_table | fkey_name | fkey_def --------------+--------------+-----------------+------------------------------------------------------------------------------------------------------ public.ref_3 | public.ref_2 | ref_3_ref2_fkey | alter table only public.ref_3 add constraint ref_3_ref2_fkey FOREIGN KEY (ref2) REFERENCES ref_2(id) (1 row) londiste-sql-3.8/expected/londiste_install.out000066400000000000000000000003001432153235300216750ustar00rootroot00000000000000\set ECHO none upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 0 (1 row) upgrade_schema ---------------- 0 (1 row) londiste-sql-3.8/expected/londiste_leaf.out000066400000000000000000000102221432153235300211420ustar00rootroot00000000000000set client_min_messages = 'warning'; \set VERBOSITY 'terse' select 1 from (select set_config(name, 'escape', false) as ignore from pg_settings where name = 'bytea_output') x where x.ignore = 'foo'; ?column? ---------- (0 rows) -- -- tables -- create table leafdata ( id serial primary key, data text ); select current_database(); current_database ------------------ regression (1 row) select * from pgq_node.register_location('leafq', 'lq_node1', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('leafq', 'lq_node2', 'dbname=db2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('leafq', 'leaf', 'lq_node2', 'londiste_leaf', 'lq_node1', 100, null::text); ret_code | ret_note ----------+---------------------------------------------------------------- 200 | Node "lq_node2" initialized for queue "leafq" with type "leaf" (1 row) select * from londiste.local_show_missing('leafq'); obj_kind | obj_name ----------+---------- (0 rows) select * from londiste.local_add_table('leafq', 'public.leafdata'); ret_code | ret_note ----------+----------------------------------------------- 404 | Table not available on queue: public.leafdata (1 row) select * from londiste.global_add_table('leafq', 'public.leafdata'); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.leafdata (1 row) select * from londiste.local_add_table('leafq', 'public.leafdata'); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.leafdata (1 row) select * from londiste.global_add_table('leafq', 'public.tmp'); ret_code | ret_note ----------+------------------------- 200 | Table added: public.tmp (1 row) select * from londiste.get_table_list('leafq'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.leafdata | t | | | | | | 0 | public.tmp | f | | | | | | 0 | (2 rows) select tgname, tgargs from pg_trigger where tgrelid = 'public.leafdata'::regclass order by 1; tgname | tgargs --------------------------+------------------- _londiste_leafq | leafq\000deny\000 _londiste_leafq_truncate | leafq\000deny\000 (2 rows) insert into leafdata values (1, 'asd'); ERROR: Table 'public.leafdata' to queue 'leafq': change not allowed (INSERT) select * from londiste.global_remove_table('leafq', 'public.tmp'); ret_code | ret_note ----------+--------------------------- 200 | Table removed: public.tmp (1 row) select * from londiste.local_remove_table('leafq', 'public.leafdata'); ret_code | ret_note ----------+-------------------------------- 200 | Table removed: public.leafdata (1 row) select * from londiste.local_remove_table('leafq', 'public.leafdata'); ret_code | ret_note ----------+----------------------------------------------- 400 | Table not registered locally: public.leafdata (1 row) select * from londiste.get_table_list('leafq'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.leafdata | f | | | | | | 0 | (1 row) select * from londiste.local_show_missing('leafq'); obj_kind | obj_name ----------+----------------- r | public.leafdata (1 row) londiste-sql-3.8/expected/londiste_merge.out000066400000000000000000000402071432153235300213400ustar00rootroot00000000000000set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- tables -- create table tblmerge ( id int4 primary key, data text ); select * from pgq_node.register_location('combined_set', 'croot', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('combined_set', 'root', 'croot', 'londiste_croot', null, null, null); ret_code | ret_note ----------+-------------------------------------------------------------------- 200 | Node "croot" initialized for queue "combined_set" with type "root" (1 row) select * from pgq_node.register_location('part1_set', 'p1root', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('part1_set', 'p1merge', 'dbname=db2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('part1_set', 'leaf', 'p1merge', 'londiste_p1merge', 'p1root', 100, 'combined_set'); ret_code | ret_note ----------+------------------------------------------------------------------- 200 | Node "p1merge" initialized for queue "part1_set" with type "leaf" (1 row) select * from pgq_node.register_location('part2_set', 'p2root', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('part2_set', 'p2merge', 'dbname=db2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('part2_set', 'leaf', 'p2merge', 'londiste_p2merge', 'p2root', 100, 'combined_set'); ret_code | ret_note ----------+------------------------------------------------------------------- 200 | Node "p2merge" initialized for queue "part2_set" with type "leaf" (1 row) select * from pgq_node.register_location('part3_set', 'p3root', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('part3_set', 'p3merge', 'dbname=db3', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('part3_set', 'leaf', 'p3merge', 'londiste_p3merge', 'p3root', 100, 'combined_set'); ret_code | ret_note ----------+------------------------------------------------------------------- 200 | Node "p3merge" initialized for queue "part3_set" with type "leaf" (1 row) select * from londiste.local_add_table('combined_set', 'tblmerge'); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.tblmerge (1 row) select * from londiste.global_add_table('part1_set', 'tblmerge'); ret_code | ret_note ----------+----------------------- 200 | Table added: tblmerge (1 row) select * from londiste.global_add_table('part2_set', 'tblmerge'); ret_code | ret_note ----------+----------------------- 200 | Table added: tblmerge (1 row) select * from londiste.local_add_table('part1_set', 'tblmerge', array['merge_all']); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.tblmerge (1 row) select * from londiste.get_table_list('part1_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | | | | | | 0 | (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | | | | | | 0 | (1 row) select * from londiste.get_table_list('combined_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | ok | | | | | 0 | (1 row) select * from londiste.local_set_table_state('part1_set', 'public.tblmerge', null, 'in-copy'); ret_code | ret_note ----------+---------------------------------------------- 200 | Table public.tblmerge state set to 'in-copy' (1 row) select * from londiste.local_set_table_state('part2_set', 'public.tblmerge', null, 'in-copy'); ret_code | ret_note ----------+---------------------------------------------- 200 | Table public.tblmerge state set to 'in-copy' (1 row) select * from londiste.get_table_list('part1_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | in-copy | | | | lead | 0 | (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | in-copy | | | | wait-copy | 1 | (1 row) select * from londiste.local_set_table_struct('part1_set', 'public.tblmerge', 'create index;'); ret_code | ret_note ----------+--------------------- 200 | Table struct stored (1 row) select * from londiste.get_table_list('part1_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+---------------+-----------+----------+------------ public.tblmerge | t | in-copy | | | create index; | lead | 0 | (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-------------+----------+------------ public.tblmerge | t | in-copy | | | | wait-replay | 1 | (1 row) select * from londiste.local_set_table_state('part2_set', 'public.tblmerge', null, 'catching-up'); ret_code | ret_note ----------+-------------------------------------------------- 200 | Table public.tblmerge state set to 'catching-up' (1 row) select * from londiste.get_table_list('part1_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+---------------+-----------+----------+------------ public.tblmerge | t | in-copy | | | create index; | lead | 0 | (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-------------+----------+------------ public.tblmerge | t | catching-up | | | | wait-replay | 0 | (1 row) select * from londiste.local_set_table_state('part1_set', 'public.tblmerge', null, 'catching-up'); ret_code | ret_note ----------+-------------------------------------------------- 200 | Table public.tblmerge state set to 'catching-up' (1 row) select * from londiste.get_table_list('part1_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+---------------+-----------+----------+------------ public.tblmerge | t | catching-up | | | create index; | | 0 | (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-------------+----------+------------ public.tblmerge | t | catching-up | | | | wait-replay | 0 | (1 row) select * from londiste.local_set_table_struct('part1_set', 'public.tblmerge', null); ret_code | ret_note ----------+--------------------- 200 | Table struct stored (1 row) select * from londiste.get_table_list('part1_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | catching-up | | | | | 0 | (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.tblmerge | t | catching-up | | | | | 0 | (1 row) -- test automatic registration on combined-root select * from londiste.global_add_table('part1_set', 'tblauto'); ret_code | ret_note ----------+---------------------- 200 | Table added: tblauto (1 row) select * from londiste.global_add_table('part2_set', 'tblauto'); ret_code | ret_note ----------+---------------------- 200 | Table added: tblauto (1 row) select * from londiste.local_add_table('part1_set', 'tblauto', array['merge_all', 'virtual_table'], 'handler=vtable'); ret_code | ret_note ----------+---------------------------------------------- 200 | Table added with no triggers: public.tblauto (1 row) select * from londiste.get_table_list('part2_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+----------------+-------------+-----------+----------+------------ public.tblmerge | t | catching-up | | | | | 0 | public.tblauto | t | ok | | handler=vtable | | | 0 | (2 rows) select * from londiste.get_table_list('combined_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+----------------+-------------+-----------+----------+------------ public.tblmerge | t | ok | | | | | 0 | public.tblauto | t | ok | | handler=vtable | | | 0 | (2 rows) -- -- Test all combinations on 3-node merge -- select * from londiste.global_add_table('part3_set', 'tblmerge'); ret_code | ret_note ----------+----------------------- 200 | Table added: tblmerge (1 row) \set ECHO none select * from testmatrix(); p1s | p2s | p3s | p1r | p2r | p3r --------------+--------------+--------------+-------------+-------------+------------- !catching-up | catching-up | catching-up | NULL | wait-replay | wait-replay !catching-up | catching-up | in-copy | wait-replay | wait-replay | wait-replay !catching-up | in-copy | catching-up | wait-replay | wait-replay | wait-replay !catching-up | in-copy | in-copy | wait-replay | wait-replay | wait-replay !in-copy | catching-up | catching-up | lead | wait-replay | wait-replay !in-copy | catching-up | in-copy | lead | wait-replay | wait-replay !in-copy | in-copy | catching-up | lead | wait-replay | wait-replay !in-copy | in-copy | in-copy | lead | wait-replay | wait-replay catching-up | !catching-up | catching-up | wait-replay | NULL | wait-replay catching-up | !catching-up | in-copy | wait-replay | wait-replay | wait-replay catching-up | !in-copy | catching-up | wait-replay | lead | wait-replay catching-up | !in-copy | in-copy | wait-replay | lead | wait-replay catching-up | catching-up | !catching-up | wait-replay | wait-replay | NULL catching-up | catching-up | !in-copy | wait-replay | wait-replay | lead catching-up | catching-up | catching-up | NULL | NULL | NULL catching-up | catching-up | in-copy | NULL | NULL | wait-replay catching-up | in-copy | !catching-up | wait-replay | wait-replay | wait-replay catching-up | in-copy | !in-copy | wait-replay | wait-replay | lead catching-up | in-copy | catching-up | NULL | wait-replay | NULL catching-up | in-copy | in-copy | NULL | wait-replay | wait-replay in-copy | !catching-up | catching-up | wait-replay | wait-replay | wait-replay in-copy | !catching-up | in-copy | wait-replay | wait-replay | wait-replay in-copy | !in-copy | catching-up | wait-replay | lead | wait-replay in-copy | !in-copy | in-copy | wait-replay | lead | wait-replay in-copy | catching-up | !catching-up | wait-replay | wait-replay | wait-replay in-copy | catching-up | !in-copy | wait-replay | wait-replay | lead in-copy | catching-up | catching-up | wait-replay | NULL | NULL in-copy | catching-up | in-copy | wait-replay | NULL | wait-replay in-copy | in-copy | !catching-up | wait-replay | wait-replay | wait-replay in-copy | in-copy | !in-copy | wait-replay | wait-replay | lead in-copy | in-copy | catching-up | wait-replay | wait-replay | NULL in-copy | in-copy | in-copy | lead | wait-copy | wait-copy (32 rows) -- test dropped ddl restore create table ddlrestore ( id int4, data1 text, data2 text ); select count(*) from pg_indexes where schemaname='public' and tablename='ddlrestore'; count ------- 0 (1 row) insert into londiste.table_info (queue_name, table_name, local, merge_state, dropped_ddl) values ('part1_set', 'public.ddlrestore', true, 'in-copy', ' ALTER TABLE ddlrestore ADD CONSTRAINT cli_pkey PRIMARY KEY (id); CREATE INDEX idx_data1 ON ddlrestore USING btree (data1); CREATE INDEX idx_data2 ON ddlrestore USING btree (data2); '); select * from londiste.local_remove_table('part1_set', 'public.ddlrestore'); ret_code | ret_note ----------+---------------------------------- 200 | Table removed: public.ddlrestore (1 row) select count(*) from pg_indexes where schemaname='public' and tablename='ddlrestore'; count ------- 3 (1 row) londiste-sql-3.8/expected/londiste_merge_fkeys.out000066400000000000000000000177461432153235300225550ustar00rootroot00000000000000set client_min_messages = 'warning'; \set VERBOSITY 'terse' \set ECHO none -- leaf1/leaf2->root \set target 'fx1-target' \set leaf1 'fx1-leaf1' \set leaf2 'fx1-leaf2' \set tbl1 'fx1.table1' \set tbl2 'fx1.table2' select mkcascade(array[:'target'], array[]::text[], null, array[:'leaf1', :'leaf2'], :'target', array[:'tbl1', :'tbl2']); mkcascade ----------- done (1 row) insert into londiste.pending_fkeys values (:'tbl1', :'tbl2', 'name', 'def'); select * from londiste.get_table_pending_fkeys(:'tbl1'); from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx1.table1 | fx1.table2 | name | def (1 row) update londiste.table_info set merge_state='ok', local=true where table_name in (:'tbl1', :'tbl2'); select * from londiste.get_valid_pending_fkeys(:'target'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx1.table1 | fx1.table2 | name | def (1 row) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state='catching-up' where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- no from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state=null, local=false where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx1.table1 | fx1.table2 | name | def (1 row) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state=null, local=false where table_name = :'tbl2' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); -- pick? from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx1.table1 | fx1.table2 | name | def (1 row) -- leaf1/leaf2->branch \set target 'fx2-branch' \set leaf1 'fx2-leaf1' \set leaf2 'fx2-leaf2' \set tbl1 'fx2.table1' \set tbl2 'fx2.table2' select mkcascade(array[]::text[], array[:'target']::text[], null, array[:'leaf1', :'leaf2'], :'target', array[:'tbl1', :'tbl2']); mkcascade ----------- done (1 row) insert into londiste.pending_fkeys values (:'tbl1', :'tbl2', 'name', 'def'); select * from londiste.get_table_pending_fkeys(:'tbl1'); from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx2.table1 | fx2.table2 | name | def (1 row) update londiste.table_info set merge_state='ok', local=true where table_name in (:'tbl1', :'tbl2'); select * from londiste.get_valid_pending_fkeys(:'target'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx2.table1 | fx2.table2 | name | def (1 row) select * from londiste.get_valid_pending_fkeys(:'leaf1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state='catching-up' where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); -- no from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state=null, local=false where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx2.table1 | fx2.table2 | name | def (1 row) select * from londiste.get_valid_pending_fkeys(:'leaf1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) -- leaf1/leaf2->no branch \set leaf1 'fx3-leaf1' \set leaf2 'fx3-leaf2' \set tbl1 'fx3.table1' \set tbl2 'fx3.table2' select mkcascade(array[]::text[], array[]::text[], null, array[:'leaf1', :'leaf2'], null, array[:'tbl1', :'tbl2']); mkcascade ----------- done (1 row) insert into londiste.pending_fkeys values (:'tbl1', :'tbl2', 'name', 'def'); select * from londiste.get_table_pending_fkeys(:'tbl1'); from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx3.table1 | fx3.table2 | name | def (1 row) update londiste.table_info set merge_state='ok', local=true where table_name in (:'tbl1', :'tbl2'); select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx3.table1 | fx3.table2 | name | def (1 row) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state='catching-up' where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'leaf1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state=null, local=false where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx3.table1 | fx3.table2 | name | def (1 row) select * from londiste.get_valid_pending_fkeys(:'leaf2'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) update londiste.table_info set merge_state=null, local=false where table_name = :'tbl2' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'leaf1'); from_table | to_table | fkey_name | fkey_def ------------+----------+-----------+---------- (0 rows) select * from londiste.get_valid_pending_fkeys(:'leaf2'); -- pick from_table | to_table | fkey_name | fkey_def ------------+------------+-----------+---------- fx3.table1 | fx3.table2 | name | def (1 row) londiste-sql-3.8/expected/londiste_provider.out000066400000000000000000000162021432153235300220710ustar00rootroot00000000000000set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- tables -- create table testdata ( id serial primary key, txt text ); create table testdata_nopk ( id serial, txt text ); select current_database(); current_database ------------------ regression (1 row) select * from pgq_node.register_location('aset', 'rnode', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('aset', 'root', 'rnode', 'londiste_root', null::text, null::int8, null::text); ret_code | ret_note ----------+------------------------------------------------------------ 200 | Node "rnode" initialized for queue "aset" with type "root" (1 row) select * from londiste.local_add_table('aset', 'public.testdata_nopk'); ret_code | ret_note ----------+---------------------------------------------------- 400 | Primary key missing on table: public.testdata_nopk (1 row) select * from londiste.local_add_table('aset', 'public.testdata'); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.testdata (1 row) select tgname from pg_trigger where tgrelid = 'public.testdata'::regclass order by 1; tgname ------------------------- _londiste_aset _londiste_aset_truncate (2 rows) insert into testdata (txt) values ('test-data'); select * from londiste.get_table_list('aset'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.testdata | t | ok | | | | | 0 | (1 row) select * from londiste.local_show_missing('aset'); obj_kind | obj_name ----------+----------------------------- S | public.testdata_id_seq S | public.testdata_nopk_id_seq r | public.testdata_nopk (3 rows) select * from londiste.local_remove_table('aset', 'public.testdata'); ret_code | ret_note ----------+-------------------------------- 200 | Table removed: public.testdata (1 row) select * from londiste.local_remove_table('aset', 'public.testdata'); ret_code | ret_note ----------+---------------------------------- 400 | Table not found: public.testdata (1 row) select tgname from pg_trigger where tgrelid = 'public.testdata'::regclass; tgname -------- (0 rows) select * from londiste.get_table_list('aset'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table ------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ (0 rows) select ev_id, ev_type, ev_data, ev_extra1 from pgq.event_template; ev_id | ev_type | ev_data | ev_extra1 -------+-----------------------+--------------------+----------------- 1 | londiste.add-table | public.testdata | 2 | I:id | id=1&txt=test-data | public.testdata 3 | londiste.remove-table | public.testdata | (3 rows) select * from londiste.local_show_missing('aset'); obj_kind | obj_name ----------+----------------------------- S | public.testdata_id_seq S | public.testdata_nopk_id_seq r | public.testdata r | public.testdata_nopk (4 rows) -- trigtest create table trg_test ( id int4 primary key, txt text ); select * from londiste.local_add_table('aset', 'public.trg_test', array['ev_extra4=''test='' || txt']); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.trg_test (1 row) select * from londiste.local_add_table('aset', 'public.trg_test'); ret_code | ret_note ----------+-------------------------------------- 200 | Table already added: public.trg_test (1 row) select * from londiste.local_add_table('aset', 'public.trg_test', array['ev_extra4=''test='' || txt'], 'handler=foobar'); ret_code | ret_note ----------+---------------------------------------------------------------- 410 | Table public.trg_test already added, but with different args: (1 row) insert into trg_test values (1, 'data'); truncate trg_test; select ev_id, ev_type, ev_data, ev_extra1, ev_extra4 from pgq.event_template where ev_extra1 = 'public.trg_test'; ev_id | ev_type | ev_data | ev_extra1 | ev_extra4 -------+---------+---------------+-----------------+----------- 5 | I:id | id=1&txt=data | public.trg_test | test=data 6 | R | | public.trg_test | (2 rows) select tgname from pg_trigger where tgrelid = 'public.trg_test'::regclass order by 1; tgname ------------------------- _londiste_aset _londiste_aset_truncate (2 rows) delete from londiste.table_info where table_name = 'public.trg_test'; select tgname from pg_trigger where tgrelid = 'public.trg_test'::regclass order by 1; tgname -------- (0 rows) -- handler test create table hdlr_test ( id int4 primary key, txt text ); select * from londiste.local_add_table('aset', 'public.hdlr_test'); ret_code | ret_note ----------+------------------------------- 200 | Table added: public.hdlr_test (1 row) insert into hdlr_test values (1, 'data'); select * from londiste.local_change_handler('aset', 'public.hdlr_test', array['ev_extra4=''test='' || txt'], 'handler=foobar'); ret_code | ret_note ----------+--------------------------------------------- 200 | Handler changed for table: public.hdlr_test (1 row) insert into hdlr_test values (2, 'data2'); select * from londiste.local_change_handler('aset', 'public.hdlr_test', '{}'::text[], ''); ret_code | ret_note ----------+--------------------------------------------- 200 | Handler changed for table: public.hdlr_test (1 row) insert into hdlr_test values (3, 'data3'); truncate hdlr_test; select ev_id, ev_type, ev_data, ev_extra1, ev_extra4 from pgq.event_template where ev_extra1 = 'public.hdlr_test'; ev_id | ev_type | ev_data | ev_extra1 | ev_extra4 -------+---------+----------------+------------------+------------ 8 | I:id | id=1&txt=data | public.hdlr_test | 9 | I:id | id=2&txt=data2 | public.hdlr_test | test=data2 10 | I:id | id=3&txt=data3 | public.hdlr_test | 11 | R | | public.hdlr_test | (4 rows) -- test proper trigger creation with add-table specific args select * from londiste.local_add_table('aset', 'public.trg_test', array['ev_extra4=''test='' || txt', 'expect_sync', 'skip']); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.trg_test (1 row) insert into trg_test values (2, 'data2'); ERROR: SKIP does not work in AFTER trigger. londiste-sql-3.8/expected/londiste_seqs.out000066400000000000000000000136551432153235300212230ustar00rootroot00000000000000set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- sequences -- create sequence masterseq; create sequence slaveseq; select * from pgq_node.register_location('seqroot', 'rnode', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('seqroot', 'root', 'rnode', 'londiste_root', null::text, null::int8, null::text); ret_code | ret_note ----------+--------------------------------------------------------------- 200 | Node "rnode" initialized for queue "seqroot" with type "root" (1 row) select * from londiste.local_add_seq('seqroot', 'masterseq'); ret_code | ret_note ----------+---------------------------------- 200 | Sequence added: public.masterseq (1 row) select * from londiste.local_add_seq('seqroot', 'masterseq'); ret_code | ret_note ----------+------------------------------------------ 201 | Sequence already added: public.masterseq (1 row) select * from londiste.root_check_seqs('seqroot'); ret_code | ret_note ----------+------------------- 100 | Sequences updated (1 row) select * from londiste.local_remove_seq('seqroot', 'masterseq'); ret_code | ret_note ----------+------------------------------------ 200 | Sequence removed: public.masterseq (1 row) select * from londiste.local_remove_seq('seqroot', 'masterseq'); ret_code | ret_note ----------+-------------------------------------- 400 | Sequence not found: public.masterseq (1 row) select * from londiste.get_seq_list('seqroot'); seq_name | last_value | local ----------+------------+------- (0 rows) select ev_id, ev_type, ev_data, ev_extra1 from pgq.event_template where ev_type like '%seq%'; ev_id | ev_type | ev_data | ev_extra1 -------+---------------------+------------------+------------------ 1 | londiste.update-seq | 30001 | public.masterseq 2 | londiste.remove-seq | public.masterseq | (2 rows) -- subscriber select * from pgq_node.register_location('seqbranch', 'subnode', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('seqbranch', 'rootnode', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('seqbranch', 'branch', 'subnode', 'londiste_branch', 'rootnode', 1, null::text); ret_code | ret_note ----------+--------------------------------------------------------------------- 200 | Node "subnode" initialized for queue "seqbranch" with type "branch" (1 row) select * from londiste.local_add_seq('seqbranch', 'masterseq'); ret_code | ret_note ----------+------------------------------------ 404 | Unknown sequence: public.masterseq (1 row) select * from londiste.global_update_seq('seqbranch', 'masterseq', 5); ret_code | ret_note ----------+------------------ 200 | Sequence updated (1 row) select * from londiste.local_add_seq('seqbranch', 'masterseq'); ret_code | ret_note ----------+---------------------------------- 200 | Sequence added: public.masterseq (1 row) select * from londiste.root_check_seqs('seqbranch'); ret_code | ret_note ----------+----------------- 402 | Not a root node (1 row) select * from londiste.get_seq_list('seqbranch'); seq_name | last_value | local ------------------+------------+------- public.masterseq | 5 | t (1 row) select * from londiste.local_remove_seq('seqbranch', 'masterseq'); ret_code | ret_note ----------+------------------------------------ 200 | Sequence removed: public.masterseq (1 row) select * from londiste.local_remove_seq('seqbranch', 'masterseq'); ret_code | ret_note ----------+-------------------------------------- 404 | Sequence not found: public.masterseq (1 row) -- seq auto-removal create table seqtable ( id1 serial primary key, id2 bigserial not null ); select * from londiste.local_add_table('seqroot', 'seqtable'); ret_code | ret_note ----------+------------------------------ 200 | Table added: public.seqtable (1 row) select * from londiste.local_add_seq('seqroot', 'seqtable_id1_seq'); ret_code | ret_note ----------+----------------------------------------- 200 | Sequence added: public.seqtable_id1_seq (1 row) select * from londiste.local_add_seq('seqroot', 'seqtable_id2_seq'); ret_code | ret_note ----------+----------------------------------------- 200 | Sequence added: public.seqtable_id2_seq (1 row) select * from londiste.get_table_list('seqroot'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table -----------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.seqtable | t | ok | | | | | 0 | (1 row) select * from londiste.get_seq_list('seqroot'); seq_name | last_value | local -------------------------+------------+------- public.seqtable_id1_seq | 30001 | t public.seqtable_id2_seq | 30001 | t (2 rows) select * from londiste.local_remove_table('seqroot', 'seqtable'); ret_code | ret_note ----------+-------------------------------- 200 | Table removed: public.seqtable (1 row) select * from londiste.get_seq_list('seqroot'); seq_name | last_value | local ----------+------------+------- (0 rows) londiste-sql-3.8/expected/londiste_subscriber.out000066400000000000000000000107271432153235300224100ustar00rootroot00000000000000set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- tables -- create table slavedata ( id serial primary key, data text ); select current_database(); current_database ------------------ regression (1 row) select * from pgq_node.register_location('branch_set', 'snode', 'dbname=db', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.register_location('branch_set', 'pnode', 'dbname=db2', false); ret_code | ret_note ----------+--------------------- 200 | Location registered (1 row) select * from pgq_node.create_node('branch_set', 'branch', 'snode', 'londiste_branch', 'pnode', 100, null::text); ret_code | ret_note ----------+-------------------------------------------------------------------- 200 | Node "snode" initialized for queue "branch_set" with type "branch" (1 row) select * from londiste.local_show_missing('branch_set'); obj_kind | obj_name ----------+---------- (0 rows) select * from londiste.local_add_table('branch_set', 'public.slavedata'); ret_code | ret_note ----------+------------------------------------------------ 404 | Table not available on queue: public.slavedata (1 row) select * from londiste.global_add_table('branch_set', 'public.slavedata'); ret_code | ret_note ----------+------------------------------- 200 | Table added: public.slavedata (1 row) select * from londiste.local_add_table('branch_set', 'public.slavedata'); ret_code | ret_note ----------+------------------------------- 200 | Table added: public.slavedata (1 row) select * from londiste.global_add_table('branch_set', 'public.tmp'); ret_code | ret_note ----------+------------------------- 200 | Table added: public.tmp (1 row) select * from londiste.get_table_list('branch_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table ------------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.slavedata | t | | | | | | 0 | public.tmp | f | | | | | | 0 | (2 rows) select * from londiste.local_set_table_state('branch_set', 'public.slavedata', null, 'in-copy'); ret_code | ret_note ----------+----------------------------------------------- 200 | Table public.slavedata state set to 'in-copy' (1 row) select * from londiste.get_table_list('branch_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table ------------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.slavedata | t | in-copy | | | | | 0 | public.tmp | f | | | | | | 0 | (2 rows) select * from londiste.global_remove_table('branch_set', 'public.tmp'); ret_code | ret_note ----------+--------------------------- 200 | Table removed: public.tmp (1 row) select * from londiste.local_remove_table('branch_set', 'public.slavedata'); ret_code | ret_note ----------+--------------------------------- 200 | Table removed: public.slavedata (1 row) select * from londiste.local_remove_table('branch_set', 'public.slavedata'); ret_code | ret_note ----------+------------------------------------------------ 400 | Table not registered locally: public.slavedata (1 row) select * from londiste.get_table_list('branch_set'); table_name | local | merge_state | custom_snapshot | table_attrs | dropped_ddl | copy_role | copy_pos | dest_table ------------------+-------+-------------+-----------------+-------------+-------------+-----------+----------+------------ public.slavedata | f | | | | | | 0 | (1 row) select * from londiste.local_show_missing('branch_set'); obj_kind | obj_name ----------+------------------ r | public.slavedata (1 row) londiste-sql-3.8/functions/000077500000000000000000000000001432153235300160135ustar00rootroot00000000000000londiste-sql-3.8/functions/londiste.create_partition.sql000066400000000000000000000205741432153235300237200ustar00rootroot00000000000000 create or replace function londiste.create_partition( i_table text, i_part text, i_pkeys text, i_part_field text, i_part_time timestamptz, i_part_period text ) returns int as $$ ------------------------------------------------------------------------ -- Function: londiste.create_partition -- -- Creates inherited child table if it does not exist by copying parent table's structure. -- Locks londiste.table_info table to avoid parallel creation of any partitions. -- -- Elements that are copied over by "LIKE x INCLUDING ALL": -- * Defaults -- * Constraints -- * Indexes -- * Storage options (9.0+) -- * Comments (9.0+) -- -- Elements that are copied over manually because LIKE ALL does not support them: -- * Grants -- * Triggers -- * Rules -- -- Parameters: -- i_table - name of parent table -- i_part - name of partition table to create -- i_pkeys - primary key fields (comma separated, used to create constraint). -- i_part_field - field used to partition table (when not partitioned by field, value is NULL) -- i_part_time - partition time -- i_part_period - period of partitioned data, current possible values are 'hour', 'day', 'month' and 'year' -- -- Example: -- select londiste.create_partition('aggregate.user_call_monthly', 'aggregate.user_call_monthly_2010_01', 'key_user', 'period_start', '2010-01-10 11:00'::timestamptz, 'month'); -- ------------------------------------------------------------------------ declare chk_start text; chk_end text; part_start timestamptz; part_end timestamptz; parent_schema text; parent_name text; parent_oid oid; part_schema text; part_name text; owner name; pos int4; fq_table text; fq_part text; q_grantee text; g record; r record; tg record; sql text; pgver integer; r_oldtbl text; r_extra text; r_sql text; begin if i_table is null or i_part is null then raise exception 'need table and part'; end if; -- load postgres version (XYYZZ). show server_version_num into pgver; -- parent table schema and name + quoted name pos := position('.' in i_table); if pos > 0 then parent_schema := substring(i_table for pos - 1); parent_name := substring(i_table from pos + 1); else parent_schema := 'public'; parent_name := i_table; end if; fq_table := quote_ident(parent_schema) || '.' || quote_ident(parent_name); -- part table schema and name + quoted name pos := position('.' in i_part); if pos > 0 then part_schema := substring(i_part for pos - 1); part_name := substring(i_part from pos + 1); else part_schema := 'public'; part_name := i_part; end if; fq_part := quote_ident(part_schema) || '.' || quote_ident(part_name); -- allow only single creation at a time, without affecting DML operations -- (changed from locking parent table to avoid deadlocks from concurrent workers) execute 'lock table londiste.table_info in share update exclusive mode'; parent_oid := fq_table::regclass::oid; -- check if part table exists perform 1 from pg_class t, pg_namespace s where t.relnamespace = s.oid and s.nspname = part_schema and t.relname = part_name; if found then return 0; end if; -- need to use 'like' to get indexes sql := 'create table ' || fq_part || ' (like ' || fq_table; if pgver >= 90000 then sql := sql || ' including all'; else sql := sql || ' including indexes including constraints including defaults'; end if; sql := sql || ') inherits (' || fq_table || ')'; execute sql; -- find out parent table owner select o.rolname into owner from pg_class t, pg_namespace s, pg_roles o where t.relnamespace = s.oid and s.nspname = parent_schema and t.relname = parent_name and t.relowner = o.oid; -- set proper part table ownership if owner != user then sql = 'alter table ' || fq_part || ' owner to ' || quote_ident(owner); execute sql; end if; -- extra check constraint if i_part_field != '' then part_start := date_trunc(i_part_period, i_part_time); part_end := part_start + ('1 ' || i_part_period)::interval; chk_start := quote_literal(to_char(part_start, 'YYYY-MM-DD HH24:MI:SS')); chk_end := quote_literal(to_char(part_end, 'YYYY-MM-DD HH24:MI:SS')); sql := 'alter table '|| fq_part || ' add check (' || quote_ident(i_part_field) || ' >= ' || chk_start || ' and ' || quote_ident(i_part_field) || ' < ' || chk_end || ')'; execute sql; end if; -- load grants from parent table for g in select grantor, grantee, privilege_type, is_grantable from information_schema.table_privileges where table_schema = parent_schema and table_name = parent_name loop if g.grantee = 'PUBLIC' then q_grantee = 'public'; else q_grantee := quote_ident(g.grantee); end if; sql := 'grant ' || g.privilege_type || ' on ' || fq_part || ' to ' || q_grantee; if g.is_grantable = 'YES' then sql := sql || ' with grant option'; end if; execute sql; end loop; -- generate triggers info query sql := 'SELECT tgname, tgenabled,' || ' pg_catalog.pg_get_triggerdef(oid) as tgdef' || ' FROM pg_catalog.pg_trigger ' || ' WHERE tgrelid = ' || parent_oid::text || ' AND '; if pgver >= 90000 then sql := sql || ' NOT tgisinternal'; else sql := sql || ' NOT tgisconstraint'; end if; -- copy triggers for tg in execute sql loop sql := regexp_replace(tg.tgdef, E' ON ([[:alnum:]_.]+|"([^"]|"")+")+ ', ' ON ' || fq_part || ' '); if sql = tg.tgdef then raise exception 'Failed to reconstruct the trigger: %', sql; end if; execute sql; if tg.tgenabled = 'O' then -- standard mode r_extra := NULL; elsif tg.tgenabled = 'D' then r_extra := ' DISABLE TRIGGER '; elsif tg.tgenabled = 'A' then r_extra := ' ENABLE ALWAYS TRIGGER '; elsif tg.tgenabled = 'R' then r_extra := ' ENABLE REPLICA TRIGGER '; else raise exception 'Unknown trigger mode: %', tg.tgenabled; end if; if r_extra is not null then sql := 'ALTER TABLE ' || fq_part || r_extra || quote_ident(tg.tgname); execute sql; end if; end loop; -- copy rules for r in select rw.rulename, rw.ev_enabled, pg_catalog.pg_get_ruledef(rw.oid) as definition from pg_catalog.pg_rewrite rw where rw.ev_class = parent_oid and rw.rulename <> '_RETURN'::name loop -- try to skip rule name r_extra := 'CREATE RULE ' || quote_ident(r.rulename) || ' AS'; r_sql := substr(r.definition, 1, char_length(r_extra)); if r_sql = r_extra then r_sql := substr(r.definition, char_length(r_extra)); else raise exception 'failed to match rule name'; end if; -- no clue what name was used in defn, so find it from sql r_oldtbl := substring(r_sql from ' TO (([[:alnum:]_.]+|"([^"]+|"")+")+)[[:space:]]'); if char_length(r_oldtbl) > 0 then sql := replace(r.definition, r_oldtbl, fq_part); else raise exception 'failed to find original table name'; end if; execute sql; -- rule flags r_extra := NULL; if r.ev_enabled = 'R' then r_extra = ' ENABLE REPLICA RULE '; elsif r.ev_enabled = 'A' then r_extra = ' ENABLE ALWAYS RULE '; elsif r.ev_enabled = 'D' then r_extra = ' DISABLE RULE '; elsif r.ev_enabled <> 'O' then raise exception 'unknown rule option: %', r.ev_enabled; end if; if r_extra is not null then sql := 'ALTER TABLE ' || fq_part || r_extra || quote_ident(r.rulename); execute sql; end if; end loop; return 1; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.create_trigger.sql000066400000000000000000000215031432153235300233430ustar00rootroot00000000000000create or replace function londiste.create_trigger( in i_queue_name text, in i_table_name text, in i_trg_args text[], in i_dest_table text, in i_node_type text, out ret_code int4, out ret_note text, out trigger_name text) as $$ ------------------------------------------------------------------------ -- Function: londiste.create_trigger(5) -- -- Create or replace londiste trigger(s) -- -- Parameters: -- i_queue_name - queue name -- i_table_name - table name -- i_trg_args - args to trigger -- i_dest_table - actual name of destination table (NULL if same as src) -- i_node_type - l3 node type -- -- Trigger args: -- See documentation for pgq triggers. -- -- Trigger creation flags (default: AIUDL): -- I - ON INSERT -- U - ON UPDATE -- D - ON DELETE -- Q - use pgq.sqltriga() as trigger function -- L - use pgq.logutriga() as trigger function -- J - use pgq.jsontriga() as trigger function -- B - BEFORE -- A - AFTER -- S - SKIP -- -- Returns: -- 200 - Ok -- 201 - Trigger not created -- 405 - Multiple SKIP triggers -- ------------------------------------------------------------------------ declare trigger_name text; lg_func text; lg_pos text; lg_event text; lg_args text[]; _old_tgargs bytea; _new_tgargs bytea; trunctrg_name text; pgversion int; sql text; arg text; i integer; _extra_args text[] := '{}'; -- skip trigger _skip_prefix text := 'zzz_'; _skip_trg_count integer; _skip_trg_name text; -- given tgflags array _tgflags char[]; -- ordinary argument array _args text[]; -- array with all valid tgflags values _valid_flags char[] := array['B','A','Q','L','J','I','U','D','S']; -- argument flags _skip boolean := false; _no_triggers boolean := false; _got_extra1 boolean := false; begin -- parse trigger args if array_lower(i_trg_args, 1) is not null then for i in array_lower(i_trg_args, 1) .. array_upper(i_trg_args, 1) loop arg := i_trg_args[i]; if arg like 'tgflags=%' then -- special flag handling arg := upper(substr(arg, 9)); for j in array_lower(_valid_flags, 1) .. array_upper(_valid_flags, 1) loop if position(_valid_flags[j] in arg) > 0 then _tgflags := array_append(_tgflags, _valid_flags[j]); end if; end loop; elsif arg = 'no_triggers' then _no_triggers := true; elsif lower(arg) = 'skip' then _skip := true; elsif arg = 'virtual_table' then _no_triggers := true; -- do not create triggers elsif arg not in ('expect_sync', 'skip_truncate', 'merge_all', 'no_merge') then -- ignore add-table args if arg like 'ev_extra1=%' then _got_extra1 := true; end if; -- ordinary arg _args = array_append(_args, quote_literal(arg)); end if; end loop; end if; if _no_triggers then select 201, 'Trigger not created' into ret_code, ret_note; return; end if; if i_dest_table <> i_table_name and not _got_extra1 then -- if renamed table, enforce trigger to put -- global table name into extra1 arg := 'ev_extra1=' || quote_literal(i_table_name); _args := array_append(_args, quote_literal(arg)); end if; trigger_name := '_londiste_' || i_queue_name; lg_func := 'pgq.logutriga'; lg_event := ''; lg_args := array[quote_literal(i_queue_name)]; lg_pos := 'after'; if array_lower(_args, 1) is not null then lg_args := lg_args || _args; end if; if 'B' = any(_tgflags) then lg_pos := 'before'; end if; if 'A' = any(_tgflags) then lg_pos := 'after'; end if; if 'Q' = any(_tgflags) then lg_func := 'pgq.sqltriga'; end if; if 'L' = any(_tgflags) then lg_func := 'pgq.logutriga'; end if; if 'J' = any(_tgflags) then lg_func := 'pgq.jsontriga'; end if; if 'I' = any(_tgflags) then lg_event := lg_event || ' or insert'; end if; if 'U' = any(_tgflags) then lg_event := lg_event || ' or update'; end if; if 'D' = any(_tgflags) then lg_event := lg_event || ' or delete'; end if; if 'S' = any(_tgflags) then _skip := true; end if; if i_node_type = 'leaf' then -- on weird leafs the trigger funcs may not exist perform 1 from pg_proc p join pg_namespace n on (n.oid = p.pronamespace) where n.nspname = 'pgq' and p.proname in ('logutriga', 'sqltriga', 'jsontriga'); if not found then select 201, 'Trigger not created' into ret_code, ret_note; return; end if; -- on regular leaf, install deny trigger _extra_args := array_append(_extra_args, quote_literal('deny')); end if; if _skip or lg_pos = 'after' then -- get count and name of existing skip triggers select count(*), min(t.tgname) into _skip_trg_count, _skip_trg_name from pg_catalog.pg_trigger t where t.tgrelid = londiste.find_table_oid(i_dest_table) and position(E'\\000SKIP\\000'::bytea in tgargs) > 0; end if; -- make sure new trigger won't be effectively inactive if lg_pos = 'after' and _skip_trg_count > 0 then select 403, 'AFTER trigger cannot work with SKIP trigger(s)' into ret_code, ret_note; return; end if; -- if skip param given, rename previous skip triggers and prefix current if _skip then -- if no previous skip triggers, prefix name and add SKIP to args if _skip_trg_count = 0 then trigger_name := _skip_prefix || trigger_name; lg_args := array_append(lg_args, quote_literal('SKIP')); -- if one previous skip trigger, check it's prefix and -- do not use SKIP on current trigger elsif _skip_trg_count = 1 then -- if not prefixed then rename if position(_skip_prefix in _skip_trg_name) != 1 then sql := 'alter trigger ' || _skip_trg_name || ' on ' || londiste.quote_fqname(i_dest_table) || ' rename to ' || _skip_prefix || _skip_trg_name; execute sql; end if; else select 405, 'Multiple SKIP triggers' into ret_code, ret_note; return; end if; end if; -- create Ins/Upd/Del trigger if it does not exists already select t.tgargs from pg_catalog.pg_trigger t where t.tgrelid = londiste.find_table_oid(i_dest_table) and t.tgname = trigger_name into _old_tgargs; if found then _new_tgargs := decode(lg_args[1], 'escape'); for i in 2 .. array_upper(lg_args, 1) loop _new_tgargs := _new_tgargs || E'\\000'::bytea || decode(lg_args[i], 'escape'); end loop; if _old_tgargs is distinct from _new_tgargs then sql := 'drop trigger if exists ' || quote_ident(trigger_name) || ' on ' || londiste.quote_fqname(i_dest_table); execute sql; end if; end if; if not found or _old_tgargs is distinct from _new_tgargs then -- finalize event lg_event := substr(lg_event, 4); -- remove ' or ' if lg_event = '' then lg_event := 'insert or update or delete'; end if; -- create trigger lg_args := lg_args || _extra_args; sql := 'create trigger ' || quote_ident(trigger_name) || ' ' || lg_pos || ' ' || lg_event || ' on ' || londiste.quote_fqname(i_dest_table) || ' for each row execute procedure ' || lg_func || '(' || array_to_string(lg_args, ', ') || ')'; execute sql; end if; -- create truncate trigger if it does not exists already show server_version_num into pgversion; if pgversion >= 80400 then trunctrg_name := '_londiste_' || i_queue_name || '_truncate'; perform 1 from pg_catalog.pg_trigger where tgrelid = londiste.find_table_oid(i_dest_table) and tgname = trunctrg_name; if not found then _extra_args := quote_literal(i_queue_name) || _extra_args; sql := 'create trigger ' || quote_ident(trunctrg_name) || ' after truncate on ' || londiste.quote_fqname(i_dest_table) || ' for each statement execute procedure pgq.sqltriga(' || array_to_string(_extra_args, ', ') || ')'; execute sql; end if; end if; select 200, 'OK' into ret_code, ret_note; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.drop_obsolete_partitions.sql000066400000000000000000000017561432153235300255010ustar00rootroot00000000000000 create or replace function londiste.drop_obsolete_partitions ( in i_parent_table text, in i_retention_period interval, in i_partition_period text ) returns setof text as $$ ------------------------------------------------------------------------------- -- Function: londiste.drop_obsolete_partitions(3) -- -- Drop obsolete partitions of partition-by-date parent table. -- -- Parameters: -- i_parent_table Master table from which partitions are inherited -- i_retention_period How long to keep partitions around -- i_partition_period One of: year, month, day, hour -- -- Returns: -- Names of partitions dropped ------------------------------------------------------------------------------- declare _part text; begin for _part in select londiste.list_obsolete_partitions (i_parent_table, i_retention_period, i_partition_period) loop execute 'drop table '|| _part; return next _part; end loop; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.drop_table_triggers.sql000066400000000000000000000034331432153235300244000ustar00rootroot00000000000000 create or replace function londiste.drop_table_triggers( in i_queue_name text, in i_table_name text) returns void as $$ -- ---------------------------------------------------------------------- -- Function: londiste.drop_table_triggers(2) -- -- Remove Londiste triggers from table. -- -- Parameters: -- i_queue_name - set name -- i_table_name - table name -- -- Returns: -- 200 - OK -- 404 - Table not found -- ---------------------------------------------------------------------- declare logtrg_name text; b_queue_name bytea; _dest_table text; begin select coalesce(dest_table, table_name) from londiste.table_info t where t.queue_name = i_queue_name and t.table_name = i_table_name into _dest_table; if not found then return; end if; -- skip if no triggers found on that table perform 1 from pg_catalog.pg_trigger where tgrelid = londiste.find_table_oid(_dest_table); if not found then return; end if; -- cast to bytea b_queue_name := decode(replace(i_queue_name, E'\\', E'\\\\'), 'escape'); -- drop all replication triggers that target our queue. -- by checking trigger func and queue name there is not -- dependency on naming standard or side-storage. for logtrg_name in select tgname from pg_catalog.pg_trigger where tgrelid = londiste.find_table_oid(_dest_table) and londiste.is_replica_func(tgfoid) and octet_length(tgargs) > 0 and substring(tgargs for (position(E'\\000'::bytea in tgargs) - 1)) = b_queue_name loop execute 'drop trigger ' || quote_ident(logtrg_name) || ' on ' || londiste.quote_fqname(_dest_table); end loop; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.execute_finish.sql000066400000000000000000000024741432153235300233650ustar00rootroot00000000000000create or replace function londiste.execute_finish( in i_queue_name text, in i_file_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.execute_finish(2) -- -- Finish execution of DDL. Should be called at the -- end of the transaction that does the SQL execution. -- -- Called-by: -- Londiste setup tool on root, replay on branches/leafs. -- -- Returns: -- 200 - Proceed. -- 404 - Current entry not found, execute_start() was not called? -- ---------------------------------------------------------------------- declare is_root boolean; sql text; attrs text; begin is_root := pgq_node.is_root_node(i_queue_name); select execute_sql, execute_attrs into sql, attrs from londiste.applied_execute where execute_file = i_file_name; if not found then select 404, 'execute_file called without execute_start' into ret_code, ret_note; return; end if; if is_root then perform pgq.insert_event(i_queue_name, 'EXECUTE', sql, i_file_name, attrs, null, null); end if; select 200, 'Execute finished: ' || i_file_name into ret_code, ret_note; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.execute_start.sql000066400000000000000000000065141432153235300232410ustar00rootroot00000000000000create or replace function londiste.execute_start( in i_queue_name text, in i_file_name text, in i_sql text, in i_expect_root boolean, in i_attrs text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.execute_start(5) -- -- Start execution of DDL. Should be called at the -- start of the transaction that does the SQL execution. -- -- Called-by: -- Londiste setup tool on root, replay on branches/leafs. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_file_name - Unique ID for SQL -- i_sql - Actual script (informative, not used here) -- i_expect_root - Is this on root? Setup tool sets this to avoid -- execution on branches. -- i_attrs - urlencoded dict of extra attributes. -- The value will be put into ev_extra2 -- field of outgoing event. -- -- Returns: -- 200 - Proceed. -- 201 - Already applied -- 401 - Not root. -- 404 - No such queue -- ---------------------------------------------------------------------- declare is_root boolean; new_row_count int4; begin is_root := pgq_node.is_root_node(i_queue_name); if i_expect_root then if not is_root then select 401, 'Node is not root node: ' || i_queue_name into ret_code, ret_note; return; end if; end if; -- this also lock against potetial parallel execute insert into londiste.applied_execute (queue_name, execute_file, execute_sql, execute_attrs) values (i_queue_name, i_file_name, i_sql, i_attrs) on conflict do nothing; get diagnostics new_row_count = row_count; if new_row_count = 0 then select 201, 'EXECUTE: "' || i_file_name || '" already applied, skipping' into ret_code, ret_note; return; end if; select 200, 'Executing: ' || i_file_name into ret_code, ret_note; return; end; $$ language plpgsql; create or replace function londiste.execute_start( in i_queue_name text, in i_file_name text, in i_sql text, in i_expect_root boolean, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.execute_start(4) -- -- Start execution of DDL. Should be called at the -- start of the transaction that does the SQL execution. -- -- Called-by: -- Londiste setup tool on root, replay on branches/leafs. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_file_name - Unique ID for SQL -- i_sql - Actual script (informative, not used here) -- i_expect_root - Is this on root? Setup tool sets this to avoid -- execution on branches. -- -- Returns: -- 200 - Proceed. -- 301 - Already applied -- 401 - Not root. -- 404 - No such queue -- ---------------------------------------------------------------------- begin select f.ret_code, f.ret_note from londiste.execute_start(i_queue_name, i_file_name, i_sql, i_expect_root, null) f into ret_code, ret_note; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.find_column_types.sql000066400000000000000000000022371432153235300241010ustar00rootroot00000000000000create or replace function londiste.find_column_types(tbl text) returns text as $plpgsql$ -- ---------------------------------------------------------------------- -- Function: londiste.find_column_types(1) -- -- Returns columnt type string for logtriga(). -- -- Parameters: -- tbl - fqname -- -- Returns: -- String of 'kv'. -- ---------------------------------------------------------------------- declare res text; col record; tbl_oid oid; begin tbl_oid := londiste.find_table_oid(tbl); res := ''; for col in SELECT CASE WHEN k.attname IS NOT NULL THEN 'k' ELSE 'v' END AS type FROM pg_attribute a LEFT JOIN ( SELECT k.attnum, k.attname FROM pg_index i, pg_attribute k WHERE i.indrelid = tbl_oid AND k.attrelid = i.indexrelid AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped ) k ON (k.attnum = a.attnum) WHERE a.attrelid = tbl_oid AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum loop res := res || col.type; end loop; return res; end; $plpgsql$ language plpgsql strict stable; londiste-sql-3.8/functions/londiste.find_table_fkeys.sql000066400000000000000000000027131432153235300236470ustar00rootroot00000000000000 create or replace function londiste.find_table_fkeys(i_table_name text) returns setof londiste.pending_fkeys as $$ -- ---------------------------------------------------------------------- -- Function: londiste.find_table_fkeys(1) -- -- Return all active fkeys. -- -- Parameters: -- i_table_name - fqname -- -- Returns: -- from_table - fqname -- to_table - fqname -- fkey_name - name -- fkey_def - full def -- ---------------------------------------------------------------------- declare fkey record; tbl_oid oid; begin select londiste.find_table_oid(i_table_name) into tbl_oid; for fkey in select n1.nspname || '.' || t1.relname as from_table, n2.nspname || '.' || t2.relname as to_table, conname::text as fkey_name, 'alter table only ' || quote_ident(n1.nspname) || '.' || quote_ident(t1.relname) || ' add constraint ' || quote_ident(conname::text) || ' ' || pg_get_constraintdef(c.oid) as fkey_def from pg_constraint c, pg_namespace n1, pg_class t1, pg_namespace n2, pg_class t2 where c.contype = 'f' and (c.conrelid = tbl_oid or c.confrelid = tbl_oid) and t1.oid = c.conrelid and n1.oid = t1.relnamespace and t2.oid = c.confrelid and n2.oid = t2.relnamespace order by 1,2,3 loop return next fkey; end loop; return; end; $$ language plpgsql strict stable; londiste-sql-3.8/functions/londiste.find_table_oid.sql000066400000000000000000000042061432153235300233000ustar00rootroot00000000000000 drop function if exists londiste.find_seq_oid(text); drop function if exists londiste.find_table_oid(text); drop function if exists londiste.find_rel_oid(text, text); create or replace function londiste.find_rel_oid(i_fqname text, i_kind text) returns oid as $$ -- ---------------------------------------------------------------------- -- Function: londiste.find_rel_oid(2) -- -- Find pg_class row oid. -- -- Parameters: -- i_fqname - fq object name -- i_kind - relkind value -- -- Returns: -- oid or exception of not found -- ---------------------------------------------------------------------- declare res oid; pos integer; schema text; name text; begin pos := position('.' in i_fqname); if pos > 0 then schema := substring(i_fqname for pos - 1); name := substring(i_fqname from pos + 1); else schema := 'public'; name := i_fqname; end if; select c.oid into res from pg_namespace n, pg_class c where c.relnamespace = n.oid and c.relkind = i_kind and n.nspname = schema and c.relname = name; if not found then res := NULL; end if; return res; end; $$ language plpgsql strict stable; create or replace function londiste.find_table_oid(tbl text) returns oid as $$ -- ---------------------------------------------------------------------- -- Function: londiste.find_table_oid(1) -- -- Find table oid based on fqname. -- -- Parameters: -- tbl - fqname -- -- Returns: -- oid -- ---------------------------------------------------------------------- begin return londiste.find_rel_oid(tbl, 'r'); end; $$ language plpgsql strict stable; create or replace function londiste.find_seq_oid(seq text) returns oid as $$ -- ---------------------------------------------------------------------- -- Function: londiste.find_seq_oid(1) -- -- Find sequence oid based on fqname. -- -- Parameters: -- seq - fqname -- -- Returns: -- oid -- ---------------------------------------------------------------------- begin return londiste.find_rel_oid(seq, 'S'); end; $$ language plpgsql strict stable; londiste-sql-3.8/functions/londiste.get_seq_list.sql000066400000000000000000000016141432153235300230400ustar00rootroot00000000000000 create or replace function londiste.get_seq_list( in i_queue_name text, out seq_name text, out last_value int8, out local boolean) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: londiste.get_seq_list(1) -- -- Returns registered seqs on this Londiste node. -- -- Result fields: -- seq_name - fully qualified name of sequence -- last_value - last globally published value -- local - is locally registered -- ---------------------------------------------------------------------- declare rec record; begin for seq_name, last_value, local in select s.seq_name, s.last_value, s.local from londiste.seq_info s where s.queue_name = i_queue_name order by s.nr, s.seq_name loop return next; end loop; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.get_table_list.sql000066400000000000000000000144471432153235300233470ustar00rootroot00000000000000 drop function if exists londiste.get_table_list(text); create or replace function londiste.get_table_list( in i_queue_name text, out table_name text, out local boolean, out merge_state text, out custom_snapshot text, out table_attrs text, out dropped_ddl text, out copy_role text, out copy_pos int4, out dest_table text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: londiste.get_table_list(1) -- -- Return info about registered tables. -- -- Parameters: -- i_queue_name - cascaded queue name -- -- Returns: -- table_name - fully-quelified table name -- local - does events needs to be applied to local table -- merge_state - show phase of initial copy -- custom_snapshot - remote snapshot of COPY transaction -- table_attrs - urlencoded dict of table attributes -- dropped_ddl - partition combining: temp place to put DDL -- copy_role - partition combining: how to handle copy -- copy_pos - position in parallel copy working order -- -- copy_role = lead: -- on copy start, drop indexes and store in dropped_ddl -- on copy finish change state to catching-up, then wait until copy_role turns to NULL -- catching-up: if dropped_ddl not NULL, restore them -- copy_role = wait-copy: -- on copy start wait, until role changes (to wait-replay) -- copy_role = wait-replay: -- on copy finish, tag as 'catching-up' -- wait until copy_role is NULL, then proceed -- ---------------------------------------------------------------------- begin for table_name, local, merge_state, custom_snapshot, table_attrs, dropped_ddl, dest_table in select t.table_name, t.local, t.merge_state, t.custom_snapshot, t.table_attrs, t.dropped_ddl, t.dest_table from londiste.table_info t where t.queue_name = i_queue_name order by t.nr, t.table_name loop copy_role := null; copy_pos := 0; if merge_state in ('in-copy', 'catching-up') then select f.copy_role, f.copy_pos from londiste._coordinate_copy(i_queue_name, table_name) f into copy_role, copy_pos; end if; return next; end loop; return; end; $$ language plpgsql strict stable; create or replace function londiste._coordinate_copy( in i_queue_name text, in i_table_name text, out copy_role text, out copy_pos int4) as $$ -- if the table is in middle of copy from multiple partitions, -- the copy processes need coordination. declare q_part1 text; q_part_ddl text; n_parts int4; n_done int4; _table_name text; n_combined_queue text; merge_state text; dest_table text; dropped_ddl text; begin copy_pos := 0; copy_role := null; select t.merge_state, t.dest_table, t.dropped_ddl, min(case when t2.local then t2.queue_name else null end) as _queue1, min(case when t2.local and t2.dropped_ddl is not null then t2.queue_name else null end) as _queue1ddl, count(case when t2.local then t2.table_name else null end) as _total, count(case when t2.local then nullif(t2.merge_state, 'in-copy') else null end) as _done, min(n.combined_queue) as _combined_queue, count(nullif(t2.queue_name < i_queue_name and t.merge_state = 'in-copy' and t2.merge_state = 'in-copy', false)) as _copy_pos from londiste.table_info t join pgq_node.node_info n on (n.queue_name = t.queue_name) left join pgq_node.node_info n2 on (n2.combined_queue = n.combined_queue or (n2.combined_queue is null and n.combined_queue is null)) left join londiste.table_info t2 on (coalesce(t2.dest_table, t2.table_name) = coalesce(t.dest_table, t.table_name) and t2.queue_name = n2.queue_name and (t2.merge_state is null or t2.merge_state != 'ok')) where t.queue_name = i_queue_name and t.table_name = i_table_name group by t.nr, t.table_name, t.local, t.merge_state, t.custom_snapshot, t.table_attrs, t.dropped_ddl, t.dest_table into merge_state, dest_table, dropped_ddl, q_part1, q_part_ddl, n_parts, n_done, n_combined_queue, copy_pos; -- q_part1, q_part_ddl, n_parts, n_done, n_combined_queue, copy_pos, dest_table -- be more robust against late joiners q_part1 := coalesce(q_part_ddl, q_part1); -- turn the logic off if no merge is happening if n_parts = 1 then q_part1 := null; end if; if q_part1 is not null then if i_queue_name = q_part1 then -- lead if merge_state = 'in-copy' then if dropped_ddl is null and n_done > 0 then -- seems late addition, let it copy with indexes copy_role := 'wait-replay'; elsif n_done < n_parts then -- show copy_role only if need to drop ddl or already did drop ddl copy_role := 'lead'; end if; -- make sure it cannot be made to wait copy_pos := 0; end if; if merge_state = 'catching-up' and dropped_ddl is not null then -- show copy_role only if need to wait for others if n_done < n_parts then copy_role := 'wait-replay'; end if; end if; else -- follow if merge_state = 'in-copy' then if q_part_ddl is not null then -- can copy, wait in replay until lead has applied ddl copy_role := 'wait-replay'; elsif n_done > 0 then -- ddl is not dropped, others are active, copy without touching ddl copy_role := 'wait-replay'; else -- wait for lead to drop ddl copy_role := 'wait-copy'; end if; elsif merge_state = 'catching-up' then -- show copy_role only if need to wait for lead if q_part_ddl is not null then copy_role := 'wait-replay'; end if; end if; end if; end if; return; end; $$ language plpgsql strict stable; londiste-sql-3.8/functions/londiste.global_add_table.sql000066400000000000000000000036341432153235300236010ustar00rootroot00000000000000 create or replace function londiste.global_add_table( in i_queue_name text, in i_table_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.global_add_table(2) -- -- Register table on Londiste set. -- -- This means its available from root, events for it appear -- in queue and nodes can attach to it. -- -- Called by: -- on root - londiste.local_add_table() -- elsewhere - londiste consumer when receives new table event -- -- Returns: -- 200 - Ok -- 400 - No such set -- ---------------------------------------------------------------------- declare fq_table_name text; _cqueue text; begin fq_table_name := londiste.make_fqname(i_table_name); select combined_queue into _cqueue from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 400, 'No such queue: ' || i_queue_name into ret_code, ret_note; return; end if; perform 1 from londiste.table_info where queue_name = i_queue_name and table_name = fq_table_name; if found then select 200, 'Table already added: ' || fq_table_name into ret_code, ret_note; return; end if; insert into londiste.table_info (queue_name, table_name) values (i_queue_name, fq_table_name); select 200, 'Table added: ' || i_table_name into ret_code, ret_note; -- let the combined node know about it too if _cqueue is not null then perform londiste.global_add_table(_cqueue, i_table_name); end if; return; exception -- seems the row was added from parallel connection (setup vs. replay) when unique_violation then select 200, 'Table already added: ' || i_table_name into ret_code, ret_note; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.global_remove_seq.sql000066400000000000000000000022031432153235300240360ustar00rootroot00000000000000 create or replace function londiste.global_remove_seq( in i_queue_name text, in i_seq_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.global_remove_seq(2) -- -- Removes sequence registration in set. -- -- Called by: -- - On root by londiste.local_remove_seq() -- - Elsewhere by consumer receiving seq remove event -- -- Returns: -- 200 - OK -- 400 - not found -- ---------------------------------------------------------------------- declare fq_name text; begin fq_name := londiste.make_fqname(i_seq_name); delete from londiste.seq_info where queue_name = i_queue_name and seq_name = fq_name; if not found then select 400, 'Sequence not found: '||fq_name into ret_code, ret_note; return; end if; if pgq_node.is_root_node(i_queue_name) then perform londiste.root_notify_change(i_queue_name, 'londiste.remove-seq', fq_name); end if; select 200, 'Sequence removed: '||fq_name into ret_code, ret_note; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.global_remove_table.sql000066400000000000000000000023641432153235300243450ustar00rootroot00000000000000 create or replace function londiste.global_remove_table( in i_queue_name text, in i_table_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.global_remove_table(2) -- -- Removes tables registration in set. -- -- Means that nodes cannot attach to this table anymore. -- -- Called by: -- - On root by londiste.local_remove_table() -- - Elsewhere by consumer receiving table remove event -- -- Returns: -- 200 - OK -- 400 - not found -- ---------------------------------------------------------------------- declare fq_table_name text; begin fq_table_name := londiste.make_fqname(i_table_name); if not pgq_node.is_root_node(i_queue_name) then perform londiste.local_remove_table(i_queue_name, fq_table_name); end if; delete from londiste.table_info where queue_name = i_queue_name and table_name = fq_table_name; if not found then select 400, 'Table not found: ' || fq_table_name into ret_code, ret_note; return; end if; select 200, 'Table removed: ' || i_table_name into ret_code, ret_note; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.global_update_seq.sql000066400000000000000000000032201432153235300240230ustar00rootroot00000000000000 create or replace function londiste.global_update_seq( in i_queue_name text, in i_seq_name text, in i_value int8, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.global_update_seq(3) -- -- Update seq. -- -- Parameters: -- i_queue_name - set name -- i_seq_name - seq name -- i_value - new published value -- -- Returns: -- 200 - OK -- ---------------------------------------------------------------------- declare n record; fqname text; seq record; begin select node_type, node_name into n from pgq_node.node_info where queue_name = i_queue_name; if not found then select 404, 'Set not found: ' || i_queue_name into ret_code, ret_note; return; end if; if n.node_type = 'root' then select 402, 'Must not run on root node' into ret_code, ret_note; return; end if; fqname := londiste.make_fqname(i_seq_name); select last_value, local from londiste.seq_info into seq where queue_name = i_queue_name and seq_name = fqname for update; if not found then insert into londiste.seq_info (queue_name, seq_name, last_value) values (i_queue_name, fqname, i_value); else update londiste.seq_info set last_value = i_value where queue_name = i_queue_name and seq_name = fqname; if seq.local then perform pgq.seq_setval(fqname, i_value); end if; end if; select 200, 'Sequence updated' into ret_code, ret_note; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.handle_fkeys.sql000066400000000000000000000144241432153235300230150ustar00rootroot00000000000000 create or replace function londiste.get_table_pending_fkeys(i_table_name text) returns setof londiste.pending_fkeys as $$ -- ---------------------------------------------------------------------- -- Function: londiste.get_table_pending_fkeys(1) -- -- Return dropped fkeys for table. -- -- Parameters: -- i_table_name - fqname -- -- Returns: -- desc -- ---------------------------------------------------------------------- declare fkeys record; begin for fkeys in select * from londiste.pending_fkeys where from_table = i_table_name or to_table = i_table_name order by 1,2,3 loop return next fkeys; end loop; return; end; $$ language plpgsql strict stable; create or replace function londiste.get_valid_pending_fkeys(i_queue_name text) returns setof londiste.pending_fkeys as $$ -- ---------------------------------------------------------------------- -- Function: londiste.get_valid_pending_fkeys(1) -- -- Returns dropped fkeys where both sides are in sync now. -- -- Parameters: -- i_queue_name - cascaded queue name -- -- Returns: -- desc -- ---------------------------------------------------------------------- declare fkeys record; from_info record; to_info record; min_queue_name text; begin for fkeys in select pf.* from londiste.pending_fkeys pf order by from_table, to_table, fkey_name loop select count(1) as num_total, sum(case when t.queue_name = i_queue_name then 1 else 0 end) as num_matching, sum(case when t.merge_state = 'ok' and t.custom_snapshot is null then 1 else 0 end) as num_ok from londiste.table_info t where coalesce(t.dest_table, t.table_name) = fkeys.from_table and t.local into from_info; -- skip fkeys without known status if from_info.num_total = 0 then continue; end if; select count(1) as num_total, sum(case when t.queue_name = i_queue_name then 1 else 0 end) as num_matching, sum(case when t.merge_state = 'ok' and t.custom_snapshot is null then 1 else 0 end) as num_ok from londiste.table_info t where coalesce(t.dest_table, t.table_name) = fkeys.to_table and t.local into to_info; -- skip fkeys without known status if to_info.num_total = 0 then continue; end if; -- skip if not all copies are finished if from_info.num_ok < from_info.num_total then continue; end if; if to_info.num_ok < to_info.num_total then continue; end if; -- skip if table is not owned by i_queue_name if from_info.num_matching = 0 and to_info.num_matching = 0 then continue; end if; -- pick right queue -- combined_root: first leaf node -- combined_branch: branch node -- default: first node select coalesce( min(case when c.node_type = 'root' then t.queue_name else null end), min(case when c.node_type = 'branch' then c.queue_name else null end), min(t.queue_name)) into min_queue_name from londiste.table_info t join pgq_node.node_info n on (n.queue_name = t.queue_name) left join pgq_node.node_info c on (c.queue_name = n.combined_queue) where coalesce(t.dest_table, t.table_name) in (fkeys.to_table, fkeys.from_table) and t.local; if i_queue_name = min_queue_name then return next fkeys; end if; end loop; return; end; $$ language plpgsql strict stable; create or replace function londiste.drop_table_fkey(i_from_table text, i_fkey_name text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: londiste.drop_table_fkey(2) -- -- Drop one fkey, save in pending table. -- ---------------------------------------------------------------------- declare fkey record; begin select * into fkey from londiste.find_table_fkeys(i_from_table) where fkey_name = i_fkey_name and from_table = i_from_table; if not found then return 0; end if; insert into londiste.pending_fkeys values (fkey.from_table, fkey.to_table, i_fkey_name, fkey.fkey_def); execute 'alter table only ' || londiste.quote_fqname(fkey.from_table) || ' drop constraint ' || quote_ident(i_fkey_name); return 1; end; $$ language plpgsql strict; drop function if exists londiste.restore_table_fkey(text, text); create or replace function londiste.restore_table_fkey(i_from_table text, i_fkey_name text, i_lazy boolean default false) returns text as $$ -- ---------------------------------------------------------------------- -- Function: londiste.restore_table_fkey(3) -- -- Restore dropped fkey. -- -- Parameters: -- i_from_table - source table -- i_fkey_name - fkey name -- i_lazy - if true, then use multi-step create -- -- Returns: -- '' - done -- sql - SQL statement to be executed -- ---------------------------------------------------------------------- declare fkey record; is_valid boolean; tbl_oid oid; begin select * into fkey from londiste.pending_fkeys where fkey_name = i_fkey_name and from_table = i_from_table for update; if not found then return ''; end if; if i_lazy then tbl_oid := londiste.find_table_oid(i_from_table); select convalidated into is_valid from pg_constraint c where c.contype = 'f' and c.conrelid = tbl_oid and c.conname = i_fkey_name; if not found then -- create fkey with NOT VALID return fkey.fkey_def || ' not valid'; elsif not is_valid then -- validate return 'alter table only ' || londiste.quote_fqname(fkey.from_table) || ' validate constraint ' || quote_ident(fkey.fkey_name); end if; else -- create fkey execute fkey.fkey_def; end if; delete from londiste.pending_fkeys where fkey_name = fkey.fkey_name and from_table = fkey.from_table; return ''; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.is_obsolete_partition.sql000066400000000000000000000034051432153235300247560ustar00rootroot00000000000000 create or replace function londiste.is_obsolete_partition ( in i_partition_table text, in i_retention_period interval, in i_partition_period text ) returns boolean as $$ ------------------------------------------------------------------------------- -- Function: londiste.is_obsolete_partition(3) -- -- Test partition name of partition-by-date parent table. -- -- Parameters: -- i_partition_table Partition table name we want to check -- i_retention_period How long to keep partitions around -- i_partition_period One of: year, month, day, hour -- -- Returns: -- True if partition is too old, false if it is not, -- null if its name does not match expected pattern. ------------------------------------------------------------------------------- declare _expr text; _dfmt text; _base text; begin if i_partition_period in ('year', 'yearly') then _expr := '_[0-9]{4}'; _dfmt := '_YYYY'; elsif i_partition_period in ('month', 'monthly') then _expr := '_[0-9]{4}_[0-9]{2}'; _dfmt := '_YYYY_MM'; elsif i_partition_period in ('day', 'daily') then _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}'; _dfmt := '_YYYY_MM_DD'; elsif i_partition_period in ('hour', 'hourly') then _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}'; _dfmt := '_YYYY_MM_DD_HH24'; else raise exception 'not supported i_partition_period: %', i_partition_period; end if; _expr = '^(.+)' || _expr || '$'; _base = substring (i_partition_table from _expr); if _base is null then return null; elsif i_partition_table < _base || to_char (now() - i_retention_period, _dfmt) then return true; else return false; end if; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.is_replica_func.sql000066400000000000000000000011661432153235300235050ustar00rootroot00000000000000 create or replace function londiste.is_replica_func(func_oid oid) returns boolean as $$ -- ---------------------------------------------------------------------- -- Function: londiste.is_replica_func(1) -- -- Returns true if function is a PgQ-based replication functions. -- This also means it takes queue name as first argument. -- ---------------------------------------------------------------------- select count(1) > 0 from pg_proc f join pg_namespace n on (n.oid = f.pronamespace) where f.oid = $1 and n.nspname = 'pgq' and f.proname in ('sqltriga', 'logutriga', 'jsontriga'); $$ language sql strict stable; londiste-sql-3.8/functions/londiste.list_obsolete_partitions.sql000066400000000000000000000040341432153235300255000ustar00rootroot00000000000000 create or replace function londiste.list_obsolete_partitions ( in i_parent_table text, in i_retention_period interval, in i_partition_period text ) returns setof text as $$ ------------------------------------------------------------------------------- -- Function: londiste.list_obsolete_partitions(3) -- -- List obsolete partitions of partition-by-date parent table. -- -- Parameters: -- i_parent_table Master table from which partitions are inherited -- i_retention_period How long to keep partitions around -- i_partition_period One of: year, month, day, hour -- -- Returns: -- Names of partitions to be dropped ------------------------------------------------------------------------------- declare _schema text not null := split_part (i_parent_table, '.', 1); _table text not null := split_part (i_parent_table, '.', 2); _part text; _expr text; _dfmt text; begin if i_partition_period in ('year', 'yearly') then _expr := '_[0-9]{4}'; _dfmt := '_YYYY'; elsif i_partition_period in ('month', 'monthly') then _expr := '_[0-9]{4}_[0-9]{2}'; _dfmt := '_YYYY_MM'; elsif i_partition_period in ('day', 'daily') then _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}'; _dfmt := '_YYYY_MM_DD'; elsif i_partition_period in ('hour', 'hourly') then _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}'; _dfmt := '_YYYY_MM_DD_HH24'; else raise exception 'not supported i_partition_period: %', i_partition_period; end if; if length (_table) = 0 then _table := _schema; _schema := 'public'; end if; for _part in select quote_ident (t.schemaname) ||'.'|| quote_ident (t.tablename) from pg_catalog.pg_tables t where t.schemaname = _schema and t.tablename ~ ('^'|| _table || _expr ||'$') and t.tablename < _table || to_char (now() - i_retention_period, _dfmt) order by 1 loop return next _part; end loop; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_add_seq.sql000066400000000000000000000047171432153235300231370ustar00rootroot00000000000000 create or replace function londiste.local_add_seq( in i_queue_name text, in i_seq_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_add_seq(2) -- -- Register sequence. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_seq_name - seq name -- -- Returns: -- 200 - OK -- 400 - Not found -- ---------------------------------------------------------------------- declare fq_seq_name text; lastval int8; seq record; begin fq_seq_name := londiste.make_fqname(i_seq_name); perform 1 from pg_class where oid = londiste.find_seq_oid(fq_seq_name); if not found then select 400, 'Sequence not found: ' || fq_seq_name into ret_code, ret_note; return; end if; if pgq_node.is_root_node(i_queue_name) then select local, last_value into seq from londiste.seq_info where queue_name = i_queue_name and seq_name = fq_seq_name for update; if found and seq.local then select 201, 'Sequence already added: ' || fq_seq_name into ret_code, ret_note; return; end if; if not seq.local then update londiste.seq_info set local = true where queue_name = i_queue_name and seq_name = fq_seq_name; else insert into londiste.seq_info (queue_name, seq_name, local, last_value) values (i_queue_name, fq_seq_name, true, 0); end if; perform * from londiste.root_check_seqs(i_queue_name); else select local, last_value into seq from londiste.seq_info where queue_name = i_queue_name and seq_name = fq_seq_name for update; if not found then select 404, 'Unknown sequence: ' || fq_seq_name into ret_code, ret_note; return; end if; if seq.local then select 201, 'Sequence already added: ' || fq_seq_name into ret_code, ret_note; return; end if; update londiste.seq_info set local = true where queue_name = i_queue_name and seq_name = fq_seq_name; perform pgq.seq_setval(fq_seq_name, seq.last_value); end if; select 200, 'Sequence added: ' || fq_seq_name into ret_code, ret_note; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_add_table.sql000066400000000000000000000351171432153235300234340ustar00rootroot00000000000000create or replace function londiste.local_add_table( in i_queue_name text, in i_table_name text, in i_trg_args text[], in i_table_attrs text, in i_dest_table text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_add_table(5) -- -- Register table on Londiste node, with customizable trigger args. -- -- Parameters: -- i_queue_name - queue name -- i_table_name - table name -- i_trg_args - args to trigger, or magic parameters. -- i_table_attrs - args to python handler -- i_dest_table - actual name of destination table (NULL if same) -- -- Trigger args: -- See documentation for pgq triggers. -- -- Magic parameters: -- no_triggers - skip trigger creation -- skip_truncate - set 'skip_truncate' table attribute -- expect_sync - set table state to 'ok' -- tgflags=X - trigger creation flags -- merge_all - merge table from all sources. required for -- multi-source table -- no_merge - do not merge tables from different sources -- skip - create skip trigger. same as S flag -- virtual_table - skips structure check and trigger creation -- -- Trigger creation flags (default: AIUDL): -- I - ON INSERT -- U - ON UPDATE -- D - ON DELETE -- Q - use pgq.sqltriga() as trigger function -- L - use pgq.logutriga() as trigger function -- J - use pgq.jsontriga() as trigger function -- B - BEFORE -- A - AFTER -- S - SKIP -- -- Example: -- > londiste.local_add_table('q', 'tbl', array['tgflags=BI', 'SKIP', 'pkey=col1,col2']) -- -- Returns: -- 200 - Ok -- 301 - Warning, trigger exists that will fire before londiste one -- 400 - No such set -- 410 - Table already exists but with different table_attrs ------------------------------------------------------------------------ declare col_types text; fq_table_name text; new_state text; pgversion int; logtrg_previous text; trigger_name text; tbl record; i integer; j integer; arg text; _node record; _tbloid oid; _combined_queue text; _combined_table text; _table_attrs text := i_table_attrs; -- check local tables from all sources _queue_name text; _local boolean; -- argument flags _expect_sync boolean := false; _merge_all boolean := false; _no_merge boolean := false; _virtual_table boolean := false; _dest_table text; _table_name2 text; _desc text; begin -------- i_trg_args ARGUMENTS PARSING (TODO: use different input param for passing extra options that have nothing to do with trigger) if array_lower(i_trg_args, 1) is not null then for i in array_lower(i_trg_args, 1) .. array_upper(i_trg_args, 1) loop arg := i_trg_args[i]; if arg = 'expect_sync' then _expect_sync := true; elsif arg = 'skip_truncate' then _table_attrs := coalesce(_table_attrs || '&skip_truncate=1', 'skip_truncate=1'); elsif arg = 'merge_all' then _merge_all = true; elsif arg = 'no_merge' then _no_merge = true; elsif arg = 'virtual_table' then _virtual_table := true; _expect_sync := true; -- do not copy end if; end loop; end if; if _merge_all and _no_merge then select 405, 'Cannot use merge-all and no-merge together' into ret_code, ret_note; return; end if; fq_table_name := londiste.make_fqname(i_table_name); _dest_table := londiste.make_fqname(coalesce(i_dest_table, i_table_name)); if _dest_table = fq_table_name then _desc := fq_table_name; else _desc := fq_table_name || '(' || _dest_table || ')'; end if; -------- TABLE STRUCTURE CHECK if not _virtual_table then _tbloid := londiste.find_table_oid(_dest_table); if _tbloid is null then select 404, 'Table does not exist: ' || _desc into ret_code, ret_note; return; end if; col_types := londiste.find_column_types(_dest_table); if position('k' in col_types) < 1 then -- allow missing primary key in case of combined table where -- pkey was removed by londiste perform 1 from londiste.table_info t, pgq_node.node_info n_this, pgq_node.node_info n_other where n_this.queue_name = i_queue_name and n_other.combined_queue = n_this.combined_queue and n_other.queue_name <> n_this.queue_name and t.queue_name = n_other.queue_name and coalesce(t.dest_table, t.table_name) = _dest_table and t.dropped_ddl is not null; if not found then select 400, 'Primary key missing on table: ' || _desc into ret_code, ret_note; return; end if; end if; end if; -------- TABLE REGISTRATION LOGIC select * from pgq_node.get_node_info(i_queue_name) into _node; if not found or _node.ret_code >= 400 then select 400, 'No such set: ' || i_queue_name into ret_code, ret_note; return; end if; select merge_state, local, table_attrs into tbl from londiste.table_info where queue_name = i_queue_name and table_name = fq_table_name; if not found then -- add to set on root if _node.node_type = 'root' then select f.ret_code, f.ret_note into ret_code, ret_note from londiste.global_add_table(i_queue_name, i_table_name) f; if ret_code <> 200 then return; end if; else select 404, 'Table not available on queue: ' || _desc into ret_code, ret_note; return; end if; -- reload info select merge_state, local, table_attrs into tbl from londiste.table_info where queue_name = i_queue_name and table_name = fq_table_name; end if; if tbl.local then if tbl.table_attrs is distinct from _table_attrs then select 410, 'Table ' || _desc || ' already added, but with different args: ' || coalesce(tbl.table_attrs, '') into ret_code, ret_note; else select 200, 'Table already added: ' || _desc into ret_code, ret_note; end if; return; end if; if _node.node_type = 'root' then new_state := 'ok'; perform londiste.root_notify_change(i_queue_name, 'londiste.add-table', fq_table_name); elsif _node.node_type = 'leaf' and _node.combined_type = 'branch' then new_state := 'ok'; elsif _expect_sync then new_state := 'ok'; else new_state := NULL; end if; update londiste.table_info set local = true, merge_state = new_state, table_attrs = coalesce(_table_attrs, table_attrs), dest_table = nullif(_dest_table, fq_table_name) where queue_name = i_queue_name and table_name = fq_table_name; if not found then raise exception 'lost table: %', fq_table_name; end if; -- merge all table sources on leaf if _node.node_type = 'leaf' and not _no_merge then for _queue_name, _table_name2, _local in select t2.queue_name, t2.table_name, t2.local from londiste.table_info t join pgq_node.node_info n on (n.queue_name = t.queue_name) left join pgq_node.node_info n2 on (n2.combined_queue = n.combined_queue or (n2.combined_queue is null and n.combined_queue is null)) left join londiste.table_info t2 on (t2.queue_name = n2.queue_name and coalesce(t2.dest_table, t2.table_name) = coalesce(t.dest_table, t.table_name)) where t.queue_name = i_queue_name and t.table_name = fq_table_name and t2.queue_name != i_queue_name -- skip self loop -- if table from some other source is already marked as local, -- raise error if _local and coalesce(new_state, 'x') <> 'ok' then select 405, 'Found local table '|| _desc || ' in queue ' || _queue_name || ', use remove-table first to remove all previous ' || 'table subscriptions' into ret_code, ret_note; return; end if; -- when table comes from multiple sources, merge_all switch is -- required if not _merge_all and coalesce(new_state, 'x') <> 'ok' then select 405, 'Found multiple sources for table '|| _desc || ', use merge-all or no-merge to continue' into ret_code, ret_note; return; end if; update londiste.table_info set local = true, merge_state = new_state, table_attrs = coalesce(_table_attrs, table_attrs) where queue_name = _queue_name and table_name = _table_name2; if not found then raise exception 'lost table: % on queue %', _table_name2, _queue_name; end if; end loop; -- if this node has combined_queue, add table there too -- note: we need to keep both table_name/dest_table values select n2.queue_name, t.table_name from pgq_node.node_info n1 join pgq_node.node_info n2 on (n2.queue_name = n1.combined_queue) left join londiste.table_info t on (t.queue_name = n2.queue_name and t.table_name = fq_table_name and t.local) where n1.queue_name = i_queue_name and (n2.node_type = 'root' or n2.node_type = 'branch') into _combined_queue, _combined_table; if found and _combined_table is null then select f.ret_code, f.ret_note from londiste.local_add_table(_combined_queue, fq_table_name, i_trg_args, _table_attrs, _dest_table) f into ret_code, ret_note; if ret_code >= 300 then return; end if; end if; end if; -- create trigger select f.ret_code, f.ret_note, f.trigger_name from londiste.create_trigger(i_queue_name, fq_table_name, i_trg_args, _dest_table, _node.node_type) f into ret_code, ret_note, trigger_name; if ret_code > 299 then ret_note := 'Trigger creation failed for table ' || _desc || ': ' || ret_note; return; elsif ret_code = 201 then select 200, 'Table added with no triggers: ' || _desc into ret_code, ret_note; return; end if; -- Check that no trigger exists on the target table that will get fired -- before londiste one (this could have londiste replicate data out-of-order) -- -- Don't report all the trigger names, 8.3 does not have array_accum available. show server_version_num into pgversion; if pgversion >= 90000 then select tg.tgname into logtrg_previous from pg_class r join pg_trigger tg on (tg.tgrelid = r.oid) where r.oid = londiste.find_table_oid(_dest_table) and not tg.tgisinternal and tg.tgname < trigger_name::name -- per-row AFTER trigger and (tg.tgtype & 3) = 1 -- bits: 0:ROW, 1:BEFORE -- current londiste and not londiste.is_replica_func(tg.tgfoid) -- old londiste and substring(tg.tgname from 1 for 10) != '_londiste_' and substring(tg.tgname from char_length(tg.tgname) - 6) != '_logger' order by 1 limit 1; else select tg.tgname into logtrg_previous from pg_class r join pg_trigger tg on (tg.tgrelid = r.oid) where r.oid = londiste.find_table_oid(_dest_table) and not tg.tgisconstraint and tg.tgname < trigger_name::name -- per-row AFTER trigger and (tg.tgtype & 3) = 1 -- bits: 0:ROW, 1:BEFORE -- current londiste and not londiste.is_replica_func(tg.tgfoid) -- old londiste and substring(tg.tgname from 1 for 10) != '_londiste_' and substring(tg.tgname from char_length(tg.tgname) - 6) != '_logger' order by 1 limit 1; end if; if logtrg_previous is not null then select 301, 'Table added: ' || _desc || ', but londiste trigger is not first: ' || logtrg_previous into ret_code, ret_note; return; end if; select 200, 'Table added: ' || _desc into ret_code, ret_note; return; end; $$ language plpgsql; create or replace function londiste.local_add_table( in i_queue_name text, in i_table_name text, in i_trg_args text[], in i_table_attrs text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_add_table(4) -- -- Register table on Londiste node. -- ---------------------------------------------------------------------- begin select f.ret_code, f.ret_note into ret_code, ret_note from londiste.local_add_table(i_queue_name, i_table_name, i_trg_args, i_table_attrs, null) f; return; end; $$ language plpgsql; create or replace function londiste.local_add_table( in i_queue_name text, in i_table_name text, in i_trg_args text[], out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_add_table(3) -- -- Register table on Londiste node. -- ---------------------------------------------------------------------- begin select f.ret_code, f.ret_note into ret_code, ret_note from londiste.local_add_table(i_queue_name, i_table_name, i_trg_args, null) f; return; end; $$ language plpgsql; create or replace function londiste.local_add_table( in i_queue_name text, in i_table_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_add_table(2) -- -- Register table on Londiste node. -- ---------------------------------------------------------------------- begin select f.ret_code, f.ret_note into ret_code, ret_note from londiste.local_add_table(i_queue_name, i_table_name, null) f; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.local_change_handler.sql000066400000000000000000000046151432153235300244560ustar00rootroot00000000000000create or replace function londiste.local_change_handler( in i_queue_name text, in i_table_name text, in i_trg_args text[], in i_table_attrs text, out ret_code int4, out ret_note text) as $$ ---------------------------------------------------------------------------------------------------- -- Function: londiste.local_change_handler(4) -- -- Change handler and rebuild trigger if needed -- -- Parameters: -- i_queue_name - set name -- i_table_name - table name -- i_trg_args - args to trigger -- i_table_attrs - args to python handler -- -- Returns: -- 200 - OK -- 400 - No such set -- 404 - Table not found -- ---------------------------------------------------------------------------------------------------- declare _dest_table text; _desc text; _node record; begin -- get node info select * from pgq_node.get_node_info(i_queue_name) into _node; if not found or _node.ret_code >= 400 then select 400, 'No such set: ' || i_queue_name into ret_code, ret_note; return; end if; -- update table_attrs with new handler info select f.ret_code, f.ret_note from londiste.local_set_table_attrs(i_queue_name, i_table_name, i_table_attrs) f into ret_code, ret_note; if ret_code <> 200 then return; end if; -- get destination table name for use in trigger creation select coalesce(ti.dest_table, i_table_name) from londiste.table_info ti where queue_name = i_queue_name and table_name = i_table_name and local into _dest_table; -- replace the trigger if needed select f.ret_code, f.ret_note from londiste.create_trigger(i_queue_name, i_table_name, i_trg_args, _dest_table, _node.node_type) f into ret_code, ret_note; if _dest_table = i_table_name then _desc := i_table_name; else _desc := i_table_name || '(' || _dest_table || ')'; end if; if ret_code > 299 then ret_note := 'Trigger creation failed for table ' || _desc || ': ' || ret_note; return; elsif ret_code = 201 then select 200, 'Table handler updated with no triggers: ' || _desc into ret_code, ret_note; return; end if; select 200, 'Handler changed for table: ' || _desc into ret_code, ret_note; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_remove_seq.sql000066400000000000000000000023001432153235300236660ustar00rootroot00000000000000 create or replace function londiste.local_remove_seq( in i_queue_name text, in i_seq_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_remove_seq(2) -- -- Remove sequence. -- -- Parameters: -- i_queue_name - set name -- i_seq_name - sequence name -- -- Returns: -- 200 - OK -- 404 - Sequence not found -- ---------------------------------------------------------------------- declare fqname text; begin fqname := londiste.make_fqname(i_seq_name); if pgq_node.is_root_node(i_queue_name) then select f.ret_code, f.ret_note into ret_code, ret_note from londiste.global_remove_seq(i_queue_name, fqname) f; return; end if; update londiste.seq_info set local = false where queue_name = i_queue_name and seq_name = fqname and local; if not found then select 404, 'Sequence not found: '||fqname into ret_code, ret_note; return; end if; select 200, 'Sequence removed: '||fqname into ret_code, ret_note; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.local_remove_table.sql000066400000000000000000000062641432153235300242020ustar00rootroot00000000000000 create or replace function londiste.local_remove_table( in i_queue_name text, in i_table_name text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_remove_table(2) -- -- Remove table. -- -- Parameters: -- i_queue_name - set name -- i_table_name - table name -- -- Returns: -- 200 - OK -- 404 - Table not found -- ---------------------------------------------------------------------- declare fq_table_name text; qtbl text; seqname text; tbl record; tbl_oid oid; pgver integer; begin fq_table_name := londiste.make_fqname(i_table_name); qtbl := londiste.quote_fqname(fq_table_name); tbl_oid := londiste.find_table_oid(i_table_name); show server_version_num into pgver; select local, dropped_ddl, merge_state into tbl from londiste.table_info where queue_name = i_queue_name and table_name = fq_table_name for update; if not found then select 400, 'Table not found: ' || fq_table_name into ret_code, ret_note; return; end if; if tbl.local then perform londiste.drop_table_triggers(i_queue_name, fq_table_name); -- restore dropped ddl if tbl.dropped_ddl is not null then -- table is not synced, drop data to make restore faster if pgver >= 80400 then execute 'TRUNCATE ONLY ' || qtbl; else execute 'TRUNCATE ' || qtbl; end if; execute tbl.dropped_ddl; end if; -- reset data update londiste.table_info set local = false, custom_snapshot = null, table_attrs = null, dropped_ddl = null, merge_state = null, dest_table = null where queue_name = i_queue_name and table_name = fq_table_name; -- drop dependent sequence for seqname in select n.nspname || '.' || s.relname from pg_catalog.pg_class s, pg_catalog.pg_namespace n, pg_catalog.pg_attribute a where a.attrelid = tbl_oid and a.atthasdef and a.atttypid::regtype::text in ('integer', 'bigint') and s.oid = pg_get_serial_sequence(qtbl, a.attname)::regclass::oid and n.oid = s.relnamespace loop perform londiste.local_remove_seq(i_queue_name, seqname); end loop; else if not pgq_node.is_root_node(i_queue_name) then select 400, 'Table not registered locally: ' || fq_table_name into ret_code, ret_note; return; end if; end if; if pgq_node.is_root_node(i_queue_name) then perform londiste.global_remove_table(i_queue_name, fq_table_name); perform londiste.root_notify_change(i_queue_name, 'londiste.remove-table', fq_table_name); end if; select 200, 'Table removed: ' || fq_table_name into ret_code, ret_note; return; end; $$ language plpgsql strict; londiste-sql-3.8/functions/londiste.local_set_skip_truncate.sql000066400000000000000000000015311432153235300252540ustar00rootroot00000000000000 create or replace function londiste.local_set_skip_truncate( in i_queue_name text, in i_table text, in i_value bool, out ret_code int4, out ret_note text) returns record as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_set_skip_truncate(3) -- -- Change skip_truncate flag for table. -- ---------------------------------------------------------------------- begin update londiste.table_info set skip_truncate = i_value where queue_name = i_queue_name and table_name = i_table; if found then select 200, 'skip_truncate=' || i_value::text into ret_code, ret_note; else select 404, 'table not found: ' || i_table into ret_code, ret_note; end if; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_set_table_attrs.sql000066400000000000000000000020051432153235300247020ustar00rootroot00000000000000 create or replace function londiste.local_set_table_attrs( in i_queue_name text, in i_table_name text, in i_table_attrs text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_set_table_attrs(3) -- -- Store urlencoded table attributes. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_table - table name -- i_table_attrs - urlencoded attributes -- ---------------------------------------------------------------------- begin update londiste.table_info set table_attrs = i_table_attrs where queue_name = i_queue_name and table_name = i_table_name and local; if found then select 200, i_table_name || ': Table attributes stored' into ret_code, ret_note; else select 404, 'no such local table: ' || i_table_name into ret_code, ret_note; end if; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_set_table_state.sql000066400000000000000000000023221432153235300246670ustar00rootroot00000000000000 create or replace function londiste.local_set_table_state( in i_queue_name text, in i_table_name text, in i_snapshot text, in i_merge_state text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_set_table_state(4) -- -- Change table state. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_table - table name -- i_snapshot - optional remote snapshot info -- i_merge_state - merge state -- ---------------------------------------------------------------------- declare _tbl text; begin _tbl = londiste.make_fqname(i_table_name); update londiste.table_info set custom_snapshot = i_snapshot, merge_state = i_merge_state where queue_name = i_queue_name and table_name = _tbl and local; if not found then select 404, 'No such table: ' || _tbl into ret_code, ret_note; return; end if; select 200, 'Table ' || _tbl || ' state set to ' || coalesce(quote_literal(i_merge_state), 'NULL') into ret_code, ret_note; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_set_table_struct.sql000066400000000000000000000017531432153235300251020ustar00rootroot00000000000000 create or replace function londiste.local_set_table_struct( in i_queue_name text, in i_table_name text, in i_dropped_ddl text, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_set_table_struct(3) -- -- Store dropped table struct temporarily. -- -- Parameters: -- i_queue_name - cascaded queue name -- i_table - table name -- i_dropped_ddl - merge state -- ---------------------------------------------------------------------- begin update londiste.table_info set dropped_ddl = i_dropped_ddl where queue_name = i_queue_name and table_name = i_table_name and local; if found then select 200, 'Table struct stored' into ret_code, ret_note; else select 404, 'no such local table: '||i_table_name into ret_code, ret_note; end if; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.local_show_missing.sql000066400000000000000000000034561432153235300242470ustar00rootroot00000000000000 create or replace function londiste.local_show_missing( in i_queue_name text, out obj_kind text, out obj_name text) returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: londiste.local_show_missing(1) -- -- Return info about missing tables. On root show tables -- not registered on set, on branch/leaf show tables -- in set but not registered locally. -- ---------------------------------------------------------------------- begin if pgq_node.is_root_node(i_queue_name) then for obj_kind, obj_name in select r.relkind, n.nspname || '.' || r.relname from pg_catalog.pg_class r, pg_catalog.pg_namespace n where n.oid = r.relnamespace and r.relkind in ('r', 'S') and n.nspname not in ('pgq', 'pgq_ext', 'pgq_node', 'londiste', 'pg_catalog', 'information_schema') and n.nspname !~ '^pg_(toast|temp)' and not exists (select 1 from londiste.table_info where queue_name = i_queue_name and local and coalesce(dest_table, table_name) = (n.nspname || '.' || r.relname)) order by 1, 2 loop return next; end loop; else for obj_kind, obj_name in select 'S', s.seq_name from londiste.seq_info s where s.queue_name = i_queue_name and not s.local union all select 'r', t.table_name from londiste.table_info t where t.queue_name = i_queue_name and not t.local order by 1, 2 loop return next; end loop; end if; return; end; $$ language plpgsql strict stable; londiste-sql-3.8/functions/londiste.make_fqname.sql000066400000000000000000000012521432153235300226200ustar00rootroot00000000000000 create or replace function londiste.make_fqname(i_name text) returns text as $$ -- ---------------------------------------------------------------------- -- Function: londiste.make_fqname(1) -- -- Make name to schema-qualified one. -- -- First dot is taken as schema separator. -- -- If schema is missing, 'public' is assumed. -- -- Parameters: -- i_name - object name. -- -- Returns: -- Schema qualified name. -- ---------------------------------------------------------------------- begin if position('.' in i_name) > 0 then return i_name; else return 'public.' || i_name; end if; end; $$ language plpgsql strict immutable; londiste-sql-3.8/functions/londiste.periodic_maintenance.sql000066400000000000000000000007761432153235300245260ustar00rootroot00000000000000 create or replace function londiste.periodic_maintenance() returns integer as $$ -- ---------------------------------------------------------------------- -- Function: londiste.periodic_maintenance(0) -- -- Clean random stuff. -- ---------------------------------------------------------------------- begin -- clean old EXECUTE entries delete from londiste.applied_execute where execute_time < now() - '3 months'::interval; return 0; end; $$ language plpgsql; -- need admin access londiste-sql-3.8/functions/londiste.quote_fqname.sql000066400000000000000000000016331432153235300230430ustar00rootroot00000000000000 create or replace function londiste.quote_fqname(i_name text) returns text as $$ -- ---------------------------------------------------------------------- -- Function: londiste.quote_fqname(1) -- -- Quete fully-qualified object name for SQL. -- -- First dot is taken as schema separator. -- -- If schema is missing, 'public' is assumed. -- -- Parameters: -- i_name - fully qualified object name. -- -- Returns: -- Quoted name. -- ---------------------------------------------------------------------- declare res text; pos integer; s text; n text; begin pos := position('.' in i_name); if pos > 0 then s := substring(i_name for pos - 1); n := substring(i_name from pos + 1); else s := 'public'; n := i_name; end if; return quote_ident(s) || '.' || quote_ident(n); end; $$ language plpgsql strict immutable; londiste-sql-3.8/functions/londiste.root_check_seqs.sql000066400000000000000000000045101432153235300235270ustar00rootroot00000000000000 create or replace function londiste.root_check_seqs( in i_queue_name text, in i_buffer int8, out ret_code int4, out ret_note text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.root_check_seqs(1) -- -- Check sequences, and publish values if needed. -- -- Parameters: -- i_queue_name - set name -- i_buffer - safety room -- -- Returns: -- 200 - OK -- 402 - Not a root node -- 404 - Queue not found -- ---------------------------------------------------------------------- declare n record; seq record; real_value int8; pub_value int8; real_buffer int8; begin if i_buffer is null or i_buffer < 10 then real_buffer := 10000; else real_buffer := i_buffer; end if; select node_type, node_name into n from pgq_node.node_info where queue_name = i_queue_name for update; if not found then select 404, 'Queue not found: ' || i_queue_name into ret_code, ret_note; return; end if; if n.node_type <> 'root' then select 402, 'Not a root node' into ret_code, ret_note; return; end if; for seq in select seq_name, last_value, londiste.quote_fqname(seq_name) as fqname from londiste.seq_info where queue_name = i_queue_name and local order by nr loop execute 'select last_value from ' || seq.fqname into real_value; if real_value + real_buffer >= seq.last_value then pub_value := real_value + real_buffer * 3; perform pgq.insert_event(i_queue_name, 'londiste.update-seq', pub_value::text, seq.seq_name, null, null, null); update londiste.seq_info set last_value = pub_value where queue_name = i_queue_name and seq_name = seq.seq_name; end if; end loop; select 100, 'Sequences updated' into ret_code, ret_note; return; end; $$ language plpgsql; create or replace function londiste.root_check_seqs( in i_queue_name text, out ret_code int4, out ret_note text) as $$ begin select f.ret_code, f.ret_note into ret_code, ret_note from londiste.root_check_seqs(i_queue_name, 10000) f; return; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.root_notify_change.sql000066400000000000000000000012421432153235300242330ustar00rootroot00000000000000 create or replace function londiste.root_notify_change(i_queue_name text, i_ev_type text, i_ev_data text) returns integer as $$ -- ---------------------------------------------------------------------- -- Function: londiste.root_notify_change(3) -- -- Send event about change in root downstream. -- ---------------------------------------------------------------------- declare que text; ntype text; begin if not coalesce(pgq_node.is_root_node(i_queue_name), false) then raise exception 'only root node can send events'; end if; perform pgq.insert_event(i_queue_name, i_ev_type, i_ev_data); return 1; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.set_session_replication_role.sql000066400000000000000000000016461432153235300263330ustar00rootroot00000000000000 create or replace function londiste.set_session_replication_role(val text, is_local bool) returns void as $$ begin if is_local then if val = 'origin' then set local session_replication_role = 'origin'; elsif val = 'replica' then set local session_replication_role = 'replica'; elsif val = 'local' then set local session_replication_role = 'local'; else raise exception 'bad value for session_replication_role'; end if; else if val = 'origin' then set session_replication_role = 'origin'; elsif val = 'replica' then set session_replication_role = 'replica'; elsif val = 'local' then set session_replication_role = 'local'; else raise exception 'bad value for session_replication_role'; end if; end if; end; $$ language plpgsql security definer; londiste-sql-3.8/functions/londiste.split_fqname.sql000066400000000000000000000015301432153235300230350ustar00rootroot00000000000000create or replace function londiste.split_fqname( in i_fqname text, out schema_part text, out name_part text) as $$ -- ---------------------------------------------------------------------- -- Function: londiste.split_fqname(1) -- -- Split fqname to schema and name parts. -- -- First dot is taken as schema separator. -- -- If schema is missing, 'public' is assumed. -- -- Parameters: -- i_fqname - object name. -- ---------------------------------------------------------------------- declare dot integer; begin dot = position('.' in i_fqname); if dot > 0 then schema_part = substring(i_fqname for dot - 1); name_part = substring(i_fqname from dot + 1); else schema_part = 'public'; name_part = i_fqname; end if; return; end; $$ language plpgsql strict immutable; londiste-sql-3.8/functions/londiste.table_info_trigger.sql000066400000000000000000000010701432153235300241770ustar00rootroot00000000000000 create or replace function londiste.table_info_trigger() returns trigger as $$ -- ---------------------------------------------------------------------- -- Function: londiste.table_info_trigger(0) -- -- Trigger on londiste.table_info. Cleans triggers from tables -- when table is removed from londiste.table_info. -- ---------------------------------------------------------------------- begin if TG_OP = 'DELETE' then perform londiste.drop_table_triggers(OLD.queue_name, OLD.table_name); end if; return OLD; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.upgrade_schema.sql000066400000000000000000000072261432153235300233320ustar00rootroot00000000000000 create or replace function londiste.upgrade_schema() returns int4 as $$ -- updates table structure if necessary declare pgversion int; cnt int4 = 0; begin show server_version_num into pgversion; -- table_info: check (dropped_ddl is null or merge_state in ('in-copy', 'catching-up')) perform 1 from information_schema.check_constraints where constraint_schema = 'londiste' and constraint_name = 'table_info_check' and position('in-copy' in check_clause) > 0 and position('catching' in check_clause) = 0; if found then alter table londiste.table_info drop constraint table_info_check; alter table londiste.table_info add constraint table_info_check check (dropped_ddl is null or merge_state in ('in-copy', 'catching-up')); cnt := cnt + 1; end if; -- table_info.dest_table perform 1 from information_schema.columns where table_schema = 'londiste' and table_name = 'table_info' and column_name = 'dest_table'; if not found then alter table londiste.table_info add column dest_table text; end if; -- table_info: change trigger timing if pgversion >= 90100 then perform 1 from information_schema.triggers where event_object_schema = 'londiste' and event_object_table = 'table_info' and trigger_name = 'table_info_trigger_sync' and action_timing = 'AFTER'; else perform 1 from information_schema.triggers where event_object_schema = 'londiste' and event_object_table = 'table_info' and trigger_name = 'table_info_trigger_sync' and condition_timing = 'AFTER'; end if; if found then drop trigger table_info_trigger_sync on londiste.table_info; create trigger table_info_trigger_sync before delete on londiste.table_info for each row execute procedure londiste.table_info_trigger(); end if; -- applied_execute.dest_table perform 1 from information_schema.columns where table_schema = 'londiste' and table_name = 'applied_execute' and column_name = 'execute_attrs'; if not found then alter table londiste.applied_execute add column execute_attrs text; end if; -- applied_execute: drop queue_name from primary key perform 1 from pg_catalog.pg_indexes where schemaname = 'londiste' and tablename = 'applied_execute' and indexname = 'applied_execute_pkey' and indexdef like '%queue_name%'; if found then alter table londiste.applied_execute drop constraint applied_execute_pkey; alter table londiste.applied_execute add constraint applied_execute_pkey primary key (execute_file); end if; -- applied_execute: drop fkey to pgq_node perform 1 from information_schema.table_constraints where constraint_schema = 'londiste' and table_schema = 'londiste' and table_name = 'applied_execute' and constraint_type = 'FOREIGN KEY' and constraint_name = 'applied_execute_queue_name_fkey'; if found then alter table londiste.applied_execute drop constraint applied_execute_queue_name_fkey; end if; -- create roles perform 1 from pg_catalog.pg_roles where rolname = 'londiste_writer'; if not found then create role londiste_writer in role pgq_admin; cnt := cnt + 1; end if; perform 1 from pg_catalog.pg_roles where rolname = 'londiste_reader'; if not found then create role londiste_reader in role pgq_reader; cnt := cnt + 1; end if; return 0; end; $$ language plpgsql; londiste-sql-3.8/functions/londiste.version.sql000066400000000000000000000007241432153235300220440ustar00rootroot00000000000000 create or replace function londiste.version() returns text as $$ -- ---------------------------------------------------------------------- -- Function: londiste.version(0) -- -- Returns version string for londiste. -- ---------------------------------------------------------------------- declare _vers text; begin select extversion from pg_catalog.pg_extension where extname = 'londiste' into _vers; return _vers; end; $$ language plpgsql; londiste-sql-3.8/londiste.control000066400000000000000000000002551432153235300172300ustar00rootroot00000000000000# Londiste extension comment = 'Londiste replication support code' default_version = '3.8' relocatable = false superuser = true schema = 'pg_catalog' requires = 'pgq_node' londiste-sql-3.8/mk/000077500000000000000000000000001432153235300144125ustar00rootroot00000000000000londiste-sql-3.8/mk/catsql.py000077500000000000000000000070161432153235300162620ustar00rootroot00000000000000#! /usr/bin/env python3 """Prints out SQL files with psql command execution. Supported psql commands: \i, \cd, \q Others are skipped. Aditionally does some pre-processing for NDoc. NDoc is looks nice but needs some hand-holding. Bug: - function def end detection searches for 'as'/'is' but does not check word boundaries - finds them even in function name. That means in main conf, as/is must be disabled and $ ' added. This script can remove the unnecessary AS from output. Niceties: - Ndoc includes function def in output only if def is after comment. But for SQL functions its better to have it after def. This script can swap comment and def. - Optionally remove CREATE FUNCTION (OR REPLACE) from def to keep it shorter in doc. Note: - NDoc compares real function name and name in comment. if differ, it decides detection failed. """ import sys, os, re, getopt def usage(x): print("usage: catsql [--ndoc] FILE [FILE ...]") sys.exit(x) # NDoc specific changes cf_ndoc = 0 # compile regexes func_re = r"create\s+(or\s+replace\s+)?function\s+" func_rc = re.compile(func_re, re.I) comm_rc = re.compile(r"^\s*([#]\s*)?(?P--.*)", re.I) end_rc = re.compile(r"\b([;]|begin|declare|end)\b", re.I) as_rc = re.compile(r"\s+as\s+", re.I) cmd_rc = re.compile(r"^\\([a-z]*)(\s+.*)?", re.I) # conversion func def fix_func(ln): # if ndoc, replace AS with ' ' if cf_ndoc: return as_rc.sub(' ', ln) else: return ln # got function def def proc_func(f, ln): # remove CREATE OR REPLACE if cf_ndoc: ln = func_rc.sub('', ln) ln = fix_func(ln) pre_list = [ln] comm_list = [] while 1: ln = f.readline() if not ln: break com = None if cf_ndoc: com = comm_rc.search(ln) if cf_ndoc and com: pos = com.start('com') comm_list.append(ln[pos:]) elif end_rc.search(ln): break elif len(comm_list) > 0: break else: pre_list.append(fix_func(ln)) if len(comm_list) > 2: for el in comm_list: sys.stdout.write(el) for el in pre_list: sys.stdout.write(el) else: for el in pre_list: sys.stdout.write(el) for el in comm_list: sys.stdout.write(el) if ln: sys.stdout.write(fix_func(ln)) def cat_file(fn): sys.stdout.write("\n") f = open(fn) while 1: ln = f.readline() if not ln: break m = cmd_rc.search(ln) if m: cmd = m.group(1) if cmd == "i": # include a file fn2 = m.group(2).strip() cat_file(fn2) elif cmd == "q": # quit sys.exit(0) elif cmd == "cd": # chdir cd_dir = m.group(2).strip() os.chdir(cd_dir) else: # skip all others pass else: if func_rc.search(ln): # function header proc_func(f, ln) else: # normal sql sys.stdout.write(ln) sys.stdout.write("\n") def main(): global cf_ndoc try: opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['ndoc']) except getopt.error as d: print(str(d)) usage(1) for o, v in opts: if o == "-h": usage(0) elif o == "--ndoc": cf_ndoc = 1 for fn in args: cat_file(fn) if __name__ == '__main__': main() londiste-sql-3.8/mk/common-pgxs.mk000066400000000000000000000104141432153235300172120ustar00rootroot00000000000000 # PGXS does not support modules that are supposed # to run on different Postgres versions very well. # Here are some workarounds for them. # Variables that are used when extensions are available Extension_data ?= Extension_data_built ?= $(EXTENSION)--$(EXT_VERSION).sql $(EXTENSION)--unpackaged--$(EXT_VERSION).sql Extension_regress ?= # Variables that are used when extensions are not available Contrib_data ?= Contrib_data_built += $(EXTENSION).sql $(EXTENSION).upgrade.sql \ structure/newgrants_$(EXTENSION).sql \ structure/oldgrants_$(EXTENSION).sql Contrib_regress ?= EXT_VERSION ?= EXT_OLD_VERSIONS ?= Extension_upgrade_files = $(if $(EXT_OLD_VERSIONS),$(foreach v,$(EXT_OLD_VERSIONS),$(EXTENSION)--$(v)--$(EXT_VERSION).sql)) Extension_data_built += $(Extension_upgrade_files) # Should the Contrib* files installed (under ../contrib/) # even when extensions are available? Contrib_install_always ?= yes # # switch variables # IfExt = $(if $(filter 8.% 9.0%,$(MAJORVERSION)8.3),$(2),$(1)) DATA = $(call IfExt,$(Extension_data),$(Contrib_data)) DATA_built = $(call IfExt,$(Extension_data_built),$(Contrib_data_built)) REGRESS = $(call IfExt,$(Extension_regress),$(Contrib_regress)) EXTRA_CLEAN += $(call IfExt,$(Contrib_data_built),$(Extension_data_built)) test.dump # have deterministic dbname for regtest database override CONTRIB_TESTDB = regression REGRESS_OPTS = --dbname=$(CONTRIB_TESTDB) # # Calculate actual sql files # GRANT_SQL = structure/newgrants_$(EXTENSION).sql SQLS = $(shell $(AWK) '/^\\i / { print $$2; }' structure/install.sql) FUNCS = $(shell $(AWK) '/^\\i / { print $$2; }' $(SQLS)) SRCS = $(SQLS) $(FUNCS) $(GRANT_SQL) # # load PGXS # PG_CONFIG ?= pg_config PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # when compiling locally and with postgres without python, # the variable may be empty PYTHON3 := $(if $(PYTHON3),$(PYTHON3),python3) # # common tools # NDOC = NaturalDocs NDOCARGS = -r -o html docs/html -p docs -i docs/sql CATSQL = $(PYTHON3) mk/catsql.py GRANTFU = $(PYTHON3) mk/grantfu.py # # build rules, in case Contrib data must be always installed # ifeq ($(call IfExt,$(Contrib_install_always),no),yes) all: $(Contrib_data) $(Contrib_data_built) installdirs: installdirs-old-contrib install: install-old-contrib installdirs-old-contrib: $(MKDIR_P) '$(DESTDIR)$(datadir)/contrib' install-old-contrib: $(Contrib_data) $(Contrib_data_built) installdirs-old-contrib $(INSTALL_DATA) $(addprefix $(srcdir)/, $(Contrib_data)) $(Contrib_data_built) '$(DESTDIR)$(datadir)/contrib/' endif # # regtest shortcuts # test: install $(MAKE) installcheck || { filterdiff --format=unified regression.diffs | less; exit 1; } #pg_dump regression > test.dump citest: checkver $(MAKE) installcheck || { filterdiff --format=unified regression.diffs; exit 1; } ack: cp results/*.out expected/ cleandox: rm -rf docs/html docs/Data docs/sql clean: cleandox .PHONY: test ack installdirs-old-contrib install-old-contrib cleandox dox # # common files # $(EXTENSION)--$(EXT_VERSION).sql: $(EXTENSION).sql structure/ext_postproc.sql $(CATSQL) $^ > $@ $(EXTENSION)--unpackaged--$(EXT_VERSION).sql: $(EXTENSION).upgrade.sql structure/ext_unpackaged.sql structure/ext_postproc.sql $(CATSQL) $^ > $@ $(EXTENSION).sql: $(SRCS) $(CATSQL) structure/install.sql $(GRANT_SQL) > $@ $(EXTENSION).upgrade.sql: $(SRCS) $(CATSQL) structure/upgrade.sql $(GRANT_SQL) > $@ ifneq ($(Extension_upgrade_files),) $(Extension_upgrade_files): $(EXTENSION).upgrade.sql cp $< $@ endif structure/newgrants_$(EXTENSION).sql: structure/grants.ini $(GRANTFU) -r -d $< > $@ structure/oldgrants_$(EXTENSION).sql: structure/grants.ini structure/grants.sql echo "begin;" > $@ $(GRANTFU) -R -o $< >> $@ cat structure/grants.sql >> $@ echo "commit;" >> $@ checkver: @echo "Checking version numbers" @grep -q "^default_version *= *'$(EXT_VERSION)'" $(EXTENSION).control \ || { echo "ERROR: $(EXTENSION).control has wrong version"; exit 1; } @test -f "docs/notes/v$(EXT_VERSION).md" \ || { echo "ERROR: notes missing: docs/notes/v$(EXT_VERSION).md"; exit 1; } all: checkver TARNAME = $(EXTENSION)-$(EXT_VERSION) dist: checkver git archive --format=tar.gz --prefix=$(TARNAME)/ -o $(TARNAME).tar.gz HEAD release: checkver git tag v$(EXT_VERSION) git push github git push github v$(EXT_VERSION):v$(EXT_VERSION) londiste-sql-3.8/mk/grantfu.py000077500000000000000000000261371432153235300164460ustar00rootroot00000000000000#! /usr/bin/env python3 # GrantFu - GRANT/REVOKE generator for Postgres # # Copyright (c) 2005 Marko Kreen # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """Generator for PostgreSQL permissions. Loads config where roles, objects and their mapping is described and generates grants based on them. ConfigParser docs: http://docs.python.org/lib/module-ConfigParser.html Example: -------------------------------------------------------------------- [DEFAULT] users = user1, user2 # users to handle groups = group1, group2 # groups to handle auto_seq = 0 # dont handle seqs (default) # '!' after a table negates this setting for a table seq_name = id # the name for serial field (default: id) seq_usage = 0 # should we grant "usage" or "select, update" # for automatically handled sequences # section names can be random, but if you want to see them # in same order as in config file, then order them alphabetically [1.section] on.tables = testtbl, testtbl_id_seq, # here we handle seq by hand table_with_seq! # handle seq automatically # (table_with_seq_id_seq) user1 = select group1 = select, insert, update # instead of 'tables', you may use 'functions', 'languages', # 'schemas', 'tablespaces' --------------------------------------------------------------------- """ import sys, os, getopt try: from configparser import ConfigParser except ImportError: from ConfigParser import SafeConfigParser as ConfigParser __version__ = "1.0" R_NEW = 0x01 R_DEFS = 0x02 G_DEFS = 0x04 R_ONLY = 0x80 def usage(err): sys.stderr.write("usage: %s [-r|-R] CONF_FILE\n" % sys.argv[0]) sys.stderr.write(" -r Generate also REVOKE commands\n") sys.stderr.write(" -R Generate only REVOKE commands\n") sys.stderr.write(" -d Also REVOKE default perms\n") sys.stderr.write(" -D Only REVOKE default perms\n") sys.stderr.write(" -o Generate default GRANTS\n") sys.stderr.write(" -v Print program version\n") sys.stderr.write(" -t Put everything in one big transaction\n") sys.exit(err) class PConf(ConfigParser): "List support for ConfigParser" def __init__(self, defaults = None): ConfigParser.__init__(self, defaults) def get_list(self, sect, key): str = self.get(sect, key).strip() res = [] if not str: return res for val in str.split(","): res.append(val.strip()) return res class GrantFu: def __init__(self, cf, revoke): self.cf = cf self.revoke = revoke # avoid putting grantfu vars into defaults, thus into every section self.group_list = [] self.user_list = [] self.auto_seq = 0 self.seq_name = "id" self.seq_usage = 0 if self.cf.has_option('GrantFu', 'groups'): self.group_list = self.cf.get_list('GrantFu', 'groups') if self.cf.has_option('GrantFu', 'users'): self.user_list += self.cf.get_list('GrantFu', 'users') if self.cf.has_option('GrantFu', 'roles'): self.user_list += self.cf.get_list('GrantFu', 'roles') if self.cf.has_option('GrantFu', 'auto_seq'): self.auto_seq = self.cf.getint('GrantFu', 'auto_seq') if self.cf.has_option('GrantFu', 'seq_name'): self.seq_name = self.cf.get('GrantFu', 'seq_name') if self.cf.has_option('GrantFu', 'seq_usage'): self.seq_usage = self.cf.getint('GrantFu', 'seq_usage') # make string of all subjects tmp = [] for g in self.group_list: tmp.append("group " + g) for u in self.user_list: tmp.append(u) self.all_subjs = ", ".join(tmp) # per-section vars self.sect = None self.seq_list = [] self.seq_allowed = [] def process(self): if len(self.user_list) == 0 and len(self.group_list) == 0: return sect_list = self.cf.sections() sect_list.sort() for self.sect in sect_list: if self.sect == "GrantFu": continue print("\n-- %s --" % self.sect) self.handle_tables() self.handle_other('on.databases', 'DATABASE') self.handle_other('on.functions', 'FUNCTION') self.handle_other('on.languages', 'LANGUAGE') self.handle_other('on.schemas', 'SCHEMA') self.handle_other('on.tablespaces', 'TABLESPACE') self.handle_other('on.sequences', 'SEQUENCE') self.handle_other('on.types', 'TYPE') self.handle_other('on.domains', 'DOMAIN') def handle_other(self, listname, obj_type): """Handle grants for all objects except tables.""" if not self.sect_hasvar(listname): return # don't parse list, as in case of functions it may be complicated obj_str = obj_type + " " + self.sect_var(listname) if self.revoke & R_NEW: self.gen_revoke(obj_str) if self.revoke & R_DEFS: self.gen_revoke_defs(obj_str, obj_type) if not self.revoke & R_ONLY: self.gen_one_type(obj_str) if self.revoke & G_DEFS: self.gen_defs(obj_str, obj_type) def handle_tables(self): """Handle grants for tables and sequences. The tricky part here is the automatic handling of sequences.""" if not self.sect_hasvar('on.tables'): return cleaned_list = [] table_list = self.sect_list('on.tables') for table in table_list: if table[-1] == '!': table = table[:-1] if not self.auto_seq: self.seq_list.append("%s_%s_seq" % (table, self.seq_name)) else: if self.auto_seq: self.seq_list.append("%s_%s_seq" % (table, self.seq_name)) cleaned_list.append(table) obj_str = "TABLE " + ", ".join(cleaned_list) if self.revoke & R_NEW: self.gen_revoke(obj_str) if self.revoke & R_DEFS: self.gen_revoke_defs(obj_str, "TABLE") if not self.revoke & R_ONLY: self.gen_one_type(obj_str) if self.revoke & G_DEFS: self.gen_defs(obj_str, "TABLE") # cleanup self.seq_list = [] self.seq_allowed = [] def gen_revoke(self, obj_str): "Generate revoke for one section / subject type (user or group)" if len(self.seq_list) > 0: obj_str += ", " + ", ".join(self.seq_list) obj_str = obj_str.strip().replace('\n', '\n ') print("REVOKE ALL ON %s\n FROM %s CASCADE;" % (obj_str, self.all_subjs)) def gen_revoke_defs(self, obj_str, obj_type): "Generate revoke defaults for one section" # process only things that have default grants to public if obj_type not in ('FUNCTION', 'DATABASE', 'LANGUAGE', 'TYPE', 'DOMAIN'): return defrole = 'public' # if the sections contains grants to 'public', dont drop if self.sect_hasvar(defrole): return obj_str = obj_str.strip().replace('\n', '\n ') print("REVOKE ALL ON %s\n FROM %s CASCADE;" % (obj_str, defrole)) def gen_defs(self, obj_str, obj_type): "Generate defaults grants for one section" if obj_type == "FUNCTION": defgrants = "execute" elif obj_type == "DATABASE": defgrants = "connect, temp" elif obj_type in ("LANGUAGE", "TYPE", "DOMAIN"): defgrants = "usage" else: return defrole = 'public' obj_str = obj_str.strip().replace('\n', '\n ') print("GRANT %s ON %s\n TO %s;" % (defgrants, obj_str, defrole)) def gen_one_subj(self, subj, fqsubj, obj_str): if not self.sect_hasvar(subj): return obj_str = obj_str.strip().replace('\n', '\n ') perm = self.sect_var(subj).strip() if perm: print("GRANT %s ON %s\n TO %s;" % (perm, obj_str, fqsubj)) # check for seq perms if len(self.seq_list) > 0: loperm = perm.lower() if loperm.find("insert") >= 0 or loperm.find("all") >= 0: self.seq_allowed.append(fqsubj) def gen_one_type(self, obj_str): "Generate GRANT for one section / one object type in section" for u in self.user_list: self.gen_one_subj(u, u, obj_str) for g in self.group_list: self.gen_one_subj(g, "group " + g, obj_str) # if there was any seq perms, generate grants if len(self.seq_allowed) > 0: seq_str = ", ".join(self.seq_list) subj_str = ", ".join(self.seq_allowed) if self.seq_usage: cmd = "GRANT usage ON SEQUENCE %s\n TO %s;" else: cmd = "GRANT select, update ON %s\n TO %s;" print(cmd % (seq_str, subj_str)) def sect_var(self, name): return self.cf.get(self.sect, name).strip() def sect_list(self, name): return self.cf.get_list(self.sect, name) def sect_hasvar(self, name): return self.cf.has_option(self.sect, name) def main(): revoke = 0 tx = False try: opts, args = getopt.getopt(sys.argv[1:], "vhrRdDot") except getopt.error as det: print("getopt error:", det) usage(1) for o, v in opts: if o == "-h": usage(0) elif o == "-r": revoke |= R_NEW elif o == "-R": revoke |= R_NEW | R_ONLY elif o == "-d": revoke |= R_DEFS elif o == "-D": revoke |= R_DEFS | R_ONLY elif o == "-o": revoke |= G_DEFS elif o == "-t": tx = True elif o == "-v": print("GrantFu version", __version__) sys.exit(0) if len(args) != 1: usage(1) # load config cf = PConf() cf.read(args[0]) if not cf.has_section("GrantFu"): print("Incorrect config file, GrantFu sction missing") sys.exit(1) if tx: print("begin;\n") # revokes and default grants if revoke & (R_NEW | R_DEFS): g = GrantFu(cf, revoke | R_ONLY) g.process() revoke = revoke & R_ONLY # grants if revoke & R_ONLY == 0: g = GrantFu(cf, revoke & G_DEFS) g.process() if tx: print("\ncommit;\n") if __name__ == '__main__': main() londiste-sql-3.8/sql/000077500000000000000000000000001432153235300146025ustar00rootroot00000000000000londiste-sql-3.8/sql/init_ext.sql000066400000000000000000000004351432153235300171500ustar00rootroot00000000000000\set ECHO none set log_error_verbosity = 'terse'; set client_min_messages = 'warning'; create extension pgq; create extension pgq_node; \set ECHO all create extension londiste; select array_length(extconfig, 1) as dumpable from pg_catalog.pg_extension where extname = 'londiste'; londiste-sql-3.8/sql/init_noext.sql000066400000000000000000000002771432153235300175110ustar00rootroot00000000000000\set ECHO none set log_error_verbosity = 'terse'; set client_min_messages = 'warning'; -- \i ../txid/txid.sql \i ../pgq/pgq.sql \i ../pgq_node/pgq_node.sql \i londiste.sql \set ECHO all londiste-sql-3.8/sql/londiste_create_part.sql000066400000000000000000000052241432153235300215200ustar00rootroot00000000000000 \set ECHO none set log_error_verbosity = 'terse'; set client_min_messages = 'warning'; \set ECHO all drop role if exists londiste_test_part1; drop role if exists londiste_test_part2; create group londiste_test_part1; create group londiste_test_part2; create table events ( id int4 primary key, txt text not null, ctime timestamptz not null default now(), someval int4 check (someval > 0) ); create index ctime_idx on events (ctime); create rule ignore_dups AS on insert to events where (exists (select 1 from events where (events.id = new.id))) do instead nothing; create or replace function "NullTrigger"() returns trigger as $$ begin return null; end; $$ language plpgsql; create trigger "Fooza" after delete on events for each row execute procedure "NullTrigger"(); alter table events enable always trigger "Fooza"; grant select,delete on events to londiste_test_part1; grant select,update,delete on events to londiste_test_part2 with grant option; grant select,insert on events to public; select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01', 'month'); select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01'::timestamptz, 'month'); select londiste.create_partition('events', 'events_2011_01', 'id', 'ctime', '2011-01-01'::timestamp, 'month'); select count(*) from pg_indexes where schemaname='public' and tablename = 'events_2011_01'; select count(*) from pg_constraint where conrelid = 'public.events_2011_01'::regclass; select count(*) from pg_rules where schemaname = 'public' and tablename = 'events_2011_01'; select trigger_name, event_manipulation, action_statement from information_schema.triggers where event_object_schema = 'public' and event_object_table = 'events_2011_01'; select tgenabled, pg_get_triggerdef(oid) from pg_trigger where tgrelid = 'events_2011_01'::regclass::oid; -- test weird quoting create table "Bad "" table '.' name!" ( id int4 primary key, txt text not null, ctime timestamptz not null default now(), someval int4 check (someval > 0) ); create rule "Ignore Dups" AS on insert to "Bad "" table '.' name!" where (exists (select 1 from "Bad "" table '.' name!" where ("Bad "" table '.' name!".id = new.id))) do instead nothing; alter table "Bad "" table '.' name!" ENABLE ALWAYS RULE "Ignore Dups"; select londiste.create_partition('public.Bad " table ''.'' name!', 'public.Bad " table ''.'' part!', 'id', 'ctime', '2011-01-01', 'month'); select count(*) from pg_rules where schemaname = 'public' and tablename ilike 'bad%'; -- \d events_2011_01 -- \dp events -- \dp events_2011_01 londiste-sql-3.8/sql/londiste_execute.sql000066400000000000000000000015371432153235300206740ustar00rootroot00000000000000 set log_error_verbosity = 'terse'; select * from londiste.execute_start('branch_set', 'DDL-A.sql', 'drop all', false); select * from londiste.execute_start('branch_set', 'DDL-A.sql', 'drop all', false); select * from londiste.execute_finish('branch_set', 'DDL-A.sql'); select * from londiste.execute_finish('branch_set', 'DDL-A.sql'); select * from londiste.execute_finish('branch_set', 'DDL-XXX.sql'); select * from londiste.execute_start('branch_set', 'DDL-B.sql', 'drop all', true); select * from londiste.execute_start('branch_set', 'DDL-B.sql', 'drop all', true); select * from londiste.execute_start('aset', 'DDL-root.sql', 'drop all', true); select * from londiste.execute_start('aset', 'DDL-root.sql', 'drop all', true); select * from londiste.execute_finish('aset', 'DDL-root.sql'); select * from londiste.execute_finish('aset', 'DDL-root.sql'); londiste-sql-3.8/sql/londiste_fkeys.sql000066400000000000000000000061531432153235300203520ustar00rootroot00000000000000 set log_error_verbosity = 'terse'; set client_min_messages = 'warning'; create table ref_1 ( id int4 primary key, val text ); create table ref_2 ( id int4 primary key, ref int4 not null references ref_1, val text ); create table ref_3 ( id int4 primary key, ref2 int4 not null references ref_2, val text ); select * from londiste.global_add_table('branch_set', 'public.ref_1'); select * from londiste.global_add_table('branch_set', 'public.ref_2'); select * from londiste.global_add_table('branch_set', 'public.ref_3'); select * from londiste.local_add_table('branch_set', 'public.ref_1'); select * from londiste.local_add_table('branch_set', 'public.ref_2'); select * from londiste.local_add_table('branch_set', 'public.ref_3'); select * from londiste.find_table_fkeys('public.ref_1'); select * from londiste.find_table_fkeys('public.ref_2'); select * from londiste.find_table_fkeys('public.ref_3'); select * from londiste.get_table_pending_fkeys('public.ref_2'); select * from londiste.get_valid_pending_fkeys('branch_set'); -- drop fkeys select * from londiste.drop_table_fkey('public.ref_2', 'ref_2_ref_fkey'); select * from londiste.find_table_fkeys('public.ref_1'); select * from londiste.find_table_fkeys('public.ref_2'); select * from londiste.find_table_fkeys('public.ref_3'); select * from londiste.drop_table_fkey('public.ref_3', 'ref_3_ref2_fkey'); -- check if dropped select * from londiste.find_table_fkeys('public.ref_1'); select * from londiste.find_table_fkeys('public.ref_2'); select * from londiste.find_table_fkeys('public.ref_3'); -- look state select * from londiste.get_table_pending_fkeys('public.ref_2'); select * from londiste.get_valid_pending_fkeys('branch_set'); -- toggle sync select * from londiste.local_set_table_state('branch_set', 'public.ref_1', null, 'ok'); select * from londiste.get_valid_pending_fkeys('branch_set'); select * from londiste.local_set_table_state('branch_set', 'public.ref_2', null, 'ok'); select * from londiste.get_valid_pending_fkeys('branch_set'); select * from londiste.local_set_table_state('branch_set', 'public.ref_3', null, 'ok'); select * from londiste.get_valid_pending_fkeys('branch_set'); -- restore: old select * from londiste.get_table_pending_fkeys('public.ref_2'); select * from londiste.restore_table_fkey('public.ref_2', 'ref_2_ref_fkey'); select * from londiste.get_table_pending_fkeys('public.ref_2'); -- restore: new select * from londiste.get_table_pending_fkeys('public.ref_3'); select londiste.restore_table_fkey('public.ref_3', 'ref_3_ref2_fkey', true) as step1; \gset :step1 ; select londiste.restore_table_fkey('public.ref_3', 'ref_3_ref2_fkey', true) as step2; \gset :step2 ; select londiste.restore_table_fkey('public.ref_3', 'ref_3_ref2_fkey', true) as step3; select * from londiste.get_table_pending_fkeys('public.ref_3'); -- look state select * from londiste.get_table_pending_fkeys('public.ref_2'); select * from londiste.get_valid_pending_fkeys('branch_set'); select * from londiste.find_table_fkeys('public.ref_1'); select * from londiste.find_table_fkeys('public.ref_2'); select * from londiste.find_table_fkeys('public.ref_3'); londiste-sql-3.8/sql/londiste_install.sql000066400000000000000000000002771432153235300207000ustar00rootroot00000000000000\set ECHO none set log_error_verbosity = 'terse'; set client_min_messages = 'warning'; -- \i ../txid/txid.sql \i ../pgq/pgq.sql \i ../pgq_node/pgq_node.sql \i londiste.sql \set ECHO all londiste-sql-3.8/sql/londiste_leaf.sql000066400000000000000000000026251432153235300201400ustar00rootroot00000000000000 set client_min_messages = 'warning'; \set VERBOSITY 'terse' select 1 from (select set_config(name, 'escape', false) as ignore from pg_settings where name = 'bytea_output') x where x.ignore = 'foo'; -- -- tables -- create table leafdata ( id serial primary key, data text ); select current_database(); select * from pgq_node.register_location('leafq', 'lq_node1', 'dbname=db', false); select * from pgq_node.register_location('leafq', 'lq_node2', 'dbname=db2', false); select * from pgq_node.create_node('leafq', 'leaf', 'lq_node2', 'londiste_leaf', 'lq_node1', 100, null::text); select * from londiste.local_show_missing('leafq'); select * from londiste.local_add_table('leafq', 'public.leafdata'); select * from londiste.global_add_table('leafq', 'public.leafdata'); select * from londiste.local_add_table('leafq', 'public.leafdata'); select * from londiste.global_add_table('leafq', 'public.tmp'); select * from londiste.get_table_list('leafq'); select tgname, tgargs from pg_trigger where tgrelid = 'public.leafdata'::regclass order by 1; insert into leafdata values (1, 'asd'); select * from londiste.global_remove_table('leafq', 'public.tmp'); select * from londiste.local_remove_table('leafq', 'public.leafdata'); select * from londiste.local_remove_table('leafq', 'public.leafdata'); select * from londiste.get_table_list('leafq'); select * from londiste.local_show_missing('leafq'); londiste-sql-3.8/sql/londiste_merge.sql000066400000000000000000000140751432153235300203320ustar00rootroot00000000000000 set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- tables -- create table tblmerge ( id int4 primary key, data text ); select * from pgq_node.register_location('combined_set', 'croot', 'dbname=db', false); select * from pgq_node.create_node('combined_set', 'root', 'croot', 'londiste_croot', null, null, null); select * from pgq_node.register_location('part1_set', 'p1root', 'dbname=db', false); select * from pgq_node.register_location('part1_set', 'p1merge', 'dbname=db2', false); select * from pgq_node.create_node('part1_set', 'leaf', 'p1merge', 'londiste_p1merge', 'p1root', 100, 'combined_set'); select * from pgq_node.register_location('part2_set', 'p2root', 'dbname=db', false); select * from pgq_node.register_location('part2_set', 'p2merge', 'dbname=db2', false); select * from pgq_node.create_node('part2_set', 'leaf', 'p2merge', 'londiste_p2merge', 'p2root', 100, 'combined_set'); select * from pgq_node.register_location('part3_set', 'p3root', 'dbname=db', false); select * from pgq_node.register_location('part3_set', 'p3merge', 'dbname=db3', false); select * from pgq_node.create_node('part3_set', 'leaf', 'p3merge', 'londiste_p3merge', 'p3root', 100, 'combined_set'); select * from londiste.local_add_table('combined_set', 'tblmerge'); select * from londiste.global_add_table('part1_set', 'tblmerge'); select * from londiste.global_add_table('part2_set', 'tblmerge'); select * from londiste.local_add_table('part1_set', 'tblmerge', array['merge_all']); select * from londiste.get_table_list('part1_set'); select * from londiste.get_table_list('part2_set'); select * from londiste.get_table_list('combined_set'); select * from londiste.local_set_table_state('part1_set', 'public.tblmerge', null, 'in-copy'); select * from londiste.local_set_table_state('part2_set', 'public.tblmerge', null, 'in-copy'); select * from londiste.get_table_list('part1_set'); select * from londiste.get_table_list('part2_set'); select * from londiste.local_set_table_struct('part1_set', 'public.tblmerge', 'create index;'); select * from londiste.get_table_list('part1_set'); select * from londiste.get_table_list('part2_set'); select * from londiste.local_set_table_state('part2_set', 'public.tblmerge', null, 'catching-up'); select * from londiste.get_table_list('part1_set'); select * from londiste.get_table_list('part2_set'); select * from londiste.local_set_table_state('part1_set', 'public.tblmerge', null, 'catching-up'); select * from londiste.get_table_list('part1_set'); select * from londiste.get_table_list('part2_set'); select * from londiste.local_set_table_struct('part1_set', 'public.tblmerge', null); select * from londiste.get_table_list('part1_set'); select * from londiste.get_table_list('part2_set'); -- test automatic registration on combined-root select * from londiste.global_add_table('part1_set', 'tblauto'); select * from londiste.global_add_table('part2_set', 'tblauto'); select * from londiste.local_add_table('part1_set', 'tblauto', array['merge_all', 'virtual_table'], 'handler=vtable'); select * from londiste.get_table_list('part2_set'); select * from londiste.get_table_list('combined_set'); -- -- Test all combinations on 3-node merge -- select * from londiste.global_add_table('part3_set', 'tblmerge'); \set ECHO none create table states ( state text ); insert into states values ('in-copy'); insert into states values ('!in-copy'); insert into states values ('catching-up'); insert into states values ('!catching-up'); create or replace function testmerge( in p1state text, in p2state text, in p3state text, out p1res text, out p2res text, out p3res text) as $$ declare p1ddl text; p2ddl text; p3ddl text; tbl text = 'public.tblmerge'; begin if position('!' in p1state) > 0 then p1ddl := 'x'; end if; if position('!' in p2state) > 0 then p2ddl := 'x'; end if; if position('!' in p3state) > 0 then p3ddl := 'x'; end if; update londiste.table_info set merge_state = replace(p1state, '!', ''), dropped_ddl = p1ddl, local = true where table_name = tbl and queue_name = 'part1_set'; update londiste.table_info set merge_state = replace(p2state, '!', ''), dropped_ddl = p2ddl, local = true where table_name = tbl and queue_name = 'part2_set'; update londiste.table_info set merge_state = replace(p3state, '!', ''), dropped_ddl = p3ddl, local = true where table_name = tbl and queue_name = 'part3_set'; select coalesce(copy_role, 'NULL') from londiste.get_table_list('part1_set') where table_name = tbl into p1res; select coalesce(copy_role, 'NULL') from londiste.get_table_list('part2_set') where table_name = tbl into p2res; select coalesce(copy_role, 'NULL') from londiste.get_table_list('part3_set') where table_name = tbl into p3res; return; end; $$ language plpgsql; create function testmatrix( out p1s text, out p2s text, out p3s text, out p1r text, out p2r text, out p3r text) returns setof record as $$ begin for p1s, p2s, p3s in select p1.state::name, p2.state::name, p3.state::name from states p1, states p2, states p3 where position('!' in p1.state) + position('!' in p2.state) + position('!' in p3.state) < 2 order by 1,2,3 loop select * from testmerge(p1s, p2s, p3s) into p1r, p2r, p3r; return next; end loop; return; end; $$ language plpgsql; \set ECHO all select * from testmatrix(); -- test dropped ddl restore create table ddlrestore ( id int4, data1 text, data2 text ); select count(*) from pg_indexes where schemaname='public' and tablename='ddlrestore'; insert into londiste.table_info (queue_name, table_name, local, merge_state, dropped_ddl) values ('part1_set', 'public.ddlrestore', true, 'in-copy', ' ALTER TABLE ddlrestore ADD CONSTRAINT cli_pkey PRIMARY KEY (id); CREATE INDEX idx_data1 ON ddlrestore USING btree (data1); CREATE INDEX idx_data2 ON ddlrestore USING btree (data2); '); select * from londiste.local_remove_table('part1_set', 'public.ddlrestore'); select count(*) from pg_indexes where schemaname='public' and tablename='ddlrestore'; londiste-sql-3.8/sql/londiste_merge_fkeys.sql000066400000000000000000000144361432153235300215340ustar00rootroot00000000000000 set client_min_messages = 'warning'; \set VERBOSITY 'terse' \set ECHO none create or replace function mkcascade( roots text[], branches text[], branch_target text, leafs text[], leaf_target text, tbls text[]) returns text as $$ declare node text; qname text; status int4; msg text; tbl text; begin for node in select unnest(roots) loop qname := node; perform pgq_node.register_location(qname, node, 'dbname=' || node, false); perform pgq_node.create_node(qname, 'root', node, node || '_worker', null, null, null); for tbl in select unnest(tbls) loop select ret_code, ret_note into status, msg from londiste.global_add_table(node, tbl); if status != 200 then raise exception 'global_add_table - %/%', status, msg; end if; end loop; end loop; for node in select unnest(branches) loop qname := node; perform pgq_node.register_location(qname, node, 'dbname=' || node, false); perform pgq_node.register_location(qname, node||'-provider', 'dbname=' || node, false); perform pgq_node.create_node(qname, 'branch', node, node || '_worker', node||'-provider', 10, branch_target); for tbl in select unnest(tbls) loop select ret_code, ret_note into status, msg from londiste.global_add_table(node, tbl); if status != 200 then raise exception 'global_add_table - %/%', status, msg; end if; end loop; end loop; for node in select unnest(leafs) loop qname := node; perform pgq_node.register_location(qname, node, 'dbname=' || node, false); perform pgq_node.register_location(qname, node||'-provider', 'dbname=' || node, false); perform pgq_node.create_node(qname, 'leaf', node, node || '_worker', node||'-provider', 11, leaf_target); for tbl in select unnest(tbls) loop select ret_code, ret_note into status, msg from londiste.global_add_table(node, tbl); if status != 200 then raise exception 'global_add_table - %/%', status, msg; end if; end loop; end loop; return 'done'; end; $$ language plpgsql; \set ECHO all -- leaf1/leaf2->root \set target 'fx1-target' \set leaf1 'fx1-leaf1' \set leaf2 'fx1-leaf2' \set tbl1 'fx1.table1' \set tbl2 'fx1.table2' select mkcascade(array[:'target'], array[]::text[], null, array[:'leaf1', :'leaf2'], :'target', array[:'tbl1', :'tbl2']); insert into londiste.pending_fkeys values (:'tbl1', :'tbl2', 'name', 'def'); select * from londiste.get_table_pending_fkeys(:'tbl1'); update londiste.table_info set merge_state='ok', local=true where table_name in (:'tbl1', :'tbl2'); select * from londiste.get_valid_pending_fkeys(:'target'); select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state='catching-up' where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- no select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state=null, local=false where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state=null, local=false where table_name = :'tbl2' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); select * from londiste.get_valid_pending_fkeys(:'leaf1'); select * from londiste.get_valid_pending_fkeys(:'leaf2'); -- pick? -- leaf1/leaf2->branch \set target 'fx2-branch' \set leaf1 'fx2-leaf1' \set leaf2 'fx2-leaf2' \set tbl1 'fx2.table1' \set tbl2 'fx2.table2' select mkcascade(array[]::text[], array[:'target']::text[], null, array[:'leaf1', :'leaf2'], :'target', array[:'tbl1', :'tbl2']); insert into londiste.pending_fkeys values (:'tbl1', :'tbl2', 'name', 'def'); select * from londiste.get_table_pending_fkeys(:'tbl1'); update londiste.table_info set merge_state='ok', local=true where table_name in (:'tbl1', :'tbl2'); select * from londiste.get_valid_pending_fkeys(:'target'); -- pick select * from londiste.get_valid_pending_fkeys(:'leaf1'); select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state='catching-up' where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); -- no select * from londiste.get_valid_pending_fkeys(:'leaf1'); select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state=null, local=false where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'target'); -- pick select * from londiste.get_valid_pending_fkeys(:'leaf1'); select * from londiste.get_valid_pending_fkeys(:'leaf2'); -- leaf1/leaf2->no branch \set leaf1 'fx3-leaf1' \set leaf2 'fx3-leaf2' \set tbl1 'fx3.table1' \set tbl2 'fx3.table2' select mkcascade(array[]::text[], array[]::text[], null, array[:'leaf1', :'leaf2'], null, array[:'tbl1', :'tbl2']); insert into londiste.pending_fkeys values (:'tbl1', :'tbl2', 'name', 'def'); select * from londiste.get_table_pending_fkeys(:'tbl1'); update londiste.table_info set merge_state='ok', local=true where table_name in (:'tbl1', :'tbl2'); select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state='catching-up' where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'leaf1'); select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state=null, local=false where table_name = :'tbl1' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'leaf1'); -- pick select * from londiste.get_valid_pending_fkeys(:'leaf2'); update londiste.table_info set merge_state=null, local=false where table_name = :'tbl2' and queue_name = :'leaf1'; select * from londiste.get_valid_pending_fkeys(:'leaf1'); select * from londiste.get_valid_pending_fkeys(:'leaf2'); -- pick londiste-sql-3.8/sql/londiste_provider.sql000066400000000000000000000054271432153235300210660ustar00rootroot00000000000000 set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- tables -- create table testdata ( id serial primary key, txt text ); create table testdata_nopk ( id serial, txt text ); select current_database(); select * from pgq_node.register_location('aset', 'rnode', 'dbname=db', false); select * from pgq_node.create_node('aset', 'root', 'rnode', 'londiste_root', null::text, null::int8, null::text); select * from londiste.local_add_table('aset', 'public.testdata_nopk'); select * from londiste.local_add_table('aset', 'public.testdata'); select tgname from pg_trigger where tgrelid = 'public.testdata'::regclass order by 1; insert into testdata (txt) values ('test-data'); select * from londiste.get_table_list('aset'); select * from londiste.local_show_missing('aset'); select * from londiste.local_remove_table('aset', 'public.testdata'); select * from londiste.local_remove_table('aset', 'public.testdata'); select tgname from pg_trigger where tgrelid = 'public.testdata'::regclass; select * from londiste.get_table_list('aset'); select ev_id, ev_type, ev_data, ev_extra1 from pgq.event_template; select * from londiste.local_show_missing('aset'); -- trigtest create table trg_test ( id int4 primary key, txt text ); select * from londiste.local_add_table('aset', 'public.trg_test', array['ev_extra4=''test='' || txt']); select * from londiste.local_add_table('aset', 'public.trg_test'); select * from londiste.local_add_table('aset', 'public.trg_test', array['ev_extra4=''test='' || txt'], 'handler=foobar'); insert into trg_test values (1, 'data'); truncate trg_test; select ev_id, ev_type, ev_data, ev_extra1, ev_extra4 from pgq.event_template where ev_extra1 = 'public.trg_test'; select tgname from pg_trigger where tgrelid = 'public.trg_test'::regclass order by 1; delete from londiste.table_info where table_name = 'public.trg_test'; select tgname from pg_trigger where tgrelid = 'public.trg_test'::regclass order by 1; -- handler test create table hdlr_test ( id int4 primary key, txt text ); select * from londiste.local_add_table('aset', 'public.hdlr_test'); insert into hdlr_test values (1, 'data'); select * from londiste.local_change_handler('aset', 'public.hdlr_test', array['ev_extra4=''test='' || txt'], 'handler=foobar'); insert into hdlr_test values (2, 'data2'); select * from londiste.local_change_handler('aset', 'public.hdlr_test', '{}'::text[], ''); insert into hdlr_test values (3, 'data3'); truncate hdlr_test; select ev_id, ev_type, ev_data, ev_extra1, ev_extra4 from pgq.event_template where ev_extra1 = 'public.hdlr_test'; -- test proper trigger creation with add-table specific args select * from londiste.local_add_table('aset', 'public.trg_test', array['ev_extra4=''test='' || txt', 'expect_sync', 'skip']); insert into trg_test values (2, 'data2'); londiste-sql-3.8/sql/londiste_seqs.sql000066400000000000000000000037771432153235300202150ustar00rootroot00000000000000 set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- sequences -- create sequence masterseq; create sequence slaveseq; select * from pgq_node.register_location('seqroot', 'rnode', 'dbname=db', false); select * from pgq_node.create_node('seqroot', 'root', 'rnode', 'londiste_root', null::text, null::int8, null::text); select * from londiste.local_add_seq('seqroot', 'masterseq'); select * from londiste.local_add_seq('seqroot', 'masterseq'); select * from londiste.root_check_seqs('seqroot'); select * from londiste.local_remove_seq('seqroot', 'masterseq'); select * from londiste.local_remove_seq('seqroot', 'masterseq'); select * from londiste.get_seq_list('seqroot'); select ev_id, ev_type, ev_data, ev_extra1 from pgq.event_template where ev_type like '%seq%'; -- subscriber select * from pgq_node.register_location('seqbranch', 'subnode', 'dbname=db', false); select * from pgq_node.register_location('seqbranch', 'rootnode', 'dbname=db', false); select * from pgq_node.create_node('seqbranch', 'branch', 'subnode', 'londiste_branch', 'rootnode', 1, null::text); select * from londiste.local_add_seq('seqbranch', 'masterseq'); select * from londiste.global_update_seq('seqbranch', 'masterseq', 5); select * from londiste.local_add_seq('seqbranch', 'masterseq'); select * from londiste.root_check_seqs('seqbranch'); select * from londiste.get_seq_list('seqbranch'); select * from londiste.local_remove_seq('seqbranch', 'masterseq'); select * from londiste.local_remove_seq('seqbranch', 'masterseq'); -- seq auto-removal create table seqtable ( id1 serial primary key, id2 bigserial not null ); select * from londiste.local_add_table('seqroot', 'seqtable'); select * from londiste.local_add_seq('seqroot', 'seqtable_id1_seq'); select * from londiste.local_add_seq('seqroot', 'seqtable_id2_seq'); select * from londiste.get_table_list('seqroot'); select * from londiste.get_seq_list('seqroot'); select * from londiste.local_remove_table('seqroot', 'seqtable'); select * from londiste.get_seq_list('seqroot'); londiste-sql-3.8/sql/londiste_subscriber.sql000066400000000000000000000025131432153235300213700ustar00rootroot00000000000000 set client_min_messages = 'warning'; \set VERBOSITY 'terse' -- -- tables -- create table slavedata ( id serial primary key, data text ); select current_database(); select * from pgq_node.register_location('branch_set', 'snode', 'dbname=db', false); select * from pgq_node.register_location('branch_set', 'pnode', 'dbname=db2', false); select * from pgq_node.create_node('branch_set', 'branch', 'snode', 'londiste_branch', 'pnode', 100, null::text); select * from londiste.local_show_missing('branch_set'); select * from londiste.local_add_table('branch_set', 'public.slavedata'); select * from londiste.global_add_table('branch_set', 'public.slavedata'); select * from londiste.local_add_table('branch_set', 'public.slavedata'); select * from londiste.global_add_table('branch_set', 'public.tmp'); select * from londiste.get_table_list('branch_set'); select * from londiste.local_set_table_state('branch_set', 'public.slavedata', null, 'in-copy'); select * from londiste.get_table_list('branch_set'); select * from londiste.global_remove_table('branch_set', 'public.tmp'); select * from londiste.local_remove_table('branch_set', 'public.slavedata'); select * from londiste.local_remove_table('branch_set', 'public.slavedata'); select * from londiste.get_table_list('branch_set'); select * from londiste.local_show_missing('branch_set'); londiste-sql-3.8/structure/000077500000000000000000000000001432153235300160435ustar00rootroot00000000000000londiste-sql-3.8/structure/ext_postproc.sql000066400000000000000000000005061432153235300213160ustar00rootroot00000000000000 -- tag data objects as dumpable SELECT pg_catalog.pg_extension_config_dump('londiste.table_info', ''); SELECT pg_catalog.pg_extension_config_dump('londiste.seq_info', ''); SELECT pg_catalog.pg_extension_config_dump('londiste.applied_execute', ''); SELECT pg_catalog.pg_extension_config_dump('londiste.pending_fkeys', ''); londiste-sql-3.8/structure/ext_unpackaged.sql000066400000000000000000000004261432153235300215500ustar00rootroot00000000000000ALTER EXTENSION londiste ADD SCHEMA londiste; ALTER EXTENSION londiste ADD TABLE londiste.table_info; ALTER EXTENSION londiste ADD TABLE londiste.seq_info; ALTER EXTENSION londiste ADD TABLE londiste.applied_execute; ALTER EXTENSION londiste ADD TABLE londiste.pending_fkeys; londiste-sql-3.8/structure/functions.sql000066400000000000000000000037451432153235300206050ustar00rootroot00000000000000-- Section: Londiste functions -- upgrade schema \i functions/londiste.upgrade_schema.sql select londiste.upgrade_schema(); -- Group: Information \i functions/londiste.get_seq_list.sql \i functions/londiste.get_table_list.sql \i functions/londiste.local_show_missing.sql -- Group: Local object registration (setup tool) \i functions/londiste.local_add_seq.sql \i functions/londiste.create_trigger.sql \i functions/londiste.local_add_table.sql \i functions/londiste.local_change_handler.sql \i functions/londiste.local_remove_seq.sql \i functions/londiste.local_remove_table.sql -- Group: Global object registrations (internal) \i functions/londiste.global_add_table.sql \i functions/londiste.global_remove_table.sql \i functions/londiste.global_update_seq.sql \i functions/londiste.global_remove_seq.sql -- Group: FKey handling \i functions/londiste.handle_fkeys.sql -- Group: Execute handling \i functions/londiste.execute_start.sql \i functions/londiste.execute_finish.sql -- Group: Internal functions \i functions/londiste.root_check_seqs.sql \i functions/londiste.root_notify_change.sql \i functions/londiste.local_set_table_state.sql \i functions/londiste.local_set_table_attrs.sql \i functions/londiste.local_set_table_struct.sql \i functions/londiste.periodic_maintenance.sql \i functions/londiste.set_session_replication_role.sql -- Group: Utility functions \i functions/londiste.find_column_types.sql \i functions/londiste.find_table_fkeys.sql \i functions/londiste.find_table_oid.sql \i functions/londiste.quote_fqname.sql \i functions/londiste.make_fqname.sql \i functions/londiste.split_fqname.sql \i functions/londiste.table_info_trigger.sql \i functions/londiste.drop_table_triggers.sql \i functions/londiste.is_replica_func.sql \i functions/londiste.version.sql -- Group: Utility functions for handlers \i functions/londiste.create_partition.sql \i functions/londiste.is_obsolete_partition.sql \i functions/londiste.list_obsolete_partitions.sql \i functions/londiste.drop_obsolete_partitions.sql londiste-sql-3.8/structure/grants.ini000066400000000000000000000061531432153235300200470ustar00rootroot00000000000000 [GrantFu] # roles that we maintain in this file roles = londiste_writer, londiste_reader, public, pgq_admin [1.tables] on.tables = londiste.table_info, londiste.seq_info, londiste.pending_fkeys, londiste.applied_execute pgq_admin = select, insert, update, delete londiste_reader = select # backwards compat, should be dropped? public = select [2.public.fns] on.functions = %(londiste_public_fns)s public = execute [3.remote.node] on.functions = %(londiste_remote_fns)s londiste_reader = execute londiste_writer = execute [4.local.node] on.functions = %(londiste_local_fns)s, %(londiste_internal_fns)s londiste_writer = execute [5.seqs] londiste_writer = usage on.sequences = londiste.table_info_nr_seq, londiste.seq_info_nr_seq [6.maint] pgq_admin = execute londiste_writer = execute on.functions = londiste.periodic_maintenance(), londiste.root_check_seqs(text), londiste.root_check_seqs(text, int8) # define various groups of functions [DEFAULT] # can be executed by everybody, read-only, not secdef londiste_public_fns = londiste.find_column_types(text), londiste.find_table_fkeys(text), londiste.find_rel_oid(text, text), londiste.find_table_oid(text), londiste.find_seq_oid(text), londiste.is_replica_func(oid), londiste.quote_fqname(text), londiste.make_fqname(text), londiste.split_fqname(text), londiste.version() # remote node uses those on provider, read local tables londiste_remote_fns = londiste.get_seq_list(text), londiste.get_table_list(text), londiste._coordinate_copy(text, text) # used by owner only londiste_internal_fns = londiste.periodic_maintenance(), londiste.set_session_replication_role(text, bool), londiste.upgrade_schema() # used by local worker, admin londiste_local_fns = londiste.local_show_missing(text), londiste.local_add_seq(text, text), londiste.local_add_table(text, text, text[], text, text), londiste.local_add_table(text, text, text[], text), londiste.local_add_table(text, text, text[]), londiste.local_add_table(text, text), londiste.local_remove_seq(text, text), londiste.local_remove_table(text, text), londiste.global_add_table(text, text), londiste.global_remove_table(text, text), londiste.global_update_seq(text, text, int8), londiste.global_remove_seq(text, text), londiste.get_table_pending_fkeys(text), londiste.get_valid_pending_fkeys(text), londiste.drop_table_fkey(text, text), londiste.restore_table_fkey(text, text, boolean), londiste.execute_start(text, text, text, boolean), londiste.execute_start(text, text, text, boolean, text), londiste.execute_finish(text, text), londiste.root_notify_change(text, text, text), londiste.local_set_table_state(text, text, text, text), londiste.local_set_table_attrs(text, text, text), londiste.local_set_table_struct(text, text, text), londiste.drop_table_triggers(text, text), londiste.table_info_trigger(), londiste.create_partition(text, text, text, text, timestamptz, text), londiste.is_obsolete_partition (text, interval, text), londiste.list_obsolete_partitions (text, interval, text), londiste.drop_obsolete_partitions (text, interval, text), londiste.create_trigger(text,text,text[],text,text) londiste-sql-3.8/structure/grants.sql000066400000000000000000000003561432153235300200660ustar00rootroot00000000000000 grant usage on schema londiste to public; grant select on londiste.table_info to public; grant select on londiste.seq_info to public; grant select on londiste.pending_fkeys to public; grant select on londiste.applied_execute to public; londiste-sql-3.8/structure/install.sql000066400000000000000000000001451432153235300202320ustar00rootroot00000000000000\i structure/tables.sql \i structure/functions.sql \i structure/triggers.sql \i structure/grants.sql londiste-sql-3.8/structure/tables.sql000066400000000000000000000137571432153235300200530ustar00rootroot00000000000000-- ---------------------------------------------------------------------- -- Section: Londiste internals -- -- Londiste storage: tables/seqs/fkeys/triggers/events. -- -- Londiste event types: -- I/U/D - partial SQL event from pgq.sqltriga() -- I:/U:/D: - urlencoded event from pgq.logutriga() -- EXECUTE - SQL script execution -- TRUNCATE - table truncation -- londiste.add-table - global table addition -- londiste.remove-table - global table removal -- londiste.update-seq - sequence update -- londiste.remove-seq - global sequence removal -- -- pgq.sqltriga() event: -- ev_type - I/U/D which means insert/update/delete -- ev_data - partial SQL -- ev_extra1 - table name -- -- Insert: ev_type = "I", ev_data = "(col1, col2) values (2, 'foo')", ev_extra1 = "public.tblname" -- -- Update: ev_type = "U", ev_data = "col2 = null where col1 = 2", ev_extra1 = "public.tblname" -- -- Delete: ev_type = "D", ev_data = "col1 = 2", ev_extra1 = "public.tblname" -- -- pgq.logutriga() event: -- ev_type - I:/U:/D: plus comma separated list of pkey columns -- ev_data - urlencoded row columns -- ev_extra1 - table name -- -- Insert: ev_type = "I:col1", ev_data = "" -- -- Truncate trigger event: -- ev_type - TRUNCATE -- ev_extra1 - table name -- -- Execute SQL event: -- ev_type - EXECUTE -- ev_data - SQL script -- ev_extra1 - Script ID -- -- Global table addition: -- ev_type - londiste.add-table -- ev_data - table name -- -- Global table removal: -- ev_type - londiste.remove-table -- ev_data - table name -- -- Global sequence update: -- ev_type - londiste.update-seq -- ev_data - seq value -- ev_extra1 - seq name --5) -- Global sequence removal: -- ev_type - londiste.remove-seq -- ev_data - seq name -- ---------------------------------------------------------------------- create schema londiste; set default_with_oids = 'off'; -- ---------------------------------------------------------------------- -- Table: londiste.table_info -- -- Info about registered tables. -- -- Columns: -- nr - number for visual ordering -- queue_name - Cascaded queue name -- table_name - fully-qualified table name -- local - Is used locally -- merge_state - State for tables -- custom_snapshot - remote snapshot for COPY command -- dropped_ddl - temp place to store ddl -- table_attrs - urlencoded dict of extra attributes -- -- Tables merge states: -- NULL - copy has not yet happened -- in-copy - ongoing bulk copy -- catching-up - copy process applies events that happened during copy -- wanna-sync:% - copy process caught up, wants to hand table over to replay -- do-sync:% - replay process is ready to accept the table -- ok - in sync, replay applies events -- ---------------------------------------------------------------------- create table londiste.table_info ( nr serial not null, queue_name text not null, table_name text not null, local boolean not null default false, merge_state text, custom_snapshot text, dropped_ddl text, table_attrs text, dest_table text, primary key (queue_name, table_name), foreign key (queue_name) references pgq_node.node_info (queue_name) on delete cascade, check (dropped_ddl is null or merge_state in ('in-copy', 'catching-up')) ); -- ---------------------------------------------------------------------- -- Table: londiste.seq_info -- -- Sequences available on this queue. -- -- Columns: -- nr - number for visual ordering -- queue_name - cascaded queue name -- seq_name - fully-qualified seq name -- local - there is actual seq on local node -- last_value - last published value from root -- ---------------------------------------------------------------------- create table londiste.seq_info ( nr serial not null, queue_name text not null, seq_name text not null, local boolean not null default false, last_value int8 not null, primary key (queue_name, seq_name), foreign key (queue_name) references pgq_node.node_info (queue_name) on delete cascade ); -- ---------------------------------------------------------------------- -- Table: londiste.applied_execute -- -- Info about EXECUTE commands that are ran. -- -- Columns: -- queue_name - cascaded queue name -- execute_file - filename / unique id -- execute_time - the time execute happened -- execute_sql - contains SQL for EXECUTE event (informative) -- ---------------------------------------------------------------------- create table londiste.applied_execute ( queue_name text not null, execute_file text not null, execute_time timestamptz not null default now(), execute_sql text not null, execute_attrs text, primary key (execute_file) ); -- ---------------------------------------------------------------------- -- Table: londiste.pending_fkeys -- -- Details on dropped fkeys. Global, not specific to any set. -- -- Columns: -- from_table - fully-qualified table name -- to_table - fully-qualified table name -- fkey_name - name of constraint -- fkey_def - full fkey definition -- ---------------------------------------------------------------------- create table londiste.pending_fkeys ( from_table text not null, to_table text not null, fkey_name text not null, fkey_def text not null, primary key (from_table, fkey_name) ); londiste-sql-3.8/structure/triggers.sql000066400000000000000000000002141432153235300204070ustar00rootroot00000000000000 create trigger table_info_trigger_sync before delete on londiste.table_info for each row execute procedure londiste.table_info_trigger(); londiste-sql-3.8/structure/upgrade.sql000066400000000000000000000000331432153235300202070ustar00rootroot00000000000000\i structure/functions.sql