pax_global_header00006660000000000000000000000064126171205040014510gustar00rootroot0000000000000052 comment=7314029a9db9923342af57198a1a185d439785ac powa-archivist-REL_3_0_0/000077500000000000000000000000001261712050400153145ustar00rootroot00000000000000powa-archivist-REL_3_0_0/.gitignore000066400000000000000000000000171261712050400173020ustar00rootroot00000000000000*.o *.so *.zip powa-archivist-REL_3_0_0/CHANGELOG.md000066400000000000000000000061371261712050400171340ustar00rootroot00000000000000## 3.0.0 (2015-11-06) Please not that there is not upgrade to switch to this version. You need to remove the old one and install the new 3.0.0 version. - Handle pg_qualtats 0.0.7 - Sample cluster wide statistics, for relations and functions - Fix the powa reset function, and rename it to powa_reset() - Add min/max records to improve performance when analyzing big time interval - Allow disabling some statistics sampling - Handle pg_track_settings extension - Add a GUC to ignore some users activity in sampled data ## 2.0.1 (2015-07-27) - Handle creation/suppression of supported extensions. - Remove then install_all script ## 2.0 (2015-02-06) Major rework of the extension. PoWA 2 is now only compatible with PostgreSQL version 9.4 and above. PoWA 2 is also now compatible with external extensions, such as [pg_qualstats](https://github.com/dalibo/pg_qualstats) or [pg_stat_kcache](https://github.com/dalibo/pg_stat_kcache). Third-part extensions can also now be implemented easily. The UI is also now in a [new repository](https://github.com/dalibo/powa-web), with more frequent release cycle. ## 1.2.1 (2015-01-16) No changes in core. New features and changes in UI : - UI is now compatible with mojolicious 5.0 and more - UI can now connect to multiple servers, and credentials can be specified for each server - Use ISO 8601 timestamp format - Add POWA_CONFIG_FILE variable to specify config file location - Better charts display on small screens When upgrading from 1.2: - No change on the extension - the format of the database section of the powa.conf has changed, to allow multiple servers specification. Please read INSTALL.md for more details about it. ## 1.2 (2014-10-27) News features and fixes in core : - Display more metrics : temporary data, I/O time, average runtime - Fix timestamp for snapshots - DEALLOCATE and BEGIN statements are now ignored - PoWA history tables are now marked as "to be dumped" by pg_dump - Improve performance for "per database aggregated stats" News features and changes in UI : - Follow the selected time interval between each page - Add a title to each page - Display metrics for each query page - Move database selector as a menu entry - Display human readable metrics - Fix empty graph bug When upgrading from older versions : - Upgrade the core with ALTER EXTENSION powa UPDATE. - The format of the database section of the powa.conf has changed. The new format is : "dbname" : "powa", "host" : "127.0.0.1", "port" : "5432", (instead of one line containing the dbi:Pg connection info) ## 1.1 (2014-08-18) **POWA is now production ready** Features: - Various UI improvments - More documentation - New demo mode - Plugin support - The code is now under the PostgreSQL license - New website - New logo Bug fixes: - Use a temporary table for unpacked records to avoid unnecessary bloat ## 1.0 (2014-06-13) **Hello World ! This is the first public release of POWA** Features: - Web UI based on Mojolicious - Graph and dynamic charts - Packed the code as an extension - PL functions powa-archivist-REL_3_0_0/CONTRIBUTORS.md000066400000000000000000000004471261712050400176000ustar00rootroot00000000000000Contributors to PoWA : * Marc Cousin * Julien Rouhaud * Damien Clochard * Thomas Reiss * Hyunjun Kim * Raghu Ram * Rodolphe Quiédeville * Ahmed Bessifi * Christopher Liu * menardorama * Victor D * Justin Miller * Arthur Lutz * Luis Pinto Da Costa * Ronan Dunklau powa-archivist-REL_3_0_0/INSTALL.md000066400000000000000000000125071261712050400167510ustar00rootroot00000000000000PostgreSQL Workload Analyzer detailled installation guide ========================================================= Read [README.md](https://github.com/dalibo/powa/blob/master/README.md) for further details about PoWA. PoWA requires PostgreSQL 9.4 or more. The following documentation describes the detailed installation steps to install PoWA. Download PoWA from the website ------------------------------ ``` wget https://github.com/dalibo/powa/archive/REL_2_0.zip ``` Unpack the downloaded file -------------------------- ``` cd /usr/src unzip powa-REL_2_0.zip ``` Compile and install the software -------------------------------- Before proceeding, be sure to have a compiler installed and the appropriate PostgreSQL development packages. Something like ``` apt-get install postgresql-server-dev-9.0 ``` or ``` yum install postgresql93-devel ``` Then: ``` cd /usr/src/powa-REL_2_0 make ``` If everything goes fine, you will have this kind of output : ``` gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -g -fpic -I. -I. -I/home/thomas/postgresql/postgresql-9.3.4/include/server -I/home/thomas/postgresql/postgresql-9.3.4/include/internal -D_GNU_SOURCE -I/usr/include/libxml2 -c -o powa.o powa.c gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -g -fpic -L/home/thomas/postgresql/postgresql-9.3.4/lib -Wl,--as-needed -Wl,-rpath,'/home/thomas/postgresql/postgresql-9.3.4/lib',--enable-new-dtags -shared -o powa.so powa.o ``` Install the software : - This step has to be made with the user that has installed PostgreSQL. If you have used a package, it will be certainly be root. If so: ``` sudo make install ``` Else, sudo into the user that owns your PostgreSQL executables, and ``` make install ``` It should output something like the following : ``` /bin/mkdir -p '/usr/pgsql-9.3/share/extension' /bin/mkdir -p '/usr/pgsql-9.3/share/extension' /bin/mkdir -p '/usr/pgsql-9.3/lib' /bin/mkdir -p '/usr/pgsql-9.3/share/doc/extension' /usr/bin/install -c -m 644 ./powa.control '/usr/pgsql-9.3/share/extension/' /usr/bin/install -c -m 644 ./powa--1.0.sql ./powa--1.1.sql ./powa--1.2.sql ./powa--1.1--1.2.sql '/usr/pgsql-9.3/share/extension/' /usr/bin/install -c -m 755 powa.so '/usr/pgsql-9.3/postgresql-9.3.4/lib/' /usr/bin/install -c -m 644 ./README.md '/usr/pgsql-9.3/share/doc/extension/' ``` Create a PoWA database and create required extensions ----------------------------------------------------- Note: if you are upgrading from a previous PoWA release, please consult the upgrading section at the end of this file. First, connect to PostgreSQL as administrator : ``` bash-4.1$ psql psql (9.3.5) Type "help" for help. postgres=# create database powa; CREATE DATABASE postgres=# \c powa You are now connected to database "powa" as user "postgres". powa=# create extension pg_stat_statements ; CREATE EXTENSION powa=# create extension btree_gist ; CREATE EXTENSION powa=# create extension powa; CREATE EXTENSION powa=# \dt List of relations Schema | Name | Type | Owner --------+---------------------------------+-------+---------- public | powa_functions | table | postgres public | powa_last_aggregation | table | postgres public | powa_last_purge | table | postgres public | powa_statements | table | postgres public | powa_statements_history | table | postgres public | powa_statements_history_current | table | postgres (6 rows) ``` Modify the configuration files ------------------------------ In `postgresql.conf`: Change the `shared_preload_libraries` appropriately : ``` shared_preload_libraries = 'powa,pg_stat_statements'# (change requires restart) ``` If possible (check with pg_test_timing), activate track_io_timing on your instance, in postgresql.conf : ``` track_io_timing = on ``` Other GUC variables are available. Read [README.md](https://github.com/dalibo/powa/blob/master/README.md) for further details. In `pg_hba.conf`: Add an entry if needed for the PostgreSQL user(s) that need to connect on the GUI. For instance, assuming a `local connection` on database `powa`, allowing any user: `host powa all 127.0.0.1/32 md5` Restart PostgreSQL ------------------ As root, run the following command : ``` service postgresql-9.3 restart ``` PostgreSQL should output the following messages in the log files : ``` 2014-07-25 03:48:20 IST LOG: registering background worker "powa" 2014-07-25 03:48:20 IST LOG: loaded library "powa" 2014-07-25 03:48:20 IST LOG: loaded library "pg_stat_statements" ``` Upgrading from a previous version of PoWA ----------------------------------------- If you already have an older PoWA installation, you can simply upgrade PoWA with the following steps : First, connect to PostgreSQL as administrator and update the extension : ``` bash-4.1$ psql powa psql (9.3.5) Type "help" for help. powa=# ALTER EXTENSION powa UPDATE ; ALTER EXTENSION ``` Next, you will need to restart PostgreSQL in order to take account of the updated background worker. As root, run the following command : ``` service postgresql-9.4 restart ``` powa-archivist-REL_3_0_0/LICENSE.md000066400000000000000000000016071261712050400167240ustar00rootroot00000000000000Copyright (c) 2014-2015, DALIBO Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL DALIBO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF DALIBO HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. DALIBO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND DALIBO HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. powa-archivist-REL_3_0_0/META.json000066400000000000000000000017151261712050400167410ustar00rootroot00000000000000{ "name": "powa", "abstract": "An extension gathering pg_stat_statements and other plugins statistics", "version": "__VERSION__", "maintainer": "Julien Rouhaud ", "license": "postgresql", "release_status": "stable", "provides": { "powa": { "abstract": "An extension gathering pg_stat_statements and other plugins statistics", "file": "powa.sql", "docfile": "README.md", "version": "__VERSION__" } }, "resources": { "bugtracker": { "web": "http://github.com/dalibo/powa/issues/" }, "repository": { "url": "git://github.com/dalibo/powa-archivist.git", "web": "http://github.com/dalibo/powa-archivist/", "type": "git" } }, "generated_by": "Julien Rouhaud", "meta-spec": { "version": "1.0.0", "url": "http://pgxn.org/meta/spec.txt" }, "tags": [ "monitoring", "workload" ] } powa-archivist-REL_3_0_0/Makefile000066400000000000000000000015151261712050400167560ustar00rootroot00000000000000EXTENSION = powa EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") TESTS = $(wildcard test/sql/*.sql) REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS)) REGRESS_OPTS = --inputdir=test DOCS = $(wildcard *.md) PG_CONFIG = pg_config MODULES = powa all: release-zip: all git archive --format zip --prefix=powa-${EXTVERSION}/ --output ./powa-${EXTVERSION}.zip HEAD unzip ./powa-$(EXTVERSION).zip rm ./powa-$(EXTVERSION).zip rm ./powa-$(EXTVERSION)/.gitignore rm ./powa-$(EXTVERSION)/reindent.sh sed -i -e "s/__VERSION__/$(EXTVERSION)/g" ./powa-$(EXTVERSION)/META.json zip -r ./powa-$(EXTVERSION).zip ./powa-$(EXTVERSION)/ rm ./powa-$(EXTVERSION) -rf DATA = $(wildcard *--*.sql) PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) powa-archivist-REL_3_0_0/PL_funcs.md000066400000000000000000000036241261712050400173540ustar00rootroot00000000000000This is a list of all functions and what they are used for: * `powa_take_snapshot`: takes a snapshot. It means calling all the **snapshot** functions registered in the **powa_functions** table, then maybe do an **aggregate** and/or a **purge**, if conditions are met (these functions are also registered in powa_functions). * `powa_take_statements_snapshot`: takes a snapshot of pg_stat_statements. This is the included **snapshot** function. * `powa_statements_purge`: does a purge of collected data from pg_stat_statements. This is the included **purge** function. * `powa_statements_aggregate`: does an aggregate (putting individual records into arrays to save space) on collected data from pg_stat_statements. This is the included **aggregate** function. * `powa_stats_reset`: cleans-up pg_stat_staments collected data. **FIXME: Should be moved to dedicated functions, and stored in powa_functions**. * `powa_kcache_register`: Add the pg_stat_kcache snapshot, aggregate and purge functions to list of powa functions if pg_stat_kcache extension exists. * `powa_kcache_unregister`: Remove the pg_stat_kcache snapshot, aggregate and purge functions from list of powa functions. * `powa_kcache_snapshot`: Take a snapshot of pg_stat_kcache. * `powa_kcache_aggregate`: Does an aggregate on collected data from pg_stat_kcache. * `powa_kcache_purge`: Does a purge of collected data from pg_stat_kcache. * `powa_qualstats_register`: Add the pg_qualstats snapshot, aggregate and purge functions to list of powa functions if pg_qualstats extension exists. * `powa_qualstats_unregister`: Remove the pg_qualstats snapshot, aggregate and purge pg_qualstats functions from list of powa functions. * `powa_qualstats_snapshot`: Take a snapshot of pg_qualstats. * `powa_qualstats_aggregate`: Does an aggregate on collected data from pg_qualstats. * `powa_qualstats_purge`: Does a purge of collected data from pg_qualstats. powa-archivist-REL_3_0_0/README.md000066400000000000000000000011211261712050400165660ustar00rootroot00000000000000 ![PostgreSQL Workload Analyzer](https://github.com/dalibo/powa/blob/master/img/powa_logo.410x161.png) PoWA Archivist ============================ This project is the core extension of the [PoWA](http://dalibo.github.io/powa/) project, a PostgreSQL Workload Analyzer that gathers performance stats and provides real-time charts and graphs to help monitor and tune your PostgreSQL servers. For more information, please read the [PoWA-archivist documentation](http://powa.readthedocs.org/en/latest/powa-archivist/index.html): http://powa.readthedocs.org/en/latest/powa-archivist/index.html powa-archivist-REL_3_0_0/powa--2.0--2.0.1.sql000066400000000000000000000230541261712050400200720ustar00rootroot00000000000000-- complain if script is sourced in psql, rather than via ALTER EXTENSION --\echo Use "ALTER EXTENSION powa" to load this file. \quit -- powa_functions now have an "unregister" operation ALTER TABLE public.powa_functions DROP CONSTRAINT powa_functions_operation_check; ALTER TABLE public.powa_functions ADD CHECK (operation IN ('snapshot','aggregate','purge','unregister')); -- Handle automatic extensions registering CREATE OR REPLACE FUNCTION public.powa_check_created_extensions() RETURNS event_trigger LANGUAGE plpgsql AS $_$ DECLARE BEGIN /* We have for now no way for a proper handling of this event, * as we don't have a table with the list of supported extensions. * So just call every powa_*_register() function we know each time an * extension is created. Powa should be in a dedicated database and the * register function handle to be called several time, so it's not critical */ PERFORM public.powa_kcache_register(); PERFORM public.powa_qualstats_register(); END; $_$; CREATE EVENT TRIGGER powa_check_created_extensions ON ddl_command_end WHEN tag IN ('CREATE EXTENSION') EXECUTE PROCEDURE public.powa_check_created_extensions() ; CREATE OR REPLACE FUNCTION public.powa_check_dropped_extensions() RETURNS event_trigger LANGUAGE plpgsql AS $_$ DECLARE funcname text; v_state text; v_msg text; v_detail text; v_hint text; v_context text; BEGIN WITH ext AS ( SELECT object_name FROM pg_event_trigger_dropped_objects() d WHERE d.object_type = 'extension' ) SELECT function_name INTO funcname FROM powa_functions f JOIN ext ON f.module = ext.object_name WHERE operation = 'unregister'; IF ( funcname IS NOT NULL ) THEN BEGIN RAISE DEBUG 'running %', funcname; EXECUTE 'SELECT ' || quote_ident(funcname) || '()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE WARNING 'powa_check_dropped_extensions(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END IF; END; $_$; -- Handle automatic extensions unregistering CREATE EVENT TRIGGER powa_check_dropped_extensions ON sql_drop WHEN tag IN ('DROP EXTENSION') EXECUTE PROCEDURE public.powa_check_dropped_extensions() ; -- New powa_kcache_register function CREATE OR REPLACE function public.powa_kcache_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_stat_kcache'; IF ( v_ext_present ) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE module = 'pg_stat_kcache'; IF ( NOT v_func_present) THEN INSERT INTO powa_functions (module, operation, function_name, added_manually) VALUES ('pg_stat_kcache', 'snapshot', 'powa_kcache_snapshot', false), ('pg_stat_kcache', 'aggregate', 'powa_kcache_aggregate', false), ('pg_stat_kcache', 'unregister', 'powa_kcache_unregister', false), ('pg_stat_kcache', 'purge', 'powa_kcache_purge', false); END IF; END IF; RETURN true; END; $_$ language plpgsql; -- New powa_qualstats_unregister function CREATE OR REPLACE function public.powa_qualstats_unregister() RETURNS bool AS $_$ BEGIN DELETE FROM public.powa_functions WHERE module = 'pg_qualstats'; RETURN true; END; $_$ language plpgsql; -- New powa_qualstats_register function CREATE OR REPLACE function public.powa_qualstats_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_qualstats'; IF ( v_ext_present) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE function_name IN ('powa_qualstats_snapshot', 'powa_qualstats_aggregate', 'powa_qualstats_purge'); IF ( NOT v_func_present) THEN INSERT INTO powa_functions (module, operation, function_name, added_manually) VALUES ('pg_qualstats', 'snapshot', 'powa_qualstats_snapshot', false), ('pg_qualstats', 'aggregate', 'powa_qualstats_aggregate', false), ('pg_qualstats', 'unregister', 'powa_qualstats_unregister', false), ('pg_qualstats', 'purge', 'powa_qualstats_purge', false); END IF; END IF; RETURN true; END; $_$ language plpgsql; -- Add the _unregister() function in powa_functions if the related extension exists WITH ext_exists AS ( SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_kcache' ) INSERT INTO public.powa_functions (module, operation, function_name, added_manually) SELECT 'pg_stat_kcache', 'unregister', 'powa_kcache_unregister', false FROM ext_exists; WITH ext_exists AS ( SELECT 1 FROM pg_extension WHERE extname = 'pg_qualstats' ) INSERT INTO public.powa_functions (module, operation, function_name, added_manually) SELECT 'pg_qualstats', 'unregister', 'powa_qualstats_unregister', false FROM ext_exists; -- Fix the "added_manually" value for pg_stat_kcache extension UPDATE public.powa_functions SET added_manually = false WHERE module = 'pg_stat_kcache'; ----------------------------------------------------------- -- Fix the tstzrange inclusive upper bounds for -- * powa_kcache_aggregate() function -- * powa_qualstats_aggregate_constvalues_current view -- * powa_qualstats_aggregate() function ----------------------------------------------------------- CREATE OR REPLACE FUNCTION powa_kcache_aggregate() RETURNS void AS $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_kcache_aggregate'; -- aggregate metrics table LOCK TABLE powa_kcache_metrics_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_kcache_metrics (coalesce_range, queryid, dbid, userid, metrics) SELECT tstzrange(min((metrics).ts), max((metrics).ts),'[]'), queryid, dbid, userid, array_agg(metrics) FROM powa_kcache_metrics_current GROUP BY queryid, dbid, userid; TRUNCATE powa_kcache_metrics_current; -- aggregate metrics_db table LOCK TABLE powa_kcache_metrics_current_db IN SHARE MODE; -- prevent any other update INSERT INTO powa_kcache_metrics_db (coalesce_range, dbid, metrics) SELECT tstzrange(min((metrics).ts), max((metrics).ts),'[]'), dbid, array_agg(metrics) FROM powa_kcache_metrics_current_db GROUP BY dbid; TRUNCATE powa_kcache_metrics_current_db; END $PROC$ language plpgsql; CREATE OR REPLACE VIEW powa_qualstats_aggregate_constvalues_current AS WITH consts AS ( SELECT qualid, queryid, dbid, userid, min(ts) as mints, max(ts) as maxts, sum(nbfiltered) as nbfiltered, sum(count) as count, constvalues FROM powa_qualstats_constvalues_history_current GROUP BY qualid, queryid, dbid, userid, constvalues ), groups AS ( SELECT qualid, queryid, dbid, userid, tstzrange(min(mints), max(maxts),'[]') FROM consts GROUP BY qualid, queryid, dbid, userid ) SELECT * FROM groups, LATERAL ( SELECT array_agg(constvalues) as mf FROM ( SELECT (constvalues, nbfiltered, count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY CASE WHEN count = 0 THEN 0 ELSE nbfiltered / count::numeric END DESC LIMIT 20 ) s ) as mf, LATERAL ( SELECT array_agg(constvalues) as lf FROM ( SELECT (constvalues, nbfiltered, count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY CASE WHEN count = 0 THEN 0 ELSE nbfiltered / count::numeric END DESC LIMIT 20 ) s ) as lf, LATERAL ( SELECT array_agg(constvalues) as me FROM ( SELECT (constvalues, nbfiltered, count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY count desc LIMIT 20 ) s ) as me; CREATE OR REPLACE FUNCTION powa_qualstats_aggregate() RETURNS void AS $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_qualstats_aggregate'; LOCK TABLE powa_qualstats_constvalues_history_current IN SHARE MODE; LOCK TABLE powa_qualstats_quals_history_current IN SHARE MODE; INSERT INTO powa_qualstats_constvalues_history ( qualid, queryid, dbid, userid, coalesce_range, most_filtering, least_filtering, most_executed) SELECT * FROM powa_qualstats_aggregate_constvalues_current; INSERT INTO powa_qualstats_quals_history (qualid, queryid, dbid, userid, coalesce_range, records) SELECT qualid, queryid, dbid, userid, tstzrange(min(ts), max(ts),'[]'), array_agg((ts, count, nbfiltered)::powa_qualstats_history_item) FROM powa_qualstats_quals_history_current GROUP BY qualid, queryid, dbid, userid; TRUNCATE powa_qualstats_constvalues_history_current; TRUNCATE powa_qualstats_quals_history_current; END $PROC$ language plpgsql; -- Try to register handled extensions SELECT * FROM public.powa_qualstats_register(); powa-archivist-REL_3_0_0/powa--2.0.1.sql000066400000000000000000000661061261712050400175250ustar00rootroot00000000000000-- complain if script is sourced in psql, rather than via CREATE EXTENSION --\echo Use "CREATE EXTENSION powa" to load this file. \quit SET statement_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SET check_function_bodies = false; SET client_min_messages = warning; SET escape_string_warning = off; SET search_path = public, pg_catalog; CREATE TYPE powa_statement_history_record AS ( ts timestamp with time zone, calls bigint, total_time double precision, rows bigint, shared_blks_hit bigint, shared_blks_read bigint, shared_blks_dirtied bigint, shared_blks_written bigint, local_blks_hit bigint, local_blks_read bigint, local_blks_dirtied bigint, local_blks_written bigint, temp_blks_read bigint, temp_blks_written bigint, blk_read_time double precision, blk_write_time double precision ); CREATE TABLE powa_last_aggregation ( aggts timestamp with time zone ); INSERT INTO powa_last_aggregation(aggts) VALUES (current_timestamp); CREATE TABLE powa_last_purge ( purgets timestamp with time zone ); INSERT INTO powa_last_purge (purgets) VALUES (current_timestamp); CREATE TABLE powa_statements ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, query text NOT NULL ); ALTER TABLE ONLY powa_statements ADD CONSTRAINT powa_statements_pkey PRIMARY KEY (queryid, dbid, userid); CREATE INDEX powa_statements_dbid_idx ON powa_statements(dbid); CREATE INDEX powa_statements_userid_idx ON powa_statements(userid); CREATE TABLE powa_statements_history ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, coalesce_range tstzrange NOT NULL, records powa_statement_history_record[] NOT NULL ); CREATE INDEX powa_statements_history_query_ts ON powa_statements_history USING gist (queryid, coalesce_range); CREATE TABLE powa_statements_history_db ( dbid oid NOT NULL, coalesce_range tstzrange NOT NULL, records powa_statement_history_record[] NOT NULL ); CREATE INDEX powa_statements_history_db_ts ON powa_statements_history_db USING gist (dbid, coalesce_range); CREATE TABLE powa_statements_history_current ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, record powa_statement_history_record NOT NULL ); CREATE TABLE powa_statements_history_current_db ( dbid oid NOT NULL, record powa_statement_history_record NOT NULL ); CREATE SEQUENCE powa_coalesce_sequence INCREMENT BY 1 START WITH 1 CYCLE; CREATE TABLE powa_functions ( module text NOT NULL, operation text NOT NULL, function_name text NOT NULL, added_manually boolean NOT NULL default true, CHECK (operation IN ('snapshot','aggregate','purge','unregister')) ); INSERT INTO powa_functions (module, operation, function_name, added_manually) VALUES ('pg_stat_statements', 'snapshot', 'powa_statements_snapshot', false), ('pg_stat_statements', 'aggregate','powa_statements_aggregate', false), ('pg_stat_statements', 'purge', 'powa_statements_purge', false); /* pg_stat_kcache integration - part 1 */ CREATE TYPE public.kcache_type AS ( ts timestamptz, reads bigint, writes bigint, user_time double precision, system_time double precision ); CREATE TABLE public.powa_kcache_metrics ( coalesce_range tstzrange NOT NULL, queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, metrics public.kcache_type[] NOT NULL, PRIMARY KEY (coalesce_range, queryid, dbid, userid) ); CREATE INDEX ON public.powa_kcache_metrics (queryid); CREATE TABLE public.powa_kcache_metrics_db ( coalesce_range tstzrange NOT NULL, dbid oid NOT NULL, metrics public.kcache_type[] NOT NULL, PRIMARY KEY (coalesce_range, dbid) ); CREATE TABLE public.powa_kcache_metrics_current ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, metrics kcache_type NULL NULL ); CREATE TABLE public.powa_kcache_metrics_current_db ( dbid oid NOT NULL, metrics kcache_type NULL NULL ); /* end of pg_stat_kcache integration - part 1 */ /* pg_qualstats integration - part 1 */ CREATE TYPE public.qual_type AS ( relid oid, attnum integer, opno oid, eval_type "char" ); CREATE TYPE public.qual_values AS ( constants text[], count bigint, nbfiltered bigint ); CREATE TYPE powa_qualstats_history_item AS ( ts timestamptz, count bigint, nbfiltered bigint ); CREATE TABLE public.powa_qualstats_quals ( qualid bigint, queryid bigint, dbid oid, userid oid, quals public.qual_type[], PRIMARY KEY (qualid, queryid, dbid, userid), FOREIGN KEY (queryid, dbid, userid) REFERENCES powa_statements(queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_quals_history ( qualid bigint, queryid bigint, dbid oid, userid oid, coalesce_range tstzrange, records powa_qualstats_history_item[], FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES public.powa_qualstats_quals (qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_quals_history_current ( qualid bigint, queryid bigint, dbid oid, userid oid, ts timestamptz, count bigint, nbfiltered bigint, FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES powa_qualstats_quals(qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_constvalues_history ( qualid bigint, queryid bigint, dbid oid, userid oid, coalesce_range tstzrange, most_filtering qual_values[], least_filtering qual_values[], most_executed qual_values[], FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES public.powa_qualstats_quals (qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_constvalues_history_current ( qualid bigint, queryid bigint, dbid oid, userid oid, ts timestamptz, constvalues text[], count bigint, nbfiltered bigint, FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES public.powa_qualstats_quals (qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX ON powa_qualstats_constvalues_history USING gist (queryid, qualid, coalesce_range); CREATE INDEX ON powa_qualstats_constvalues_history (qualid, queryid); CREATE INDEX ON powa_qualstats_quals(queryid); /* end of pg_qualstats_integration - part 1 */ -- Mark all of powa's tables as "to be dumped" SELECT pg_catalog.pg_extension_config_dump('powa_statements',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history_current_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_functions','WHERE added_manually'); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics',''); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics_current_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_quals',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_quals_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_quals_history_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_constvalues_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_constvalues_history_current',''); CREATE OR REPLACE FUNCTION public.powa_check_created_extensions() RETURNS event_trigger LANGUAGE plpgsql AS $_$ DECLARE BEGIN /* We have for now no way for a proper handling of this event, * as we don't have a table with the list of supported extensions. * So just call every powa_*_register() function we know each time an * extension is created. Powa should be in a dedicated database and the * register function handle to be called several time, so it's not critical */ PERFORM public.powa_kcache_register(); PERFORM public.powa_qualstats_register(); END; $_$; CREATE EVENT TRIGGER powa_check_created_extensions ON ddl_command_end WHEN tag IN ('CREATE EXTENSION') EXECUTE PROCEDURE public.powa_check_created_extensions() ; CREATE OR REPLACE FUNCTION public.powa_check_dropped_extensions() RETURNS event_trigger LANGUAGE plpgsql AS $_$ DECLARE funcname text; v_state text; v_msg text; v_detail text; v_hint text; v_context text; BEGIN WITH ext AS ( SELECT object_name FROM pg_event_trigger_dropped_objects() d WHERE d.object_type = 'extension' ) SELECT function_name INTO funcname FROM powa_functions f JOIN ext ON f.module = ext.object_name WHERE operation = 'unregister'; IF ( funcname IS NOT NULL ) THEN BEGIN RAISE DEBUG 'running %', funcname; EXECUTE 'SELECT ' || quote_ident(funcname) || '()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE WARNING 'powa_check_dropped_extensions(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END IF; END; $_$; CREATE EVENT TRIGGER powa_check_dropped_extensions ON sql_drop WHEN tag IN ('DROP EXTENSION') EXECUTE PROCEDURE public.powa_check_dropped_extensions() ; CREATE OR REPLACE FUNCTION powa_take_snapshot() RETURNS void AS $PROC$ DECLARE purgets timestamp with time zone; purge_seq bigint; funcname text; v_state text; v_msg text; v_detail text; v_hint text; v_context text; BEGIN -- For all snapshot functions in the powa_functions table, execute FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='snapshot' LOOP -- Call all of them, with no parameter RAISE debug 'fonction: %',funcname; BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_take_snapshot(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; -- Coalesce datas if needed SELECT nextval('powa_coalesce_sequence'::regclass) INTO purge_seq; IF ( purge_seq % current_setting('powa.coalesce')::bigint ) = 0 THEN FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='aggregate' LOOP -- Call all of them, with no parameter BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_take_snapshot(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; UPDATE powa_last_aggregation SET aggts = now(); END IF; -- Once every 10 packs, we also purge IF ( purge_seq % (current_setting('powa.coalesce')::bigint *10) ) = 0 THEN FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='purge' LOOP -- Call all of them, with no parameter BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_take_snapshot(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; UPDATE powa_last_purge SET purgets = now(); END IF; END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_statements_snapshot() RETURNS void AS $PROC$ DECLARE result boolean; ignore_regexp text:='^[[:space:]]*(BEGIN)'; -- Ignore begin at beginning of statement BEGIN -- In this function, we capture statements, and also aggregate counters by database -- so that the first screens of powa stay reactive even though there may be thousands -- of different statements RAISE DEBUG 'running powa_statements_snapshot'; WITH capture AS( SELECT pg_stat_statements.* FROM pg_stat_statements WHERE pg_stat_statements.query !~* ignore_regexp ), missing_statements AS( INSERT INTO powa_statements (queryid, dbid, userid, query) SELECT queryid, dbid, userid, query FROM capture c WHERE NOT EXISTS (SELECT 1 FROM powa_statements ps WHERE ps.queryid = c.queryid AND ps.dbid = c.dbid AND ps.userid = c.userid ) ), by_query AS ( INSERT INTO powa_statements_history_current SELECT queryid, dbid, userid, ROW( now(), calls, total_time, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time, blk_write_time )::powa_statement_history_record AS record FROM capture ), by_database AS ( INSERT INTO powa_statements_history_current_db SELECT dbid, ROW( now(), sum(calls), sum(total_time), sum(rows), sum(shared_blks_hit), sum(shared_blks_read), sum(shared_blks_dirtied), sum(shared_blks_written), sum(local_blks_hit), sum(local_blks_read), sum(local_blks_dirtied), sum(local_blks_written), sum(temp_blks_read), sum(temp_blks_written), sum(blk_read_time), sum(blk_write_time) )::powa_statement_history_record AS record FROM capture GROUP BY dbid ) SELECT true::boolean INTO result; -- For now we don't care. What could we do on error except crash anyway? END; $PROC$ language plpgsql; CREATE OR REPLACE FUNCTION powa_statements_purge() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_statements_purge'; -- Delete obsolete datas. We only bother with already coalesced data DELETE FROM powa_statements_history WHERE upper(coalesce_range)< (now() - current_setting('powa.retention')::interval); DELETE FROM powa_statements_history_db WHERE upper(coalesce_range)< (now() - current_setting('powa.retention')::interval); -- FIXME maybe we should cleanup the powa_statements table ? But it will take a while: unnest all records... END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_statements_aggregate() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_statements_aggregate'; -- aggregate statements table LOCK TABLE powa_statements_history_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_statements_history SELECT queryid, dbid, userid, tstzrange(min((record).ts), max((record).ts),'[]'), array_agg(record) FROM powa_statements_history_current GROUP BY queryid, dbid, userid; TRUNCATE powa_statements_history_current; -- aggregate db table LOCK TABLE powa_statements_history_current_db IN SHARE MODE; -- prevent any other update INSERT INTO powa_statements_history_db SELECT dbid, tstzrange(min((record).ts), max((record).ts),'[]'), array_agg(record) FROM powa_statements_history_current_db GROUP BY dbid; TRUNCATE powa_statements_history_current_db; END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION public.powa_stats_reset() RETURNS boolean LANGUAGE plpgsql AS $function$ BEGIN TRUNCATE TABLE powa_statements_history; TRUNCATE TABLE powa_statements_history_current; TRUNCATE TABLE powa_statements_history_db; TRUNCATE TABLE powa_statements_history_current_db; TRUNCATE TABLE powa_statements; RETURN true; END; $function$; /* pg_stat_kcache integration - part 2 */ /* * register pg_stat_kcache extension */ CREATE OR REPLACE function public.powa_kcache_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_stat_kcache'; IF ( v_ext_present ) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE module = 'pg_stat_kcache'; IF ( NOT v_func_present) THEN INSERT INTO powa_functions (module, operation, function_name, added_manually) VALUES ('pg_stat_kcache', 'snapshot', 'powa_kcache_snapshot', false), ('pg_stat_kcache', 'aggregate', 'powa_kcache_aggregate', false), ('pg_stat_kcache', 'unregister', 'powa_kcache_unregister', false), ('pg_stat_kcache', 'purge', 'powa_kcache_purge', false); END IF; END IF; RETURN true; END; $_$ language plpgsql; /* * unregister pg_stat_kcache extension */ CREATE OR REPLACE function public.powa_kcache_unregister() RETURNS bool AS $_$ BEGIN DELETE FROM public.powa_functions WHERE module = 'pg_stat_kcache'; RETURN true; END; $_$ language plpgsql; /* * powa_kcache snapshot collection. */ CREATE OR REPLACE FUNCTION powa_kcache_snapshot() RETURNS void as $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_kcache_snapshot'; WITH capture AS ( SELECT * FROM pg_stat_kcache() ), by_query AS ( INSERT INTO powa_kcache_metrics_current (queryid, dbid, userid, metrics) SELECT queryid, dbid, userid, (now(), reads, writes, user_time, system_time)::kcache_type FROM capture ), by_database AS ( INSERT INTO powa_kcache_metrics_current_db (dbid, metrics) SELECT dbid, (now(), sum(reads), sum(writes), sum(user_time), sum(system_time))::kcache_type FROM capture GROUP BY dbid ) SELECT true into result; END $PROC$ language plpgsql; /* * powa_kcache aggregation */ CREATE OR REPLACE FUNCTION powa_kcache_aggregate() RETURNS void AS $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_kcache_aggregate'; -- aggregate metrics table LOCK TABLE powa_kcache_metrics_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_kcache_metrics (coalesce_range, queryid, dbid, userid, metrics) SELECT tstzrange(min((metrics).ts), max((metrics).ts),'[]'), queryid, dbid, userid, array_agg(metrics) FROM powa_kcache_metrics_current GROUP BY queryid, dbid, userid; TRUNCATE powa_kcache_metrics_current; -- aggregate metrics_db table LOCK TABLE powa_kcache_metrics_current_db IN SHARE MODE; -- prevent any other update INSERT INTO powa_kcache_metrics_db (coalesce_range, dbid, metrics) SELECT tstzrange(min((metrics).ts), max((metrics).ts),'[]'), dbid, array_agg(metrics) FROM powa_kcache_metrics_current_db GROUP BY dbid; TRUNCATE powa_kcache_metrics_current_db; END $PROC$ language plpgsql; /* * powa_kcache purge */ CREATE OR REPLACE FUNCTION powa_kcache_purge() RETURNS void as $PROC$ BEGIN RAISE DEBUG 'running powa_kcache_purge'; DELETE FROM powa_kcache_metrics WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); DELETE FROM powa_kcache_metrics_db WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); END; $PROC$ language plpgsql; -- By default, try to register pg_stat_kcache, in case it's alreay here SELECT * FROM public.powa_kcache_register(); /* end of pg_stat_kcache integration - part 2 */ /* pg_qualstats integration - part 2 */ /* * powa_qualstats_register */ CREATE OR REPLACE function public.powa_qualstats_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_qualstats'; IF ( v_ext_present) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE function_name IN ('powa_qualstats_snapshot', 'powa_qualstats_aggregate', 'powa_qualstats_purge'); IF ( NOT v_func_present) THEN INSERT INTO powa_functions (module, operation, function_name, added_manually) VALUES ('pg_qualstats', 'snapshot', 'powa_qualstats_snapshot', false), ('pg_qualstats', 'aggregate', 'powa_qualstats_aggregate', false), ('pg_qualstats', 'unregister', 'powa_qualstats_unregister', false), ('pg_qualstats', 'purge', 'powa_qualstats_purge', false); END IF; END IF; RETURN true; END; $_$ language plpgsql; /* * powa_qualstats utility view for aggregating constvalues */ CREATE OR REPLACE VIEW powa_qualstats_aggregate_constvalues_current AS WITH consts AS ( SELECT qualid, queryid, dbid, userid, min(ts) as mints, max(ts) as maxts, sum(nbfiltered) as nbfiltered, sum(count) as count, constvalues FROM powa_qualstats_constvalues_history_current GROUP BY qualid, queryid, dbid, userid, constvalues ), groups AS ( SELECT qualid, queryid, dbid, userid, tstzrange(min(mints), max(maxts),'[]') FROM consts GROUP BY qualid, queryid, dbid, userid ) SELECT * FROM groups, LATERAL ( SELECT array_agg(constvalues) as mf FROM ( SELECT (constvalues, nbfiltered, count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY CASE WHEN count = 0 THEN 0 ELSE nbfiltered / count::numeric END DESC LIMIT 20 ) s ) as mf, LATERAL ( SELECT array_agg(constvalues) as lf FROM ( SELECT (constvalues, nbfiltered, count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY CASE WHEN count = 0 THEN 0 ELSE nbfiltered / count::numeric END DESC LIMIT 20 ) s ) as lf, LATERAL ( SELECT array_agg(constvalues) as me FROM ( SELECT (constvalues, nbfiltered, count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY count desc LIMIT 20 ) s ) as me; CREATE OR REPLACE FUNCTION powa_qualstats_snapshot() RETURNS void as $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_qualstats_snaphot'; WITH capture AS ( SELECT pgqs.*, s.query FROM pg_qualstats_by_query pgqs JOIN powa_statements s USING(queryid, dbid, userid) ), missing_quals AS ( INSERT INTO powa_qualstats_quals (qualid, queryid, dbid, userid, quals) SELECT DISTINCT qs.qualnodeid, qs.queryid, qs.dbid, qs.userid, array_agg(DISTINCT q::qual_type) FROM capture qs, LATERAL (SELECT (unnest(quals)).*) as q WHERE NOT EXISTS ( SELECT 1 FROM powa_qualstats_quals nh WHERE nh.qualid = qs.qualnodeid AND nh.queryid = qs.queryid AND nh.dbid = qs.dbid AND nh.userid = qs.userid ) GROUP BY qualnodeid, queryid, dbid, userid RETURNING * ), by_qual AS ( INSERT INTO powa_qualstats_quals_history_current (qualid, queryid, dbid, userid, ts, count, nbfiltered) SELECT qs.qualnodeid, qs.queryid, qs.dbid, qs.userid, now(), sum(count), sum(nbfiltered) FROM capture as qs GROUP BY qualnodeid, qs.queryid, qs.dbid, qs.userid RETURNING * ), by_qual_with_const AS ( INSERT INTO powa_qualstats_constvalues_history_current(qualid, queryid, dbid, userid, ts, count, nbfiltered, constvalues) SELECT qualnodeid, qs.queryid, qs.dbid, qs.userid, now(), count, nbfiltered, constvalues FROM capture as qs ) SELECT true into result; PERFORM pg_qualstats_reset(); END $PROC$ language plpgsql; /* * powa_qualstats aggregate */ CREATE OR REPLACE FUNCTION powa_qualstats_aggregate() RETURNS void AS $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_qualstats_aggregate'; LOCK TABLE powa_qualstats_constvalues_history_current IN SHARE MODE; LOCK TABLE powa_qualstats_quals_history_current IN SHARE MODE; INSERT INTO powa_qualstats_constvalues_history ( qualid, queryid, dbid, userid, coalesce_range, most_filtering, least_filtering, most_executed) SELECT * FROM powa_qualstats_aggregate_constvalues_current; INSERT INTO powa_qualstats_quals_history (qualid, queryid, dbid, userid, coalesce_range, records) SELECT qualid, queryid, dbid, userid, tstzrange(min(ts), max(ts),'[]'), array_agg((ts, count, nbfiltered)::powa_qualstats_history_item) FROM powa_qualstats_quals_history_current GROUP BY qualid, queryid, dbid, userid; TRUNCATE powa_qualstats_constvalues_history_current; TRUNCATE powa_qualstats_quals_history_current; END $PROC$ language plpgsql; /* * powa_qualstats_purge */ CREATE OR REPLACE FUNCTION powa_qualstats_purge() RETURNS void as $PROC$ BEGIN RAISE DEBUG 'running powa_qualstats_purge'; DELETE FROM powa_qualstats_constvalues_history WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); DELETE FROM powa_qualstats_quals_history WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); END; $PROC$ language plpgsql; /* * powa_qualstats_unregister */ CREATE OR REPLACE function public.powa_qualstats_unregister() RETURNS bool AS $_$ BEGIN DELETE FROM public.powa_functions WHERE module = 'pg_qualstats'; RETURN true; END; $_$ language plpgsql; SELECT * FROM public.powa_qualstats_register(); /* end of pg_qualstats_integration - part 2 */ powa-archivist-REL_3_0_0/powa--3.0.0.sql000066400000000000000000001341071261712050400175220ustar00rootroot00000000000000-- complain if script is sourced in psql, rather than via CREATE EXTENSION --\echo Use "CREATE EXTENSION powa" to load this file. \quit SET statement_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SET client_min_messages = warning; SET escape_string_warning = off; SET search_path = public, pg_catalog; CREATE FUNCTION powa_stat_user_functions(IN dbid oid, OUT funcid oid, OUT calls bigint, OUT total_time double precision, OUT self_time double precision) RETURNS SETOF record LANGUAGE c COST 100 AS '$libdir/powa', 'powa_stat_user_functions'; CREATE FUNCTION powa_stat_all_rel(IN dbid oid, OUT relid oid, OUT numscan bigint, OUT tup_returned bigint, OUT tup_fetched bigint, OUT n_tup_ins bigint, OUT n_tup_upd bigint, OUT n_tup_del bigint, OUT n_tup_hot_upd bigint, OUT n_liv_tup bigint, OUT n_dead_tup bigint, OUT n_mod_since_analyze bigint, OUT blks_read bigint, OUT blks_hit bigint, OUT last_vacuum timestamp with time zone, OUT vacuum_count bigint, OUT last_autovacuum timestamp with time zone, OUT autovacuum_count bigint, OUT last_analyze timestamp with time zone, OUT analyze_count bigint, OUT last_autoanalyze timestamp with time zone, OUT autoanalyze_count bigint) RETURNS SETOF record LANGUAGE c COST 100 AS '$libdir/powa', 'powa_stat_all_rel'; CREATE TYPE powa_statements_history_record AS ( ts timestamp with time zone, calls bigint, total_time double precision, rows bigint, shared_blks_hit bigint, shared_blks_read bigint, shared_blks_dirtied bigint, shared_blks_written bigint, local_blks_hit bigint, local_blks_read bigint, local_blks_dirtied bigint, local_blks_written bigint, temp_blks_read bigint, temp_blks_written bigint, blk_read_time double precision, blk_write_time double precision ); CREATE TYPE powa_user_functions_history_record AS ( ts timestamp with time zone, calls bigint, total_time double precision, self_time double precision ); CREATE TYPE powa_all_relations_history_record AS ( ts timestamp with time zone, numscan bigint, tup_returned bigint, tup_fetched bigint, n_tup_ins bigint, n_tup_upd bigint, n_tup_del bigint, n_tup_hot_upd bigint, n_liv_tup bigint, n_dead_tup bigint, n_mod_since_analyze bigint, blks_read bigint, blks_hit bigint, last_vacuum timestamp with time zone, vacuum_count bigint, last_autovacuum timestamp with time zone, autovacuum_count bigint, last_analyze timestamp with time zone, analyze_count bigint, last_autoanalyze timestamp with time zone, autoanalyze_count bigint ); CREATE TABLE powa_last_aggregation ( aggts timestamp with time zone ); INSERT INTO powa_last_aggregation(aggts) VALUES (current_timestamp); CREATE TABLE powa_last_purge ( purgets timestamp with time zone ); INSERT INTO powa_last_purge (purgets) VALUES (current_timestamp); CREATE TABLE powa_statements ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, query text NOT NULL ); ALTER TABLE ONLY powa_statements ADD CONSTRAINT powa_statements_pkey PRIMARY KEY (queryid, dbid, userid); CREATE INDEX powa_statements_dbid_idx ON powa_statements(dbid); CREATE INDEX powa_statements_userid_idx ON powa_statements(userid); CREATE TABLE powa_statements_history ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, coalesce_range tstzrange NOT NULL, records powa_statements_history_record[] NOT NULL, mins_in_range powa_statements_history_record NOT NULL, maxs_in_range powa_statements_history_record NOT NULL ); CREATE INDEX powa_statements_history_query_ts ON powa_statements_history USING gist (queryid, coalesce_range); CREATE TABLE powa_statements_history_db ( dbid oid NOT NULL, coalesce_range tstzrange NOT NULL, records powa_statements_history_record[] NOT NULL, mins_in_range powa_statements_history_record NOT NULL, maxs_in_range powa_statements_history_record NOT NULL ); CREATE INDEX powa_statements_history_db_ts ON powa_statements_history_db USING gist (dbid, coalesce_range); CREATE TABLE powa_statements_history_current ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, record powa_statements_history_record NOT NULL ); CREATE TABLE powa_statements_history_current_db ( dbid oid NOT NULL, record powa_statements_history_record NOT NULL ); CREATE TABLE powa_user_functions_history ( dbid oid NOT NULL, funcid oid NOT NULL, coalesce_range tstzrange NOT NULL, records powa_user_functions_history_record[] NOT NULL, mins_in_range powa_user_functions_history_record NOT NULL, maxs_in_range powa_user_functions_history_record NOT NULL ); CREATE INDEX powa_user_functions_history_funcid_ts ON powa_user_functions_history USING gist (funcid, coalesce_range); CREATE TABLE powa_user_functions_history_current ( dbid oid NOT NULL, funcid oid NOT NULL, record powa_user_functions_history_record NOT NULL ); CREATE TABLE powa_all_relations_history ( dbid oid NOT NULL, relid oid NOT NULL, coalesce_range tstzrange NOT NULL, records powa_all_relations_history_record[] NOT NULL, mins_in_range powa_all_relations_history_record NOT NULL, maxs_in_range powa_all_relations_history_record NOT NULL ); CREATE INDEX powa_all_relations_history_relid_ts ON powa_all_relations_history USING gist (relid, coalesce_range); CREATE TABLE powa_all_relations_history_current ( dbid oid NOT NULL, relid oid NOT NULL, record powa_all_relations_history_record NOT NULL ); CREATE SEQUENCE powa_coalesce_sequence INCREMENT BY 1 START WITH 1 CYCLE; CREATE TABLE powa_functions ( module text NOT NULL, operation text NOT NULL, function_name text NOT NULL, added_manually boolean NOT NULL default true, enabled boolean NOT NULL default true, CHECK (operation IN ('snapshot','aggregate','purge','unregister','reset')) ); INSERT INTO powa_functions (module, operation, function_name, added_manually, enabled) VALUES ('pg_stat_statements', 'snapshot', 'powa_statements_snapshot', false, true), ('powa_stat_user_functions', 'snapshot', 'powa_user_functions_snapshot', false, true), ('powa_stat_all_relations', 'snapshot', 'powa_all_relations_snapshot', false, true), ('pg_stat_statements', 'aggregate','powa_statements_aggregate', false, true), ('powa_stat_user_functions', 'aggregate','powa_user_functions_aggregate', false, true), ('powa_stat_all_relations', 'aggregate','powa_all_relations_aggregate', false, true), ('pg_stat_statements', 'purge', 'powa_statements_purge', false, true), ('powa_stat_user_functions', 'purge', 'powa_user_functions_purge', false, true), ('powa_stat_all_relations', 'purge', 'powa_all_relations_purge', false, true), ('pg_stat_statements', 'reset', 'powa_statements_reset', false, true), ('powa_stat_user_functions', 'reset', 'powa_user_functions_reset', false, true), ('powa_stat_all_relations', 'reset', 'powa_all_relations_reset', false, true); /* pg_stat_kcache integration - part 1 */ CREATE TYPE public.kcache_type AS ( ts timestamptz, reads bigint, writes bigint, user_time double precision, system_time double precision ); CREATE TABLE public.powa_kcache_metrics ( coalesce_range tstzrange NOT NULL, queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, metrics public.kcache_type[] NOT NULL, mins_in_range public.kcache_type NOT NULL, maxs_in_range public.kcache_type NOT NULL, PRIMARY KEY (coalesce_range, queryid, dbid, userid) ); CREATE INDEX ON public.powa_kcache_metrics (queryid); CREATE TABLE public.powa_kcache_metrics_db ( coalesce_range tstzrange NOT NULL, dbid oid NOT NULL, metrics public.kcache_type[] NOT NULL, mins_in_range public.kcache_type NOT NULL, maxs_in_range public.kcache_type NOT NULL, PRIMARY KEY (coalesce_range, dbid) ); CREATE TABLE public.powa_kcache_metrics_current ( queryid bigint NOT NULL, dbid oid NOT NULL, userid oid NOT NULL, metrics kcache_type NULL NULL ); CREATE TABLE public.powa_kcache_metrics_current_db ( dbid oid NOT NULL, metrics kcache_type NULL NULL ); /* end of pg_stat_kcache integration - part 1 */ /* pg_qualstats integration - part 1 */ CREATE TYPE public.qual_type AS ( relid oid, attnum integer, opno oid, eval_type "char" ); CREATE TYPE public.qual_values AS ( constants text[], occurences bigint, execution_count bigint, nbfiltered bigint ); CREATE TYPE powa_qualstats_history_item AS ( ts timestamptz, occurences bigint, execution_count bigint, nbfiltered bigint ); CREATE TABLE public.powa_qualstats_quals ( qualid bigint, queryid bigint, dbid oid, userid oid, quals public.qual_type[], PRIMARY KEY (qualid, queryid, dbid, userid), FOREIGN KEY (queryid, dbid, userid) REFERENCES powa_statements(queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_quals_history ( qualid bigint, queryid bigint, dbid oid, userid oid, coalesce_range tstzrange, records powa_qualstats_history_item[], mins_in_range powa_qualstats_history_item, maxs_in_range powa_qualstats_history_item, FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES public.powa_qualstats_quals (qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_quals_history_current ( qualid bigint, queryid bigint, dbid oid, userid oid, ts timestamptz, occurences bigint, execution_count bigint, nbfiltered bigint, FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES powa_qualstats_quals(qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_constvalues_history ( qualid bigint, queryid bigint, dbid oid, userid oid, coalesce_range tstzrange, most_used qual_values[], most_filtering qual_values[], least_filtering qual_values[], most_executed qual_values[], FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES public.powa_qualstats_quals (qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE TABLE public.powa_qualstats_constvalues_history_current ( qualid bigint, queryid bigint, dbid oid, userid oid, ts timestamptz, constvalues text[], occurences bigint, execution_count bigint, nbfiltered bigint, FOREIGN KEY (qualid, queryid, dbid, userid) REFERENCES public.powa_qualstats_quals (qualid, queryid, dbid, userid) MATCH FULL ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX ON powa_qualstats_constvalues_history USING gist (queryid, qualid, coalesce_range); CREATE INDEX ON powa_qualstats_constvalues_history (qualid, queryid); CREATE INDEX ON powa_qualstats_quals(queryid); /* end of pg_qualstats_integration - part 1 */ -- Mark all of powa's tables as "to be dumped" SELECT pg_catalog.pg_extension_config_dump('powa_statements',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_statements_history_current_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_user_functions_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_user_functions_history_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_all_relations_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_all_relations_history_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_functions','WHERE added_manually'); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics',''); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_kcache_metrics_current_db',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_quals',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_quals_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_quals_history_current',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_constvalues_history',''); SELECT pg_catalog.pg_extension_config_dump('powa_qualstats_constvalues_history_current',''); CREATE OR REPLACE FUNCTION public.powa_check_created_extensions() RETURNS event_trigger LANGUAGE plpgsql AS $_$ DECLARE BEGIN /* We have for now no way for a proper handling of this event, * as we don't have a table with the list of supported extensions. * So just call every powa_*_register() function we know each time an * extension is created. Powa should be in a dedicated database and the * register function handle to be called several time, so it's not critical */ PERFORM public.powa_kcache_register(); PERFORM public.powa_qualstats_register(); PERFORM public.powa_track_settings_register(); END; $_$; CREATE EVENT TRIGGER powa_check_created_extensions ON ddl_command_end WHEN tag IN ('CREATE EXTENSION') EXECUTE PROCEDURE public.powa_check_created_extensions() ; CREATE OR REPLACE FUNCTION public.powa_check_dropped_extensions() RETURNS event_trigger LANGUAGE plpgsql AS $_$ DECLARE funcname text; v_state text; v_msg text; v_detail text; v_hint text; v_context text; BEGIN -- We unregister extensions regardless the "enabled" field WITH ext AS ( SELECT object_name FROM pg_event_trigger_dropped_objects() d WHERE d.object_type = 'extension' ) SELECT function_name INTO funcname FROM powa_functions f JOIN ext ON f.module = ext.object_name WHERE operation = 'unregister'; IF ( funcname IS NOT NULL ) THEN BEGIN RAISE DEBUG 'running %', funcname; EXECUTE 'SELECT ' || quote_ident(funcname) || '()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE WARNING 'powa_check_dropped_extensions(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END IF; END; $_$; CREATE EVENT TRIGGER powa_check_dropped_extensions ON sql_drop WHEN tag IN ('DROP EXTENSION') EXECUTE PROCEDURE public.powa_check_dropped_extensions() ; CREATE OR REPLACE FUNCTION powa_take_snapshot() RETURNS void AS $PROC$ DECLARE purgets timestamp with time zone; purge_seq bigint; funcname text; v_state text; v_msg text; v_detail text; v_hint text; v_context text; BEGIN -- For all enabled snapshot functions in the powa_functions table, execute FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='snapshot' AND enabled LOOP -- Call all of them, with no parameter RAISE debug 'fonction: %',funcname; BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_take_snapshot(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; -- Coalesce datas if needed SELECT nextval('powa_coalesce_sequence'::regclass) INTO purge_seq; IF ( purge_seq % current_setting('powa.coalesce')::bigint ) = 0 THEN FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='aggregate' AND enabled LOOP -- Call all of them, with no parameter BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_take_snapshot(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; UPDATE powa_last_aggregation SET aggts = now(); END IF; -- Once every 10 packs, we also purge IF ( purge_seq % (current_setting('powa.coalesce')::bigint *10) ) = 0 THEN FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='purge' AND enabled LOOP -- Call all of them, with no parameter BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_take_snapshot(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; UPDATE powa_last_purge SET purgets = now(); END IF; END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_statements_snapshot() RETURNS void AS $PROC$ DECLARE result boolean; ignore_regexp text:='^[[:space:]]*(BEGIN)'; -- Ignore begin at beginning of statement BEGIN -- In this function, we capture statements, and also aggregate counters by database -- so that the first screens of powa stay reactive even though there may be thousands -- of different statements RAISE DEBUG 'running powa_statements_snapshot'; WITH capture AS( SELECT pgss.* FROM pg_stat_statements pgss JOIN pg_roles r ON pgss.userid = r.oid WHERE pgss.query !~* ignore_regexp AND NOT (r.rolname = ANY (string_to_array(current_setting('powa.ignored_users'),','))) ), missing_statements AS( INSERT INTO powa_statements (queryid, dbid, userid, query) SELECT queryid, dbid, userid, query FROM capture c WHERE NOT EXISTS (SELECT 1 FROM powa_statements ps WHERE ps.queryid = c.queryid AND ps.dbid = c.dbid AND ps.userid = c.userid ) ), by_query AS ( INSERT INTO powa_statements_history_current SELECT queryid, dbid, userid, ROW( now(), calls, total_time, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time, blk_write_time )::powa_statements_history_record AS record FROM capture ), by_database AS ( INSERT INTO powa_statements_history_current_db SELECT dbid, ROW( now(), sum(calls), sum(total_time), sum(rows), sum(shared_blks_hit), sum(shared_blks_read), sum(shared_blks_dirtied), sum(shared_blks_written), sum(local_blks_hit), sum(local_blks_read), sum(local_blks_dirtied), sum(local_blks_written), sum(temp_blks_read), sum(temp_blks_written), sum(blk_read_time), sum(blk_write_time) )::powa_statements_history_record AS record FROM capture GROUP BY dbid ) SELECT true::boolean INTO result; -- For now we don't care. What could we do on error except crash anyway? END; $PROC$ language plpgsql; CREATE OR REPLACE FUNCTION powa_user_functions_snapshot() RETURNS void AS $PROC$ DECLARE result boolean; BEGIN RAISE DEBUG 'running powa_user_functions_snapshot'; -- Insert cluster-wide user function statistics WITH func(dbid,funcid, r) AS ( SELECT oid, (powa_stat_user_functions(oid)).funcid, powa_stat_user_functions(oid) FROM pg_database ) INSERT INTO powa_user_functions_history_current SELECT dbid, funcid, ROW(now(), (r).calls, (r).total_time, (r).self_time)::powa_user_functions_history_record AS record FROM func; result := true; END; $PROC$ language plpgsql; CREATE OR REPLACE FUNCTION powa_all_relations_snapshot() RETURNS void AS $PROC$ DECLARE result boolean; BEGIN RAISE DEBUG 'running powa_all_relations_snapshot'; -- Insert cluster-wide relation statistics WITH rel(dbid, relid, r) AS ( SELECT oid, (powa_stat_all_rel(oid)).relid, powa_stat_all_rel(oid) FROM pg_database ) INSERT INTO powa_all_relations_history_current SELECT dbid, relid, ROW(now(),(r).numscan, (r).tup_returned, (r).tup_fetched, (r).n_tup_ins, (r).n_tup_upd, (r).n_tup_del, (r).n_tup_hot_upd, (r).n_liv_tup, (r).n_dead_tup, (r).n_mod_since_analyze, (r).blks_read, (r).blks_hit, (r).last_vacuum, (r).vacuum_count, (r).last_autovacuum, (r).autovacuum_count, (r).last_analyze, (r).analyze_count, (r).last_autoanalyze, (r).autoanalyze_count)::powa_all_relations_history_record AS record FROM rel; result := true; END; $PROC$ language plpgsql; CREATE OR REPLACE FUNCTION powa_statements_purge() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_statements_purge'; -- Delete obsolete datas. We only bother with already coalesced data DELETE FROM powa_statements_history WHERE upper(coalesce_range)< (now() - current_setting('powa.retention')::interval); DELETE FROM powa_statements_history_db WHERE upper(coalesce_range)< (now() - current_setting('powa.retention')::interval); -- FIXME maybe we should cleanup the powa_*_history tables ? But it will take a while: unnest all records... END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_user_functions_purge() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_user_functions_purge'; -- Delete obsolete datas. We only bother with already coalesced data DELETE FROM powa_user_functions_history WHERE upper(coalesce_range)< (now() - current_setting('powa.retention')::interval); -- FIXME maybe we should cleanup the powa_*_history tables ? But it will take a while: unnest all records... END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_all_relations_purge() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_all_relations_purge'; -- Delete obsolete datas. We only bother with already coalesced data DELETE FROM powa_all_relations_history WHERE upper(coalesce_range)< (now() - current_setting('powa.retention')::interval); -- FIXME maybe we should cleanup the powa_*_history tables ? But it will take a while: unnest all records... END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_statements_aggregate() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_statements_aggregate'; -- aggregate statements table LOCK TABLE powa_statements_history_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_statements_history SELECT queryid, dbid, userid, tstzrange(min((record).ts), max((record).ts),'[]'), array_agg(record), ROW(min((record).ts), min((record).calls),min((record).total_time),min((record).rows), min((record).shared_blks_hit),min((record).shared_blks_read), min((record).shared_blks_dirtied),min((record).shared_blks_written), min((record).local_blks_hit),min((record).local_blks_read), min((record).local_blks_dirtied),min((record).local_blks_written), min((record).temp_blks_read),min((record).temp_blks_written), min((record).blk_read_time),min((record).blk_write_time))::powa_statements_history_record, ROW(max((record).ts), max((record).calls),max((record).total_time),max((record).rows), max((record).shared_blks_hit),max((record).shared_blks_read), max((record).shared_blks_dirtied),max((record).shared_blks_written), max((record).local_blks_hit),max((record).local_blks_read), max((record).local_blks_dirtied),max((record).local_blks_written), max((record).temp_blks_read),max((record).temp_blks_written), max((record).blk_read_time),max((record).blk_write_time))::powa_statements_history_record FROM powa_statements_history_current GROUP BY queryid, dbid, userid; TRUNCATE powa_statements_history_current; -- aggregate db table LOCK TABLE powa_statements_history_current_db IN SHARE MODE; -- prevent any other update INSERT INTO powa_statements_history_db SELECT dbid, tstzrange(min((record).ts), max((record).ts),'[]'), array_agg(record), ROW(min((record).ts), min((record).calls),min((record).total_time),min((record).rows), min((record).shared_blks_hit),min((record).shared_blks_read), min((record).shared_blks_dirtied),min((record).shared_blks_written), min((record).local_blks_hit),min((record).local_blks_read), min((record).local_blks_dirtied),min((record).local_blks_written), min((record).temp_blks_read),min((record).temp_blks_written), min((record).blk_read_time),min((record).blk_write_time))::powa_statements_history_record, ROW(max((record).ts), max((record).calls),max((record).total_time),max((record).rows), max((record).shared_blks_hit),max((record).shared_blks_read), max((record).shared_blks_dirtied),max((record).shared_blks_written), max((record).local_blks_hit),max((record).local_blks_read), max((record).local_blks_dirtied),max((record).local_blks_written), max((record).temp_blks_read),max((record).temp_blks_written), max((record).blk_read_time),max((record).blk_write_time))::powa_statements_history_record FROM powa_statements_history_current_db GROUP BY dbid; TRUNCATE powa_statements_history_current_db; END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_user_functions_aggregate() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_user_functions_aggregate'; -- aggregate user_functions table LOCK TABLE powa_user_functions_history_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_user_functions_history SELECT dbid, funcid, tstzrange(min((record).ts), max((record).ts),'[]'), array_agg(record), ROW(min((record).ts), min((record).calls),min((record).total_time), min((record).self_time))::powa_user_functions_history_record, ROW(max((record).ts), max((record).calls),max((record).total_time), max((record).self_time))::powa_user_functions_history_record FROM powa_user_functions_history_current GROUP BY dbid, funcid; TRUNCATE powa_user_functions_history_current; END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION powa_all_relations_aggregate() RETURNS void AS $PROC$ BEGIN RAISE DEBUG 'running powa_all_relations_aggregate'; -- aggregate all_relations table LOCK TABLE powa_all_relations_history_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_all_relations_history SELECT dbid, relid, tstzrange(min((record).ts), max((record).ts),'[]'), array_agg(record), ROW(min((record).ts), min((record).numscan),min((record).tup_returned),min((record).tup_fetched), min((record).n_tup_ins),min((record).n_tup_upd), min((record).n_tup_del),min((record).n_tup_hot_upd), min((record).n_liv_tup),min((record).n_dead_tup), min((record).n_mod_since_analyze),min((record).blks_read), min((record).blks_hit),min((record).last_vacuum), min((record).vacuum_count),min((record).last_autovacuum), min((record).autovacuum_count),min((record).last_analyze), min((record).analyze_count),min((record).last_autoanalyze), min((record).autoanalyze_count))::powa_all_relations_history_record, ROW(max((record).ts), max((record).numscan),max((record).tup_returned),max((record).tup_fetched), max((record).n_tup_ins),max((record).n_tup_upd), max((record).n_tup_del),max((record).n_tup_hot_upd), max((record).n_liv_tup),max((record).n_dead_tup), max((record).n_mod_since_analyze),max((record).blks_read), max((record).blks_hit),max((record).last_vacuum), max((record).vacuum_count),max((record).last_autovacuum), max((record).autovacuum_count),max((record).last_analyze), max((record).analyze_count),max((record).last_autoanalyze), max((record).autoanalyze_count))::powa_all_relations_history_record FROM powa_all_relations_history_current GROUP BY dbid, relid; TRUNCATE powa_all_relations_history_current; END; $PROC$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION public.powa_reset() RETURNS boolean LANGUAGE plpgsql AS $function$ DECLARE funcname text; v_state text; v_msg text; v_detail text; v_hint text; v_context text; BEGIN -- Find reset function for every supported datasource, including pgss -- Also call reset function even if they're not enabled FOR funcname IN SELECT function_name FROM powa_functions WHERE operation='reset' LOOP -- Call all of them, with no parameter BEGIN EXECUTE 'SELECT ' || quote_ident(funcname)||'()'; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS v_state = RETURNED_SQLSTATE, v_msg = MESSAGE_TEXT, v_detail = PG_EXCEPTION_DETAIL, v_hint = PG_EXCEPTION_HINT, v_context = PG_EXCEPTION_CONTEXT; RAISE warning 'powa_reset(): function "%" failed: state : % message: % detail : % hint : % context: %', funcname, v_state, v_msg, v_detail, v_hint, v_context; END; END LOOP; RETURN true; END; $function$; CREATE OR REPLACE FUNCTION public.powa_statements_reset() RETURNS boolean LANGUAGE plpgsql AS $function$ BEGIN TRUNCATE TABLE powa_statements_history; TRUNCATE TABLE powa_statements_history_current; TRUNCATE TABLE powa_statements_history_db; TRUNCATE TABLE powa_statements_history_current_db; -- if 3rd part datasource has FK on it, throw everything away TRUNCATE TABLE powa_statements CASCADE; RETURN true; END; $function$; CREATE OR REPLACE FUNCTION public.powa_user_functions_reset() RETURNS boolean LANGUAGE plpgsql AS $function$ BEGIN TRUNCATE TABLE powa_user_functions_history; TRUNCATE TABLE powa_user_functions_history_current; RETURN true; END; $function$; CREATE OR REPLACE FUNCTION public.powa_all_relations_reset() RETURNS boolean LANGUAGE plpgsql AS $function$ BEGIN TRUNCATE TABLE powa_all_relations_history; TRUNCATE TABLE powa_all_relations_history_current; RETURN true; END; $function$; /* pg_stat_kcache integration - part 2 */ /* * register pg_stat_kcache extension */ CREATE OR REPLACE function public.powa_kcache_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_stat_kcache'; IF ( v_ext_present ) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE module = 'pg_stat_kcache'; IF ( NOT v_func_present) THEN INSERT INTO powa_functions (module, operation, function_name, added_manually, enabled) VALUES ('pg_stat_kcache', 'snapshot', 'powa_kcache_snapshot', false, true), ('pg_stat_kcache', 'aggregate', 'powa_kcache_aggregate', false, true), ('pg_stat_kcache', 'unregister', 'powa_kcache_unregister', false, true), ('pg_stat_kcache', 'purge', 'powa_kcache_purge', false, true), ('pg_stat_kcache', 'reset', 'powa_kcache_reset', false, true); END IF; END IF; RETURN true; END; $_$ language plpgsql; /* * unregister pg_stat_kcache extension */ CREATE OR REPLACE function public.powa_kcache_unregister() RETURNS bool AS $_$ BEGIN DELETE FROM public.powa_functions WHERE module = 'pg_stat_kcache'; RETURN true; END; $_$ language plpgsql; /* * powa_kcache snapshot collection. */ CREATE OR REPLACE FUNCTION powa_kcache_snapshot() RETURNS void as $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_kcache_snapshot'; WITH capture AS ( SELECT * FROM pg_stat_kcache() k JOIN pg_roles r ON r.oid = k.userid WHERE NOT (r.rolname = ANY (string_to_array(current_setting('powa.ignored_users'),','))) ), by_query AS ( INSERT INTO powa_kcache_metrics_current (queryid, dbid, userid, metrics) SELECT queryid, dbid, userid, (now(), reads, writes, user_time, system_time)::kcache_type FROM capture ), by_database AS ( INSERT INTO powa_kcache_metrics_current_db (dbid, metrics) SELECT dbid, (now(), sum(reads), sum(writes), sum(user_time), sum(system_time))::kcache_type FROM capture GROUP BY dbid ) SELECT true into result; END $PROC$ language plpgsql; /* * powa_kcache aggregation */ CREATE OR REPLACE FUNCTION powa_kcache_aggregate() RETURNS void AS $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_kcache_aggregate'; -- aggregate metrics table LOCK TABLE powa_kcache_metrics_current IN SHARE MODE; -- prevent any other update INSERT INTO powa_kcache_metrics (coalesce_range, queryid, dbid, userid, metrics, mins_in_range, maxs_in_range) SELECT tstzrange(min((metrics).ts), max((metrics).ts),'[]'), queryid, dbid, userid, array_agg(metrics), ROW(min((metrics).ts), min((metrics).reads),min((metrics).writes),min((metrics).user_time), min((metrics).system_time))::kcache_type, ROW(max((metrics).ts), max((metrics).reads),max((metrics).writes),max((metrics).user_time), max((metrics).system_time))::kcache_type FROM powa_kcache_metrics_current GROUP BY queryid, dbid, userid; TRUNCATE powa_kcache_metrics_current; -- aggregate metrics_db table LOCK TABLE powa_kcache_metrics_current_db IN SHARE MODE; -- prevent any other update INSERT INTO powa_kcache_metrics_db (coalesce_range, dbid, metrics, mins_in_range, maxs_in_range) SELECT tstzrange(min((metrics).ts), max((metrics).ts),'[]'), dbid, array_agg(metrics), ROW(min((metrics).ts), min((metrics).reads),min((metrics).writes),min((metrics).user_time), min((metrics).system_time))::kcache_type, ROW(max((metrics).ts), max((metrics).reads),max((metrics).writes),max((metrics).user_time), max((metrics).system_time))::kcache_type FROM powa_kcache_metrics_current_db GROUP BY dbid; TRUNCATE powa_kcache_metrics_current_db; END $PROC$ language plpgsql; /* * powa_kcache purge */ CREATE OR REPLACE FUNCTION powa_kcache_purge() RETURNS void as $PROC$ BEGIN RAISE DEBUG 'running powa_kcache_purge'; DELETE FROM powa_kcache_metrics WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); DELETE FROM powa_kcache_metrics_db WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); END; $PROC$ language plpgsql; /* * powa_kcache reset */ CREATE OR REPLACE FUNCTION powa_kcache_reset() RETURNS void as $PROC$ BEGIN RAISE DEBUG 'running powa_kcache_reset'; TRUNCATE TABLE powa_kcache_metrics; TRUNCATE TABLE powa_kcache_metrics_db; TRUNCATE TABLE powa_kcache_metrics_current; TRUNCATE TABLE powa_kcache_metrics_current_db; END; $PROC$ language plpgsql; -- By default, try to register pg_stat_kcache, in case it's alreay here SELECT * FROM public.powa_kcache_register(); /* end of pg_stat_kcache integration - part 2 */ /* pg_qualstats integration - part 2 */ /* * powa_qualstats_register */ CREATE OR REPLACE function public.powa_qualstats_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_qualstats'; IF ( v_ext_present) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE function_name IN ('powa_qualstats_snapshot', 'powa_qualstats_aggregate', 'powa_qualstats_purge'); IF ( NOT v_func_present) THEN INSERT INTO powa_functions (module, operation, function_name, added_manually, enabled) VALUES ('pg_qualstats', 'snapshot', 'powa_qualstats_snapshot', false, true), ('pg_qualstats', 'aggregate', 'powa_qualstats_aggregate', false, true), ('pg_qualstats', 'unregister', 'powa_qualstats_unregister', false, true), ('pg_qualstats', 'purge', 'powa_qualstats_purge', false, true), ('pg_qualstats', 'reset', 'powa_qualstats_reset', false, true); END IF; END IF; RETURN true; END; $_$ language plpgsql; /* * powa_qualstats utility view for aggregating constvalues */ CREATE OR REPLACE VIEW powa_qualstats_aggregate_constvalues_current AS WITH consts AS ( SELECT qualid, queryid, dbid, userid, min(ts) as mints, max(ts) as maxts, sum(occurences) as occurences, sum(nbfiltered) as nbfiltered, sum(execution_count) as execution_count, constvalues FROM powa_qualstats_constvalues_history_current GROUP BY qualid, queryid, dbid, userid, constvalues ), groups AS ( SELECT qualid, queryid, dbid, userid, tstzrange(min(mints), max(maxts),'[]') FROM consts GROUP BY qualid, queryid, dbid, userid ) SELECT * FROM groups, LATERAL ( SELECT array_agg(constvalues) as mu FROM ( SELECT (constvalues, occurences, nbfiltered, execution_count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY occurences desc LIMIT 20 ) s ) as mu, LATERAL ( SELECT array_agg(constvalues) as mf FROM ( SELECT (constvalues, occurences, nbfiltered, execution_count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY CASE WHEN execution_count = 0 THEN 0 ELSE nbfiltered / execution_count::numeric END DESC LIMIT 20 ) s ) as mf, LATERAL ( SELECT array_agg(constvalues) as lf FROM ( SELECT (constvalues, occurences, nbfiltered, execution_count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY CASE WHEN execution_count = 0 THEN 0 ELSE nbfiltered / execution_count::numeric END DESC LIMIT 20 ) s ) as lf, LATERAL ( SELECT array_agg(constvalues) as me FROM ( SELECT (constvalues, occurences, nbfiltered, execution_count)::qual_values as constvalues FROM consts WHERE consts.qualid = groups.qualid AND consts.queryid = groups.queryid AND consts.dbid = groups.dbid AND consts.userid = groups.userid ORDER BY execution_count desc LIMIT 20 ) s ) as me; CREATE OR REPLACE FUNCTION powa_qualstats_snapshot() RETURNS void as $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_qualstats_snaphot'; WITH capture AS ( SELECT pgqs.*, s.query FROM pg_qualstats_by_query pgqs JOIN powa_statements s USING(queryid, dbid, userid) JOIN pg_roles r ON s.userid = r.oid AND NOT (r.rolname = ANY (string_to_array(current_setting('powa.ignored_users'),','))) ), missing_quals AS ( INSERT INTO powa_qualstats_quals (qualid, queryid, dbid, userid, quals) SELECT DISTINCT qs.qualnodeid, qs.queryid, qs.dbid, qs.userid, array_agg(DISTINCT q::qual_type) FROM capture qs, LATERAL (SELECT (unnest(quals)).*) as q WHERE NOT EXISTS ( SELECT 1 FROM powa_qualstats_quals nh WHERE nh.qualid = qs.qualnodeid AND nh.queryid = qs.queryid AND nh.dbid = qs.dbid AND nh.userid = qs.userid ) GROUP BY qualnodeid, queryid, dbid, userid RETURNING * ), by_qual AS ( INSERT INTO powa_qualstats_quals_history_current (qualid, queryid, dbid, userid, ts, occurences, execution_count, nbfiltered) SELECT qs.qualnodeid, qs.queryid, qs.dbid, qs.userid, now(), sum(occurences), sum(execution_count), sum(nbfiltered) FROM capture as qs GROUP BY qualnodeid, qs.queryid, qs.dbid, qs.userid RETURNING * ), by_qual_with_const AS ( INSERT INTO powa_qualstats_constvalues_history_current(qualid, queryid, dbid, userid, ts, occurences, execution_count, nbfiltered, constvalues) SELECT qualnodeid, qs.queryid, qs.dbid, qs.userid, now(), occurences, execution_count, nbfiltered, constvalues FROM capture as qs ) SELECT true into result; PERFORM pg_qualstats_reset(); END $PROC$ language plpgsql; /* * powa_qualstats aggregate */ CREATE OR REPLACE FUNCTION powa_qualstats_aggregate() RETURNS void AS $PROC$ DECLARE result bool; BEGIN RAISE DEBUG 'running powa_qualstats_aggregate'; LOCK TABLE powa_qualstats_constvalues_history_current IN SHARE MODE; LOCK TABLE powa_qualstats_quals_history_current IN SHARE MODE; INSERT INTO powa_qualstats_constvalues_history ( qualid, queryid, dbid, userid, coalesce_range, most_used, most_filtering, least_filtering, most_executed) SELECT * FROM powa_qualstats_aggregate_constvalues_current; INSERT INTO powa_qualstats_quals_history (qualid, queryid, dbid, userid, coalesce_range, records, mins_in_range, maxs_in_range) SELECT qualid, queryid, dbid, userid, tstzrange(min(ts), max(ts),'[]'), array_agg((ts, occurences, execution_count, nbfiltered)::powa_qualstats_history_item), ROW(min(ts), min(occurences), min(execution_count), min(nbfiltered))::powa_qualstats_history_item, ROW(max(ts), max(occurences), max(execution_count), max(nbfiltered))::powa_qualstats_history_item FROM powa_qualstats_quals_history_current GROUP BY qualid, queryid, dbid, userid; TRUNCATE powa_qualstats_constvalues_history_current; TRUNCATE powa_qualstats_quals_history_current; END $PROC$ language plpgsql; /* * powa_qualstats_purge */ CREATE OR REPLACE FUNCTION powa_qualstats_purge() RETURNS void as $PROC$ BEGIN RAISE DEBUG 'running powa_qualstats_purge'; DELETE FROM powa_qualstats_constvalues_history WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); DELETE FROM powa_qualstats_quals_history WHERE upper(coalesce_range) < (now() - current_setting('powa.retention')::interval); END; $PROC$ language plpgsql; /* * powa_qualstats_reset */ CREATE OR REPLACE FUNCTION powa_qualstats_reset() RETURNS void as $PROC$ BEGIN RAISE DEBUG 'running powa_qualstats_reset'; TRUNCATE TABLE powa_qualstats_quals CASCADE; -- cascaded : -- powa_qualstats_quals_history -- powa_qualstats_quals_history_current -- powa_qualstats_constvalues_history -- powa_qualstats_constvalues_history_current END; $PROC$ language plpgsql; /* * powa_qualstats_unregister */ CREATE OR REPLACE function public.powa_qualstats_unregister() RETURNS bool AS $_$ BEGIN DELETE FROM public.powa_functions WHERE module = 'pg_qualstats'; RETURN true; END; $_$ language plpgsql; SELECT * FROM public.powa_qualstats_register(); /* end of pg_qualstats_integration - part 2 */ /* pg_track_settings integration */ CREATE OR REPLACE FUNCTION powa_track_settings_register() RETURNS bool AS $_$ DECLARE v_func_present bool; v_ext_present bool; BEGIN SELECT COUNT(*) = 1 INTO v_ext_present FROM pg_extension WHERE extname = 'pg_track_settings'; IF ( v_ext_present ) THEN SELECT COUNT(*) > 0 INTO v_func_present FROM public.powa_functions WHERE module = 'pg_track_settings'; IF ( NOT v_func_present) THEN -- This extension handles its own storage, just its snapshot -- function and an unregister function. INSERT INTO powa_functions (module, operation, function_name, added_manually, enabled) VALUES ('pg_track_settings', 'snapshot', 'pg_track_settings_snapshot', false, true), ('pg_track_settings', 'unregister', 'powa_track_settings_unregister', false, true); END IF; END IF; RETURN true; END; $_$ language plpgsql; CREATE OR REPLACE function public.powa_track_settings_unregister() RETURNS bool AS $_$ BEGIN DELETE FROM public.powa_functions WHERE module = 'pg_track_settings'; RETURN true; END; $_$ language plpgsql; -- By default, try to register pg_track_settings, in case it's alreay here SELECT * FROM public.powa_track_settings_register(); /* end pg_track_settings integration */ powa-archivist-REL_3_0_0/powa.c000066400000000000000000000312721261712050400164330ustar00rootroot00000000000000/*------------------------------------------------------------------------- * * powa.c: PoWA background worker * * This program is open source, licensed under the PostgreSQL license. * For license terms, see the LICENSE file. * *------------------------------------------------------------------------- */ #include "postgres.h" /* For a bgworker */ #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/ipc.h" #include "storage/latch.h" #include "storage/lwlock.h" #include "storage/proc.h" #include "storage/shmem.h" /* Access a database */ #include "access/xact.h" #include "executor/spi.h" #include "fmgr.h" #include "lib/stringinfo.h" #include "utils/builtins.h" #include "utils/snapmgr.h" /* Some catalog elements */ #include "catalog/pg_type.h" #include "utils/timestamp.h" /* There is a GUC */ #include "utils/guc.h" /* We use tuplestore */ #include "funcapi.h" /* pgsats access */ #include "pgstat.h" PG_MODULE_MAGIC; #define POWA_STAT_FUNC_COLS 4 /* # of cols for functions stat SRF */ #define POWA_STAT_TAB_COLS 21 /* # of cols for relations stat SRF */ typedef enum { POWA_STAT_FUNCTION, POWA_STAT_TABLE } PowaStatKind; void _PG_init(void); void die_on_too_small_frequency(void); Datum powa_stat_user_functions(PG_FUNCTION_ARGS); Datum powa_stat_all_rel(PG_FUNCTION_ARGS); static Datum powa_stat_common(PG_FUNCTION_ARGS, PowaStatKind kind); PG_FUNCTION_INFO_V1(powa_stat_user_functions); PG_FUNCTION_INFO_V1(powa_stat_all_rel); static bool got_sigterm = false; static void powa_main(Datum main_arg); static void powa_sigterm(SIGNAL_ARGS); static void powa_sighup(SIGNAL_ARGS); static int powa_frequency; static int min_powa_frequency = 5000; static int powa_retention; static int powa_coalesce; static char *powa_database = NULL; static char *powa_ignored_users = NULL; void die_on_too_small_frequency(void) { if (powa_frequency > 0 && powa_frequency < min_powa_frequency) { elog(LOG, "POWA frequency cannot be smaller than %i milliseconds", min_powa_frequency); exit(1); } } void _PG_init(void) { BackgroundWorker worker; DefineCustomIntVariable("powa.frequency", "Defines the frequency in seconds of the snapshots", NULL, &powa_frequency, 300000, -1, INT_MAX / 1000, PGC_SUSET, GUC_UNIT_MS, NULL, NULL, NULL); DefineCustomIntVariable("powa.coalesce", "Defines the amount of records to group together in the table (more compact)", NULL, &powa_coalesce, 100, 5, INT_MAX, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("powa.retention", "Automatically purge data older than N minutes", NULL, &powa_retention, HOURS_PER_DAY * MINS_PER_HOUR, 0, INT_MAX / SECS_PER_MINUTE, PGC_SUSET, GUC_UNIT_MIN, NULL, NULL, NULL); DefineCustomStringVariable("powa.database", "Defines the database of the workload repository", NULL, &powa_database, "powa", PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomStringVariable("powa.ignored_users", "Defines a coma-separated list of users to ignore when taking activity snapshot", NULL, &powa_ignored_users, NULL, PGC_SIGHUP, 0, NULL, NULL, NULL); /* Register the worker processes */ worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_start_time = BgWorkerStart_RecoveryFinished; /* Must write to the database */ worker.bgw_main = powa_main; snprintf(worker.bgw_name, BGW_MAXLEN, "powa"); worker.bgw_restart_time = 10; worker.bgw_main_arg = (Datum) 0; #if (PG_VERSION_NUM >= 90400) worker.bgw_notify_pid = 0; #endif RegisterBackgroundWorker(&worker); } static void powa_main(Datum main_arg) { char *q1 = "SELECT powa_take_snapshot()"; static char *q2 = "SET application_name = 'POWA collector'"; instr_time begin; instr_time end; long time_to_wait; die_on_too_small_frequency(); /* Set up signal handlers, then unblock signalsl */ pqsignal(SIGHUP, powa_sighup); pqsignal(SIGTERM, powa_sigterm); BackgroundWorkerUnblockSignals(); /* We only connect when powa_frequency >0. If not, powa has been deactivated */ if (powa_frequency < 0) { elog(LOG, "POWA is deactivated (powa.frequency = %i), exiting", powa_frequency); exit(1); } // We got here: it means powa_frequency > 0. Let's connect /* Connect to POWA database */ BackgroundWorkerInitializeConnection(powa_database, NULL); elog(LOG, "POWA connected to %s", powa_database); StartTransactionCommand(); SetCurrentStatementStartTimestamp(); SPI_connect(); PushActiveSnapshot(GetTransactionSnapshot()); SPI_execute(q2, false, 0); SPI_finish(); PopActiveSnapshot(); CommitTransactionCommand(); /* let's store the current time. It will be used to calculate a quite stable interval between each measure */ while (!got_sigterm) { /* We can get here with a new value of powa_frequency because of a reload. Let's suicide to disconnect if this value is <0 */ if (powa_frequency < 0) { elog(LOG, "POWA exits to disconnect from the database now"); exit(1); } INSTR_TIME_SET_CURRENT(begin); ResetLatch(&MyProc->procLatch); SetCurrentStatementStartTimestamp(); StartTransactionCommand(); SPI_connect(); PushActiveSnapshot(GetTransactionSnapshot()); SPI_execute(q1, false, 0); SPI_finish(); PopActiveSnapshot(); CommitTransactionCommand(); INSTR_TIME_SET_CURRENT(end); INSTR_TIME_SUBTRACT(end, begin); /* Wait powa.frequency, compensate for work time of last snapshot */ /* If we got off schedule (because of a compact or delete, just do another operation right now */ time_to_wait = powa_frequency - INSTR_TIME_GET_MILLISEC(end); elog(DEBUG1, "Waiting for %li milliseconds", time_to_wait); if (time_to_wait > 0) { WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, time_to_wait); } } proc_exit(0); } static void powa_sigterm(SIGNAL_ARGS) { int save_errno = errno; got_sigterm = true; if (MyProc) SetLatch(&MyProc->procLatch); errno = save_errno; } static void powa_sighup(SIGNAL_ARGS) { ProcessConfigFile(PGC_SIGHUP); die_on_too_small_frequency(); } Datum powa_stat_user_functions(PG_FUNCTION_ARGS) { return powa_stat_common(fcinfo, POWA_STAT_FUNCTION); } Datum powa_stat_all_rel(PG_FUNCTION_ARGS) { return powa_stat_common(fcinfo, POWA_STAT_TABLE); } static Datum powa_stat_common(PG_FUNCTION_ARGS, PowaStatKind kind) { Oid dbid = PG_GETARG_OID(0); Oid currentdbid = MyDatabaseId; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; MemoryContext per_query_ctx; MemoryContext oldcontext; TupleDesc tupdesc; Tuplestorestate *tupstore; PgStat_StatDBEntry *dbentry; HASH_SEQ_STATUS hash_seq; /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("set-valued function called in context that cannot accept a set"))); if (!(rsinfo->allowedModes & SFRM_Materialize)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("materialize mode required, but it is not " \ "allowed in this context"))); per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; oldcontext = MemoryContextSwitchTo(per_query_ctx); /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); tupstore = tuplestore_begin_heap(true, false, work_mem); rsinfo->returnMode = SFRM_Materialize; rsinfo->setResult = tupstore; rsinfo->setDesc = tupdesc; MemoryContextSwitchTo(oldcontext); /* * Lookup the requested database, then retrieve all functions stats. This * function will read the statistic collector stats file if not already * done in the transaction. As we may (and probably) have to access * statistics on multiple databases, force a cluster wide and deep stats * retrieval, by setting MyDatabaseId to InvalidOid. However, by doing so we * won't force a fresh statsfile read. Having slighty outdated stat is not * an issue, but after a cluster restart, it may take some time before * having any data returned. */ MyDatabaseId = InvalidOid; dbentry = pgstat_fetch_stat_dbentry(dbid); /* And restore it */ MyDatabaseId = currentdbid; if (dbentry != NULL && dbentry->functions != NULL) { switch (kind) { case POWA_STAT_FUNCTION: { PgStat_StatFuncEntry *funcentry = NULL; hash_seq_init(&hash_seq, dbentry->functions); while ((funcentry = hash_seq_search(&hash_seq)) != NULL) { Datum values[POWA_STAT_FUNC_COLS]; bool nulls[POWA_STAT_FUNC_COLS]; int i = 0; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[i++] = ObjectIdGetDatum(funcentry->functionid); values[i++] = Int64GetDatum(funcentry->f_numcalls); values[i++] = Float8GetDatum(((double) funcentry->f_total_time) / 1000.0); values[i++] = Float8GetDatum(((double) funcentry->f_self_time) / 1000.0); Assert(i == POWA_STAT_FUNC_COLS); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } break; } case POWA_STAT_TABLE: { PgStat_StatTabEntry *tabentry = NULL; hash_seq_init(&hash_seq, dbentry->tables); while ((tabentry = hash_seq_search(&hash_seq)) != NULL) { Datum values[POWA_STAT_TAB_COLS]; bool nulls[POWA_STAT_TAB_COLS]; int i = 0; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); /* Oid of the table (or index) */ values[i++] = ObjectIdGetDatum(tabentry->tableid); values[i++] = Int64GetDatum((int64) tabentry->numscans); values[i++] = Int64GetDatum((int64) tabentry->tuples_returned); values[i++] = Int64GetDatum((int64) tabentry->tuples_fetched); values[i++] = Int64GetDatum((int64) tabentry->tuples_inserted); values[i++] = Int64GetDatum((int64) tabentry->tuples_updated); values[i++] = Int64GetDatum((int64) tabentry->tuples_deleted); values[i++] = Int64GetDatum((int64) tabentry->tuples_hot_updated); values[i++] = Int64GetDatum((int64) tabentry->n_live_tuples); values[i++] = Int64GetDatum((int64) tabentry->n_dead_tuples); values[i++] = Int64GetDatum((int64) tabentry->changes_since_analyze); values[i++] = Int64GetDatum((int64) (tabentry->blocks_fetched - tabentry->blocks_hit)); values[i++] = Int64GetDatum((int64) tabentry->blocks_hit); /* last vacuum */ if (tabentry->vacuum_timestamp == 0) nulls[i++] = true; else values[i++] = TimestampTzGetDatum(tabentry->vacuum_timestamp); values[i++] = Int64GetDatum((int64) tabentry->vacuum_count); /* last_autovacuum */ if (tabentry->autovac_vacuum_timestamp == 0) nulls[i++] = true; else values[i++] = TimestampTzGetDatum(tabentry->autovac_vacuum_timestamp); values[i++] = Int64GetDatum((int64) tabentry->autovac_vacuum_count); /* last_analyze */ if (tabentry->analyze_timestamp == 0) nulls[i++] = true; else values[i++] = TimestampTzGetDatum(tabentry->analyze_timestamp); values[i++] = Int64GetDatum((int64) tabentry->analyze_count); /* last_autoanalyze */ if (tabentry->autovac_analyze_timestamp == 0) nulls[i++] = true; else values[i++] = TimestampTzGetDatum(tabentry->autovac_analyze_timestamp); values[i++] = Int64GetDatum((int64) tabentry->autovac_analyze_count); Assert(i == POWA_STAT_TAB_COLS); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } break; } } } /* clean up and return the tuplestore */ tuplestore_donestoring(tupstore); return (Datum) 0; } powa-archivist-REL_3_0_0/powa.control000066400000000000000000000003461261712050400176670ustar00rootroot00000000000000# powa extension comment = 'PostgreSQL Workload Analyser-core' default_version = '3.0.0' module_pathname = '$libdir/powa' requires = 'plpgsql, pg_stat_statements, btree_gist' superuser = true relocatable = false schema = 'public' powa-archivist-REL_3_0_0/reindent.sh000077500000000000000000000005571261712050400174720ustar00rootroot00000000000000#/bin/sh # This reindents all C files in the directory following PG rules (ported to gnu indent) # as some of us have tab=8 spaces, and others tab=4 spaces indent -bad -bap -bbo -bbb -bc -bl -brs -c33 -cd33 -cdb -nce -ci4 -cli0 \ -cp33 -cs -d0 -di12 -nfc1 -nfca -nfc1 -i4 -nip -l79 -lp -nip -npcs \ -nprs -npsl -saf -sai -saw -nsc -nsob -nss -nut *.c