pax_global_header00006660000000000000000000000064141713354640014522gustar00rootroot0000000000000052 comment=3b2f1293764bb1bb7767aa02f159f2d24b53cb1c pg_checksums-1.1/000077500000000000000000000000001417133546400140365ustar00rootroot00000000000000pg_checksums-1.1/.gitlab-ci.yml000066400000000000000000000006121417133546400164710ustar00rootroot00000000000000image: "debian:latest" build: stage: build before_script: - apt-get update -qq - apt-get install -y --no-install-recommends build-essential postgresql-server-dev-all postgresql libipc-run-perl xsltproc - useradd ci-runner script: - make - make install - su ci-runner -c 'make installcheck' || (cat tmp_check/log/*; exit 1) pg_checksums-1.1/COPYRIGHT000066400000000000000000000022501417133546400153300ustar00rootroot00000000000000PostgreSQL Database Management System (formerly known as Postgres, then as Postgres95) Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. pg_checksums-1.1/Makefile000066400000000000000000000022221417133546400154740ustar00rootroot00000000000000#------------------------------------------------------------------------- # # Makefile for pg_checksums_ext # # Copyright (c) 1998-2022, PostgreSQL Global Development Group # # pg_checksums_ext/Makefile # #------------------------------------------------------------------------- PROGRAM = pg_checksums_ext PGFILEDESC = "pg_checksums_ext - Checks, enables or disables page level checksums for a cluster" PGAPPICON=win32 OBJS= pg_checksums_ext.o port.o $(WIN32RES) EXTRA_CLEAN = tmp_check doc/man1 PG_CONFIG ?= pg_config PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # avoid linking against all libs that the server links against (xml, selinux, ...) LIBS = $(libpq_pgport) PROVE_FLAGS += -I./t/perl all: pg_checksums_ext man: doc/man1/pg_checksums_ext.1 doc/man1/pg_checksums_ext.1: doc/pg_checksums.sgml (cd doc && xsltproc stylesheet-man.xsl pg_checksums.sgml) prove_installcheck: rm -rf $(CURDIR)/tmp_check cd $(srcdir) && TESTDIR='$(CURDIR)' PATH="$(bindir):$$PATH" PGPORT='6$(DEF_PGPORT)' PG_REGRESS='$(top_builddir)/src/test/regress/pg_regress' $(PROVE) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),t/*.pl) installcheck: prove_installcheck pg_checksums-1.1/README.md000066400000000000000000000016601417133546400153200ustar00rootroot00000000000000pg_checksums_ext - Activate/deactivate/verify checksums in PostgreSQL clusters ============================================================================== `pg_checksums_ext` is based on the `pg_verify_checksums` and `pg_checksums` programs available in PostgreSQL version 11 and from 12, respectively. It can verify, activate or deactivate checksums. Activating requires all database blocks to be read and all page headers to be updated, so can take a long time on a large database. The database cluster needs to be shutdown cleanly in the case of checksum activation or deactivation, while checksum verification can be performed online, contrary to PostgreSQL's `pg_checksums`. Other changes include the possibility to toggle progress reporting via the `SIGUSR1` signal, more fine-grained progress reporting and I/O rate limiting. PostgreSQL versions since 9.3 are supported, the November 8th 2018 PostgreSQL point release is required. pg_checksums-1.1/doc/000077500000000000000000000000001417133546400146035ustar00rootroot00000000000000pg_checksums-1.1/doc/pg_checksums.sgml000066400000000000000000000170431417133546400201470ustar00rootroot00000000000000 pg_checksums pg_checksums_ext 1 Application pg_checksums_ext enable, disable or check data checksums in a PostgreSQL database cluster pg_checksums_ext option datadir Description pg_checksums_ext checks, enables or disables data checksums in a PostgreSQL cluster. The server must be shut down when enabling or disabling checksums, while verifying checksums can be done online. When verifying checksums, the exit status is zero if there are no checksum errors, and nonzero if at least one checksum failure is detected. When enabling or disabling checksums, the exit status is nonzero if the operation failed. When verifying checksums, every file in the cluster is scanned. When enabling checksums, each relation file block with a changed checksum is rewritten in-place. Disabling checksums only updates the file pg_control. Options The following command-line options are available: Specifies the directory where the database cluster is stored. Checks checksums. This is the default mode if nothing else is specified. Disables checksums. Enables checksums. Only validate checksums in the relation with filenode filenode. By default, pg_checksums_ext will wait for all files to be written safely to disk. This option causes pg_checksums_ext to return without waiting, which is faster, but means that a subsequent operating system crash can leave the updated data directory corrupt. Generally, this option is useful for testing but should not be used on a production installation. This option has no effect when using --check. Enable progress reporting. Turning this on will deliver a progress report while checking or enabling checksums. Progress reporting can also be triggered by the SIGUSR1 signal. Set a maximum I/O rate (in MB/s) while checking or enabling checksums. Enable verbose output. Lists all checked files. Enable debug output. Print the pg_checksums_ext version and exit. Show help about pg_checksums_ext command line arguments, and exit. Environment PGDATA Specifies the directory where the database cluster is stored; can be overridden using the option. PG_COLOR Specifies whether to use color in diagnostics messages. Possible values are always, auto, never. Notes Enabling checksums in a large cluster can potentially take a long time. During this operation, the cluster or other programs that write to the data directory must not be started or else data loss may occur. When using a replication setup with tools which perform direct copies of relation file blocks (for example pg_rewind), enabling or disabling checksums can lead to page corruptions in the shape of incorrect checksums if the operation is not done consistently across all nodes. When enabling or disabling checksums in a replication setup, it is thus recommended to stop all the clusters before switching them all consistently. Destroying all standbys, performing the operation on the primary and finally recreating the standbys from scratch is also safe. If pg_checksums_ext is aborted or killed while enabling or disabling checksums, the cluster's data checksum configuration remains unchanged, and pg_checksums_ext can be re-run to perform the same operation. pg_checksums-1.1/doc/stylesheet-common.xsl000066400000000000000000000050371417133546400210170ustar00rootroot00000000000000 no 2 1 ? ? pg_checksums-1.1/doc/stylesheet-man.xsl000066400000000000000000000207551417133546400203060ustar00rootroot00000000000000 0 0 0 32 40 < > ^ ( ) 0 Note: Note: (soelim stub) : pg_checksums-1.1/doc/stylesheet-speedup-common.xsl000066400000000000000000000100261417133546400224540ustar00rootroot00000000000000 en pg_checksums-1.1/pg_checksums_ext.c000066400000000000000000000656301417133546400175470ustar00rootroot00000000000000/*------------------------------------------------------------------------- * * pg_checksums_ext.c * Checks, enables or disables page level checksums for a cluster * * Copyright (c) 2010-2022, PostgreSQL Global Development Group * * IDENTIFICATION * pg_checksums_ext.c * *------------------------------------------------------------------------- */ #define PG_CHECKSUMS_VERSION "1.1" #include "postgres_fe.h" #include "port.h" #include #include #include #include #include #include "catalog/pg_control.h" #include "portability/instr_time.h" #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/checksum_impl.h" #if PG_VERSION_NUM <= 110000 #include "catalog/catalog.h" #endif static int64 files_scanned = 0; static int64 files_written = 0; static int64 skippedfiles = 0; static int64 blocks_scanned = 0; static int64 blocks_written = 0; static int64 skippedblocks = 0; static int64 badblocks = 0; static double maxrate = 0; static ControlFileData *ControlFile; static XLogRecPtr checkpointLSN; static char *only_filenode = NULL; static bool do_sync = true; static bool debug = false; static bool verbose = false; static bool showprogress = false; static bool online = false; char *DataDir = NULL; typedef enum { PG_MODE_CHECK, PG_MODE_DISABLE, PG_MODE_ENABLE } PgChecksumMode; /* * Filename components. * * XXX: fd.h is not declared here as frontend side code is not able to * interact with the backend-side definitions for the various fsync * wrappers. */ #define PG_TEMP_FILES_DIR "pgsql_tmp" #define PG_TEMP_FILE_PREFIX "pgsql_tmp" static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; /* * Progress status information. */ int64 total_size = 0; int64 current_size = 0; instr_time last_progress_report; instr_time last_throttle; instr_time scan_started; static void usage(void) { printf(_("%s enables, disables, or verifies data checksums in a PostgreSQL database cluster.\n\n"), progname); printf(_("Usage:\n")); printf(_(" %s [OPTION]... [DATADIR]\n"), progname); printf(_("\nOptions:\n")); printf(_(" [-D, --pgdata=]DATADIR data directory\n")); printf(_(" -c, --check check data checksums (default)\n")); printf(_(" -d, --disable disable data checksums\n")); printf(_(" -e, --enable enable data checksums\n")); printf(_(" -f, --filenode=FILENODE check only relation with specified filenode\n")); printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" --max-rate=RATE maximum I/O rate to verify or enable checksums\n")); printf(_(" (in MB/s)\n")); printf(_(" --debug debug output\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nIf no data directory (DATADIR) is specified, " "the environment variable PGDATA\nis used.\n\n")); printf(_("Report bugs to https://github.com/credativ/pg_checksums/issues/new.\n")); } /* * Definition of one element part of an exclusion list, used for files * to exclude from checksum validation. "name" is the name of the file * or path to check for exclusion. If "match_prefix" is true, any items * matching the name as prefix are excluded. */ struct exclude_list_item { const char *name; bool match_prefix; }; /* * List of files excluded from checksum validation. */ static const struct exclude_list_item skip[] = { {"pg_control", false}, {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, #ifdef EXEC_BACKEND {"config_exec_params", true}, #endif {NULL, false} }; static void update_checkpoint_lsn(void) { #if PG_VERSION_NUM >= 100000 bool crc_ok; #if PG_VERSION_NUM >= 120000 ControlFile = get_controlfile(DataDir, &crc_ok); #else ControlFile = get_controlfile(DataDir, progname, &crc_ok); #endif if (!crc_ok) { pg_log_error("pg_control CRC value is incorrect"); exit(1); } #elif PG_VERSION_NUM >= 90600 ControlFile = get_controlfile(DataDir, progname); #else ControlFile = getControlFile(DataDir); #endif /* Update checkpointLSN with the current value */ checkpointLSN = ControlFile->checkPoint; } static void toggle_progress_report(int signum) { /* we handle SIGUSR1 only, and toggle the value of showprogress */ if (signum == SIGUSR1) showprogress = !showprogress; } /* * Report current progress status and/or throttle. Parts borrowed from * PostgreSQL's src/bin/pg_basebackup.c. */ static void progress_report_or_throttle(bool finished) { double elapsed; double wait; double percent; double current_rate; bool skip_progress = false; char total_size_str[32]; char current_size_str[32]; instr_time now; Assert(showprogress); INSTR_TIME_SET_CURRENT(now); /* Make sure we throttle at most once every 50 milliseconds */ if ((INSTR_TIME_GET_MILLISEC(now) - INSTR_TIME_GET_MILLISEC(last_throttle) < 50) && !finished) return; /* Make sure we report at most once every 250 milliseconds */ if ((INSTR_TIME_GET_MILLISEC(now) - INSTR_TIME_GET_MILLISEC(last_progress_report) < 250) && !finished) skip_progress = true; /* Save current time */ last_throttle = now; /* Elapsed time in milliseconds since start of scan */ elapsed = (INSTR_TIME_GET_MILLISEC(now) - INSTR_TIME_GET_MILLISEC(scan_started)); /* Adjust total size if current_size is larger */ if (current_size > total_size) total_size = current_size; /* Calculate current percentage of size done */ percent = total_size ? 100.0 * current_size / total_size : 0.0; #define MEGABYTES (1024 * 1024) /* * Calculate current speed, converting current_size from bytes to megabytes * and elapsed from milliseconds to seconds. */ current_rate = (current_size / MEGABYTES) / (elapsed / 1000); snprintf(total_size_str, sizeof(total_size_str), INT64_FORMAT, total_size / MEGABYTES); snprintf(current_size_str, sizeof(current_size_str), INT64_FORMAT, current_size / MEGABYTES); /* Throttle if desired */ if (maxrate > 0 && current_rate > maxrate) { /* * Calculate time to sleep in milliseconds. Convert maxrate to MB/ms * in order to get a better resolution. */ wait = (current_size / MEGABYTES / (maxrate / 1000)) - elapsed; if (debug) pg_log_debug("waiting for %f ms due to throttling", wait); pg_usleep(wait * 1000); /* Recalculate current rate */ INSTR_TIME_SET_CURRENT(now); elapsed = INSTR_TIME_GET_MILLISEC(now) - INSTR_TIME_GET_MILLISEC(scan_started); current_rate = (int64)(current_size / MEGABYTES) / (elapsed / 1000); } /* Report progress if desired */ if (showprogress && !skip_progress) { /* * Print five blanks at the end so the end of previous lines which were * longer don't remain partly visible. */ fprintf(stderr, "%s/%s MB (%d%%, %.0f MB/s)%5s", current_size_str, total_size_str, (int)percent, current_rate, ""); /* * Stay on the same line if reporting to a terminal and we're not done * yet. */ fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr); last_progress_report = now; } } static bool skipfile(const char *fn) { int excludeIdx; for (excludeIdx = 0; skip[excludeIdx].name != NULL; excludeIdx++) { int cmplen = strlen(skip[excludeIdx].name); if (!skip[excludeIdx].match_prefix) cmplen++; if (strncmp(skip[excludeIdx].name, fn, cmplen) == 0) return true; } return false; } static void scan_file(const char *fn, int segmentno) { PGAlignedBlock buf; PageHeader header = (PageHeader) buf.data; int i; int f; BlockNumber blockno; int flags; int64 blocks_written_in_file = 0; bool block_retry = false; bool all_zeroes; size_t *pagebytes; Assert(mode == PG_MODE_ENABLE || mode == PG_MODE_CHECK); flags = (mode == PG_MODE_ENABLE) ? O_RDWR : O_RDONLY; f = open(fn, PG_BINARY | flags, 0); if (f < 0) { if (online && errno == ENOENT) { /* File was removed in the meantime */ return; } pg_log_error("could not open file \"%s\": %m", fn); exit(1); } files_scanned++; for (blockno = 0;; blockno++) { uint16 csum; int r = read(f, buf.data, BLCKSZ); if (debug && block_retry) pg_log_debug("retrying block %u in file \"%s\"", blockno, fn); if (r == 0) break; if (r != BLCKSZ) { if (r < 0) { skippedfiles++; pg_log_error("could not read block %u in file \"%s\": %m", blockno, fn); return; } else { if (online) { if (block_retry) { /* We already tried once to reread the block, skip to the next block */ skippedblocks++; if (debug) pg_log_debug("retrying block %u in file \"%s\" failed, skipping to next block", blockno, fn); if (lseek(f, BLCKSZ-r, SEEK_CUR) == -1) { pg_log_error("could not lseek to next block in file \"%s\": %m", fn); return; } continue; } /* * Retry the block. It's possible that we read the block while it * was extended or shrinked, so it it ends up looking torn to us. */ /* * Seek back by the amount of bytes we read to the beginning of * the failed block. */ if (lseek(f, -r, SEEK_CUR) == -1) { skippedfiles++; pg_log_error("could not lseek to in file \"%s\": %m", fn); return; } /* Set flag so we know a retry was attempted */ block_retry = true; /* Reset loop to validate the block again */ blockno--; continue; } else { /* Directly skip file if offline */ skippedfiles++; pg_log_error("could not read block %u in file \"%s\": read %d of %d", blockno, fn, r, BLCKSZ); return; } } } blocks_scanned++; /* * Since the file size is counted as total_size for progress status * information, the sizes of all pages including new ones in the file * should be counted as current_size. Otherwise the progress reporting * calculated using those counters may not reach 100%. */ current_size += r; /* New pages have no checksum yet */ if (PageIsNew(header)) { /* Check for an all-zeroes page */ all_zeroes = true; pagebytes = (size_t *) buf.data; for (i = 0; i < (BLCKSZ / sizeof(size_t)); i++) { if (pagebytes[i] != 0) { all_zeroes = false; break; } } if (!all_zeroes) { pg_log_error("checksum verification failed in file \"%s\", block %u: pd_upper is zero but block is not all-zero", fn, blockno); badblocks++; } else { if (debug && block_retry) pg_log_debug("block %u in file \"%s\" is new, ignoring", blockno, fn); skippedblocks++; } continue; } csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE); if (mode == PG_MODE_CHECK) { if (csum != header->pd_checksum) { if (online) { /* * Retry the block on the first failure if online. If the * verification is done while the instance is online, it is * possible that we read the first 4K page of the block * just before postgres updated the entire block so it ends * up looking torn to us. We only need to retry once * because the LSN should be updated to something we can * ignore on the next pass. If the error happens again * then it is a true validation failure. */ if (!block_retry) { /* Seek to the beginning of the failed block */ if (lseek(f, -BLCKSZ, SEEK_CUR) == -1) { skippedfiles++; pg_log_error("could not lseek in file \"%s\": %m", fn); return; } /* Set flag so we know a retry was attempted */ block_retry = true; if (debug) pg_log_debug("checksum verification failed on first attempt in file \"%s\", block %u: calculated checksum %X but block contains %X", fn, blockno, csum, header->pd_checksum); /* Reset loop to validate the block again */ blockno--; blocks_scanned--; current_size -= r; /* * Update the checkpoint LSN now. If we get a failure * on re-read, we would need to do this anyway, and * doing it now lowers the probability that we see the * same torn page on re-read. */ update_checkpoint_lsn(); continue; } /* * The checksum verification failed on retry as well. Check if * the page has been modified since the checkpoint and skip it * in this case. As a sanity check, demand that the upper * 32 bits of the LSN are identical in order to skip as a * guard against a corrupted LSN in the pageheader. */ if ((PageGetLSN(buf.data) > checkpointLSN) && (PageGetLSN(buf.data) >> 32 == checkpointLSN >> 32)) { if (debug) pg_log_debug("block %u in file \"%s\" with LSN %X/%X is newer than checkpoint LSN %X/%X, ignoring", blockno, fn, (uint32) (PageGetLSN(buf.data) >> 32), (uint32) PageGetLSN(buf.data), (uint32) (checkpointLSN >> 32), (uint32) checkpointLSN); block_retry = false; skippedblocks++; continue; } } if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) pg_log_error("checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X", fn, blockno, csum, header->pd_checksum); badblocks++; } else if (block_retry && debug) pg_log_debug("block %u in file \"%s\" verified ok on recheck", blockno, fn); block_retry = false; } else if (mode == PG_MODE_ENABLE) { int w; /* * Do not rewrite if the checksum is already set to the expected * value. */ if (header->pd_checksum == csum) continue; blocks_written_in_file++; /* Set checksum in page header */ header->pd_checksum = csum; /* Seek back to beginning of block */ if (lseek(f, -BLCKSZ, SEEK_CUR) < 0) { pg_log_error("seek failed for block %u in file \"%s\": %m", blockno, fn); exit(1); } /* Write block with checksum */ w = write(f, buf.data, BLCKSZ); if (w != BLCKSZ) { if (w < 0) pg_log_error("could not write block %u in file \"%s\": %m", blockno, fn); else pg_log_error("could not write block %u in file \"%s\": wrote %d of %d", blockno, fn, w, BLCKSZ); exit(1); } } /* Report progress or throttle every 1024 blocks */ if ((showprogress || maxrate > 0) && (blockno % 1024 == 0)) progress_report_or_throttle(false); } if (verbose) { if (mode == PG_MODE_CHECK) pg_log_info("checksums verified in file \"%s\"", fn); if (mode == PG_MODE_ENABLE) pg_log_info("checksums enabled in file \"%s\"", fn); } /* Make sure progress is reported at least once per file */ if (showprogress || maxrate > 0) progress_report_or_throttle(false); /* Update write counters if any write activity has happened */ if (blocks_written_in_file > 0) { files_written++; blocks_written += blocks_written_in_file; } close(f); } /* * Scan the given directory for items which can be checksummed and * operate on each one of them. If "sizeonly" is true, the size of * all the items which have checksums is computed and returned back * to the caller without operating on the files. This is used to compile * the total size of the data directory for progress reports. */ static int64 scan_directory(const char *basedir, const char *subdir, bool sizeonly) { int64 dirsize = 0; char path[MAXPGPATH]; DIR *dir; struct dirent *de; snprintf(path, sizeof(path), "%s/%s", basedir, subdir); dir = opendir(path); if (!dir) { pg_log_error("could not open directory \"%s\": %m", path); exit(1); } while ((de = readdir(dir)) != NULL) { char fn[MAXPGPATH]; struct stat st; if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; /* Skip temporary files */ if (strncmp(de->d_name, PG_TEMP_FILE_PREFIX, strlen(PG_TEMP_FILE_PREFIX)) == 0) continue; /* Skip temporary folders */ if (strncmp(de->d_name, PG_TEMP_FILES_DIR, strlen(PG_TEMP_FILES_DIR)) == 0) continue; snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name); if (lstat(fn, &st) < 0) { if (online && errno == ENOENT) { /* File was removed in the meantime */ if (debug) pg_log_debug("ignoring deleted file \"%s\"", fn); continue; } pg_log_error("could not stat file \"%s\": %m", fn); exit(1); } if (S_ISREG(st.st_mode)) { char fnonly[MAXPGPATH]; char *forkpath, *segmentpath; int segmentno = 0; if (skipfile(de->d_name)) continue; /* * Cut off at the segment boundary (".") to get the segment number * in order to mix it into the checksum. Then also cut off at the * fork boundary, to get the filenode the file belongs to for * filtering. */ strlcpy(fnonly, de->d_name, sizeof(fnonly)); segmentpath = strchr(fnonly, '.'); if (segmentpath != NULL) { *segmentpath++ = '\0'; segmentno = atoi(segmentpath); if (segmentno == 0) continue; } forkpath = strchr(fnonly, '_'); if (forkpath != NULL) *forkpath++ = '\0'; if (only_filenode && strcmp(only_filenode, fnonly) != 0) /* filenode not to be included */ continue; dirsize += st.st_size; /* * No need to work on the file when calculating only the size of * the items in the data folder. */ if (!sizeonly) scan_file(fn, segmentno); } #ifndef WIN32 else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)) #else else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn)) #endif { /* * If going through the entries of pg_tblspc, we assume to operate * on tablespace locations where only TABLESPACE_VERSION_DIRECTORY * is valid, resolving the linked locations and dive into them * directly. */ if (strncmp("pg_tblspc", subdir, strlen("pg_tblspc")) == 0) { char tblspc_path[MAXPGPATH]; struct stat tblspc_st; /* * Resolve tablespace location path and check whether * TABLESPACE_VERSION_DIRECTORY exists. Not finding a valid * location is unexpected, since there should be no orphaned * links and no links pointing to something else than a * directory. */ snprintf(tblspc_path, sizeof(tblspc_path), "%s/%s/%s", path, de->d_name, TABLESPACE_VERSION_DIRECTORY); if (lstat(tblspc_path, &tblspc_st) < 0) { pg_log_error("could not stat file \"%s\": %m", tblspc_path); exit(1); } /* * Move backwards once as the scan needs to happen for the * contents of TABLESPACE_VERSION_DIRECTORY. */ snprintf(tblspc_path, sizeof(tblspc_path), "%s/%s", path, de->d_name); /* Looks like a valid tablespace location */ dirsize += scan_directory(tblspc_path, TABLESPACE_VERSION_DIRECTORY, sizeonly); } else { dirsize += scan_directory(path, de->d_name, sizeonly); } } } closedir(dir); return dirsize; } int main(int argc, char *argv[]) { static struct option long_options[] = { {"check", no_argument, NULL, 'c'}, {"pgdata", required_argument, NULL, 'D'}, {"disable", no_argument, NULL, 'd'}, {"enable", no_argument, NULL, 'e'}, {"filenode", required_argument, NULL, 'f'}, {"no-sync", no_argument, NULL, 'N'}, {"progress", no_argument, NULL, 'P'}, {"max-rate", required_argument, NULL, 1}, {"verbose", no_argument, NULL, 'v'}, {"debug", no_argument, NULL, 2}, {NULL, 0, NULL, 0} }; int c; int option_index; #if PG_VERSION_NUM >= 100000 bool crc_ok; #endif pg_logging_init(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_checksums_ext")); progname = get_progname(argv[0]); if (argc > 1) { if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) { usage(); exit(0); } if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) { puts("pg_checksums_ext " PG_CHECKSUMS_VERSION " (PostgreSQL " PG_MAJORVERSION ")"); exit(0); } } while ((c = getopt_long(argc, argv, "abcD:def:NPv", long_options, &option_index)) != -1) { switch (c) { case 'a': mode = PG_MODE_ENABLE; /* compat */ break; case 'b': mode = PG_MODE_DISABLE; /* compat */ break; case 'c': mode = PG_MODE_CHECK; break; case 'd': mode = PG_MODE_DISABLE; break; case 'e': mode = PG_MODE_ENABLE; break; case 'f': if (atoi(optarg) == 0) { pg_log_error("invalid filenode specification, must be numeric: %s", optarg); exit(1); } only_filenode = pstrdup(optarg); break; case 'N': do_sync = false; break; case 'v': verbose = true; break; case 'D': DataDir = optarg; break; case 'P': showprogress = true; break; case 1: if (atof(optarg) == 0) { pg_log_error("invalid max-rate specification, must be numeric: %s", optarg); exit(1); } maxrate = atof(optarg); break; case 2: __pg_log_level = PG_LOG_DEBUG; debug = true; verbose = true; break; default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } } if (DataDir == NULL) { if (optind < argc) DataDir = argv[optind++]; else DataDir = getenv("PGDATA"); /* If no DataDir was specified, and none could be found, error out */ if (DataDir == NULL) { pg_log_error("no data directory specified"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } } /* Complain if any arguments remain */ if (optind < argc) { pg_log_error("too many command-line arguments (first is \"%s\")", argv[optind]); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } /* filenode checking only works in --check mode */ if (mode != PG_MODE_CHECK && only_filenode) { pg_log_error("option -f/--filenode can only be used with --check"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } /* Check major version compatibility */ CheckDataVersion(DataDir); /* Read the control file and check compatibility */ #if PG_VERSION_NUM >= 100000 #if PG_VERSION_NUM >= 120000 ControlFile = get_controlfile(DataDir, &crc_ok); #else ControlFile = get_controlfile(DataDir, progname, &crc_ok); #endif if (!crc_ok) { pg_log_error("pg_control CRC value is incorrect"); exit(1); } #elif PG_VERSION_NUM >= 90600 ControlFile = get_controlfile(DataDir, progname); #else ControlFile = getControlFile(DataDir); #endif if (ControlFile->pg_control_version != PG_CONTROL_VERSION) { pg_log_error("cluster is not compatible with this version of pg_checksums_ext"); exit(1); } if (ControlFile->blcksz != BLCKSZ) { pg_log_error("database cluster is not compatible"); fprintf(stderr, _("The database cluster was initialized with block size %u, but pg_checksums_ext was compiled with block size %u.\n"), ControlFile->blcksz, BLCKSZ); exit(1); } /* * Cluster must be shut down for activation/deactivation of checksums, but * online verification is supported. */ if (ControlFile->state != DB_SHUTDOWNED && ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY) { if (mode != PG_MODE_CHECK) { pg_log_error("cluster must be shut down"); exit(1); } online = true; } if (debug) { if (online) pg_log_debug("online mode"); else pg_log_debug("offline mode"); } if (ControlFile->data_checksum_version == 0 && mode == PG_MODE_CHECK) { pg_log_error("data checksums are not enabled in cluster"); exit(1); } if (ControlFile->data_checksum_version == 0 && mode == PG_MODE_DISABLE) { pg_log_error("data checksums are already disabled in cluster"); exit(1); } if (ControlFile->data_checksum_version > 0 && mode == PG_MODE_ENABLE) { pg_log_error("data checksums are already enabled in cluster"); exit(1); } /* Get checkpoint LSN */ checkpointLSN = ControlFile->checkPoint; /* Operate on all files if checking or enabling checksums */ if (mode == PG_MODE_CHECK || mode == PG_MODE_ENABLE) { #ifndef WIN32 /* * Assign SIGUSR1 signal handler to toggle progress status information. */ pqsignal(SIGUSR1, toggle_progress_report); #endif /* * As progress status information may be requested even after start of * operation, we need to scan the directory tree(s) twice, once to get * the idea how much data we need to scan and finally to do the real * legwork. */ if (debug) pg_log_debug("acquiring data for progress reporting"); total_size = scan_directory(DataDir, "global", true); total_size += scan_directory(DataDir, "base", true); total_size += scan_directory(DataDir, "pg_tblspc", true); /* * Remember start time. Required to calculate the current rate in * progress_report_or_throttle(). */ if (debug) pg_log_debug("starting scan"); INSTR_TIME_SET_CURRENT(scan_started); (void) scan_directory(DataDir, "global", false); (void) scan_directory(DataDir, "base", false); (void) scan_directory(DataDir, "pg_tblspc", false); /* * Done. Move to next line in case progress information was shown. * Otherwise we clutter the summary output. */ if (showprogress) { progress_report_or_throttle(true); if (isatty(fileno(stderr))) fprintf(stderr, "\n"); } printf(_("Checksum operation completed\n")); printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files_scanned)); if (skippedfiles > 0) printf(_("Files skipped: %s\n"), psprintf(INT64_FORMAT, skippedfiles)); printf(_("Blocks scanned: %s\n"), psprintf(INT64_FORMAT, blocks_scanned)); if (skippedblocks > 0) printf(_("Blocks skipped: %s\n"), psprintf(INT64_FORMAT, skippedblocks));; if (mode == PG_MODE_CHECK) { printf(_("Bad checksums: %s\n"), psprintf(INT64_FORMAT, badblocks)); printf(_("Data checksum version: %u\n"), ControlFile->data_checksum_version); if (badblocks > 0) exit(1); /* skipped blocks or files are considered an error if offline */ if (!online) if (skippedblocks > 0 || skippedfiles > 0) exit(1); } else if (mode == PG_MODE_ENABLE) { printf(_("Files written: %s\n"), psprintf(INT64_FORMAT, files_written)); printf(_("Blocks written: %s\n"), psprintf(INT64_FORMAT, blocks_written)); } } /* * Finally make the data durable on disk if enabling or disabling * checksums. Flush first the data directory for safety, and then update * the control file to keep the switch consistent. */ if (mode == PG_MODE_ENABLE || mode == PG_MODE_DISABLE) { ControlFile->data_checksum_version = (mode == PG_MODE_ENABLE) ? PG_DATA_CHECKSUM_VERSION : 0; if (do_sync) { pg_log_info("syncing data directory"); #if PG_VERSION_NUM >= 120000 fsync_pgdata(DataDir, PG_VERSION_NUM); #else fsync_pgdata(DataDir, progname, PG_VERSION_NUM); #endif } pg_log_info("updating control file"); updateControlFile(DataDir, ControlFile, do_sync); if (verbose) printf(_("Data checksum version: %u\n"), ControlFile->data_checksum_version); if (mode == PG_MODE_ENABLE) printf(_("Checksums enabled in cluster\n")); else printf(_("Checksums disabled in cluster\n")); } return 0; } pg_checksums-1.1/port.c000066400000000000000000000367011417133546400151750ustar00rootroot00000000000000/* * pg_checksums_ext * * Verifies/enables/disables data checksums * * Copyright (c) 2010-2022, PostgreSQL Global Development Group * * port.c */ #include "postgres_fe.h" #include "port.h" #include #include #if PG_VERSION_NUM < 90600 /* * Read in the control file. */ ControlFileData * getControlFile(char *DataDir) { ControlFileData *ControlFile; int fd; char ControlFilePath[MAXPGPATH]; ControlFile = palloc(sizeof(ControlFileData)); snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir); if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1) { fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"), progname, ControlFilePath, strerror(errno)); exit(1); } if (read(fd, ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) { fprintf(stderr, _("%s: could not read file \"%s\": %s\n"), progname, ControlFilePath, strerror(errno)); exit(1); } close(fd); return ControlFile; } #endif /* PG_VERSION_NUM < 90600 */ /* * Update the control file. */ void updateControlFile(char *DataDir, ControlFileData *ControlFile, bool do_sync) { int fd; char buffer[PG_CONTROL_FILE_SIZE]; char ControlFilePath[MAXPGPATH]; /* * For good luck, apply the same static assertions as in backend's * WriteControlFile(). */ #if PG_VERSION_NUM >= 100000 StaticAssertStmt(sizeof(ControlFileData) <= PG_CONTROL_MAX_SAFE_SIZE, "pg_control is too large for atomic disk writes"); #endif StaticAssertStmt(sizeof(ControlFileData) <= PG_CONTROL_FILE_SIZE, "sizeof(ControlFileData) exceeds PG_CONTROL_FILE_SIZE"); /* Recalculate CRC of control file */ INIT_CRC32C(ControlFile->crc); COMP_CRC32C(ControlFile->crc, (char *) ControlFile, offsetof(ControlFileData, crc)); FIN_CRC32C(ControlFile->crc); /* * Write out PG_CONTROL_FILE_SIZE bytes into pg_control by zero-padding * the excess over sizeof(ControlFileData), to avoid premature EOF related * errors when reading it. */ memset(buffer, 0, PG_CONTROL_FILE_SIZE); memcpy(buffer, ControlFile, sizeof(ControlFileData)); snprintf(ControlFilePath, sizeof(ControlFilePath), "%s/%s", DataDir, XLOG_CONTROL_FILE); fd = open(ControlFilePath, O_WRONLY | PG_BINARY, pg_file_create_mode); if (fd < 0) { fprintf(stderr, _("%s: could not create pg_control file: %s\n"), progname, strerror(errno)); exit(1); } errno = 0; if (write(fd, buffer, PG_CONTROL_FILE_SIZE) != PG_CONTROL_FILE_SIZE) { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; fprintf(stderr, _("%s: could not write pg_control file: %s\n"), progname, strerror(errno)); exit(1); } if (do_sync) { if (fsync(fd) != 0) { fprintf(stderr, _("%s: fsync error: %s\n"), progname, strerror(errno)); exit(1); } } if (close(fd) < 0) { fprintf(stderr, _("%s: could not close control file: %s\n"), progname, strerror(errno)); exit(1); } } #if PG_VERSION_NUM < 100000 /* * fsync_fname -- Try to fsync a file or directory * * Ignores errors trying to open unreadable files, or trying to fsync * directories on systems where that isn't allowed/required. Reports * other errors non-fatally. */ int fsync_fname(const char *fname, bool isdir, const char *progname) { int fd; int flags; int returncode; /* * Some OSs require directories to be opened read-only whereas other * systems don't allow us to fsync files opened read-only; so we need both * cases here. Using O_RDWR will cause us to fail to fsync files that are * not writable by our userid, but we assume that's OK. */ flags = PG_BINARY; if (!isdir) flags |= O_RDWR; else flags |= O_RDONLY; /* * Open the file, silently ignoring errors about unreadable files (or * unsupported operations, e.g. opening a directory under Windows), and * logging others. */ fd = open(fname, flags, 0); if (fd < 0) { if (errno == EACCES || (isdir && errno == EISDIR)) return 0; fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), progname, fname, strerror(errno)); return -1; } returncode = fsync(fd); /* * Some OSes don't allow us to fsync directories at all, so we can ignore * those errors. Anything else needs to be reported. */ if (returncode != 0 && !(isdir && (errno == EBADF || errno == EINVAL))) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, fname, strerror(errno)); (void) close(fd); return -1; } (void) close(fd); return 0; } /* * walkdir: recursively walk a directory, applying the action to each * regular file and directory (including the named directory itself). * * If process_symlinks is true, the action and recursion are also applied * to regular files and directories that are pointed to by symlinks in the * given directory; otherwise symlinks are ignored. Symlinks are always * ignored in subdirectories, ie we intentionally don't pass down the * process_symlinks flag to recursive calls. * * Errors are reported but not considered fatal. * * See also walkdir in fd.c, which is a backend version of this logic. */ void walkdir(const char *path, int (*action) (const char *fname, bool isdir, const char *progname), bool process_symlinks, const char *progname) { DIR *dir; struct dirent *de; dir = opendir(path); if (dir == NULL) { fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), progname, path, strerror(errno)); return; } while (errno = 0, (de = readdir(dir)) != NULL) { char subpath[MAXPGPATH * 2]; struct stat fst; int sret; if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name); if (process_symlinks) sret = stat(subpath, &fst); else sret = lstat(subpath, &fst); if (sret < 0) { fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), progname, subpath, strerror(errno)); continue; } if (S_ISREG(fst.st_mode)) (*action) (subpath, false, progname); else if (S_ISDIR(fst.st_mode)) walkdir(subpath, action, false, progname); } if (errno) fprintf(stderr, _("%s: could not read directory \"%s\": %s\n"), progname, path, strerror(errno)); (void) closedir(dir); /* * It's important to fsync the destination directory itself as individual * file fsyncs don't guarantee that the directory entry for the file is * synced. Recent versions of ext4 have made the window much wider but * it's been an issue for ext3 and other filesystems in the past. */ (*action) (path, true, progname); } /* * Issue fsync recursively on PGDATA and all its contents. * * We fsync regular files and directories wherever they are, but we follow * symlinks only for pg_wal (or pg_xlog) and immediately under pg_tblspc. * Other symlinks are presumed to point at files we're not responsible for * fsyncing, and might not have privileges to write at all. * * serverVersion indicates the version of the server to be fsync'd. * * Errors are reported but not considered fatal. */ void fsync_pgdata(const char *pg_data, const char *progname, int serverVersion) { bool xlog_is_symlink; char pg_wal[MAXPGPATH]; char pg_tblspc[MAXPGPATH]; /* handle renaming of pg_xlog to pg_wal in post-10 clusters */ snprintf(pg_wal, MAXPGPATH, "%s/%s", pg_data, serverVersion < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal"); snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pg_data); /* * If pg_wal is a symlink, we'll need to recurse into it separately, * because the first walkdir below will ignore it. */ xlog_is_symlink = false; #ifndef WIN32 { struct stat st; if (lstat(pg_wal, &st) < 0) fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), progname, pg_wal, strerror(errno)); else if (S_ISLNK(st.st_mode)) xlog_is_symlink = true; } #else if (pgwin32_is_junction(pg_wal)) xlog_is_symlink = true; #endif /* * If possible, hint to the kernel that we're soon going to fsync the data * directory and its contents. */ #ifdef PG_FLUSH_DATA_WORKS walkdir(pg_data, pre_sync_fname, false, progname); if (xlog_is_symlink) walkdir(pg_wal, pre_sync_fname, false, progname); walkdir(pg_tblspc, pre_sync_fname, true, progname); #endif /* * Now we do the fsync()s in the same order. * * The main call ignores symlinks, so in addition to specially processing * pg_wal if it's a symlink, pg_tblspc has to be visited separately with * process_symlinks = true. Note that if there are any plain directories * in pg_tblspc, they'll get fsync'd twice. That's not an expected case * so we don't worry about optimizing it. */ walkdir(pg_data, fsync_fname, false, progname); if (xlog_is_symlink) walkdir(pg_wal, fsync_fname, false, progname); walkdir(pg_tblspc, fsync_fname, true, progname); } #endif /* PG_VERSION_NUM < 100000 */ #if PG_VERSION_NUM < 120000 enum pg_log_level __pg_log_level; static int log_flags; static void (*log_pre_callback) (void); static void (*log_locus_callback) (const char **, uint64 *); static const char *sgr_error = NULL; static const char *sgr_warning = NULL; static const char *sgr_locus = NULL; #define SGR_ERROR_DEFAULT "01;31" #define SGR_WARNING_DEFAULT "01;35" #define SGR_LOCUS_DEFAULT "01" #define ANSI_ESCAPE_FMT "\x1b[%sm" #define ANSI_ESCAPE_RESET "\x1b[0m" /* * This should be called before any output happens. */ void pg_logging_init(const char *argv0) { const char *pg_color_env = getenv("PG_COLOR"); char *token; bool log_color = false; /* usually the default, but not on Windows */ setvbuf(stderr, NULL, _IONBF, 0); progname = get_progname(argv0); __pg_log_level = PG_LOG_INFO; if (pg_color_env) { if (strcmp(pg_color_env, "always") == 0 || (strcmp(pg_color_env, "auto") == 0 && isatty(fileno(stderr)))) log_color = true; } if (log_color) { const char *pg_colors_env = getenv("PG_COLORS"); if (pg_colors_env) { char *colors = strdup(pg_colors_env); if (colors) { for (token = strtok(colors, ":"); token; token = strtok(NULL, ":")) { char *e = strchr(token, '='); if (e) { char *name; char *value; *e = '\0'; name = token; value = e + 1; if (strcmp(name, "error") == 0) sgr_error = strdup(value); if (strcmp(name, "warning") == 0) sgr_warning = strdup(value); if (strcmp(name, "locus") == 0) sgr_locus = strdup(value); } } free(colors); } } else { sgr_error = SGR_ERROR_DEFAULT; sgr_warning = SGR_WARNING_DEFAULT; sgr_locus = SGR_LOCUS_DEFAULT; } } } void pg_log_generic(enum pg_log_level level, const char *pg_restrict fmt,...) { va_list ap; va_start(ap, fmt); pg_log_generic_v(level, fmt, ap); va_end(ap); } void pg_log_generic_v(enum pg_log_level level, const char *pg_restrict fmt, va_list ap) { int save_errno = errno; const char *filename = NULL; uint64 lineno = 0; va_list ap2; size_t required_len; char *buf; Assert(progname); Assert(level); Assert(fmt); Assert(fmt[strlen(fmt) - 1] != '\n'); /* * Flush stdout before output to stderr, to ensure sync even when stdout * is buffered. */ fflush(stdout); if (log_pre_callback) log_pre_callback(); if (log_locus_callback) log_locus_callback(&filename, &lineno); fmt = _(fmt); if (!(log_flags & PG_LOG_FLAG_TERSE) || filename) { if (sgr_locus) fprintf(stderr, ANSI_ESCAPE_FMT, sgr_locus); if (!(log_flags & PG_LOG_FLAG_TERSE)) fprintf(stderr, "%s:", progname); if (filename) { fprintf(stderr, "%s:", filename); if (lineno > 0) fprintf(stderr, UINT64_FORMAT ":", lineno); } fprintf(stderr, " "); if (sgr_locus) fprintf(stderr, ANSI_ESCAPE_RESET); } if (!(log_flags & PG_LOG_FLAG_TERSE)) { switch (level) { case PG_LOG_FATAL: if (sgr_error) fprintf(stderr, ANSI_ESCAPE_FMT, sgr_error); fprintf(stderr, _("fatal: ")); if (sgr_error) fprintf(stderr, ANSI_ESCAPE_RESET); break; case PG_LOG_ERROR: if (sgr_error) fprintf(stderr, ANSI_ESCAPE_FMT, sgr_error); fprintf(stderr, _("error: ")); if (sgr_error) fprintf(stderr, ANSI_ESCAPE_RESET); break; case PG_LOG_WARNING: if (sgr_warning) fprintf(stderr, ANSI_ESCAPE_FMT, sgr_warning); fprintf(stderr, _("warning: ")); if (sgr_warning) fprintf(stderr, ANSI_ESCAPE_RESET); break; default: break; } } errno = save_errno; va_copy(ap2, ap); required_len = vsnprintf(NULL, 0, fmt, ap2) + 1; va_end(ap2); buf = pg_malloc(required_len); errno = save_errno; /* malloc might change errno */ if (!buf) { /* memory trouble, just print what we can and get out of here */ vfprintf(stderr, fmt, ap); return; } vsnprintf(buf, required_len, fmt, ap); /* strip one newline, for PQerrorMessage() */ if (required_len >= 2 && buf[required_len - 2] == '\n') buf[required_len - 2] = '\0'; fprintf(stderr, "%s\n", buf); free(buf); } #endif /* PG_VERSION_NUM < 120000 */ /* * Look at the version string stored in PG_VERSION and decide if this utility * can be run safely or not (adpated from pg_resetwal). */ void CheckDataVersion(char *DataDir) { const char *ver_file = "PG_VERSION"; char ver_filepath[MAXPGPATH]; FILE *ver_fd; char rawline[64]; int len; strcpy(ver_filepath, DataDir); strcat(ver_filepath, "/"); strcat(ver_filepath, ver_file); if ((ver_fd = fopen(ver_filepath, "r")) == NULL) { pg_log_error("could not open file \"%s\" for reading: %m", ver_filepath); exit(1); } /* version number has to be the first line read */ if (!fgets(rawline, sizeof(rawline), ver_fd)) { if (!ferror(ver_fd)) pg_log_error("unexpected empty file \"%s\"", ver_filepath); else pg_log_error("could not read file \"%s\": %m", ver_filepath); exit(1); } /* strip trailing newline and carriage return */ len = strlen(rawline); if (len > 0 && rawline[len - 1] == '\n') { rawline[--len] = '\0'; if (len > 0 && rawline[len - 1] == '\r') rawline[--len] = '\0'; } if (strcmp(rawline, PG_MAJORVERSION) != 0) { pg_log_error("data directory is of wrong version"); pg_log_info("File \"%s\" contains \"%s\", which is not compatible with this program's version \"%s\".", ver_file, rawline, PG_MAJORVERSION); exit(1); } fclose(ver_fd); } pg_checksums-1.1/port.h000066400000000000000000000071061417133546400151770ustar00rootroot00000000000000/* * pg_checksums_ext * * Verifies/enables/disables data checksums * * Copyright (c) 2010-2022, PostgreSQL Global Development Group * * port.h */ #include #include "catalog/pg_control.h" #if PG_VERSION_NUM < 90400 #include #include extern char *optarg; #else #include "pg_getopt.h" #endif #if PG_VERSION_NUM >= 90600 #include "common/controldata_utils.h" #include "common/relpath.h" #endif #if PG_VERSION_NUM >= 110000 #include "common/file_perm.h" #else #define pg_file_create_mode 0600 #endif #if PG_VERSION_NUM >= 100000 #include "common/file_utils.h" #endif #if PG_VERSION_NUM >= 120000 #include "common/logging.h" #endif #if PG_VERSION_NUM < 100000 #define PG_CONTROL_FILE_SIZE PG_CONTROL_SIZE #endif #if PG_VERSION_NUM < 90500 #define INIT_CRC32C INIT_CRC32 #define COMP_CRC32C COMP_CRC32 #define FIN_CRC32C FIN_CRC32 #define INT64_MODIFIER "l" #define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) #endif #if PG_VERSION_NUM < 90600 typedef enum ForkNumber { InvalidForkNumber = -1, MAIN_FORKNUM = 0, FSM_FORKNUM, VISIBILITYMAP_FORKNUM, INIT_FORKNUM } ForkNumber; #endif #if PG_VERSION_NUM < 100000 #if __GNUC__ >= 3 #define likely(x) __builtin_expect((x) != 0, 1) #define unlikely(x) __builtin_expect((x) != 0, 0) #else #define likely(x) ((x) != 0) #define unlikely(x) ((x) != 0) #endif #endif #if PG_VERSION_NUM < 110000 #define pg_restrict __restrict #endif /* * pg_xlog has been renamed to pg_wal in version 10. */ #define MINIMUM_VERSION_FOR_PG_WAL 100000 /* * The control file (relative to $PGDATA) */ #define XLOG_CONTROL_FILE "global/pg_control" extern char *DataDir; static const char *progname; void updateControlFile(char *DataDir, ControlFileData *ControlFile, bool do_sync); #if PG_VERSION_NUM < 90600 extern int forkname_chars(const char *str, ForkNumber *fork); #endif #if PG_VERSION_NUM < 90600 ControlFileData *getControlFile(char *DataDir); #endif #if PG_VERSION_NUM < 100000 int fsync_fname(const char *fname, bool isdir, const char *progname); void walkdir(const char *path, int (*action) (const char *fname, bool isdir, const char *progname), bool process_symlinks, const char *progname); void fsync_pgdata(const char *pg_data, const char *progname, int serverVersion); #endif #if PG_VERSION_NUM < 120000 enum pg_log_level { PG_LOG_NOTSET = 0, PG_LOG_DEBUG, PG_LOG_INFO, PG_LOG_WARNING, PG_LOG_ERROR, PG_LOG_FATAL, PG_LOG_OFF, }; extern enum pg_log_level __pg_log_level; #define PG_LOG_FLAG_TERSE 1 void pg_logging_init(const char *argv0); void pg_log_generic(enum pg_log_level level, const char *pg_restrict fmt,...) pg_attribute_printf(2, 3); void pg_log_generic_v(enum pg_log_level level, const char *pg_restrict fmt, va_list ap) pg_attribute_printf(2, 0); #define pg_log_fatal(...) do { \ if (likely(__pg_log_level <= PG_LOG_FATAL)) pg_log_generic(PG_LOG_FATAL, __VA_ARGS__); \ } while(0) #define pg_log_error(...) do { \ if (likely(__pg_log_level <= PG_LOG_ERROR)) pg_log_generic(PG_LOG_ERROR, __VA_ARGS__); \ } while(0) #define pg_log_warning(...) do { \ if (likely(__pg_log_level <= PG_LOG_WARNING)) pg_log_generic(PG_LOG_WARNING, __VA_ARGS__); \ } while(0) #define pg_log_info(...) do { \ if (likely(__pg_log_level <= PG_LOG_INFO)) pg_log_generic(PG_LOG_INFO, __VA_ARGS__); \ } while(0) #define pg_log_debug(...) do { \ if (unlikely(__pg_log_level <= PG_LOG_DEBUG)) pg_log_generic(PG_LOG_DEBUG, __VA_ARGS__); \ } while(0) #endif void CheckDataVersion(char *DataDir); pg_checksums-1.1/t/000077500000000000000000000000001417133546400143015ustar00rootroot00000000000000pg_checksums-1.1/t/001_checksums.pl000066400000000000000000000203321417133546400172030ustar00rootroot00000000000000#!/usr/bin/env perl use strict; use warnings; use Cwd; use Config; use PostgresNode; use TestLib; use Test::More tests => 121; program_help_ok('pg_checksums_ext'); program_version_ok('pg_checksums_ext'); program_options_handling_ok('pg_checksums_ext'); my $tempdir = TestLib::tempdir; # Initialize node with checksums disabled. my $node = get_new_node('node_checksum'); $node->init; $node->start; my $pgdata = $node->data_dir; $node->command_fails(['pg_checksums_ext', '-c'], 'pg_checksums_ext needs needs target directory specified'); $node->command_fails(['pg_checksums_ext', '-a', '-D', $pgdata], 'pg_checksums_ext -a needs to run against offfline cluster'); my $checksum = $node->safe_psql('postgres', 'SHOW data_checksums;'); is($checksum, 'off', 'checksums are disabled'); $node->stop; $node->command_ok(['pg_checksums_ext', '-a', '-N', '-D', $pgdata], 'pg_checksums_ext are activated in offline cluster'); $node->start; $checksum = $node->safe_psql('postgres', 'SHOW data_checksums;'); is($checksum, 'on', 'checksums are enabled'); # Add set of dummy files with some contents. These should not be scanned # by the tool. append_to_file "$pgdata/global/123.", "foo"; append_to_file "$pgdata/global/123_", "foo"; append_to_file "$pgdata/global/123_.", "foo"; append_to_file "$pgdata/global/123.12t", "foo"; append_to_file "$pgdata/global/foo", "foo2"; append_to_file "$pgdata/global/t123", "bar"; append_to_file "$pgdata/global/123a", "bar2"; append_to_file "$pgdata/global/.123", "foobar"; append_to_file "$pgdata/global/_fsm", "foobar2"; append_to_file "$pgdata/global/_init", "foobar3"; append_to_file "$pgdata/global/_vm.123", "foohoge"; append_to_file "$pgdata/global/123_vm.123t", "foohoge2"; # Those are correct but empty files, so they should pass through. append_to_file "$pgdata/global/99999", ""; append_to_file "$pgdata/global/99999.123", ""; append_to_file "$pgdata/global/99999_fsm", ""; append_to_file "$pgdata/global/99999_init", ""; append_to_file "$pgdata/global/99999_vm", ""; append_to_file "$pgdata/global/99999_init.123", ""; append_to_file "$pgdata/global/99999_fsm.123", ""; append_to_file "$pgdata/global/99999_vm.123", ""; $node->stop; $node->command_ok(['pg_checksums_ext', '-b', '-N', '-D', $pgdata], 'pg_checksums_ext are deactivated in offline cluster'); $node->start; $checksum = $node->safe_psql('postgres', 'SHOW data_checksums;'); is($checksum, 'off', 'checksums are disabled'); $node->stop; $node->command_ok(['pg_checksums_ext', '-a', '-N', '-D', $pgdata], 'pg_checksums_ext are again activated in offline cluster'); $node->start; $node->command_ok(['pg_checksums_ext', '-c', '-D', $pgdata], 'pg_checksums_ext can be verified in online cluster'); # Set page header and block size my $pageheader_size = 24; my $block_size = $node->safe_psql('postgres', 'SHOW block_size;'); # Check corruption of table on default tablespace. check_relation_corruption($node, 'corrupt1', 'pg_default', $pageheader_size, "\0\0\0\0\0\0\0\0\0", "on tablespace pg_default"); # Create tablespace to check corruptions in a non-default tablespace. my $basedir = $node->basedir; my $tablespace_dir = "$basedir/ts_corrupt_dir"; mkdir ($tablespace_dir); $tablespace_dir = TestLib::real_dir($tablespace_dir); $node->safe_psql('postgres', "CREATE TABLESPACE ts_corrupt LOCATION '$tablespace_dir';"); check_relation_corruption($node, 'corrupt2', 'ts_corrupt', $pageheader_size, "\0\0\0\0\0\0\0\0\0", "on tablespace ts_corrupt"); # Check corruption in the pageheader with random data in it my $random_data = join '', map { ("a".."z")[rand 26] } 1 .. $pageheader_size; check_relation_corruption($node, 'corrupt1', 'pg_default', 0, $random_data, "with random data in pageheader"); # Check corruption when the pageheader has been zeroed-out completely my $zero_data = "\0"x$pageheader_size; check_relation_corruption($node, 'corrupt1', 'pg_default', 0, $zero_data, "with zeroed-out pageheader"); # Utility routine to create and check a table with corrupted checksums # on a wanted tablespace. Note that this stops and starts the node # multiple times to perform the checks, leaving the node started # at the end. sub check_relation_corruption { my $node = shift; my $table = shift; my $tablespace = shift; my $offset = shift; my $corrupted_data = shift; my $description = shift; my $pgdata = $node->data_dir; $node->safe_psql('postgres', "CREATE TABLE $table AS SELECT a FROM generate_series(1,10000) AS a; ALTER TABLE $table SET (autovacuum_enabled=false);"); $node->safe_psql('postgres', "ALTER TABLE ".$table." SET TABLESPACE ".$tablespace.";"); my $file_corrupted = $node->safe_psql('postgres', "SELECT pg_relation_filepath('$table');"); my $relfilenode_corrupted = $node->safe_psql('postgres', "SELECT relfilenode FROM pg_class WHERE relname = '$table';"); $node->stop; # Checksums are correct for single relfilenode as the table is not # corrupted yet. command_ok(['pg_checksums_ext', '-c', '-D', $pgdata, '-f', $relfilenode_corrupted], "succeeds for single relfilenode $description with offline cluster"); # Time to create some corruption open my $file, '+<', "$pgdata/$file_corrupted"; seek($file, $offset, 0); syswrite($file, $corrupted_data); close $file; # Checksum checks on single relfilenode fail $node->command_checks_all([ 'pg_checksums_ext', '-c', '-D', $pgdata, '-f', $relfilenode_corrupted], 1, [qr/Bad checksums:.*1/], [qr/checksum verification failed/], "fails with corrupted data $description"); # Global checksum checks fail as well $node->command_checks_all([ 'pg_checksums_ext', '-c', '-D', $pgdata], 1, [qr/Bad checksums:.*1/], [qr/checksum verification failed/], "fails with corrupted data for single relfilenode on tablespace $tablespace"); # Now check online as well $node->start; # Checksum checks on single relfilenode fail $node->command_checks_all([ 'pg_checksums_ext', '-c', '-D', $pgdata, '-f', $relfilenode_corrupted], 1, [qr/Bad checksums:.*1/], [qr/checksum verification failed/], "fails with corrupted data $description"); # Global checksum checks fail as well $node->command_checks_all([ 'pg_checksums_ext', '-c', '-D', $pgdata], 1, [qr/Bad checksums:.*1/], [qr/checksum verification failed/], "fails with corrupted data for single relfilenode on tablespace $tablespace"); # Drop corrupted table again and make sure there is no more corruption. $node->safe_psql('postgres', "DROP TABLE $table;"); $node->command_ok(['pg_checksums_ext', '-c', '-D', $pgdata], "succeeds again after table drop on tablespace $tablespace"); return; } # Utility routine to check that pg_checksums_ext is able to detect # correctly-named relation files filled with some corrupted data. sub fail_corrupt { my $node = shift; my $file = shift; my $pgdata = $node->data_dir; # Create the file with some dummy data in it. my $file_name = "$pgdata/global/$file"; append_to_file $file_name, "foo"; $node->stop; # If the instance is offline, the whole file is skipped and this is # considered to be an error. $node->command_checks_all([ 'pg_checksums_ext', '-c', '-D', $pgdata], 1, [qr/Files skipped:.*1/], [qr/could not read block 0 in file.*$file\":/], "skips file for corrupted data in $file when offline"); $node->start; # If the instance is online, the block is skipped and this is not # considered to be an error $node->command_checks_all([ 'pg_checksums_ext', '-c', '-D', $pgdata], 0, [qr/Blocks skipped:.*1/], [qr/^$/], "skips block for corrupted data in $file when online"); # Remove file to prevent future lookup errors on conflicts. unlink $file_name; return; } # Authorized relation files filled with corrupted data cause the files to be # skipped and, if the instance is offline, a non-zero exit status. Make sure # to use file names different than the previous ones. fail_corrupt($node, "99990"); fail_corrupt($node, "99990.123"); fail_corrupt($node, "99990_fsm"); fail_corrupt($node, "99990_init"); fail_corrupt($node, "99990_vm"); fail_corrupt($node, "99990_init.123"); fail_corrupt($node, "99990_fsm.123"); fail_corrupt($node, "99990_vm.123"); # Stop node again at the end of tests $node->stop; pg_checksums-1.1/t/perl/000077500000000000000000000000001417133546400152435ustar00rootroot00000000000000pg_checksums-1.1/t/perl/PostgresNode.pm000066400000000000000000001224551417133546400202260ustar00rootroot00000000000000 =pod =head1 NAME PostgresNode - class representing PostgreSQL server instance =head1 SYNOPSIS use PostgresNode; my $node = PostgresNode->get_new_node('mynode'); # Create a data directory with initdb $node->init(); # Start the PostgreSQL server $node->start(); # Change a setting and restart $node->append_conf('postgresql.conf', 'hot_standby = on'); $node->restart(); # run a query with psql, like: # echo 'SELECT 1' | psql -qAXt postgres -v ON_ERROR_STOP=1 $psql_stdout = $node->safe_psql('postgres', 'SELECT 1'); # Run psql with a timeout, capturing stdout and stderr # as well as the psql exit code. Pass some extra psql # options. If there's an error from psql raise an exception. my ($stdout, $stderr, $timed_out); my $cmdret = $node->psql('postgres', 'SELECT pg_sleep(60)', stdout => \$stdout, stderr => \$stderr, timeout => 30, timed_out => \$timed_out, extra_params => ['--single-transaction'], on_error_die => 1) print "Sleep timed out" if $timed_out; # Similar thing, more convenient in common cases my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT 1'); # run query every second until it returns 't' # or times out $node->poll_query_until('postgres', q|SELECT random() < 0.1;|') or die "timed out"; # Do an online pg_basebackup my $ret = $node->backup('testbackup1'); # Take a backup of a running server my $ret = $node->backup_fs_hot('testbackup2'); # Take a backup of a stopped server $node->stop; my $ret = $node->backup_fs_cold('testbackup3') # Restore it to create a new independent node (not a replica) my $replica = get_new_node('replica'); $replica->init_from_backup($node, 'testbackup'); $replica->start; # Stop the server $node->stop('fast'); =head1 DESCRIPTION PostgresNode contains a set of routines able to work on a PostgreSQL node, allowing to start, stop, backup and initialize it with various options. The set of nodes managed by a given test is also managed by this module. In addition to node management, PostgresNode instances have some wrappers around Test::More functions to run commands with an environment set up to point to the instance. The IPC::Run module is required. =cut package PostgresNode; use strict; use warnings; use Carp; use Config; use Cwd; use Exporter 'import'; use Fcntl qw(:mode); use File::Basename; use File::Path qw(rmtree); use File::Spec; use File::stat qw(stat); use File::Temp (); use IPC::Run; use RecursiveCopy; use Socket; use Test::More; use TestLib (); use Time::HiRes qw(usleep); use Scalar::Util qw(blessed); our @EXPORT = qw( get_new_node ); our ($test_localhost, $test_pghost, $last_port_assigned, @all_nodes, $died); # Windows path to virtual file system root our $vfs_path = ''; if ($Config{osname} eq 'msys') { $vfs_path = `cd / && pwd -W`; chomp $vfs_path; } INIT { # PGHOST is set once and for all through a single series of tests when # this module is loaded. $test_localhost = "127.0.0.1"; $test_pghost = $TestLib::windows_os ? $test_localhost : TestLib::tempdir_short; $ENV{PGHOST} = $test_pghost; $ENV{PGDATABASE} = 'postgres'; # Tracking of last port value assigned to accelerate free port lookup. $last_port_assigned = int(rand() * 16384) + 49152; } =pod =head1 METHODS =over =item PostgresNode::new($class, $name, $pghost, $pgport) Create a new PostgresNode instance. Does not initdb or start it. You should generally prefer to use get_new_node() instead since it takes care of finding port numbers, registering instances for cleanup, etc. =cut sub new { my ($class, $name, $pghost, $pgport) = @_; my $testname = basename($0); $testname =~ s/\.[^.]+$//; my $self = { _port => $pgport, _host => $pghost, _basedir => "$TestLib::tmp_check/t_${testname}_${name}_data", _name => $name, _logfile => "$TestLib::log_path/${testname}_${name}.log" }; bless $self, $class; mkdir $self->{_basedir} or BAIL_OUT("could not create data directory \"$self->{_basedir}\": $!"); $self->dump_info; return $self; } =pod =item $node->port() Get the port number assigned to the host. This won't necessarily be a TCP port open on the local host since we prefer to use unix sockets if possible. Use $node->connstr() if you want a connection string. =cut sub port { my ($self) = @_; return $self->{_port}; } =pod =item $node->host() Return the host (like PGHOST) for this instance. May be a UNIX socket path. Use $node->connstr() if you want a connection string. =cut sub host { my ($self) = @_; return $self->{_host}; } =pod =item $node->basedir() The directory all the node's files will be within - datadir, archive directory, backups, etc. =cut sub basedir { my ($self) = @_; return $self->{_basedir}; } =pod =item $node->name() The name assigned to the node at creation time. =cut sub name { my ($self) = @_; return $self->{_name}; } =pod =item $node->logfile() Path to the PostgreSQL log file for this instance. =cut sub logfile { my ($self) = @_; return $self->{_logfile}; } =pod =item $node->connstr() Get a libpq connection string that will establish a connection to this node. Suitable for passing to psql, DBD::Pg, etc. =cut sub connstr { my ($self, $dbname) = @_; my $pgport = $self->port; my $pghost = $self->host; if (!defined($dbname)) { return "port=$pgport host=$pghost"; } # Escape properly the database string before using it, only # single quotes and backslashes need to be treated this way. $dbname =~ s#\\#\\\\#g; $dbname =~ s#\'#\\\'#g; return "port=$pgport host=$pghost dbname='$dbname'"; } =pod =item $node->group_access() Does the data dir allow group access? =cut sub group_access { my ($self) = @_; my $dir_stat = stat($self->data_dir); defined($dir_stat) or die('unable to stat ' . $self->data_dir); return (S_IMODE($dir_stat->mode) == 0750); } =pod =item $node->data_dir() Returns the path to the data directory. postgresql.conf and pg_hba.conf are always here. =cut sub data_dir { my ($self) = @_; my $res = $self->basedir; return "$res/pgdata"; } =pod =item $node->archive_dir() If archiving is enabled, WAL files go here. =cut sub archive_dir { my ($self) = @_; my $basedir = $self->basedir; return "$basedir/archives"; } =pod =item $node->backup_dir() The output path for backups taken with $node->backup() =cut sub backup_dir { my ($self) = @_; my $basedir = $self->basedir; return "$basedir/backup"; } =pod =item $node->info() Return a string containing human-readable diagnostic information (paths, etc) about this node. =cut sub info { my ($self) = @_; my $_info = ''; open my $fh, '>', \$_info or die; print $fh "Name: " . $self->name . "\n"; print $fh "Data directory: " . $self->data_dir . "\n"; print $fh "Backup directory: " . $self->backup_dir . "\n"; print $fh "Archive directory: " . $self->archive_dir . "\n"; print $fh "Connection string: " . $self->connstr . "\n"; print $fh "Log file: " . $self->logfile . "\n"; close $fh or die; return $_info; } =pod =item $node->dump_info() Print $node->info() =cut sub dump_info { my ($self) = @_; print $self->info; } # Internal method to set up trusted pg_hba.conf for replication. Not # documented because you shouldn't use it, it's called automatically if needed. sub set_replication_conf { my ($self) = @_; my $pgdata = $self->data_dir; $self->host eq $test_pghost or croak "set_replication_conf only works with the default host"; open my $hba, '>>', "$pgdata/pg_hba.conf"; print $hba "\n# Allow replication (set up by PostgresNode.pm)\n"; if ($TestLib::windows_os) { print $hba "host replication all $test_localhost/32 sspi include_realm=1 map=regress\n"; } close $hba; } =pod =item $node->init(...) Initialize a new cluster for testing. Authentication is set up so that only the current OS user can access the cluster. On Unix, we use Unix domain socket connections, with the socket in a directory that's only accessible to the current user to ensure that. On Windows, we use SSPI authentication to ensure the same (by pg_regress --config-auth). WAL archiving can be enabled on this node by passing the keyword parameter has_archiving => 1. This is disabled by default. postgresql.conf can be set up for replication by passing the keyword parameter allows_streaming => 'logical' or 'physical' (passing 1 will also suffice for physical replication) depending on type of replication that should be enabled. This is disabled by default. The new node is set up in a fast but unsafe configuration where fsync is disabled. =cut sub init { my ($self, %params) = @_; my $port = $self->port; my $pgdata = $self->data_dir; my $host = $self->host; $params{allows_streaming} = 0 unless defined $params{allows_streaming}; $params{has_archiving} = 0 unless defined $params{has_archiving}; mkdir $self->backup_dir; mkdir $self->archive_dir; TestLib::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N', @{ $params{extra} }); TestLib::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata); open my $conf, '>>', "$pgdata/postgresql.conf"; print $conf "\n# Added by PostgresNode.pm\n"; print $conf "fsync = off\n"; print $conf "restart_after_crash = off\n"; print $conf "log_line_prefix = '%m [%p] %q%a '\n"; print $conf "log_statement = all\n"; print $conf "log_min_messages = debug1\n"; #print $conf "log_replication_commands = on\n"; #print $conf "wal_retrieve_retry_interval = '500ms'\n"; print $conf "port = $port\n"; if ($params{allows_streaming}) { if ($params{allows_streaming} eq "logical") { print $conf "wal_level = logical\n"; } else { print $conf "wal_level = replica\n"; } print $conf "max_wal_senders = 5\n"; print $conf "max_replication_slots = 5\n"; #print $conf "max_wal_size = 128MB\n"; print $conf "shared_buffers = 1MB\n"; print $conf "wal_log_hints = on\n"; print $conf "hot_standby = on\n"; print $conf "max_connections = 10\n"; } else { print $conf "wal_level = minimal\n"; print $conf "max_wal_senders = 0\n"; } if ($TestLib::windows_os) { print $conf "listen_addresses = '$host'\n"; } else { print $conf "unix_socket_directories = '$host'\n"; print $conf "listen_addresses = ''\n"; } close $conf; chmod($self->group_access ? 0640 : 0600, "$pgdata/postgresql.conf") or die("unable to set permissions for $pgdata/postgresql.conf"); $self->set_replication_conf if $params{allows_streaming}; $self->enable_archiving if $params{has_archiving}; } =pod =item $node->append_conf(filename, str) A shortcut method to append to files like pg_hba.conf and postgresql.conf. Does no validation or sanity checking. Does not reload the configuration after writing. A newline is automatically appended to the string. =cut sub append_conf { my ($self, $filename, $str) = @_; my $conffile = $self->data_dir . '/' . $filename; TestLib::append_to_file($conffile, $str . "\n"); chmod($self->group_access() ? 0640 : 0600, $conffile) or die("unable to set permissions for $conffile"); } =pod =item $node->backup(backup_name) Create a hot backup with B in subdirectory B of B<< $node->backup_dir >>, including the WAL. WAL files fetched at the end of the backup, not streamed. You'll have to configure a suitable B on the target server since it isn't done by default. =cut sub backup { my ($self, $backup_name) = @_; my $backup_path = $self->backup_dir . '/' . $backup_name; my $port = $self->port; my $name = $self->name; print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-p', $port, '--no-sync'); print "# Backup finished\n"; } =item $node->backup_fs_hot(backup_name) Create a backup with a filesystem level copy in subdirectory B of B<< $node->backup_dir >>, including WAL. Archiving must be enabled, as B and B are used. This is not checked or enforced. The backup name is passed as the backup label to B. =cut sub backup_fs_hot { my ($self, $backup_name) = @_; $self->_backup_fs($backup_name, 1); } =item $node->backup_fs_cold(backup_name) Create a backup with a filesystem level copy in subdirectory B of B<< $node->backup_dir >>, including WAL. The server must be stopped as no attempt to handle concurrent writes is made. Use B or B if you want to back up a running server. =cut sub backup_fs_cold { my ($self, $backup_name) = @_; $self->_backup_fs($backup_name, 0); } # Common sub of backup_fs_hot and backup_fs_cold sub _backup_fs { my ($self, $backup_name, $hot) = @_; my $backup_path = $self->backup_dir . '/' . $backup_name; my $port = $self->port; my $name = $self->name; print "# Taking filesystem backup $backup_name from node \"$name\"\n"; if ($hot) { my $stdout = $self->safe_psql('postgres', "SELECT * FROM pg_start_backup('$backup_name');"); print "# pg_start_backup: $stdout\n"; } RecursiveCopy::copypath( $self->data_dir, $backup_path, filterfn => sub { my $src = shift; return ($src ne 'log' and $src ne 'postmaster.pid'); }); if ($hot) { # We ignore pg_stop_backup's return value. We also assume archiving # is enabled; otherwise the caller will have to copy the remaining # segments. my $stdout = $self->safe_psql('postgres', 'SELECT * FROM pg_stop_backup();'); print "# pg_stop_backup: $stdout\n"; } print "# Backup finished\n"; } =pod =item $node->init_from_backup(root_node, backup_name) Initialize a node from a backup, which may come from this node or a different node. root_node must be a PostgresNode reference, backup_name the string name of a backup previously created on that node with $node->backup. Does not start the node after initializing it. A recovery.conf is not created. Streaming replication can be enabled on this node by passing the keyword parameter has_streaming => 1. This is disabled by default. Restoring WAL segments from archives using restore_command can be enabled by passing the keyword parameter has_restoring => 1. This is disabled by default. The backup is copied, leaving the original unmodified. pg_hba.conf is unconditionally set to enable replication connections. =cut sub init_from_backup { my ($self, $root_node, $backup_name, %params) = @_; my $backup_path = $root_node->backup_dir . '/' . $backup_name; my $port = $self->port; my $node_name = $self->name; my $root_name = $root_node->name; $params{has_streaming} = 0 unless defined $params{has_streaming}; $params{has_restoring} = 0 unless defined $params{has_restoring}; print "# Initializing node \"$node_name\" from backup \"$backup_name\" of node \"$root_name\"\n"; croak "Backup \"$backup_name\" does not exist at $backup_path" unless -d $backup_path; mkdir $self->backup_dir; mkdir $self->archive_dir; my $data_path = $self->data_dir; rmdir($data_path); RecursiveCopy::copypath($backup_path, $data_path); chmod(0700, $data_path); # Base configuration for this node $self->append_conf( 'postgresql.conf', qq( port = $port )); $self->enable_streaming($root_node) if $params{has_streaming}; $self->enable_restoring($root_node) if $params{has_restoring}; } =pod =item $node->start() Wrapper for pg_ctl start Start the node and wait until it is ready to accept connections. =cut sub start { my ($self) = @_; my $port = $self->port; my $pgdata = $self->data_dir; my $name = $self->name; BAIL_OUT("node \"$name\" is already running") if defined $self->{_pid}; print("### Starting node \"$name\"\n"); my $ret = TestLib::system_log('pg_ctl', '-D', $self->data_dir, '-l', $self->logfile, '-w', 'start'); if ($ret != 0) { print "# pg_ctl start failed; logfile:\n"; print TestLib::slurp_file($self->logfile); BAIL_OUT("pg_ctl start failed"); } $self->_update_pid(1); } =pod =item $node->stop(mode) Stop the node using pg_ctl -m $mode and wait for it to stop. Note: if the node is already known stopped, this does nothing. However, if we think it's running and it's not, it's important for this to fail. Otherwise, tests might fail to detect server crashes. =cut sub stop { my ($self, $mode) = @_; my $port = $self->port; my $pgdata = $self->data_dir; my $name = $self->name; $mode = 'fast' unless defined $mode; return unless defined $self->{_pid}; print "### Stopping node \"$name\" using mode $mode\n"; TestLib::system_or_bail('pg_ctl', '-D', $pgdata, '-m', $mode, 'stop'); $self->_update_pid(0); } =pod =item $node->reload() Reload configuration parameters on the node. =cut sub reload { my ($self) = @_; my $port = $self->port; my $pgdata = $self->data_dir; my $name = $self->name; print "### Reloading node \"$name\"\n"; TestLib::system_or_bail('pg_ctl', '-D', $pgdata, 'reload'); } =pod =item $node->restart() Wrapper for pg_ctl restart =cut sub restart { my ($self) = @_; my $port = $self->port; my $pgdata = $self->data_dir; my $logfile = $self->logfile; my $name = $self->name; print "### Restarting node \"$name\"\n"; TestLib::system_or_bail('pg_ctl', '-D', $pgdata, '-l', $logfile, 'restart'); $self->_update_pid(1); } =pod =item $node->promote() Wrapper for pg_ctl promote =cut sub promote { my ($self) = @_; my $port = $self->port; my $pgdata = $self->data_dir; my $logfile = $self->logfile; my $name = $self->name; print "### Promoting node \"$name\"\n"; TestLib::system_or_bail('pg_ctl', '-D', $pgdata, '-l', $logfile, 'promote'); } # Internal routine to enable streaming replication on a standby node. sub enable_streaming { my ($self, $root_node) = @_; my $root_connstr = $root_node->connstr; my $name = $self->name; print "### Enabling streaming replication for node \"$name\"\n"; $self->append_conf( 'recovery.conf', qq( primary_conninfo='$root_connstr application_name=$name' standby_mode=on )); } # Internal routine to enable archive recovery command on a standby node sub enable_restoring { my ($self, $root_node) = @_; my $path = $vfs_path . $root_node->archive_dir; my $name = $self->name; print "### Enabling WAL restore for node \"$name\"\n"; # On Windows, the path specified in the restore command needs to use # double back-slashes to work properly and to be able to detect properly # the file targeted by the copy command, so the directory value used # in this routine, using only one back-slash, need to be properly changed # first. Paths also need to be double-quoted to prevent failures where # the path contains spaces. $path =~ s{\\}{\\\\}g if ($TestLib::windows_os); my $copy_command = $TestLib::windows_os ? qq{copy "$path\\\\%f" "%p"} : qq{cp "$path/%f" "%p"}; $self->append_conf( 'recovery.conf', qq( restore_command = '$copy_command' standby_mode = on )); } # Internal routine to enable archiving sub enable_archiving { my ($self) = @_; my $path = $vfs_path . $self->archive_dir; my $name = $self->name; print "### Enabling WAL archiving for node \"$name\"\n"; # On Windows, the path specified in the restore command needs to use # double back-slashes to work properly and to be able to detect properly # the file targeted by the copy command, so the directory value used # in this routine, using only one back-slash, need to be properly changed # first. Paths also need to be double-quoted to prevent failures where # the path contains spaces. $path =~ s{\\}{\\\\}g if ($TestLib::windows_os); my $copy_command = $TestLib::windows_os ? qq{copy "%p" "$path\\\\%f"} : qq{cp "%p" "$path/%f"}; # Enable archive_mode and archive_command on node $self->append_conf( 'postgresql.conf', qq( archive_mode = on archive_command = '$copy_command' )); } # Internal method sub _update_pid { my ($self, $is_running) = @_; my $name = $self->name; # If we can open the PID file, read its first line and that's the PID we # want. if (open my $pidfile, '<', $self->data_dir . "/postmaster.pid") { chomp($self->{_pid} = <$pidfile>); print "# Postmaster PID for node \"$name\" is $self->{_pid}\n"; close $pidfile; # If we found a pidfile when there shouldn't be one, complain. BAIL_OUT("postmaster.pid unexpectedly present") unless $is_running; return; } $self->{_pid} = undef; print "# No postmaster PID for node \"$name\"\n"; # Complain if we expected to find a pidfile. BAIL_OUT("postmaster.pid unexpectedly not present") if $is_running; } =pod =item PostgresNode->get_new_node(node_name) Build a new object of class C (or of a subclass, if you have one), assigning a free port number. Remembers the node, to prevent its port number from being reused for another node, and to ensure that it gets shut down when the test script exits. You should generally use this instead of C. For backwards compatibility, it is also exported as a standalone function, which can only create objects of class C. =cut sub get_new_node { my $class = 'PostgresNode'; $class = shift if 1 < scalar @_; my $name = shift; my $found = 0; my $port = $last_port_assigned; while ($found == 0) { # advance $port, wrapping correctly around range end $port = 49152 if ++$port >= 65536; print "# Checking port $port\n"; # Check first that candidate port number is not included in # the list of already-registered nodes. $found = 1; foreach my $node (@all_nodes) { $found = 0 if ($node->port == $port); } # Check to see if anything else is listening on this TCP port. # This is *necessary* on Windows, and seems like a good idea # on Unixen as well, even though we don't ask the postmaster # to open a TCP port on Unix. if ($found == 1) { my $iaddr = inet_aton($test_localhost); my $paddr = sockaddr_in($port, $iaddr); my $proto = getprotobyname("tcp"); socket(SOCK, PF_INET, SOCK_STREAM, $proto) or die "socket failed: $!"; # As in postmaster, don't use SO_REUSEADDR on Windows setsockopt(SOCK, SOL_SOCKET, SO_REUSEADDR, pack("l", 1)) unless $TestLib::windows_os; (bind(SOCK, $paddr) && listen(SOCK, SOMAXCONN)) or $found = 0; close(SOCK); } } print "# Found free port $port\n"; # Lock port number found by creating a new node my $node = $class->new($name, $test_pghost, $port); # Add node to list of nodes push(@all_nodes, $node); # And update port for next time $last_port_assigned = $port; return $node; } # Retain the errno on die() if set, else assume a generic errno of 1. # This will instruct the END handler on how to handle artifacts left # behind from tests. $SIG{__DIE__} = sub { if ($!) { $died = $!; } else { $died = 1; } }; # Automatically shut down any still-running nodes when the test script exits. # Note that this just stops the postmasters (in the same order the nodes were # created in). Any temporary directories are deleted, in an unspecified # order, later when the File::Temp objects are destroyed. END { # take care not to change the script's exit value my $exit_code = $?; foreach my $node (@all_nodes) { $node->teardown_node; # skip clean if we are requested to retain the basedir next if defined $ENV{'PG_TEST_NOCLEAN'}; # clean basedir on clean test invocation $node->clean_node if TestLib::all_tests_passing() && !defined $died && !$exit_code; } $? = $exit_code; } =pod =item $node->teardown_node() Do an immediate stop of the node =cut sub teardown_node { my $self = shift; $self->stop('immediate'); } =pod =item $node->clean_node() Remove the base directory of the node if the node has been stopped. =cut sub clean_node { my $self = shift; rmtree $self->{_basedir} unless defined $self->{_pid}; } =pod =item $node->safe_psql($dbname, $sql) => stdout Invoke B to run B on B and return its stdout on success. Die if the SQL produces an error. Runs with B set. Takes optional extra params like timeout and timed_out parameters with the same options as psql. =cut sub safe_psql { my ($self, $dbname, $sql, %params) = @_; my ($stdout, $stderr); my $ret = $self->psql( $dbname, $sql, %params, stdout => \$stdout, stderr => \$stderr, on_error_die => 1, on_error_stop => 1); # psql can emit stderr from NOTICEs etc if ($stderr ne "") { print "#### Begin standard error\n"; print $stderr; print "\n#### End standard error\n"; } $stdout =~ s/\r//g if $TestLib::windows_os; return $stdout; } =pod =item $node->psql($dbname, $sql, %params) => psql_retval Invoke B to execute B<$sql> on B<$dbname> and return the return value from B, which is run with on_error_stop by default so that it will stop running sql and return 3 if the passed SQL results in an error. As a convenience, if B is called in array context it returns an array containing ($retval, $stdout, $stderr). psql is invoked in tuples-only unaligned mode with reading of B<.psqlrc> disabled. That may be overridden by passing extra psql parameters. stdout and stderr are transformed to UNIX line endings if on Windows. Any trailing newline is removed. Dies on failure to invoke psql but not if psql exits with a nonzero return code (unless on_error_die specified). If psql exits because of a signal, an exception is raised. =over =item stdout => \$stdout B, if given, must be a scalar reference to which standard output is written. If not given, standard output is not redirected and will be printed unless B is called in array context, in which case it's captured and returned. =item stderr => \$stderr Same as B but gets standard error. If the same scalar is passed for both B and B the results may be interleaved unpredictably. =item on_error_stop => 1 By default, the B method invokes the B program with ON_ERROR_STOP=1 set, so SQL execution is stopped at the first error and exit code 2 is returned. Set B to 0 to ignore errors instead. =item on_error_die => 0 By default, this method returns psql's result code. Pass on_error_die to instead die with an informative message. =item timeout => 'interval' Set a timeout for the psql call as an interval accepted by B (integer seconds is fine). This method raises an exception on timeout, unless the B parameter is also given. =item timed_out => \$timed_out If B is set and this parameter is given, the scalar it references is set to true if the psql call times out. =item extra_params => ['--single-transaction'] If given, it must be an array reference containing additional parameters to B. =back e.g. my ($stdout, $stderr, $timed_out); my $cmdret = $node->psql('postgres', 'SELECT pg_sleep(60)', stdout => \$stdout, stderr => \$stderr, timeout => 30, timed_out => \$timed_out, extra_params => ['--single-transaction']) will set $cmdret to undef and $timed_out to a true value. $node->psql('postgres', $sql, on_error_die => 1); dies with an informative message if $sql fails. =cut sub psql { my ($self, $dbname, $sql, %params) = @_; my $stdout = $params{stdout}; my $stderr = $params{stderr}; my $timeout = undef; my $timeout_exception = 'psql timed out'; my @psql_params = ('psql', '-XAtq', '-d', $self->connstr($dbname), '-f', '-'); # If the caller wants an array and hasn't passed stdout/stderr # references, allocate temporary ones to capture them so we # can return them. Otherwise we won't redirect them at all. if (wantarray) { if (!defined($stdout)) { my $temp_stdout = ""; $stdout = \$temp_stdout; } if (!defined($stderr)) { my $temp_stderr = ""; $stderr = \$temp_stderr; } } $params{on_error_stop} = 1 unless defined $params{on_error_stop}; $params{on_error_die} = 0 unless defined $params{on_error_die}; push @psql_params, '-v', 'ON_ERROR_STOP=1' if $params{on_error_stop}; push @psql_params, @{ $params{extra_params} } if defined $params{extra_params}; $timeout = IPC::Run::timeout($params{timeout}, exception => $timeout_exception) if (defined($params{timeout})); ${ $params{timed_out} } = 0 if defined $params{timed_out}; # IPC::Run would otherwise append to existing contents: $$stdout = "" if ref($stdout); $$stderr = "" if ref($stderr); my $ret; # Run psql and capture any possible exceptions. If the exception is # because of a timeout and the caller requested to handle that, just return # and set the flag. Otherwise, and for any other exception, rethrow. # # For background, see # http://search.cpan.org/~ether/Try-Tiny-0.24/lib/Try/Tiny.pm do { local $@; eval { my @ipcrun_opts = (\@psql_params, '<', \$sql); push @ipcrun_opts, '>', $stdout if defined $stdout; push @ipcrun_opts, '2>', $stderr if defined $stderr; push @ipcrun_opts, $timeout if defined $timeout; IPC::Run::run @ipcrun_opts; $ret = $?; }; my $exc_save = $@; if ($exc_save) { # IPC::Run::run threw an exception. re-throw unless it's a # timeout, which we'll handle by testing is_expired die $exc_save if (blessed($exc_save) || $exc_save !~ /^\Q$timeout_exception\E/); $ret = undef; die "Got timeout exception '$exc_save' but timer not expired?!" unless $timeout->is_expired; if (defined($params{timed_out})) { ${ $params{timed_out} } = 1; } else { die "psql timed out: stderr: '$$stderr'\n" . "while running '@psql_params'"; } } }; if (defined $$stdout) { chomp $$stdout; $$stdout =~ s/\r//g if $TestLib::windows_os; } if (defined $$stderr) { chomp $$stderr; $$stderr =~ s/\r//g if $TestLib::windows_os; } # See http://perldoc.perl.org/perlvar.html#%24CHILD_ERROR # We don't use IPC::Run::Simple to limit dependencies. # # We always die on signal. my $core = $ret & 128 ? " (core dumped)" : ""; die "psql exited with signal " . ($ret & 127) . "$core: '$$stderr' while running '@psql_params'" if $ret & 127; $ret = $ret >> 8; if ($ret && $params{on_error_die}) { die "psql error: stderr: '$$stderr'\nwhile running '@psql_params'" if $ret == 1; die "connection error: '$$stderr'\nwhile running '@psql_params'" if $ret == 2; die "error running SQL: '$$stderr'\nwhile running '@psql_params' with sql '$sql'" if $ret == 3; die "psql returns $ret: '$$stderr'\nwhile running '@psql_params'"; } if (wantarray) { return ($ret, $$stdout, $$stderr); } else { return $ret; } } =pod =item $node->poll_query_until($dbname, $query [, $expected ]) Run B<$query> repeatedly, until it returns the B<$expected> result ('t', or SQL boolean true, by default). Continues polling if B returns an error result. Times out after 180 seconds. Returns 1 if successful, 0 if timed out. =cut sub poll_query_until { my ($self, $dbname, $query, $expected) = @_; $expected = 't' unless defined($expected); # default value my $cmd = [ 'psql', '-XAt', '-c', $query, '-d', $self->connstr($dbname) ]; my ($stdout, $stderr); my $max_attempts = 180 * 10; my $attempts = 0; while ($attempts < $max_attempts) { my $result = IPC::Run::run $cmd, '>', \$stdout, '2>', \$stderr; chomp($stdout); $stdout =~ s/\r//g if $TestLib::windows_os; if ($stdout eq $expected) { return 1; } # Wait 0.1 second before retrying. usleep(100_000); $attempts++; } # The query result didn't change in 180 seconds. Give up. Print the stderr # from the last attempt, hopefully that's useful for debugging. diag $stderr; return 0; } =pod =item $node->command_ok(...) Runs a shell command like TestLib::command_ok, but with PGPORT set so that the command will default to connecting to this PostgresNode. =cut sub command_ok { my $self = shift; local $ENV{PGPORT} = $self->port; TestLib::command_ok(@_); } =pod =item $node->command_fails(...) TestLib::command_fails with our PGPORT. See command_ok(...) =cut sub command_fails { my $self = shift; local $ENV{PGPORT} = $self->port; TestLib::command_fails(@_); } =pod =item $node->command_like(...) TestLib::command_like with our PGPORT. See command_ok(...) =cut sub command_like { my $self = shift; local $ENV{PGPORT} = $self->port; TestLib::command_like(@_); } =pod =item $node->command_checks_all(...) TestLib::command_checks_all with our PGPORT. See command_ok(...) =cut sub command_checks_all { my $self = shift; local $ENV{PGPORT} = $self->port; TestLib::command_checks_all(@_); } =pod =item $node->issues_sql_like(cmd, expected_sql, test_name) Run a command on the node, then verify that $expected_sql appears in the server log file. Reads the whole log file so be careful when working with large log outputs. The log file is truncated prior to running the command, however. =cut sub issues_sql_like { my ($self, $cmd, $expected_sql, $test_name) = @_; local $ENV{PGPORT} = $self->port; truncate $self->logfile, 0; my $result = TestLib::run_log($cmd); ok($result, "@$cmd exit code 0"); my $log = TestLib::slurp_file($self->logfile); like($log, $expected_sql, "$test_name: SQL found in server log"); } =pod =item $node->run_log(...) Runs a shell command like TestLib::run_log, but with PGPORT set so that the command will default to connecting to this PostgresNode. =cut sub run_log { my $self = shift; local $ENV{PGPORT} = $self->port; TestLib::run_log(@_); } =pod =item $node->lsn(mode) Look up WAL locations on the server: * insert location (master only, error on replica) * write location (master only, error on replica) * flush location (master only, error on replica) * receive location (always undef on master) * replay location (always undef on master) mode must be specified. =cut sub lsn { my ($self, $mode) = @_; my %modes = ( 'insert' => 'pg_current_wal_insert_lsn()', 'flush' => 'pg_current_wal_flush_lsn()', 'write' => 'pg_current_wal_lsn()', 'receive' => 'pg_last_wal_receive_lsn()', 'replay' => 'pg_last_wal_replay_lsn()'); $mode = '' if !defined($mode); croak "unknown mode for 'lsn': '$mode', valid modes are " . join(', ', keys %modes) if !defined($modes{$mode}); my $result = $self->safe_psql('postgres', "SELECT $modes{$mode}"); chomp($result); if ($result eq '') { return; } else { return $result; } } =pod =item $node->wait_for_catchup(standby_name, mode, target_lsn) Wait for the node with application_name standby_name (usually from node->name, also works for logical subscriptions) until its replication location in pg_stat_replication equals or passes the upstream's WAL insert point at the time this function is called. By default the replay_lsn is waited for, but 'mode' may be specified to wait for any of sent|write|flush|replay. If there is no active replication connection from this peer, waits until poll_query_until timeout. Requires that the 'postgres' db exists and is accessible. target_lsn may be any arbitrary lsn, but is typically $master_node->lsn('insert'). If omitted, pg_current_wal_lsn() is used. This is not a test. It die()s on failure. =cut sub wait_for_catchup { my ($self, $standby_name, $mode, $target_lsn) = @_; $mode = defined($mode) ? $mode : 'replay'; my %valid_modes = ('sent' => 1, 'write' => 1, 'flush' => 1, 'replay' => 1); croak "unknown mode $mode for 'wait_for_catchup', valid modes are " . join(', ', keys(%valid_modes)) unless exists($valid_modes{$mode}); # Allow passing of a PostgresNode instance as shorthand if (blessed($standby_name) && $standby_name->isa("PostgresNode")) { $standby_name = $standby_name->name; } my $lsn_expr; if (defined($target_lsn)) { $lsn_expr = "'$target_lsn'"; } else { $lsn_expr = 'pg_current_wal_lsn()' } print "Waiting for replication conn " . $standby_name . "'s " . $mode . "_lsn to pass " . $lsn_expr . " on " . $self->name . "\n"; my $query = qq[SELECT $lsn_expr <= ${mode}_lsn FROM pg_catalog.pg_stat_replication WHERE application_name = '$standby_name';]; $self->poll_query_until('postgres', $query) or croak "timed out waiting for catchup"; print "done\n"; } =pod =item $node->wait_for_slot_catchup(slot_name, mode, target_lsn) Wait for the named replication slot to equal or pass the supplied target_lsn. The location used is the restart_lsn unless mode is given, in which case it may be 'restart' or 'confirmed_flush'. Requires that the 'postgres' db exists and is accessible. This is not a test. It die()s on failure. If the slot is not active, will time out after poll_query_until's timeout. target_lsn may be any arbitrary lsn, but is typically $master_node->lsn('insert'). Note that for logical slots, restart_lsn is held down by the oldest in-progress tx. =cut sub wait_for_slot_catchup { my ($self, $slot_name, $mode, $target_lsn) = @_; $mode = defined($mode) ? $mode : 'restart'; if (!($mode eq 'restart' || $mode eq 'confirmed_flush')) { croak "valid modes are restart, confirmed_flush"; } croak 'target lsn must be specified' unless defined($target_lsn); print "Waiting for replication slot " . $slot_name . "'s " . $mode . "_lsn to pass " . $target_lsn . " on " . $self->name . "\n"; my $query = qq[SELECT '$target_lsn' <= ${mode}_lsn FROM pg_catalog.pg_replication_slots WHERE slot_name = '$slot_name';]; $self->poll_query_until('postgres', $query) or croak "timed out waiting for catchup"; print "done\n"; } =pod =item $node->query_hash($dbname, $query, @columns) Execute $query on $dbname, replacing any appearance of the string __COLUMNS__ within the query with a comma-separated list of @columns. If __COLUMNS__ does not appear in the query, its result columns must EXACTLY match the order and number (but not necessarily alias) of supplied @columns. The query must return zero or one rows. Return a hash-ref representation of the results of the query, with any empty or null results as defined keys with an empty-string value. There is no way to differentiate between null and empty-string result fields. If the query returns zero rows, return a hash with all columns empty. There is no way to differentiate between zero rows returned and a row with only null columns. =cut sub query_hash { my ($self, $dbname, $query, @columns) = @_; croak 'calls in array context for multi-row results not supported yet' if (wantarray); # Replace __COLUMNS__ if found substr($query, index($query, '__COLUMNS__'), length('__COLUMNS__')) = join(', ', @columns) if index($query, '__COLUMNS__') >= 0; my $result = $self->safe_psql($dbname, $query); # hash slice, see http://stackoverflow.com/a/16755894/398670 . # # Fills the hash with empty strings produced by x-operator element # duplication if result is an empty row # my %val; @val{@columns} = $result ne '' ? split(qr/\|/, $result, -1) : ('',) x scalar(@columns); return \%val; } =pod =item $node->slot(slot_name) Return hash-ref of replication slot data for the named slot, or a hash-ref with all values '' if not found. Does not differentiate between null and empty string for fields, no field is ever undef. The restart_lsn and confirmed_flush_lsn fields are returned verbatim, and also as a 2-list of [highword, lowword] integer. Since we rely on Perl 5.8.8 we can't "use bigint", it's from 5.20, and we can't assume we have Math::Bigint from CPAN either. =cut sub slot { my ($self, $slot_name) = @_; my @columns = ( 'plugin', 'slot_type', 'datoid', 'database', 'active', 'active_pid', 'xmin', 'catalog_xmin', 'restart_lsn'); return $self->query_hash( 'postgres', "SELECT __COLUMNS__ FROM pg_catalog.pg_replication_slots WHERE slot_name = '$slot_name'", @columns); } =pod =item $node->pg_recvlogical_upto(self, dbname, slot_name, endpos, timeout_secs, ...) Invoke pg_recvlogical to read from slot_name on dbname until LSN endpos, which corresponds to pg_recvlogical --endpos. Gives up after timeout (if nonzero). Disallows pg_recvlogical from internally retrying on error by passing --no-loop. Plugin options are passed as additional keyword arguments. If called in scalar context, returns stdout, and die()s on timeout or nonzero return. If called in array context, returns a tuple of (retval, stdout, stderr, timeout). timeout is the IPC::Run::Timeout object whose is_expired method can be tested to check for timeout. retval is undef on timeout. =cut sub pg_recvlogical_upto { my ($self, $dbname, $slot_name, $endpos, $timeout_secs, %plugin_options) = @_; my ($stdout, $stderr); my $timeout_exception = 'pg_recvlogical timed out'; croak 'slot name must be specified' unless defined($slot_name); croak 'endpos must be specified' unless defined($endpos); my @cmd = ( 'pg_recvlogical', '-S', $slot_name, '--dbname', $self->connstr($dbname)); push @cmd, '--endpos', $endpos; push @cmd, '-f', '-', '--no-loop', '--start'; while (my ($k, $v) = each %plugin_options) { croak "= is not permitted to appear in replication option name" if ($k =~ qr/=/); push @cmd, "-o", "$k=$v"; } my $timeout; $timeout = IPC::Run::timeout($timeout_secs, exception => $timeout_exception) if $timeout_secs; my $ret = 0; do { local $@; eval { IPC::Run::run(\@cmd, ">", \$stdout, "2>", \$stderr, $timeout); $ret = $?; }; my $exc_save = $@; if ($exc_save) { # IPC::Run::run threw an exception. re-throw unless it's a # timeout, which we'll handle by testing is_expired die $exc_save if (blessed($exc_save) || $exc_save !~ qr/$timeout_exception/); $ret = undef; die "Got timeout exception '$exc_save' but timer not expired?!" unless $timeout->is_expired; die "$exc_save waiting for endpos $endpos with stdout '$stdout', stderr '$stderr'" unless wantarray; } }; $stdout =~ s/\r//g if $TestLib::windows_os; $stderr =~ s/\r//g if $TestLib::windows_os; if (wantarray) { return ($ret, $stdout, $stderr, $timeout); } else { die "pg_recvlogical exited with code '$ret', stdout '$stdout' and stderr '$stderr'" if $ret; return $stdout; } } =pod =back =cut 1; pg_checksums-1.1/t/perl/RecursiveCopy.pm000066400000000000000000000075111417133546400204070ustar00rootroot00000000000000 =pod =head1 NAME RecursiveCopy - simple recursive copy implementation =head1 SYNOPSIS use RecursiveCopy; RecursiveCopy::copypath($from, $to, filterfn => sub { return 1; }); RecursiveCopy::copypath($from, $to); =cut package RecursiveCopy; use strict; use warnings; use Carp; use File::Basename; use File::Copy; =pod =head1 DESCRIPTION =head2 copypath($from, $to, %params) Recursively copy all files and directories from $from to $to. Does not preserve file metadata (e.g., permissions). Only regular files and subdirectories are copied. Trying to copy other types of directory entries raises an exception. Raises an exception if a file would be overwritten, the source directory can't be read, or any I/O operation fails. However, we silently ignore ENOENT on open, because when copying from a live database it's possible for a file/dir to be deleted after we see its directory entry but before we can open it. Always returns true. If the B parameter is given, it must be a subroutine reference. This subroutine will be called for each entry in the source directory with its relative path as only parameter; if the subroutine returns true the entry is copied, otherwise the file is skipped. On failure the target directory may be in some incomplete state; no cleanup is attempted. =head1 EXAMPLES RecursiveCopy::copypath('/some/path', '/empty/dir', filterfn => sub { # omit log/ and contents my $src = shift; return $src ne 'log'; } ); =cut sub copypath { my ($base_src_dir, $base_dest_dir, %params) = @_; my $filterfn; if (defined $params{filterfn}) { croak "if specified, filterfn must be a subroutine reference" unless defined(ref $params{filterfn}) and (ref $params{filterfn} eq 'CODE'); $filterfn = $params{filterfn}; } else { $filterfn = sub { return 1; }; } # Complain if original path is bogus, because _copypath_recurse won't. croak "\"$base_src_dir\" does not exist" if !-e $base_src_dir; # Start recursive copy from current directory return _copypath_recurse($base_src_dir, $base_dest_dir, "", $filterfn); } # Recursive private guts of copypath sub _copypath_recurse { my ($base_src_dir, $base_dest_dir, $curr_path, $filterfn) = @_; my $srcpath = "$base_src_dir/$curr_path"; my $destpath = "$base_dest_dir/$curr_path"; # invoke the filter and skip all further operation if it returns false return 1 unless &$filterfn($curr_path); # Check for symlink -- needed only on source dir # (note: this will fall through quietly if file is already gone) croak "Cannot operate on symlink \"$srcpath\"" if -l $srcpath; # Abort if destination path already exists. Should we allow directories # to exist already? croak "Destination path \"$destpath\" already exists" if -e $destpath; # If this source path is a file, simply copy it to destination with the # same name and we're done. if (-f $srcpath) { my $fh; unless (open($fh, '<', $srcpath)) { return 1 if ($!{ENOENT}); die "open($srcpath) failed: $!"; } copy($fh, $destpath) or die "copy $srcpath -> $destpath failed: $!"; close $fh; return 1; } # If it's a directory, create it on dest and recurse into it. if (-d $srcpath) { my $directory; unless (opendir($directory, $srcpath)) { return 1 if ($!{ENOENT}); die "opendir($srcpath) failed: $!"; } mkdir($destpath) or die "mkdir($destpath) failed: $!"; while (my $entry = readdir($directory)) { next if ($entry eq '.' or $entry eq '..'); _copypath_recurse($base_src_dir, $base_dest_dir, $curr_path eq '' ? $entry : "$curr_path/$entry", $filterfn) or die "copypath $srcpath/$entry -> $destpath/$entry failed"; } closedir($directory); return 1; } # If it disappeared from sight, that's OK. return 1 if !-e $srcpath; # Else it's some weird file type; complain. croak "Source path \"$srcpath\" is not a regular file or directory"; } 1; pg_checksums-1.1/t/perl/SimpleTee.pm000066400000000000000000000011361417133546400174710ustar00rootroot00000000000000# A simple 'tee' implementation, using perl tie. # # Whenever you print to the handle, it gets forwarded to a list of # handles. The list of output filehandles is passed to the constructor. # # This is similar to IO::Tee, but only used for output. Only the PRINT # method is currently implemented; that's all we need. We don't want to # depend on IO::Tee just for this. package SimpleTee; use strict; sub TIEHANDLE { my $self = shift; bless \@_, $self; } sub PRINT { my $self = shift; my $ok = 1; for my $fh (@$self) { print $fh @_ or $ok = 0; $fh->flush or $ok = 0; } return $ok; } 1; pg_checksums-1.1/t/perl/TestLib.pm000066400000000000000000000277701417133546400171640ustar00rootroot00000000000000# TestLib, low-level routines and actions regression tests. # # This module contains a set of routines dedicated to environment setup for # a PostgreSQL regression test run and includes some low-level routines # aimed at controlling command execution, logging and test functions. This # module should never depend on any other PostgreSQL regression test modules. package TestLib; use strict; use warnings; use Config; use Cwd; use Exporter 'import'; use Fcntl qw(:mode); use File::Basename; use File::Find; use File::Spec; use File::stat qw(stat); use File::Temp (); use IPC::Run; use SimpleTee; # specify a recent enough version of Test::More to support the done_testing() function use Test::More 0.87; our @EXPORT = qw( generate_ascii_string slurp_dir slurp_file append_to_file check_mode_recursive chmod_recursive check_pg_config system_or_bail system_log run_log command_ok command_fails command_exit_is program_help_ok program_version_ok program_options_handling_ok command_like command_like_safe command_fails_like command_checks_all $windows_os ); our ($windows_os, $tmp_check, $log_path, $test_logfile); BEGIN { # Set to untranslated messages, to be able to compare program output # with expected strings. delete $ENV{LANGUAGE}; delete $ENV{LC_ALL}; $ENV{LC_MESSAGES} = 'C'; delete $ENV{PGCONNECT_TIMEOUT}; delete $ENV{PGDATA}; delete $ENV{PGDATABASE}; delete $ENV{PGHOSTADDR}; delete $ENV{PGREQUIRESSL}; delete $ENV{PGSERVICE}; delete $ENV{PGSSLMODE}; delete $ENV{PGUSER}; delete $ENV{PGPORT}; delete $ENV{PGHOST}; $ENV{PGAPPNAME} = basename($0); # Must be set early $windows_os = $Config{osname} eq 'MSWin32' || $Config{osname} eq 'msys'; } INIT { # Return EPIPE instead of killing the process with SIGPIPE. An affected # test may still fail, but it's more likely to report useful facts. $SIG{PIPE} = 'IGNORE'; # Determine output directories, and create them. The base path is the # TESTDIR environment variable, which is normally set by the invoking # Makefile. $tmp_check = $ENV{TESTDIR} ? "$ENV{TESTDIR}/tmp_check" : "tmp_check"; $log_path = "$tmp_check/log"; mkdir $tmp_check; mkdir $log_path; # Open the test log file, whose name depends on the test name. $test_logfile = basename($0); $test_logfile =~ s/\.[^.]+$//; $test_logfile = "$log_path/regress_log_$test_logfile"; open my $testlog, '>', $test_logfile or die "could not open STDOUT to logfile \"$test_logfile\": $!"; # Hijack STDOUT and STDERR to the log file open(my $orig_stdout, '>&', \*STDOUT); open(my $orig_stderr, '>&', \*STDERR); open(STDOUT, '>&', $testlog); open(STDERR, '>&', $testlog); # The test output (ok ...) needs to be printed to the original STDOUT so # that the 'prove' program can parse it, and display it to the user in # real time. But also copy it to the log file, to provide more context # in the log. my $builder = Test::More->builder; my $fh = $builder->output; tie *$fh, "SimpleTee", $orig_stdout, $testlog; $fh = $builder->failure_output; tie *$fh, "SimpleTee", $orig_stderr, $testlog; # Enable auto-flushing for all the file handles. Stderr and stdout are # redirected to the same file, and buffering causes the lines to appear # in the log in confusing order. autoflush STDOUT 1; autoflush STDERR 1; autoflush $testlog 1; } END { # Preserve temporary directory for this test on failure $File::Temp::KEEP_ALL = 1 unless all_tests_passing(); } sub all_tests_passing { my $fail_count = 0; foreach my $status (Test::More->builder->summary) { return 0 unless $status; } return 1; } # # Helper functions # sub tempdir { my ($prefix) = @_; $prefix = "tmp_test" unless defined $prefix; return File::Temp::tempdir( $prefix . '_XXXX', DIR => $tmp_check, CLEANUP => 1); } sub tempdir_short { # Use a separate temp dir outside the build tree for the # Unix-domain socket, to avoid file name length issues. return File::Temp::tempdir(CLEANUP => 1); } # Return the real directory for a virtual path directory under msys. # The directory must exist. If it's not an existing directory or we're # not under msys, return the input argument unchanged. sub real_dir { my $dir = "$_[0]"; return $dir unless -d $dir; return $dir unless $Config{osname} eq 'msys'; my $here = cwd; chdir $dir; # this odd way of calling 'pwd -W' is the only way that seems to work. $dir = qx{sh -c "pwd -W"}; chomp $dir; chdir $here; return $dir; } sub system_log { print("# Running: " . join(" ", @_) . "\n"); return system(@_); } sub system_or_bail { if (system_log(@_) != 0) { BAIL_OUT("system $_[0] failed"); } } sub run_log { print("# Running: " . join(" ", @{ $_[0] }) . "\n"); return IPC::Run::run(@_); } # Generate a string made of the given range of ASCII characters sub generate_ascii_string { my ($from_char, $to_char) = @_; my $res; for my $i ($from_char .. $to_char) { $res .= sprintf("%c", $i); } return $res; } sub slurp_dir { my ($dir) = @_; opendir(my $dh, $dir) or die "could not opendir \"$dir\": $!"; my @direntries = readdir $dh; closedir $dh; return @direntries; } sub slurp_file { my ($filename) = @_; local $/; open(my $in, '<', $filename) or die "could not read \"$filename\": $!"; my $contents = <$in>; close $in; $contents =~ s/\r//g if $Config{osname} eq 'msys'; return $contents; } sub append_to_file { my ($filename, $str) = @_; open my $fh, ">>", $filename or die "could not write \"$filename\": $!"; print $fh $str; close $fh; } # Check that all file/dir modes in a directory match the expected values, # ignoring the mode of any specified files. sub check_mode_recursive { my ($dir, $expected_dir_mode, $expected_file_mode, $ignore_list) = @_; # Result defaults to true my $result = 1; find ( {follow_fast => 1, wanted => sub { my $file_stat = stat($File::Find::name); # Is file in the ignore list? foreach my $ignore ($ignore_list ? @{$ignore_list} : []) { if ("$dir/$ignore" eq $File::Find::name) { return; } } defined($file_stat) or die("unable to stat $File::Find::name"); my $file_mode = S_IMODE($file_stat->mode); # Is this a file? if (S_ISREG($file_stat->mode)) { if ($file_mode != $expected_file_mode) { print(*STDERR, sprintf("$File::Find::name mode must be %04o\n", $expected_file_mode)); $result = 0; return; } } # Else a directory? elsif (S_ISDIR($file_stat->mode)) { if ($file_mode != $expected_dir_mode) { print(*STDERR, sprintf("$File::Find::name mode must be %04o\n", $expected_dir_mode)); $result = 0; return; } } # Else something we can't handle else { die "unknown file type for $File::Find::name"; } }}, $dir ); return $result; } # Change mode recursively on a directory sub chmod_recursive { my ($dir, $dir_mode, $file_mode) = @_; find ( {follow_fast => 1, wanted => sub { my $file_stat = stat($File::Find::name); if (defined($file_stat)) { chmod(S_ISDIR($file_stat->mode) ? $dir_mode : $file_mode, $File::Find::name) or die "unable to chmod $File::Find::name"; } }}, $dir ); } # Check presence of a given regexp within pg_config.h for the installation # where tests are running, returning a match status result depending on # that. sub check_pg_config { my ($regexp) = @_; my ($stdout, $stderr); my $result = IPC::Run::run [ 'pg_config', '--includedir' ], '>', \$stdout, '2>', \$stderr or die "could not execute pg_config"; chomp($stdout); open my $pg_config_h, '<', "$stdout/pg_config.h" or die "$!"; my $match = (grep {/^$regexp/} <$pg_config_h>); close $pg_config_h; return $match; } # # Test functions # sub command_ok { my ($cmd, $test_name) = @_; my $result = run_log($cmd); ok($result, $test_name); } sub command_fails { my ($cmd, $test_name) = @_; my $result = run_log($cmd); ok(!$result, $test_name); } sub command_exit_is { my ($cmd, $expected, $test_name) = @_; print("# Running: " . join(" ", @{$cmd}) . "\n"); my $h = IPC::Run::start $cmd; $h->finish(); # On Windows, the exit status of the process is returned directly as the # process's exit code, while on Unix, it's returned in the high bits # of the exit code (see WEXITSTATUS macro in the standard # header file). IPC::Run's result function always returns exit code >> 8, # assuming the Unix convention, which will always return 0 on Windows as # long as the process was not terminated by an exception. To work around # that, use $h->full_result on Windows instead. my $result = ($Config{osname} eq "MSWin32") ? ($h->full_results)[0] : $h->result(0); is($result, $expected, $test_name); } sub program_help_ok { my ($cmd) = @_; my ($stdout, $stderr); print("# Running: $cmd --help\n"); my $result = IPC::Run::run [ $cmd, '--help' ], '>', \$stdout, '2>', \$stderr; ok($result, "$cmd --help exit code 0"); isnt($stdout, '', "$cmd --help goes to stdout"); is($stderr, '', "$cmd --help nothing to stderr"); } sub program_version_ok { my ($cmd) = @_; my ($stdout, $stderr); print("# Running: $cmd --version\n"); my $result = IPC::Run::run [ $cmd, '--version' ], '>', \$stdout, '2>', \$stderr; ok($result, "$cmd --version exit code 0"); isnt($stdout, '', "$cmd --version goes to stdout"); is($stderr, '', "$cmd --version nothing to stderr"); } sub program_options_handling_ok { my ($cmd) = @_; my ($stdout, $stderr); print("# Running: $cmd --not-a-valid-option\n"); my $result = IPC::Run::run [ $cmd, '--not-a-valid-option' ], '>', \$stdout, '2>', \$stderr; ok(!$result, "$cmd with invalid option nonzero exit code"); isnt($stderr, '', "$cmd with invalid option prints error message"); } sub command_like { my ($cmd, $expected_stdout, $test_name) = @_; my ($stdout, $stderr); print("# Running: " . join(" ", @{$cmd}) . "\n"); my $result = IPC::Run::run $cmd, '>', \$stdout, '2>', \$stderr; ok($result, "$test_name: exit code 0"); is($stderr, '', "$test_name: no stderr"); like($stdout, $expected_stdout, "$test_name: matches"); } sub command_like_safe { # Doesn't rely on detecting end of file on the file descriptors, # which can fail, causing the process to hang, notably on Msys # when used with 'pg_ctl start' my ($cmd, $expected_stdout, $test_name) = @_; my ($stdout, $stderr); my $stdoutfile = File::Temp->new(); my $stderrfile = File::Temp->new(); print("# Running: " . join(" ", @{$cmd}) . "\n"); my $result = IPC::Run::run $cmd, '>', $stdoutfile, '2>', $stderrfile; $stdout = slurp_file($stdoutfile); $stderr = slurp_file($stderrfile); ok($result, "$test_name: exit code 0"); is($stderr, '', "$test_name: no stderr"); like($stdout, $expected_stdout, "$test_name: matches"); } sub command_fails_like { my ($cmd, $expected_stderr, $test_name) = @_; my ($stdout, $stderr); print("# Running: " . join(" ", @{$cmd}) . "\n"); my $result = IPC::Run::run $cmd, '>', \$stdout, '2>', \$stderr; ok(!$result, "$test_name: exit code not 0"); like($stderr, $expected_stderr, "$test_name: matches"); } # Run a command and check its status and outputs. # The 5 arguments are: # - cmd: ref to list for command, options and arguments to run # - ret: expected exit status # - out: ref to list of re to be checked against stdout (all must match) # - err: ref to list of re to be checked against stderr (all must match) # - test_name: name of test sub command_checks_all { my ($cmd, $expected_ret, $out, $err, $test_name) = @_; # run command my ($stdout, $stderr); print("# Running: " . join(" ", @{$cmd}) . "\n"); IPC::Run::run($cmd, '>', \$stdout, '2>', \$stderr); # See http://perldoc.perl.org/perlvar.html#%24CHILD_ERROR my $ret = $?; die "command exited with signal " . ($ret & 127) if $ret & 127; $ret = $ret >> 8; # check status ok($ret == $expected_ret, "$test_name status (got $ret vs expected $expected_ret)"); # check stdout for my $re (@$out) { like($stdout, $re, "$test_name stdout /$re/"); } # check stderr for my $re (@$err) { like($stderr, $re, "$test_name stderr /$re/"); } } 1;