pax_global_header00006660000000000000000000000064144726624530014527gustar00rootroot0000000000000052 comment=f589aa03e2e53397da8c925a750b4a089a63cc60 python-pgq-3.8/000077500000000000000000000000001447266245300135075ustar00rootroot00000000000000python-pgq-3.8/.coveragerc000066400000000000000000000000731447266245300156300ustar00rootroot00000000000000[report] exclude_lines = ^try: ^except pragma: no cover python-pgq-3.8/.github/000077500000000000000000000000001447266245300150475ustar00rootroot00000000000000python-pgq-3.8/.github/workflows/000077500000000000000000000000001447266245300171045ustar00rootroot00000000000000python-pgq-3.8/.github/workflows/ci.yml000066400000000000000000000070531447266245300202270ustar00rootroot00000000000000# # https://docs.github.com/en/actions/reference # https://github.com/actions # # uses: https://github.com/actions/checkout @v3 # uses: https://github.com/actions/setup-python @v4 # uses: https://github.com/actions/download-artifact @v3 # uses: https://github.com/actions/upload-artifact @v3 # name: CI on: pull_request: {} push: {} jobs: check: name: "Check" runs-on: ubuntu-latest strategy: matrix: test: - {PY: "3.10", TOXENV: "lint"} steps: - name: "Checkout" uses: actions/checkout@v3 - name: "Setup Python ${{matrix.test.PY}}" uses: actions/setup-python@v4 with: python-version: ${{matrix.test.PY}} - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check - name: "Test" env: TOXENV: ${{matrix.test.TOXENV}} run: python3 -m tox -r database: name: "Python ${{matrix.test.PY}} + PostgreSQL ${{matrix.test.PG}}" runs-on: ubuntu-latest strategy: matrix: test: - {PY: "3.7", PG: "11", TOXENV: "py37"} - {PY: "3.8", PG: "12", TOXENV: "py38"} - {PY: "3.9", PG: "13", TOXENV: "py39"} - {PY: "3.10", PG: "14", TOXENV: "py310"} - {PY: "3.11", PG: "15", TOXENV: "py311"} steps: - name: "Checkout" uses: actions/checkout@v3 - name: "Setup Python ${{matrix.test.PY}}" uses: actions/setup-python@v4 with: python-version: ${{matrix.test.PY}} - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check - name: "InstallDB" run: | echo "::group::apt-get-update" sudo -nH apt-get -q update sudo -nH apt-get -q install curl ca-certificates gnupg curl https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | gpg --dearmor \ | sudo -nH tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main ${{matrix.test.PG}}" \ | sudo -nH tee /etc/apt/sources.list.d/pgdg.list sudo -nH apt-get -q update echo "::endgroup::" echo "::group::apt-get-install" # disable new cluster creation sudo -nH mkdir -p /etc/postgresql-common/createcluster.d echo "create_main_cluster = false" | sudo -nH tee /etc/postgresql-common/createcluster.d/no-main.conf sudo -nH apt-get -qyu install postgresql-${{matrix.test.PG}} pgqd postgresql-${{matrix.test.PG}}-pgq-node echo "::endgroup::" # tune environment echo "/usr/lib/postgresql/${{matrix.test.PG}}/bin" >> $GITHUB_PATH echo "PGHOST=/tmp" >> $GITHUB_ENV - name: "StartDB" run: | rm -rf data log mkdir -p log LANG=C initdb data sed -ri -e "s,^[# ]*(unix_socket_directories).*,\\1='/tmp'," data/postgresql.conf pg_ctl -D data -l log/pg.log start || { cat log/pg.log ; exit 1; } sleep 1 - name: "CreateDB" run: | psql -d postgres -c "create database testdb" psql -d testdb -c "create extension pgq; create extension pgq_node;" psql -d testdb -c "select pgq.create_queue('testq')" - name: "Test" env: TOXENV: ${{matrix.test.TOXENV}} TEST_Q_NAME: testq PGDATABASE: testdb run: | python3 -m tox -r -- --color=yes - name: "StopDB" run: | pg_ctl -D data stop rm -rf data log /tmp/.s.PGSQL* python-pgq-3.8/.github/workflows/release.yml000066400000000000000000000053221447266245300212510ustar00rootroot00000000000000# # This runs when version tag is pushed # name: REL on: push: tags: ["v[0-9]*"] jobs: release: name: Release runs-on: ubuntu-latest steps: - name: Checkout code id: checkout uses: actions/checkout@v3 - name: "Setup Python" uses: actions/setup-python@v4 with: python-version: "3.11" - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check - name: "Get files" run: | python3 setup.py sdist python3 -m pip wheel --disable-pip-version-check --no-deps -w dist dist/* ls -l dist - name: "Install pandoc" run: | sudo -nH apt-get -u -y install pandoc pandoc --version - name: "Prepare" run: | PACKAGE=$(python3 setup.py --name) VERSION=$(python3 setup.py --version) TGZ="${PACKAGE}-${VERSION}.tar.gz" # default - gh:release, pypi # PRERELEASE - gh:prerelease, pypi # DRAFT - gh:draft,prerelease, testpypi PRERELEASE="false"; DRAFT="false" case "${VERSION}" in *[ab]*|*rc*) PRERELEASE="true";; *dev*) PRERELEASE="true"; DRAFT="true";; esac test "${{github.ref}}" = "refs/tags/v${VERSION}" || { echo "ERR: tag mismatch"; exit 1; } test -f "dist/${TGZ}" || { echo "ERR: sdist failed"; exit 1; } echo "PACKAGE=${PACKAGE}" >> $GITHUB_ENV echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "TGZ=${TGZ}" >> $GITHUB_ENV echo "PRERELEASE=${PRERELEASE}" >> $GITHUB_ENV echo "DRAFT=${DRAFT}" >> $GITHUB_ENV mkdir -p tmp make -s shownote > tmp/note.md cat tmp/note.md ls -l dist - name: "Create Github release" env: GH_TOKEN: ${{secrets.GITHUB_TOKEN}} run: | title="${PACKAGE} v${VERSION}" ghf="--notes-file=./tmp/note.md" if test "${DRAFT}" = "true"; then ghf="${ghf} --draft"; fi if test "${PRERELEASE}" = "true"; then ghf="${ghf} --prerelease"; fi gh release create "v${VERSION}" "dist/${TGZ}" --title="${title}" ${ghf} - name: "Upload to PYPI" id: pypi_upload env: PYPI_TOKEN: ${{secrets.PYPI_TOKEN}} PYPI_TEST_TOKEN: ${{secrets.PYPI_TEST_TOKEN}} run: | ls -l dist if test "${DRAFT}" = "false"; then python -m twine upload -u __token__ -p ${PYPI_TOKEN} \ --repository pypi --disable-progress-bar dist/* else python -m twine upload -u __token__ -p ${PYPI_TEST_TOKEN} \ --repository testpypi --disable-progress-bar dist/* fi python-pgq-3.8/.gitignore000066400000000000000000000002711447266245300154770ustar00rootroot00000000000000__pycache__ *.pyc *.swp *.o *.so *.egg-info *.debhelper *.log *.substvars *-stamp debian/files debian/python-* debian/python3-* cover dist .tox .coverage .pybuild MANIFEST build tmp python-pgq-3.8/MANIFEST.in000066400000000000000000000002111447266245300152370ustar00rootroot00000000000000include tests/*.py tests/*.ini include tox.ini .pylintrc .coveragerc include MANIFEST.in include README.rst NEWS.md include pgq/py.typed python-pgq-3.8/Makefile000066400000000000000000000015771447266245300151610ustar00rootroot00000000000000 VERSION = $(shell python3 setup.py --version) RXVERSION = $(shell python3 setup.py --version | sed 's/\./[.]/g') TAG = v$(VERSION) NEWS = NEWS.rst all: clean: rm -rf build *.egg-info */__pycache__ tests/*.pyc rm -rf .pybuild MANIFEST lint: tox -q -e lint xlint: tox -q -e xlint xclean: clean rm -rf .tox dist sdist: python3 setup.py -q sdist test: PGDATABASE=testdb TEST_Q_NAME=testq PGHOST=/tmp PGPORT=5120 tox -e py38 checkver: @echo "Checking version" @grep -q '^pgq $(RXVERSION)\b' $(NEWS) \ || { echo "Version '$(VERSION)' not in $(NEWS)"; exit 1; } @echo "Checking git repo" @git diff --stat --exit-code || { echo "ERROR: Unclean repo"; exit 1; } release: checkver git tag $(TAG) git push github $(TAG):$(TAG) unrelease: git push github :$(TAG) git tag -d $(TAG) shownote: awk -v VER="$(VERSION)" -f etc/note.awk $(NEWS) \ | pandoc -f rst -t gfm --wrap=none python-pgq-3.8/NEWS.rst000066400000000000000000000025161447266245300150210ustar00rootroot00000000000000NEWS ==== pgq 3.8 ------- * build: convert to pyproject.toml * typing: add full typing * ci: drop unmaintained actions pgq 3.7.3 --------- Fixes: * worker: another refactor for wait-behind - use dedicated code path for it, following main path makes things too complicated. Fixes the problem of missing node location events. * nodeinfo: set more fields on dead node pgq 3.7.2 --------- Fixes: * status: fix crash on dead node pgq 3.7.1 --------- Fixes: * worker: fix merge-leaf watermark pgq 3.7 ------- Fixes: * worker: another wait-behind fix - avoid too eager switch. Cleanups: * Additional typing improvements. pgq 3.6.2 --------- Fixes: * worker: detect if wait-behind is finished pgq 3.6.1 --------- Fixes: * status: handle root with random provider pgq 3.6 ------- Features: * takeover: handle merge workers on takeover * takeover: move switch of other subscriber before takeover pgq 3.5.2 --------- Features: * status command improvements: - Switch --compact for shorter output. - Coloring when lagging. - Order by node name. Cleanups: * Disable "universal" build (py2+3) * Fix release uploads. pgq 3.5.1 --------- Cleanups: * Drop Debian packaging * style: use compact super() * ci: use new path/env syntax * Add py.typed pgq 3.5 ------- * Enable Github actions * Drop Py2 support. * Upgrade tox setup. python-pgq-3.8/README.rst000066400000000000000000000003641447266245300152010ustar00rootroot00000000000000PgQ client library for Python ============================= Features -------- Simple consumers on top of ``pgq``: * Event producing * Base classes for consumers Cascaded consumers on top of ``pgq_node``: * Worker base class * Admin tools python-pgq-3.8/etc/000077500000000000000000000000001447266245300142625ustar00rootroot00000000000000python-pgq-3.8/etc/note.awk000066400000000000000000000004601447266245300157330ustar00rootroot00000000000000# extract version notes for version VER /^[-_0-9a-zA-Z]+ v?[0-9]/ { if ($2 == VER) { good = 1 next } else { good = 0 } } /^(===|---)/ { next } { if (good) { # also remove sphinx syntax print gensub(/:(\w+):`~?([^`]+)`/, "``\\2``", "g") } } python-pgq-3.8/etc/requirements.build.txt000066400000000000000000000000631447266245300206430ustar00rootroot00000000000000setuptools>=67 wheel>=0.41 twine==4.0.2 tox==4.8.0 python-pgq-3.8/pgq/000077500000000000000000000000001447266245300142765ustar00rootroot00000000000000python-pgq-3.8/pgq/__init__.py000066400000000000000000000016151447266245300164120ustar00rootroot00000000000000"""PgQ framework for Python.""" from pgq.cascade.admin import CascadeAdmin from pgq.cascade.consumer import CascadedConsumer from pgq.cascade.nodeinfo import MemberInfo, NodeInfo, QueueInfo from pgq.cascade.worker import CascadedWorker from pgq.baseconsumer import EventList, BatchInfo from pgq.consumer import Consumer from pgq.coopconsumer import CoopConsumer from pgq.event import Event from pgq.localconsumer import LocalConsumer from pgq.producer import bulk_insert_events, insert_event from pgq.remoteconsumer import RemoteConsumer, SerialConsumer from pgq.status import PGQStatus __all__ = [ 'Event', 'Consumer', 'CoopConsumer', 'LocalConsumer', 'bulk_insert_events', 'insert_event', 'RemoteConsumer', 'SerialConsumer', 'PGQStatus', 'CascadeAdmin', 'CascadedConsumer', 'CascadedWorker', 'MemberInfo', 'NodeInfo', 'QueueInfo', 'EventList', 'BatchInfo', ] __version__ = '3.8' python-pgq-3.8/pgq/baseconsumer.py000066400000000000000000000304611447266245300173420ustar00rootroot00000000000000"""PgQ consumer framework for Python. todo: - pgq.next_batch_details() - tag_done() by default """ from typing import Optional, Sequence, List, Iterator, Union, Dict, Any import logging import sys import time import optparse import skytools from skytools.basetypes import Cursor, Connection, DictRow from pgq.event import Event __all__ = ['BaseConsumer', 'BaseBatchWalker'] EventList = Union[List[Event], "BaseBatchWalker"] BatchInfo = Dict[str, Any] class BaseBatchWalker(object): """Lazy iterator over batch events. Events are loaded using cursor. It will be given as ev_list to process_batch(). It allows: - one for loop over events - len() after that """ queue_name: str fetch_size: int sql_cursor: str curs: Cursor length: int batch_id: int fetch_status: int consumer_filter: Optional[str] log = logging.getLogger("pgq.BaseBatchWalker") def __init__(self, curs: Cursor, batch_id: int, queue_name: str, fetch_size: int = 300, consumer_filter: Optional[str] = None) -> None: self.queue_name = queue_name self.fetch_size = fetch_size self.sql_cursor = "batch_walker" self.curs = curs self.length = 0 self.batch_id = batch_id self.fetch_status = 0 # 0-not started, 1-in-progress, 2-done self.consumer_filter = consumer_filter def _make_event(self, queue_name: str, row: DictRow) -> Event: return Event(queue_name, row) def __iter__(self) -> Iterator[Event]: if self.fetch_status: raise Exception("BatchWalker: double fetch? (%d)" % self.fetch_status) self.fetch_status = 1 q = "select * from pgq.get_batch_cursor(%s, %s, %s, %s)" self.curs.execute(q, [self.batch_id, self.sql_cursor, self.fetch_size, self.consumer_filter]) # this will return first batch of rows q = "fetch %d from %s" % (self.fetch_size, self.sql_cursor) while True: rows = self.curs.fetchall() self.log.debug("BaseBatchWalker.iter: fetch batch=%r rows=%r", self.batch_id, len(rows)) if not len(rows): break self.length += len(rows) for row in rows: ev = self._make_event(self.queue_name, row) yield ev # if less rows than requested, it was final block if len(rows) < self.fetch_size: break # request next block of rows self.curs.execute(q) self.curs.execute("close %s" % self.sql_cursor) self.fetch_status = 2 def __len__(self) -> int: return self.length class BaseConsumer(skytools.DBScript): """Consumer base class. Do not subclass directly (use pgq.Consumer or pgq.LocalConsumer instead) Config template:: ## Parameters for pgq.Consumer ## # queue name to read from queue_name = # override consumer name #consumer_name = %(job_name)s # filter out only events for specific tables #table_filter = table1, table2 # whether to use cursor to fetch events (0 disables) #pgq_lazy_fetch = 300 # whether to read from source size in autocommmit mode # not compatible with pgq_lazy_fetch # the actual user script on top of pgq.Consumer must also support it #pgq_autocommit = 0 # whether to wait for specified number of events, # before assigning a batch (0 disables) #pgq_batch_collect_events = 0 # whether to wait specified amount of time, # before assigning a batch (postgres interval) #pgq_batch_collect_interval = # whether to stay behind queue top (postgres interval) #pgq_keep_lag = # in how many seconds to write keepalive stats for idle consumers # this stats is used for detecting that consumer is still running #keepalive_stats = 300 """ # by default, use cursor-based fetch default_lazy_fetch: int = 300 # should reader connection be used in autocommit mode pgq_autocommit: int = 0 # proper variables consumer_name: str queue_name: str # compat variables pgq_queue_name: Optional[str] = None pgq_consumer_id: Optional[str] = None pgq_lazy_fetch: Optional[int] = None pgq_min_count: Optional[int] = None pgq_min_interval: Optional[str] = None pgq_min_lag: Optional[str] = None batch_info: Optional[BatchInfo] = None consumer_filter: Optional[str] = None keepalive_stats: int # statistics: time spent waiting for events idle_start: float stat_batch_start: float _batch_walker_class = BaseBatchWalker def __init__(self, service_name: str, db_name: str, args: Sequence[str]) -> None: """Initialize new consumer. @param service_name: service_name for DBScript @param db_name: name of database for get_database() @param args: cmdline args for DBScript """ super().__init__(service_name, args) self.db_name = db_name # compat params self.consumer_name = self.cf.get("pgq_consumer_id", '') self.queue_name = self.cf.get("pgq_queue_name", '') # proper params if not self.consumer_name: self.consumer_name = self.cf.get("consumer_name", self.job_name) if not self.queue_name: self.queue_name = self.cf.get("queue_name") self.stat_batch_start = 0 # compat vars self.pgq_queue_name = self.queue_name self.consumer_id = self.consumer_name # set default just once self.pgq_autocommit = self.cf.getint("pgq_autocommit", self.pgq_autocommit) if self.pgq_autocommit and self.pgq_lazy_fetch: raise skytools.UsageError("pgq_autocommit is not compatible with pgq_lazy_fetch") self.set_database_defaults(self.db_name, autocommit=self.pgq_autocommit) self.idle_start = time.time() def reload(self) -> None: skytools.DBScript.reload(self) self.pgq_lazy_fetch = self.cf.getint("pgq_lazy_fetch", self.default_lazy_fetch) # set following ones to None if not set self.pgq_min_count = self.cf.getint("pgq_batch_collect_events", 0) or None self.pgq_min_interval = self.cf.get("pgq_batch_collect_interval", '') or None self.pgq_min_lag = self.cf.get("pgq_keep_lag", '') or None # filter out specific tables only tfilt = [] for t in self.cf.getlist('table_filter', []): tfilt.append(skytools.quote_literal(skytools.fq_name(t))) if len(tfilt) > 0: expr = "ev_extra1 in (%s)" % ','.join(tfilt) self.consumer_filter = expr self.keepalive_stats = self.cf.getint("keepalive_stats", 300) def startup(self) -> None: """Handle commands here. __init__ does not have error logging.""" if self.options.register: self.register_consumer() sys.exit(0) if self.options.unregister: self.unregister_consumer() sys.exit(0) return skytools.DBScript.startup(self) def init_optparse(self, parser: Optional[optparse.OptionParser] = None) -> optparse.OptionParser: p = super().init_optparse(parser) p.add_option('--register', action='store_true', help='register consumer on queue') p.add_option('--unregister', action='store_true', help='unregister consumer from queue') return p def process_event(self, db: Connection, event: Event) -> None: """Process one event. Should be overridden by user code. """ raise Exception("needs to be implemented") def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: """Process all events in batch. By default calls process_event for each. Can be overridden by user code. """ for ev in event_list: self.process_event(db, ev) def work(self) -> Optional[int]: """Do the work loop, once (internal). Returns: true if wants to be called again, false if script can sleep. """ db = self.get_database(self.db_name) curs = db.cursor() self.stat_start() # acquire batch batch_id = self._load_next_batch(curs) db.commit() if batch_id is None: return 0 # load events ev_list = self._load_batch_events(curs, batch_id) db.commit() # process events self._launch_process_batch(db, batch_id, ev_list) # done self._finish_batch(curs, batch_id, ev_list) db.commit() self.stat_end(len(ev_list)) return 1 def register_consumer(self) -> int: self.log.info("Registering consumer on source queue") db = self.get_database(self.db_name) cx = db.cursor() cx.execute("select pgq.register_consumer(%s, %s)", [self.queue_name, self.consumer_name]) res = cx.fetchone()[0] db.commit() return res def unregister_consumer(self) -> None: self.log.info("Unregistering consumer from source queue") db = self.get_database(self.db_name) cx = db.cursor() cx.execute("select pgq.unregister_consumer(%s, %s)", [self.queue_name, self.consumer_name]) db.commit() def _launch_process_batch(self, db: Connection, batch_id: int, ev_list: EventList) -> None: self.process_batch(db, batch_id, ev_list) def _make_event(self, queue_name: str, row: DictRow) -> Event: return Event(queue_name, row) def _load_batch_events_old(self, curs: Cursor, batch_id: int) -> List[Event]: """Fetch all events for this batch.""" # load events sql = "select * from pgq.get_batch_events(%d)" % batch_id if self.consumer_filter is not None: sql += " where %s" % self.consumer_filter curs.execute(sql) rows = curs.fetchall() # map them to python objects ev_list = [] for r in rows: ev = self._make_event(self.queue_name, r) ev_list.append(ev) self.log.debug("BaseConsumer._load_batch_events_old: id=%r num=%r", batch_id, len(ev_list)) return ev_list def _load_batch_events(self, curs: Cursor, batch_id: int) -> EventList: """Fetch all events for this batch.""" if self.pgq_lazy_fetch: return self._batch_walker_class(curs, batch_id, self.queue_name, self.pgq_lazy_fetch, self.consumer_filter) else: return self._load_batch_events_old(curs, batch_id) def _load_next_batch(self, curs: Cursor) -> Optional[int]: """Allocate next batch. (internal)""" q = """select * from pgq.next_batch_custom(%s, %s, %s, %s, %s)""" curs.execute(q, [self.queue_name, self.consumer_name, self.pgq_min_lag, self.pgq_min_count, self.pgq_min_interval]) inf = dict(curs.fetchone().items()) inf['tick_id'] = inf['cur_tick_id'] inf['batch_end'] = inf['cur_tick_time'] inf['batch_start'] = inf['prev_tick_time'] inf['seq_start'] = inf['prev_tick_event_seq'] inf['seq_end'] = inf['cur_tick_event_seq'] self.batch_info = inf self.log.debug( "BaseConsumer._load_next_batch: id=%r prev=%r cur=%r", inf['batch_id'], inf['prev_tick_id'], inf['cur_tick_id'], ) return self.batch_info['batch_id'] def _finish_batch(self, curs: Cursor, batch_id: int, ev_list: EventList) -> None: """Tag events and notify that the batch is done.""" self.log.debug("BaseConsumer._finish_batch: id=%r", batch_id) curs.execute("select pgq.finish_batch(%s)", [batch_id]) def stat_start(self) -> None: t = time.time() self.stat_batch_start = t if self.stat_batch_start - self.idle_start > self.keepalive_stats: self.stat_put('idle', round(self.stat_batch_start - self.idle_start, 4)) self.idle_start = t def stat_end(self, count: int) -> None: t = time.time() self.stat_put('count', count) self.stat_put('duration', round(t - self.stat_batch_start, 4)) if count > 0: # reset timer if we got some events self.stat_put('idle', round(self.stat_batch_start - self.idle_start, 4)) self.idle_start = t python-pgq-3.8/pgq/cascade/000077500000000000000000000000001447266245300156615ustar00rootroot00000000000000python-pgq-3.8/pgq/cascade/__init__.py000066400000000000000000000000401447266245300177640ustar00rootroot00000000000000"""Cascaded Queue support. """ python-pgq-3.8/pgq/cascade/admin.py000066400000000000000000001727131447266245300173360ustar00rootroot00000000000000"""Cascaded queue administration. londiste.py INI pause [NODE [CONS]] setadm.py INI pause NODE [CONS] """ ## NB: not all commands work ## import optparse import os.path import queue import sys import threading import time from typing import Dict, List, Sequence, Optional, Tuple, Any import skytools from skytools import DBError, UsageError from skytools.basetypes import Connection, Cursor, DictRow from pgq.cascade.nodeinfo import NodeInfo, QueueInfo __all__ = ['CascadeAdmin'] RESURRECT_DUMP_FILE = "resurrect-lost-events.json" command_usage = """\ %prog [options] INI CMD [subcmd args] Node Initialization: create-root NAME [PUBLIC_CONNSTR] create-branch NAME [PUBLIC_CONNSTR] --provider= create-leaf NAME [PUBLIC_CONNSTR] --provider= All of the above initialize a node Node Administration: pause Pause node worker resume Resume node worker wait-root Wait until node has caught up with root wait-provider Wait until node has caught up with provider status Show cascade state node-status Show status of local node members Show members in set Cascade layout change: change-provider --provider NEW_NODE Change where worker reads from takeover FROM_NODE [--all] [--dead] Take other node position drop-node NAME Remove node from cascade tag-dead NODE .. Tag node as dead tag-alive NODE .. Tag node as alive """ standalone_usage = """ setadm extra switches: pause/resume/change-provider: --node=NODE_NAME | --consumer=CONSUMER_NAME create-root/create-branch/create-leaf: --worker=WORKER_NAME """ class CascadeAdmin(skytools.AdminScript): """Cascaded PgQ administration.""" queue_name: Optional[str] = None queue_info: Optional[QueueInfo] = None extra_objs: List[skytools.DBObject] = [] local_node: str = '?' root_node_name: Optional[str] = None commands_without_pidfile = ['status', 'node-status', 'node-info'] def __init__(self, svc_name: str, dbname: str, args: Sequence[str], worker_setup: bool = False) -> None: super().__init__(svc_name, args) self.initial_db_name = dbname if worker_setup: self.options.worker = self.job_name self.options.consumer = self.job_name def init_optparse(self, parser: Optional[optparse.OptionParser] = None) -> optparse.OptionParser: """Add SetAdmin switches to parser.""" p = super().init_optparse(parser) usage = command_usage + standalone_usage p.set_usage(usage.strip()) g = optparse.OptionGroup(p, "actual queue admin options") g.add_option("--connstr", action="store_true", help="initial connect string") g.add_option("--provider", help="init: connect string for provider") g.add_option("--queue", help="specify queue name") g.add_option("--worker", help="create: specify worker name") g.add_option("--node", help="specify node name") g.add_option("--consumer", help="specify consumer name") g.add_option("--target", help="takeover: specify node to take over") g.add_option("--merge", help="create-node: combined queue name") g.add_option("--dead", action="append", help="tag some node as dead") g.add_option("--dead-root", action="store_true", help="tag some node as dead") g.add_option("--dead-branch", action="store_true", help="tag some node as dead") g.add_option("--sync-watermark", help="list of node names to sync with") g.add_option("--nocheck", action="store_true", help="create: do not check public connect string") g.add_option("--compact", action="store_true", help="status: print tree in compact format") p.add_option_group(g) return p def reload(self) -> None: """Reload config.""" skytools.AdminScript.reload(self) if self.options.queue: self.queue_name = self.options.queue else: self.queue_name = self.cf.get('queue_name', '') if not self.queue_name: self.queue_name = self.cf.get('pgq_queue_name', '') if not self.queue_name: raise Exception('"queue_name" not specified in config') # # Node initialization. # def cmd_install(self) -> None: db = self.get_database(self.initial_db_name) self.install_code(db) def cmd_create_root(self, *args: str) -> None: return self.create_node('root', args) def cmd_create_branch(self, *args: str) -> None: return self.create_node('branch', args) def cmd_create_leaf(self, *args: str) -> None: return self.create_node('leaf', args) def create_node(self, node_type: str, args: Sequence[str]) -> None: """Generic node init.""" if node_type not in ('root', 'branch', 'leaf'): raise Exception('unknown node type') # load node name if len(args) > 0: node_name = args[0] else: node_name = self.cf.get('node_name', '') if not node_name: raise UsageError('Node name must be given either in command line or config') # load node public location if len(args) > 1: node_location = args[1] else: node_location = self.cf.get('public_node_location', '') if not node_location: raise UsageError('Node public location must be given either in command line or config') if len(args) > 2: raise UsageError('Too many args, only node name and public connect string allowed') # load provider provider_loc = self.options.provider if not provider_loc: provider_loc = self.cf.get('initial_provider_location', '') # check if sane ok = 0 for k, _ in skytools.parse_connect_string(node_location): if k in ('host', 'service'): ok = 1 break if not ok: self.log.warning('No host= in public connect string, bad idea') # connect to database db = self.get_database(self.initial_db_name) # check if code is installed self.install_code(db) # query current status res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)", [self.queue_name]) info = res[0] if info['node_type'] is not None: self.log.info("Node is already initialized as %s", info['node_type']) return # check if public connstr is sane self.check_public_connstr(db, node_location) self.log.info("Initializing node") node_attrs = {} worker_name = self.options.worker if not worker_name: raise Exception('--worker required') combined_queue = self.options.merge if combined_queue and node_type != 'leaf': raise Exception('--merge can be used only for leafs') if self.options.sync_watermark: if node_type != 'branch': raise UsageError('--sync-watermark can be used only for branch nodes') node_attrs['sync_watermark'] = self.options.sync_watermark # register member if node_type == 'root': global_watermark = None combined_queue = None provider_name = None self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) self.exec_cmd(db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)", [self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue]) self.extra_init(node_type, db, None) else: if not provider_loc: raise Exception('Please specify --provider') root_db = self.find_root_db(provider_loc) queue_info = self.load_queue_info(root_db) # check if member already exists if queue_info.get_member(node_name) is not None: self.log.error("Node '%s' already exists", node_name) sys.exit(1) provider_db = self.get_database('provider_db', connstr=provider_loc, profile='remote') q = "select node_type, node_name from pgq_node.get_node_info(%s)" res = self.exec_query(provider_db, q, [self.queue_name]) row = res[0] if not row['node_name']: raise Exception("provider node not found") provider_name = row['node_name'] # register member on root self.exec_cmd(root_db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) # lookup provider provider = queue_info.get_member(provider_name) if not provider: self.log.error("Node %s does not exist", provider_name) sys.exit(1) # register on provider self.exec_cmd(provider_db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) rows = self.exec_cmd(provider_db, "select * from pgq_node.register_subscriber(%s, %s, %s, null)", [self.queue_name, node_name, worker_name]) global_watermark = rows[0]['global_watermark'] # initialize node itself # insert members self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) for m in queue_info.member_map.values(): self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, %s)", [self.queue_name, m.name, m.location, m.dead]) # real init self.exec_cmd(db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)", [self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue]) self.extra_init(node_type, db, provider_db) if node_attrs: s_attrs = skytools.db_urlencode(node_attrs) self.exec_cmd(db, "select * from pgq_node.set_node_attrs(%s, %s)", [self.queue_name, s_attrs]) self.log.info("Done") def check_public_connstr(self, db: Connection, pub_connstr: str) -> None: """Look if public and local connect strings point to same db's. """ if self.options.nocheck: return pub_db = self.get_database("pub_db", connstr=pub_connstr, profile='remote') curs1 = db.cursor() curs2 = pub_db.cursor() q = "select oid, datname, txid_current() as txid, txid_current_snapshot() as snap"\ " from pg_catalog.pg_database where datname = current_database()" curs1.execute(q) res1 = curs1.fetchone() db.commit() curs2.execute(q) res2 = curs2.fetchone() pub_db.commit() curs1.execute(q) res3 = curs1.fetchone() db.commit() self.close_database("pub_db") failure = 0 if (res1['oid'], res1['datname']) != (res2['oid'], res2['datname']): failure += 1 sn1 = skytools.Snapshot(res1['snap']) tx = res2['txid'] sn2 = skytools.Snapshot(res3['snap']) if sn1.contains(tx): failure += 2 elif not sn2.contains(tx): failure += 4 if failure: raise UsageError("Public connect string points to different database" " than local connect string (fail=%d)" % failure) def extra_init(self, node_type: str, node_db: Connection, provider_db: Optional[Connection]) -> None: """Callback to do specific init.""" pass def find_root_db(self, initial_loc: Optional[str] = None) -> Connection: """Find root node, having start point.""" if initial_loc: loc = initial_loc db = self.get_database('root_db', connstr=loc, profile='remote') else: loc = self.cf.get(self.initial_db_name) db = self.get_database('root_db', connstr=loc) while True: # query current status res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)", [self.queue_name]) info = res[0] node_type = info['node_type'] if node_type is None: self.log.info("Root node not initialized?") sys.exit(1) self.log.debug("db='%s' -- type='%s' provider='%s'", loc, node_type, info['provider_location']) # configured db may not be root anymore, walk upwards then if node_type in ('root', 'combined-root'): db.commit() self.root_node_name = info['node_name'] return db self.close_database('root_db') if loc == info['provider_location']: raise Exception("find_root_db: got loop: %s" % loc) loc = info['provider_location'] if loc is None: self.log.error("Sub node provider not initialized?") sys.exit(1) db = self.get_database('root_db', connstr=loc, profile='remote') raise Exception('process canceled') def find_root_node(self) -> str: self.find_root_db() return self.root_node_name or '' def find_consumer_check(self, node: str, consumer: str) -> bool: cmap = self.get_node_consumer_map(node) return consumer in cmap def find_consumer(self, node: str = '', consumer: str = '') -> Tuple[str, str]: if not node and not consumer: node = self.options.node consumer = self.options.consumer if not node and not consumer: raise Exception('Need either --node or --consumer') # specific node given if node: if consumer: if not self.find_consumer_check(node, consumer): raise Exception('Consumer not found') else: state = self.get_node_info(node) consumer = state.worker_name or '' return (node, consumer) # global consumer search if self.local_node and self.find_consumer_check(self.local_node, consumer): return (self.local_node, consumer) # fixme: dead node handling? nodelist = self.queue_info.member_map.keys() if self.queue_info else [] for xnode in nodelist: if xnode == self.local_node: continue if self.find_consumer_check(xnode, consumer): return (xnode, consumer) raise Exception('Consumer not found') def install_code(self, db: Connection) -> None: """Install cascading code to db.""" objs = [ skytools.DBLanguage("plpgsql"), #skytools.DBFunction("txid_current_snapshot", 0, sql_file="txid.sql"), skytools.DBSchema("pgq", sql="create extension pgq"), #skytools.DBFunction("pgq.get_batch_cursor", 3, sql_file="pgq.upgrade.2to3.sql"), #skytools.DBSchema("pgq_ext", sql_file="pgq_ext.sql"), # not needed actually skytools.DBSchema("pgq_node", sql="create extension pgq_node"), ] objs += self.extra_objs skytools.db_install(db.cursor(), objs, self.log) db.commit() # # Print status of whole set. # def cmd_status(self) -> None: """Show set status.""" queue_info = self.load_local_info() # prepare data for workers members: queue.Queue = queue.Queue() for m in queue_info.member_map.values(): cstr = self.add_connect_string_profile(m.location or '', 'remote') members.put((m.name, cstr)) num_nodes = len(queue_info.member_map) # launch workers and wait nodes: queue.Queue = queue.Queue() tlist = [] num_threads = max(min(num_nodes // 4, 100), 1) for _ in range(num_threads): t = threading.Thread(target=self._cmd_status_worker, args=(members, nodes)) t.daemon = True t.start() tlist.append(t) #members.join() for t in tlist: t.join() while True: try: node = nodes.get_nowait() except queue.Empty: break queue_info.add_node(node) if self.options.compact: queue_info.print_tree_compact() else: queue_info.print_tree() def _cmd_status_worker(self, members: queue.Queue, nodes: queue.Queue) -> None: # members in, nodes out, both thread-safe while True: try: node_name, node_connstr = members.get_nowait() except queue.Empty: break node = self.load_node_status(node_name, node_connstr) nodes.put(node) members.task_done() def load_node_status(self, name: str, location: str) -> NodeInfo: """ Load node info & status """ # must be thread-safe (!) if not self.node_alive(name): node = NodeInfo(self.queue_name or '', None, node_name=name, location=location) return node db = None try: db = skytools.connect_database(location) db.set_isolation_level(skytools.I_AUTOCOMMIT) curs = db.cursor() curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name]) node = NodeInfo(self.queue_name or '', curs.fetchone(), location=location) node.load_status(curs) self.load_extra_status(curs, node) except DBError as d: msg = str(d).strip().split('\n', 1)[0].strip() print('Node %r failure: %s' % (name, msg)) node = NodeInfo(self.queue_name or '', None, node_name=name, location=location) finally: if db: db.close() return node def cmd_node_status(self) -> None: """ Show status of a local node. """ queue_info = self.load_local_info() db = self.get_node_database(self.local_node) assert db curs = db.cursor() node = queue_info.local_node node.load_status(curs) self.load_extra_status(curs, node) subscriber_nodes = self.get_node_subscriber_list(self.local_node or '') offset = 4 * ' ' print(node.get_title()) print(offset + 'Provider: %s' % node.provider_node) print(offset + 'Subscribers: %s' % ', '.join(subscriber_nodes)) for l in node.get_infolines(): print(offset + l) def load_extra_status(self, curs: Cursor, node: NodeInfo) -> None: """Fetch extra info.""" # must be thread-safe (!) pass # # Normal commands. # def cmd_change_provider(self) -> None: """Change node provider.""" self.load_local_info() self.change_provider(node=self.options.node, consumer=self.options.consumer, new_provider=self.options.provider) def node_change_provider(self, node: str, new_provider: str) -> None: self.change_provider(node, new_provider=new_provider) def change_provider(self, node: str = '', consumer: str = '', new_provider: str = '') -> None: old_provider = None if not new_provider: raise Exception('Please give --provider') if not node or not consumer: node, consumer = self.find_consumer(node=node, consumer=consumer) if node == new_provider: raise UsageError("cannot subscribe to itself") cmap = self.get_node_consumer_map(node) cinfo = cmap[consumer] old_provider = cinfo['provider_node'] if old_provider == new_provider: self.log.info("Consumer '%s' at node '%s' has already '%s' as provider", consumer, node, new_provider) return # pause target node self.pause_consumer(node, consumer) # reload node info node_db = self.get_node_database(node) assert node_db qinfo = self.load_queue_info(node_db) ninfo = qinfo.local_node nmember = qinfo.get_member(node) node_location = nmember.location if nmember else '' # reload consumer info cmap = self.get_node_consumer_map(node) cinfo = cmap[consumer] # is it node worker or plain consumer? is_worker = ninfo.worker_name == consumer # fixme: expect the node to be described already q = "select * from pgq_node.register_location(%s, %s, %s, false)" self.node_cmd(new_provider, q, [self.queue_name, node, node_location]) # subscribe on new provider if is_worker: q = 'select * from pgq_node.register_subscriber(%s, %s, %s, %s)' self.node_cmd(new_provider, q, [self.queue_name, node, consumer, cinfo['last_tick_id']]) else: q = 'select * from pgq.register_consumer_at(%s, %s, %s)' self.node_cmd(new_provider, q, [self.queue_name, consumer, cinfo['last_tick_id']]) # change provider on target node q = 'select * from pgq_node.change_consumer_provider(%s, %s, %s)' self.node_cmd(node, q, [self.queue_name, consumer, new_provider]) # done self.resume_consumer(node, consumer) # unsubscribe from old provider try: if is_worker: q = "select * from pgq_node.unregister_subscriber(%s, %s)" self.node_cmd(old_provider, q, [self.queue_name, node]) else: q = "select * from pgq.unregister_consumer(%s, %s)" self.node_cmd(old_provider, q, [self.queue_name, consumer]) except skytools.DBError as d: self.log.warning("failed to unregister from old provider (%s): %s", old_provider, str(d)) def cmd_rename_node(self, old_name: str, new_name: str) -> None: """Rename node.""" self.load_local_info() root_db = self.find_root_db() # pause target node self.pause_node(old_name) node = self.load_node_info(old_name) assert node provider_node = node.provider_node or '' subscriber_list = self.get_node_subscriber_list(old_name) # create copy of member info / subscriber+queue info step1 = 'select * from pgq_node.rename_node_step1(%s, %s, %s)' # rename node itself, drop copies step2 = 'select * from pgq_node.rename_node_step2(%s, %s, %s)' # step1 self.exec_cmd(root_db, step1, [self.queue_name, old_name, new_name]) self.node_cmd(provider_node, step1, [self.queue_name, old_name, new_name]) self.node_cmd(old_name, step1, [self.queue_name, old_name, new_name]) for child in subscriber_list: self.node_cmd(child, step1, [self.queue_name, old_name, new_name]) # step1 self.node_cmd(old_name, step2, [self.queue_name, old_name, new_name]) self.node_cmd(provider_node, step1, [self.queue_name, old_name, new_name]) for child in subscriber_list: self.node_cmd(child, step2, [self.queue_name, old_name, new_name]) self.exec_cmd(root_db, step2, [self.queue_name, old_name, new_name]) # resume node self.resume_node(old_name) def cmd_drop_node(self, node_name: str) -> None: """Drop a node.""" queue_info = self.load_local_info() node = None try: node = self.load_node_info(node_name) if node: # see if we can safely drop subscriber_list = self.get_node_subscriber_list(node_name) if subscriber_list: raise UsageError('node still has subscribers') except skytools.DBError: pass try: # unregister node location from root node (event will be added to queue) if node and node.type == 'root': pass else: root_db = self.find_root_db() q = "select * from pgq_node.unregister_location(%s, %s)" self.exec_cmd(root_db, q, [self.queue_name, node_name]) except skytools.DBError as d: self.log.warning("Unregister from root failed: %s", str(d)) try: # drop node info db = self.get_node_database(node_name) args = [self.queue_name, node_name] if db: q = "select * from pgq_node.drop_node(%s, %s)" self.exec_cmd(db, q, args) else: self.log.warning("ignoring cmd for dead node '%s': %s", node_name, skytools.quote_statement(q, args)) except skytools.DBError as d: self.log.warning("Local drop failure: %s", str(d)) # brute force removal for n in queue_info.member_map.values(): try: q = "select * from pgq_node.drop_node(%s, %s)" self.node_cmd(n.name, q, [self.queue_name, node_name]) except skytools.DBError as d: self.log.warning("Failed to remove from '%s': %s", n.name, str(d)) def node_depends(self, sub_node: str, top_node: str) -> bool: cur_node = sub_node # walk upstream while True: info = self.get_node_info(cur_node) if cur_node == top_node: # yes, top_node is sub_node's provider return True if info.type == 'root': # found root, no dependancy return False # step upwards cur_node = info.provider_node or '' def demote_node(self, oldnode: str, step: int, newnode: str) -> Optional[int]: """Downgrade old root?""" q = "select * from pgq_node.demote_root(%s, %s, %s)" res = self.node_cmd(oldnode, q, [self.queue_name, step, newnode]) if res: return res[0]['last_tick'] return None def promote_branch(self, node: str) -> None: """Promote old branch as root.""" q = "select * from pgq_node.promote_branch(%s)" self.node_cmd(node, q, [self.queue_name]) def wait_for_catchup(self, new: str, last_tick: int) -> NodeInfo: """Wait until new_node catches up to old_node.""" # wait for it on subscriber info = self.load_node_info(new) assert info if info.completed_tick >= last_tick: self.log.info('tick already exists') return info if info.paused: self.log.info('new node seems paused, resuming') self.resume_node(new) while True: self.log.debug('waiting for catchup: need=%d, cur=%d', last_tick, info.completed_tick) time.sleep(1) info = self.load_node_info(new) assert info if info.completed_tick >= last_tick: return info def pause_and_wait_merge_workers(self, old_info: NodeInfo, new_info: NodeInfo) -> None: if not old_info.target_for or not new_info.target_for: if not old_info.target_for and not new_info.target_for: return raise Exception('Inconsistent targets: old-node=%r new-node=%r' % ( old_info.target_for, new_info.target_for)) old_target_for = sorted(old_info.target_for) new_target_for = sorted(new_info.target_for) if old_target_for != new_target_for: raise Exception('Inconsistent targets: old-node=%r new-node=%r' % ( old_target_for, new_target_for)) self.log.info('%s: pausing merge workers', old_info.name) old_merges = self.load_merge_queues(old_info.name, old_target_for) for other_queue_name, other_info in old_merges.items(): assert other_info self.set_paused(old_info.name, other_info.worker_name, True, queue_name=other_queue_name) # load final state old_merges = self.load_merge_queues(old_info.name, old_target_for) self.log.info('%s: waiting for merge position to catch up', new_info.name) while True: time.sleep(2) in_sync = True new_merges = self.load_merge_queues(new_info.name, new_target_for) for source_queue, old_merge_info in old_merges.items(): assert old_merge_info new_merge_info = new_merges[source_queue] if new_merge_info.last_tick != old_merge_info.last_tick: in_sync = False if in_sync: break def resume_merge_workers(self, old_node_name: str) -> None: old_info = self.get_node_info_opt(old_node_name) if not old_info or not old_info.target_for: return old_target_for = sorted(old_info.target_for) old_merges = self.load_merge_queues(old_info.name, old_target_for) for other_queue_name, other_info in old_merges.items(): self.set_paused(old_info.name, other_info.worker_name, False, queue_name=other_queue_name) self.log.info('%s: merge workers resumed', old_info.name) def load_merge_queues(self, node_name: str, queue_list: Sequence[str]) -> Dict[str, NodeInfo]: res = {} for queue_name in queue_list: node_info = self.load_other_node_info(node_name, queue_name) if node_info: res[queue_name] = node_info return res def takeover_root(self, old_node_name: str, new_node_name: str, failover: bool = False) -> None: """Root switchover.""" new_info = self.get_node_info(new_node_name) old_info = None if self.node_alive(old_node_name): # old root works, switch properly old_info = self.get_node_info(old_node_name) self.pause_and_wait_merge_workers(old_info, new_info) self.pause_node(old_node_name) self.demote_node(old_node_name, 1, new_node_name) last_tick = self.demote_node(old_node_name, 2, new_node_name) if last_tick: self.wait_for_catchup(new_node_name, last_tick) else: # find latest tick on local node q = "select * from pgq.get_queue_info(%s)" db = self.get_node_database(new_node_name) assert db curs = db.cursor() curs.execute(q, [self.queue_name]) row = curs.fetchone() last_tick = row['last_tick_id'] db.commit() # find if any other node has more ticks other_node = None other_tick = last_tick sublist = self.find_subscribers_for(old_node_name) for n in sublist: q = "select * from pgq_node.get_node_info(%s)" rows = self.node_cmd(n, q, [self.queue_name]) info = rows[0] if info['worker_last_tick'] > other_tick: other_tick = info['worker_last_tick'] other_node = n # if yes, load batches from there if other_node and other_tick: self.change_provider(new_node_name, new_provider=other_node) self.wait_for_catchup(new_node_name, other_tick) last_tick = other_tick # promote new root self.pause_node(new_node_name) self.promote_branch(new_node_name) # register old root on new root as subscriber if self.node_alive(old_node_name) and old_info: old_worker_name = old_info.worker_name else: old_worker_name = self.failover_consumer_name(old_node_name) q = 'select * from pgq_node.register_subscriber(%s, %s, %s, %s)' self.node_cmd(new_node_name, q, [self.queue_name, old_node_name, old_worker_name, last_tick]) # unregister new root from old root if new_info.provider_node: q = "select * from pgq_node.unregister_subscriber(%s, %s)" self.node_cmd(new_info.provider_node, q, [self.queue_name, new_node_name]) # launch new node self.resume_node(new_node_name) # demote & launch old node if self.node_alive(old_node_name): self.demote_node(old_node_name, 3, new_node_name) self.resume_node(old_node_name) self.resume_merge_workers(old_node_name) def takeover_nonroot(self, old_node_name: str, new_node_name: str, failover: bool) -> None: """Non-root switchover.""" if self.node_depends(new_node_name, old_node_name): # yes, old_node is new_nodes provider, # switch it around pnode = self.find_provider(old_node_name) self.node_change_provider(new_node_name, pnode) self.node_change_provider(old_node_name, new_node_name) def cmd_takeover(self, old_node_name: str) -> None: """Generic node switchover.""" self.log.info("old: %s", old_node_name) queue_info = self.load_local_info() new_node_name = self.options.node if not new_node_name: worker = self.options.consumer if not worker: raise UsageError('old node not given') if queue_info.local_node.worker_name != worker: raise UsageError('old node not given') new_node_name = self.local_node if not old_node_name: raise UsageError('old node not given') if old_node_name not in queue_info.member_map: raise UsageError('Unknown node: %s' % old_node_name) if self.options.dead_root: otype = 'root' failover = True elif self.options.dead_branch: otype = 'branch' failover = True else: onode = self.get_node_info(old_node_name) otype = onode.type failover = False if failover: self.cmd_tag_dead(old_node_name) new_node = self.get_node_info(new_node_name) if old_node_name == new_node.name: self.log.info("same node?") return # switch subscribers around if self.options.all or failover: for n in self.find_subscribers_for(old_node_name): if n != new_node_name: self.node_change_provider(n, new_node_name) # actual switch if otype == 'root': self.takeover_root(old_node_name, new_node_name, failover) else: self.takeover_nonroot(old_node_name, new_node_name, failover) def find_provider(self, node_name: str) -> str: if self.node_alive(node_name): info = self.get_node_info(node_name) return info.provider_node or '' if self.queue_info: nodelist = self.queue_info.member_map.keys() for n in nodelist: if n == node_name: continue if not self.node_alive(n): continue if node_name in self.get_node_subscriber_list(n): return n return self.find_root_node() def find_subscribers_for(self, parent_node_name: str) -> List[str]: """Find subscribers for particular node.""" if not self.queue_info: return [] # use dict to eliminate duplicates res = {} nodelist = self.queue_info.member_map.keys() for node_name in nodelist: if node_name == parent_node_name: continue if not self.node_alive(node_name): continue n = self.get_node_info_opt(node_name) if not n: continue if n.provider_node == parent_node_name: res[n.name] = 1 return list(sorted(res.keys())) def cmd_tag_dead(self, dead_node_name: str) -> None: queue_info = self.load_local_info() # tag node dead in memory self.log.info("Tagging node '%s' as dead", dead_node_name) queue_info.tag_dead(dead_node_name) # tag node dead in local node q = "select * from pgq_node.register_location(%s, %s, null, true)" self.node_cmd(self.local_node, q, [self.queue_name, dead_node_name]) # tag node dead in other nodes nodelist = queue_info.member_map.keys() for node_name in nodelist: if not self.node_alive(node_name): continue if node_name == dead_node_name: continue if node_name == self.local_node: continue try: q = "select * from pgq_node.register_location(%s, %s, null, true)" self.node_cmd(node_name, q, [self.queue_name, dead_node_name]) except DBError as d: msg = str(d).strip().split('\n', 1)[0] print('Node %s failure: %s' % (node_name, msg)) self.close_node_database(node_name) def cmd_pause(self) -> None: """Pause a node""" self.load_local_info() node, consumer = self.find_consumer() self.pause_consumer(node, consumer) def cmd_resume(self) -> None: """Resume a node from pause.""" self.load_local_info() node, consumer = self.find_consumer() self.resume_consumer(node, consumer) def cmd_members(self) -> None: """Show member list.""" self.load_local_info() db = self.get_database(self.initial_db_name) desc = 'Member info on %s@%s:' % (self.local_node, self.queue_name) q = "select node_name, dead, node_location"\ " from pgq_node.get_queue_locations(%s) order by 1" self.display_table(db, desc, q, [self.queue_name]) def cmd_node_info(self) -> None: q = self.load_local_info() n = q.local_node m = q.get_member(n.name) stlist = [] if m and m.dead: stlist.append('DEAD') if n and n.paused: stlist.append("PAUSED") if n and not n.uptodate: stlist.append("NON-UP-TO-DATE") st = ', '.join(stlist) if not st: st = 'OK' print('Node: %s Type: %s Queue: %s' % (n.name, n.type, q.queue_name)) print('Status: %s' % st) if n.type != 'root': print('Provider: %s' % n.provider_node) else: print('Provider: --') print('Connect strings:') print(' Local : %s' % self.cf.get('db')) if m: print(' Public : %s' % m.location) if n.type != 'root': print(' Provider: %s' % n.provider_location) if n.combined_queue: print('Combined Queue: %s (node type: %s)' % (n.combined_queue, n.combined_type)) def cmd_wait_root(self) -> None: """Wait for next tick from root.""" queue_info = self.load_local_info() if queue_info.local_node.type == 'root': self.log.info("Current node is root, no need to wait") return self.log.info("Finding root node") root_node = self.find_root_node() self.log.info("Root is %s", root_node) dst_db = self.get_database(self.initial_db_name) self.wait_for_node(dst_db, root_node) def cmd_wait_provider(self) -> None: """Wait for next tick from provider.""" queue_info = self.load_local_info() if queue_info.local_node.type == 'root': self.log.info("Current node is root, no need to wait") return dst_db = self.get_database(self.initial_db_name) node = queue_info.local_node.provider_node if node: self.log.info("Provider is %s", node) self.wait_for_node(dst_db, node) def wait_for_node(self, dst_db: Connection, node_name: str) -> None: """Core logic for waiting.""" self.log.info("Fetching last tick for %s", node_name) node_info = self.load_node_info(node_name) assert node_info tick_id = node_info.last_tick self.log.info("Waiting for tick > %d", tick_id) q = "select * from pgq_node.get_node_info(%s)" dst_curs = dst_db.cursor() while True: dst_curs.execute(q, [self.queue_name]) row = dst_curs.fetchone() dst_db.commit() if row['ret_code'] >= 300: self.log.warning("Problem: [%s] %s", row['ret_code'], row['ret_note']) return if row['worker_last_tick'] > tick_id: self.log.info("Got tick %d, exiting", row['worker_last_tick']) break self.sleep(2) def cmd_resurrect(self) -> None: """Convert out-of-sync old root to branch and sync queue contents. """ queue_info = self.load_local_info() db = self.get_database(self.initial_db_name) curs = db.cursor() # stop if leaf if queue_info.local_node.type == 'leaf': self.log.info("Current node is leaf, nothing to do") return # stop if dump file exists if os.path.lexists(RESURRECT_DUMP_FILE): self.log.error("Dump file exists, cannot perform resurrection: %s", RESURRECT_DUMP_FILE) sys.exit(1) # # Find failover position # self.log.info("** Searching for gravestone **") # load subscribers sub_list = [] q = "select * from pgq_node.get_subscriber_info(%s)" curs.execute(q, [self.queue_name]) for row in curs.fetchall(): sub_list.append(row['node_name']) db.commit() # find backup subscription this_node = queue_info.local_node.name failover_cons = self.failover_consumer_name(this_node) full_list = list(queue_info.member_map.keys()) done_nodes = {this_node: 1} prov_node = None root_node = None for node_name in sub_list + full_list: if node_name in done_nodes: continue done_nodes[node_name] = 1 if not self.node_alive(node_name): self.log.info('Node %s is dead, skipping', node_name) continue self.log.info('Looking on node %s', node_name) node_db = None try: node_db = self.get_node_database(node_name) assert node_db node_curs = node_db.cursor() node_curs.execute("select * from pgq.get_consumer_info(%s, %s)", [self.queue_name, failover_cons]) cons_rows = node_curs.fetchall() node_curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name]) node_info = node_curs.fetchone() node_db.commit() if len(cons_rows) == 1: if prov_node: raise Exception('Unexpected situation: there are two gravestones' ' - on nodes %s and %s' % (prov_node, node_name)) prov_node = node_name failover_tick = cons_rows[0]['last_tick'] self.log.info("Found gravestone on node: %s", node_name) if node_info['node_type'] == 'root': self.log.info("Found new root node: %s", node_name) root_node = node_name self.close_node_database(node_name) node_db = None if root_node and prov_node: break except skytools.DBError: self.log.warning("failed to check node %s", node_name) if node_db: self.close_node_database(node_name) node_db = None if not root_node: self.log.error("Cannot find new root node") sys.exit(1) if not prov_node: self.log.error("Cannot find failover position (%s)", failover_cons) sys.exit(1) # load worker state q = "select * from pgq_node.get_worker_state(%s)" rows = self.exec_cmd(db, q, [self.queue_name]) state = rows[0] # demote & pause self.log.info("** Demote & pause local node **") if queue_info.local_node.type == 'root': self.log.info('Node %s is root, demoting', this_node) q = "select * from pgq_node.demote_root(%s, %s, %s)" self.exec_cmd(db, q, [self.queue_name, 1, prov_node]) self.exec_cmd(db, q, [self.queue_name, 2, prov_node]) # change node type and set worker paused in same TX curs = db.cursor() self.exec_cmd(curs, q, [self.queue_name, 3, prov_node]) q = "select * from pgq_node.set_consumer_paused(%s, %s, true)" self.exec_cmd(curs, q, [self.queue_name, state['worker_name']]) db.commit() elif not state['paused']: # pause worker, don't wait for reaction, as it may be dead self.log.info('Node %s is branch, pausing worker: %s', this_node, state['worker_name']) q = "select * from pgq_node.set_consumer_paused(%s, %s, true)" self.exec_cmd(db, q, [self.queue_name, state['worker_name']]) else: self.log.info('Node %s is branch and worker is paused', this_node) # # Drop old consumers and subscribers # self.log.info("** Dropping old subscribers and consumers **") # unregister subscriber nodes q = "select pgq_node.unregister_subscriber(%s, %s)" for node_name in sub_list: self.log.info("Dropping old subscriber node: %s", node_name) curs.execute(q, [self.queue_name, node_name]) # unregister consumers q = "select consumer_name from pgq.get_consumer_info(%s)" curs.execute(q, [self.queue_name]) for row in curs.fetchall(): cname = row['consumer_name'] if cname[0] == '.': self.log.info("Keeping consumer: %s", cname) continue self.log.info("Dropping old consumer: %s", cname) q = "pgq.unregister_consumer(%s, %s)" curs.execute(q, [self.queue_name, cname]) db.commit() # dump events self.log.info("** Dump & delete lost events **") stats = self.resurrect_process_lost_events(db, failover_tick) self.log.info("** Subscribing %s to %s **", this_node, prov_node) # set local position self.log.info("Reset local completed pos") q = "select * from pgq_node.set_consumer_completed(%s, %s, %s)" self.exec_cmd(db, q, [self.queue_name, state['worker_name'], failover_tick]) # rename gravestone self.log.info("Rename gravestone to worker: %s", state['worker_name']) prov_db = self.get_node_database(prov_node) assert prov_db prov_curs = prov_db.cursor() q = "select * from pgq_node.unregister_subscriber(%s, %s)" self.exec_cmd(prov_curs, q, [self.queue_name, this_node], quiet=True) q = "select ret_code, ret_note, global_watermark"\ " from pgq_node.register_subscriber(%s, %s, %s, %s)" res = self.exec_cmd(prov_curs, q, [self.queue_name, this_node, state['worker_name'], failover_tick], quiet=True) global_wm = res[0]['global_watermark'] prov_db.commit() # import new global watermark self.log.info("Reset global watermark") q = "select * from pgq_node.set_global_watermark(%s, %s)" self.exec_cmd(db, q, [self.queue_name, global_wm], quiet=True) # show stats if stats: self.log.info("** Statistics **") klist = sorted(stats.keys()) for k in klist: v = stats[k] self.log.info(" %s: %s", k, v) self.log.info("** Resurrection done, worker paused **") def resurrect_process_lost_events(self, db: Connection, failover_tick: int) -> Dict[str, int]: curs = db.cursor() assert self.queue_info this_node = self.queue_info.local_node.name cons_name = this_node + '.dumper' self.log.info("Dumping lost events") # register temp consumer on queue q = "select pgq.register_consumer_at(%s, %s, %s)" curs.execute(q, [self.queue_name, cons_name, failover_tick]) db.commit() # process events as usual total_count = 0 final_tick_id = -1 stats: Dict[str, int] = {} while True: q = "select * from pgq.next_batch_info(%s, %s)" curs.execute(q, [self.queue_name, cons_name]) b = curs.fetchone() batch_id = b['batch_id'] if batch_id is None: break final_tick_id = b['cur_tick_id'] q = "select * from pgq.get_batch_events(%s)" curs.execute(q, [batch_id]) cnt = 0 for ev in curs.fetchall(): cnt += 1 total_count += 1 self.resurrect_dump_event(ev, stats, b) q = "select pgq.finish_batch(%s)" curs.execute(q, [batch_id]) if cnt > 0: db.commit() stats['dumped_count'] = total_count self.resurrect_dump_finish() self.log.info("%s events dumped", total_count) # unregiser consumer q = "select pgq.unregister_consumer(%s, %s)" curs.execute(q, [self.queue_name, cons_name]) db.commit() if failover_tick == final_tick_id: self.log.info("No batches found") return {} # # Delete the events from queue # # This is done snapshots, to make sure we delete only events # that were dumped out previously. This uses the long-tx # resistant logic described in pgq.batch_event_sql(). # # find snapshots q = "select t1.tick_snapshot as s1, t2.tick_snapshot as s2"\ " from pgq.tick t1, pgq.tick t2"\ " where t1.tick_id = %s"\ " and t2.tick_id = %s" curs.execute(q, [failover_tick, final_tick_id]) ticks = curs.fetchone() s1 = skytools.Snapshot(ticks['s1']) s2 = skytools.Snapshot(ticks['s2']) xlist = [] for tx in s1.txid_list: if s2.contains(tx): xlist.append(str(tx)) # create where clauses where1 = None if len(xlist) > 0: where1 = "ev_txid in (%s)" % (",".join(xlist),) where2 = ("ev_txid >= %d AND ev_txid <= %d" # noqa " and not txid_visible_in_snapshot(ev_txid, '%s')" " and txid_visible_in_snapshot(ev_txid, '%s')" % ( s1.xmax, s2.xmax, ticks['s1'], ticks['s2'])) # loop over all queue data tables q = "select * from pgq.queue where queue_name = %s" curs.execute(q, [self.queue_name]) row = curs.fetchone() ntables = row['queue_ntables'] tbl_pfx = row['queue_data_pfx'] schema, table = tbl_pfx.split('.') total_del_count = 0 self.log.info("Deleting lost events") for i in range(ntables): del_count = 0 self.log.debug("Deleting events from table %d", i) qtbl = "%s.%s" % (skytools.quote_ident(schema), skytools.quote_ident(table + '_' + str(i))) q = "delete from " + qtbl + " where " if where1: self.log.debug(q + where1) curs.execute(q + where1) if curs.rowcount and curs.rowcount > 0: del_count += curs.rowcount self.log.debug(q + where2) curs.execute(q + where2) if curs.rowcount and curs.rowcount > 0: del_count += curs.rowcount total_del_count += del_count self.log.debug('%d events deleted', del_count) self.log.info('%d events deleted', total_del_count) stats['deleted_count'] = total_del_count # delete new ticks q = "delete from pgq.tick t using pgq.queue q"\ " where q.queue_name = %s"\ " and t.tick_queue = q.queue_id"\ " and t.tick_id > %s"\ " and t.tick_id <= %s" curs.execute(q, [self.queue_name, failover_tick, final_tick_id]) self.log.info("%s ticks deleted", curs.rowcount) db.commit() return stats _json_dump_file = None def resurrect_dump_event(self, ev: DictRow, stats: Dict[str, Any], batch_info: DictRow) -> None: if self._json_dump_file is None: self._json_dump_file = open(RESURRECT_DUMP_FILE, "w", encoding="utf8") # pylint: disable=consider-using-with sep = '[' else: sep = ',' # create ordinary dict to avoid problems with row class and datetime d = { 'ev_id': ev['ev_id'], 'ev_type': ev['ev_type'], 'ev_data': ev['ev_data'], 'ev_extra1': ev['ev_extra1'], 'ev_extra2': ev['ev_extra2'], 'ev_extra3': ev['ev_extra3'], 'ev_extra4': ev['ev_extra4'], 'ev_time': ev['ev_time'].isoformat(), 'ev_txid': ev['ev_txid'], 'ev_retry': ev['ev_retry'], 'tick_id': batch_info['cur_tick_id'], 'prev_tick_id': batch_info['prev_tick_id'], } jsev = skytools.json_encode(d) s = sep + '\n' + jsev self._json_dump_file.write(s) def resurrect_dump_finish(self) -> None: if self._json_dump_file: self._json_dump_file.write('\n]\n') self._json_dump_file.close() self._json_dump_file = None def failover_consumer_name(self, node_name: str) -> str: return node_name + ".gravestone" # # Shortcuts for operating on nodes. # def load_local_info(self) -> QueueInfo: """fetch set info from local node.""" db = self.get_database(self.initial_db_name) queue_info = self.load_queue_info(db) self.queue_info = queue_info self.local_node = self.queue_info.local_node.name return queue_info def get_node_database(self, node_name: str) -> Optional[Connection]: """Connect to node.""" assert self.queue_info if node_name == self.queue_info.local_node.name: db = self.get_database(self.initial_db_name) else: m = self.queue_info.get_member(node_name) if not m: self.log.error("get_node_database: cannot resolve %s", node_name) sys.exit(1) #self.log.info("%s: dead=%s", m.name, m.dead) if m.dead: return None loc = m.location db = self.get_database('node.' + node_name, connstr=loc, profile='remote') return db def node_alive(self, node_name: str) -> bool: m = self.queue_info.get_member(node_name) if self.queue_info else None if not m: res = False elif m.dead: res = False else: res = True #self.log.warning('node_alive(%s) = %s', node_name, res) return res def close_node_database(self, node_name: str) -> None: """Disconnect node's connection.""" if self.queue_info and node_name == self.queue_info.local_node.name: self.close_database(self.initial_db_name) else: self.close_database("node." + node_name) def node_cmd(self, node_name: str, sql: str, args: List[Any], quiet: bool = False) -> Sequence[DictRow]: """Execute SQL command on particular node.""" db = self.get_node_database(node_name) if not db: self.log.warning("ignoring cmd for dead node '%s': %s", node_name, skytools.quote_statement(sql, args)) return [] return self.exec_cmd(db, sql, args, quiet=quiet, prefix=node_name) # # Various operation on nodes. # def set_paused(self, node: str, consumer: str, pause_flag: bool, queue_name: Optional[str] = None) -> None: """Set node pause flag and wait for confirmation.""" if not queue_name: queue_name = self.queue_name q = "select * from pgq_node.set_consumer_paused(%s, %s, %s)" self.node_cmd(node, q, [queue_name, consumer, pause_flag]) self.log.info('Waiting for worker to accept') while True: q = "select * from pgq_node.get_consumer_state(%s, %s)" stat = self.node_cmd(node, q, [queue_name, consumer], quiet=True)[0] if stat['paused'] != pause_flag: raise Exception('operation canceled? %s <> %s' % (repr(stat['paused']), repr(pause_flag))) if stat['uptodate']: op = pause_flag and "paused" or "resumed" self.log.info("Consumer '%s' on node '%s' %s", consumer, node, op) return time.sleep(1) raise Exception('process canceled') def pause_consumer(self, node: str, consumer: str) -> None: """Shortcut for pausing by name.""" self.set_paused(node, consumer, True) def resume_consumer(self, node: str, consumer: str) -> None: """Shortcut for resuming by name.""" self.set_paused(node, consumer, False) def pause_node(self, node: str) -> None: """Shortcut for pausing by name.""" state = self.get_node_info(node) self.pause_consumer(node, state.worker_name) def resume_node(self, node: str) -> None: """Shortcut for resuming by name.""" state = self.get_node_info(node) self.resume_consumer(node, state.worker_name) def subscribe_node(self, target_node: str, subscriber_node: str, tick_pos: int) -> None: """Subscribing one node to another.""" q = "select * from pgq_node.subscribe_node(%s, %s, %s)" self.node_cmd(target_node, q, [self.queue_name, subscriber_node, tick_pos]) def unsubscribe_node(self, target_node: str, subscriber_node: str) -> None: """Unsubscribing one node from another.""" q = "select * from pgq_node.unsubscribe_node(%s, %s)" self.node_cmd(target_node, q, [self.queue_name, subscriber_node]) _node_cache: Dict[str, Optional[NodeInfo]] = {} def get_node_info_opt(self, node_name: str) -> Optional[NodeInfo]: """Cached node info lookup.""" if node_name in self._node_cache: return self._node_cache[node_name] inf = self.load_node_info(node_name) self._node_cache[node_name] = inf return inf def get_node_info(self, node_name: str) -> NodeInfo: """Cached node info lookup.""" inf = self.get_node_info_opt(node_name) if not inf: raise UsageError('Unknown node: %s' % node_name) return inf def load_node_info(self, node_name: str) -> Optional[NodeInfo]: """Non-cached node info lookup.""" db = self.get_node_database(node_name) if not db: self.log.warning('load_node_info(%s): ignoring dead node', node_name) return None q = "select * from pgq_node.get_node_info(%s)" rows = self.exec_query(db, q, [self.queue_name]) m = self.queue_info.get_member(node_name) if self.queue_info else None return NodeInfo(self.queue_name or '', rows[0], location=m.location if m else None) def load_other_node_info(self, our_node_name: str, other_queue_name: str) -> Optional[NodeInfo]: """Load node info from other queue. our_node_name - node name in local queue other_queue_name - queue name located in database of our_node_name """ db = self.get_node_database(our_node_name) if not db: self.log.warning('load_node_info(%s): ignoring dead node', our_node_name) return None q = "select * from pgq_node.get_node_info(%s)" rows = self.exec_query(db, q, [other_queue_name]) return NodeInfo(other_queue_name, rows[0], location=None) def load_queue_info(self, db: Connection) -> QueueInfo: """Non-cached set info lookup.""" res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)", [self.queue_name]) info = res[0] q = "select * from pgq_node.get_queue_locations(%s)" member_list = self.exec_query(db, q, [self.queue_name]) assert self.queue_name qinf = QueueInfo(self.queue_name, info, member_list) if self.options.dead: for node in self.options.dead: self.log.info("Assuming node '%s' as dead", node) qinf.tag_dead(node) return qinf def get_node_subscriber_list(self, node_name: str) -> List[str]: """Fetch subscriber list from a node.""" q = "select node_name, node_watermark from pgq_node.get_subscriber_info(%s)" db = self.get_node_database(node_name) if db: rows = self.exec_query(db, q, [self.queue_name]) return [r['node_name'] for r in rows] return [] def get_node_consumer_map(self, node_name: str) -> Dict[str, DictRow]: """Fetch consumer list from a node.""" q = "select consumer_name, provider_node, last_tick_id from pgq_node.get_consumer_info(%s)" db = self.get_node_database(node_name) if not db: return {} rows = self.exec_query(db, q, [self.queue_name]) res = {} for r in rows: res[r['consumer_name']] = r return res if __name__ == '__main__': script = CascadeAdmin('setadm', 'node_db', sys.argv[1:], worker_setup=False) script.start() python-pgq-3.8/pgq/cascade/consumer.py000066400000000000000000000251501447266245300200710ustar00rootroot00000000000000"""Cascaded consumer. Does not maintain node, but is able to pause, resume and switch provider. """ from typing import Optional, Any, Sequence import sys import time import optparse from skytools.basetypes import Cursor, Connection, DictRow from pgq.baseconsumer import BaseConsumer, EventList, BatchInfo from pgq.event import Event PDB = '_provider_db' __all__ = ['CascadedConsumer'] class CascadedConsumer(BaseConsumer): """CascadedConsumer base class. Loads provider from target node, accepts pause/resume commands. """ _consumer_state: Optional[DictRow] target_db: str provider_connstr: Optional[str] def __init__(self, service_name: str, db_name: str, args: Sequence[str]) -> None: """Initialize new consumer. @param service_name: service_name for DBScript @param db_name: target database name for get_database() @param args: cmdline args for DBScript """ super().__init__(service_name, PDB, args) self.log.debug("__init__") self.target_db = db_name self.provider_connstr = None self._consumer_state = None def init_optparse(self, parser: Optional[optparse.OptionParser] = None) -> optparse.OptionParser: p = super().init_optparse(parser) p.add_option("--provider", help="provider location for --register") p.add_option("--rewind", action="store_true", help="change queue position according to destination") p.add_option("--reset", action="store_true", help="reset queue position on destination side") return p def startup(self) -> None: if self.options.rewind: self.rewind() sys.exit(0) if self.options.reset: self.dst_reset() sys.exit(0) return super().startup() def register_consumer(self, provider_loc: Optional[str] = None) -> int: """Register consumer on source node first, then target node.""" if not provider_loc: provider_loc = self.options.provider if not provider_loc: self.log.error('Please give provider location with --provider=') sys.exit(1) dst_db = self.get_database(self.target_db) #dst_curs = dst_db.cursor() src_db = self.get_database(PDB, connstr=provider_loc, profile='remote') src_curs = src_db.cursor() # check target info q = "select * from pgq_node.get_node_info(%s)" res = self.exec_cmd(src_db, q, [self.queue_name]) pnode = res[0]['node_name'] if not pnode: raise Exception('parent node not initialized?') # source queue super().register_consumer() # fetch pos q = "select last_tick from pgq.get_consumer_info(%s, %s)" src_curs.execute(q, [self.queue_name, self.consumer_name]) last_tick = src_curs.fetchone()['last_tick'] if not last_tick: raise Exception('registration failed?') src_db.commit() # target node q = "select * from pgq_node.register_consumer(%s, %s, %s, %s)" self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name, pnode, last_tick]) return 1 def get_consumer_state(self) -> DictRow: dst_db = self.get_database(self.target_db) q = "select * from pgq_node.get_consumer_state(%s, %s)" rows = self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name]) state = rows[0] self.log.debug("CascadedConsumer.get_consumer_state: state=%r", state) return state def get_provider_db(self, state: DictRow) -> Connection: provider_loc = state['provider_location'] return self.get_database(PDB, connstr=provider_loc, profile='remote') def unregister_consumer(self) -> None: dst_db = self.get_database(self.target_db) state = self.get_consumer_state() self.get_provider_db(state) # unregister on provider super().unregister_consumer() # unregister on subscriber q = "select * from pgq_node.unregister_consumer(%s, %s)" self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name]) def rewind(self) -> None: self.log.info("Rewinding queue") dst_db = self.get_database(self.target_db) state = self.get_consumer_state() src_db = self.get_provider_db(state) src_curs = src_db.cursor() dst_tick = state['completed_tick'] if dst_tick: q = "select pgq.register_consumer_at(%s, %s, %s)" src_curs.execute(q, [self.queue_name, self.consumer_name, dst_tick]) else: self.log.warning('No tick found on dst side') dst_db.commit() src_db.commit() def dst_reset(self) -> None: self.log.info("Resetting queue tracking on dst side") dst_db = self.get_database(self.target_db) dst_curs = dst_db.cursor() state = self.get_consumer_state() src_db = self.get_provider_db(state) src_curs = src_db.cursor() # fetch last tick from source q = "select last_tick from pgq.get_consumer_info(%s, %s)" src_curs.execute(q, [self.queue_name, self.consumer_name]) row = src_curs.fetchone() src_db.commit() # on root node we dont have consumer info if not row: self.log.info("No info about consumer, cannot reset") return # set on destination last_tick = row['last_tick'] q = "select * from pgq_node.set_consumer_completed(%s, %s, %s)" dst_curs.execute(q, [self.queue_name, self.consumer_name, last_tick]) dst_db.commit() def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: state = self._consumer_state dst_db = self.get_database(self.target_db) assert self.batch_info assert state if self.is_batch_done(state, self.batch_info, dst_db): return tick_id = self.batch_info['tick_id'] self.process_remote_batch(db, tick_id, event_list, dst_db) # this also commits self.finish_remote_batch(db, dst_db, tick_id) def process_root_node(self, dst_db: Connection) -> None: """This is called on root node, where no processing should happen. """ # extra sleep time.sleep(10 * self.loop_delay) self.log.info('{standby: 1}') def work(self) -> Optional[int]: """Refresh state before calling Consumer.work().""" dst_db = self.get_database(self.target_db) self._consumer_state = self.refresh_state(dst_db) if self._consumer_state['node_type'] == 'root': self.process_root_node(dst_db) return 0 if not self.provider_connstr: raise Exception('provider_connstr not set') self.get_provider_db(self._consumer_state) return super().work() def refresh_state(self, dst_db: Connection, full_logic: bool = True) -> DictRow: """Fetch consumer state from target node. This also sleeps if pause is set and updates "uptodate" flag to notify that data is refreshed. """ while True: q = "select * from pgq_node.get_consumer_state(%s, %s)" rows = self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name]) state = rows[0] self.log.debug("CascadedConsumer.refresh_state: state=%r", state) # tag refreshed if not state['uptodate'] and full_logic: q = "select * from pgq_node.set_consumer_uptodate(%s, %s, true)" self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name]) if state['cur_error'] and self.work_state != -1: q = "select * from pgq_node.set_consumer_error(%s, %s, NULL)" self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name]) if not state['paused'] or not full_logic: break time.sleep(self.loop_delay) # update connection loc = state['provider_location'] if self.provider_connstr != loc: self.close_database(PDB) self.provider_connstr = loc # re-initialize provider connection self.get_provider_db(state) return state def is_batch_done(self, state: DictRow, batch_info: BatchInfo, dst_db: Connection) -> bool: cur_tick = batch_info['tick_id'] prev_tick = batch_info['prev_tick_id'] dst_tick = state['completed_tick'] if not dst_tick: raise Exception('dst_tick NULL?') if prev_tick == dst_tick: # on track return False if cur_tick == dst_tick: # current batch is already applied, skip it return True # anything else means problems raise Exception('Lost position: batch %s..%s, dst has %s' % ( prev_tick, cur_tick, dst_tick)) def process_remote_batch(self, src_db: Connection, tick_id: int, event_list: EventList, dst_db: Connection) -> None: """Per-batch callback. By default just calls process_remote_event() in loop.""" src_curs = src_db.cursor() dst_curs = dst_db.cursor() for ev in event_list: self.process_remote_event(src_curs, dst_curs, ev) def process_remote_event(self, src_curs: Cursor, dst_curs: Cursor, ev: Event) -> None: """Per-event callback. By default ignores cascading events and gives error on others. Can be called from user handler to finish unprocessed events. """ if ev.ev_type[:4] == "pgq.": # ignore cascading events pass else: raise Exception('Unhandled event type in queue: %s' % ev.ev_type) def finish_remote_batch(self, src_db: Connection, dst_db: Connection, tick_id: int) -> None: """Called after event processing. This should finish work on remote db and commit there. """ self.log.debug("CascadedConsumer.finish_remote_batch: tick_id=%r", tick_id) # this also commits q = "select * from pgq_node.set_consumer_completed(%s, %s, %s)" self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name, tick_id]) def exception_hook(self, det: Any, emsg: str) -> None: try: dst_db = self.get_database(self.target_db) q = "select * from pgq_node.set_consumer_error(%s, %s, %s)" self.exec_cmd(dst_db, q, [self.queue_name, self.consumer_name, emsg]) except BaseException: self.log.warning("Failure to call pgq_node.set_consumer_error()") self.reset() super().exception_hook(det, emsg) python-pgq-3.8/pgq/cascade/nodeinfo.py000066400000000000000000000273361447266245300200470ustar00rootroot00000000000000"""Info about node/set/members. For admin tool. """ from typing import Dict, List, Optional, Sequence, Tuple, Mapping, Any, Union import datetime import re import sys from skytools.basetypes import DictRow, Cursor import skytools __all__ = ['MemberInfo', 'NodeInfo', 'QueueInfo'] # node types ROOT = 'root' BRANCH = 'branch' LEAF = 'leaf' class MemberInfo(object): """Info about set member.""" name: str location: Optional[str] dead: bool def __init__(self, row: Union[DictRow, Mapping[str, Any]]) -> None: self.name = row['node_name'] self.location = row['node_location'] self.dead = row['dead'] def ival2str(iv: datetime.timedelta) -> str: res = "" tmp, secs = divmod(iv.seconds, 60) hrs, mins = divmod(tmp, 60) if iv.days: res += "%dd" % iv.days if hrs: res += "%dh" % hrs if mins: res += "%dm" % mins res += "%ds" % secs return res class NodeInfo(object): """Detailed info about set node.""" name: str type: str global_watermark: int local_watermark: int completed_tick: int provider_node: Optional[str] provider_location: Optional[str] consumer_name: str worker_name: str paused: bool uptodate: bool combined_queue: Optional[str] combined_type: Optional[str] target_for: Optional[List[str]] last_tick: Optional[str] node_attrs: Dict[str, Optional[str]] service: Optional[str] consumer_map: Dict[str, DictRow] queue_info: Optional[DictRow] _info_lines: List[str] cascaded_consumer_map: Dict[str, DictRow] parent: Optional["NodeInfo"] child_list: List["NodeInfo"] total_childs: int levels: int def __init__(self, queue_name: str, row: Optional[DictRow], main_worker: bool = True, node_name: Optional[str] = None, location: Optional[str] = None) -> None: self.queue_name = queue_name self.main_worker = main_worker self.parent = None self.consumer_map = {} self.queue_info = None self._info_lines = [] self.cascaded_consumer_map = {} self.node_attrs = {} self.child_list = [] self.total_childs = 0 self.levels = 0 self.service = None self.provider_node = None self._row = row if location: m = re.search(r'service=(\S+)', location) if m: self.service = m.group(1) if not row: assert node_name self.name = node_name self.type = 'dead' self.last_tick = None self.paused = True self.uptodate = False return self.name = row['node_name'] self.type = row['node_type'] self.global_watermark = row['global_watermark'] self.local_watermark = row['local_watermark'] self.completed_tick = row['worker_last_tick'] self.provider_node = row['provider_node'] self.provider_location = row['provider_location'] self.consumer_name = row['worker_name'] self.worker_name = row['worker_name'] self.paused = row['worker_paused'] self.uptodate = row['worker_uptodate'] self.combined_queue = row['combined_queue'] self.combined_type = row['combined_type'] self.last_tick = row['worker_last_tick'] try: target_for = row['target_for'] if isinstance(target_for, str): nodes = skytools.parse_pgarray(target_for) if nodes: self.target_for = [n for n in nodes if n] else: self.target_for = target_for except KeyError: pass self.node_attrs = {} if 'node_attrs' in row: a = row['node_attrs'] if a: self.node_attrs = skytools.db_urldecode(a) def get_title(self) -> str: if self.service: return "%s (%s, %s)" % (self.name, self.type, self.service) return "%s (%s)" % (self.name, self.type) def get_infolines(self) -> List[str]: lst = self._info_lines lag = None if self.parent: root = self.parent while root.parent: root = root.parent if self.consumer_name: cinfo = self.parent.consumer_map.get(self.consumer_name) else: cinfo = None if cinfo and root.queue_info: tick_time = cinfo['tick_time'] root_time = root.queue_info['now'] if root_time < tick_time: # ignore negative lag - probably due to info gathering # taking long time lag = datetime.timedelta(0) else: lag = root_time - tick_time elif self.queue_info: lag = self.queue_info['ticker_lag'] c1 = "" c2 = "" use_colors = sys.stdout.isatty() if use_colors: if sys.platform.startswith('win'): use_colors = False if use_colors: if lag and lag > datetime.timedelta(minutes=1): c1 = "\033[31m" # red c2 = "\033[0m" txt = "Lag: %s%s%s" % (c1, lag and ival2str(lag) or "(n/a)", c2) if self.last_tick: txt += ", Tick: %s" % self.last_tick if self.paused: txt += ", PAUSED" if not self.uptodate: txt += ", NOT UPTODATE" lst.append(txt) for k, v in self.node_attrs.items(): txt = "Attr: %s=%s" % (k, v) lst.append(txt) for cname, row in self.cascaded_consumer_map.items(): err = row['cur_error'] if err: # show only first line pos = err.find('\n') if pos > 0: err = err[:pos] lst.append("ERR: %s: %s" % (cname, err)) return lst def add_info_line(self, ln: str) -> None: self._info_lines.append(ln) def load_status(self, curs: Cursor) -> None: self.consumer_map = {} self.queue_info = None self.cascaded_consumer_map = {} if self.queue_name: q = "select consumer_name, current_timestamp - lag as tick_time,"\ " lag, last_seen, last_tick "\ "from pgq.get_consumer_info(%s)" curs.execute(q, [self.queue_name]) for row in curs.fetchall(): cname = row['consumer_name'] self.consumer_map[cname] = row q = "select current_timestamp - ticker_lag as tick_time,"\ " ticker_lag, current_timestamp as now "\ "from pgq.get_queue_info(%s)" curs.execute(q, [self.queue_name]) self.queue_info = curs.fetchone() q = "select * from pgq_node.get_consumer_info(%s)" curs.execute(q, [self.queue_name]) for row in curs.fetchall(): cname = row['consumer_name'] self.cascaded_consumer_map[cname] = row class QueueInfo(object): """Info about cascaded queue. Slightly broken, as all info is per-node. """ member_map: Dict[str, MemberInfo] local_node: NodeInfo queue_name: str node_map: Dict[str, NodeInfo] def __init__(self, queue_name: str, info_row: DictRow, member_rows: Sequence[DictRow]) -> None: self.member_map = {} for r in member_rows: m = MemberInfo(r) self._add_member(m) cur = self.member_map.get(info_row['node_name']) self.local_node = NodeInfo(queue_name, info_row, location=cur and cur.location or None) self.queue_name = queue_name self.node_map = {} self.add_node(self.local_node) def _add_member(self, member: MemberInfo) -> None: self.member_map[member.name] = member def get_member(self, name: str) -> Optional[MemberInfo]: return self.member_map.get(name) def get_node(self, name: str) -> Optional[NodeInfo]: return self.node_map.get(name) def add_node(self, node: NodeInfo) -> None: if node.name: self.node_map[node.name] = node def tag_dead(self, node_name: str) -> None: if node_name in self.node_map: self.member_map[node_name].dead = True else: row = {'node_name': node_name, 'node_location': None, 'dead': True} m = MemberInfo(row) self.member_map[node_name] = m # # Rest is about printing the tree # _DATAFMT = "%-30s%s" def print_tree(self) -> None: """Print ascii-tree for set. Expects that data for all nodes is filled in.""" print('Queue: %s Local node: %s' % (self.queue_name, self.local_node.name)) print('') root_list = self._prepare_tree() for root in root_list: self._tree_calc(root) datalines = self._print_node(root, '', []) for ln in datalines: print(self._DATAFMT % (' ', ln)) def print_tree_compact(self) -> None: """Print ascii-tree for set in compact format. Expects that data for all nodes is filled in.""" print('Queue: %s Local node: %s' % (self.queue_name, self.local_node.name)) print('') root_list = self._prepare_tree() for root in root_list: self._tree_calc(root) self._print_node_compact(root, '') def _print_node_compact(self, node: NodeInfo, pfx: str) -> None: print(pfx + node.get_title().ljust(60-len(pfx)) + ''.join(ln.ljust(30) for ln in node.get_infolines())) for n in node.child_list: self._print_node_compact(n, pfx + ' ') def _print_node(self, node: NodeInfo, pfx: str, datalines: List[str]) -> List[str]: # print a tree fragment for node and info # returns list of unprinted data rows for ln in datalines: print(self._DATAFMT % (_setpfx(pfx, '|'), ln)) datalines = node.get_infolines() print("%s%s" % (_setpfx(pfx, '+--: '), node.get_title())) for i, n in enumerate(node.child_list): sfx = ((i < len(node.child_list) - 1) and ' |' or ' ') datalines = self._print_node(n, pfx + sfx, datalines) return datalines def _prepare_tree(self) -> List[NodeInfo]: # reset vars, fill parent and child_list for each node # returns list of root nodes (mostly 1) for node in self.node_map.values(): node.total_childs = 0 node.levels = 0 node.child_list = [] node.parent = None root_list = [] for node in self.node_map.values(): if node.type != 'root' \ and node.provider_node \ and node.provider_node != node.name \ and node.provider_node in self.node_map: p = self.node_map[node.provider_node] p.child_list.append(node) node.parent = p else: node.parent = None root_list.append(node) return root_list def _tree_calc(self, node: NodeInfo) -> None: # calculate levels and count total childs # sort the tree based on them total = len(node.child_list) levels = 1 for subnode in node.child_list: self._tree_calc(subnode) total += subnode.total_childs if levels < subnode.levels + 1: levels = subnode.levels + 1 node.total_childs = total node.levels = levels node.child_list.sort(key=_node_key) def _setpfx(pfx: str, sfx: str) -> str: if pfx: pfx = pfx[:-1] + sfx return pfx def _node_key(n: NodeInfo) -> Tuple[int, int, str, str]: return (n.levels, n.total_childs, n.service or '', n.name or '') python-pgq-3.8/pgq/cascade/worker.py000066400000000000000000000434371447266245300175570ustar00rootroot00000000000000"""Cascaded worker. CascadedConsumer that also maintains node. """ import time import datetime from typing import Sequence, List, Dict, Optional, cast import skytools from skytools.basetypes import Cursor, Connection, DictRow from pgq.cascade.consumer import CascadedConsumer from pgq.baseconsumer import BatchInfo, EventList from pgq.event import Event from pgq.producer import bulk_insert_events __all__ = ['CascadedWorker'] class WorkerState(object): """Depending on node state decides on actions worker needs to do.""" # node_type, # node_name, provider_node, # global_watermark, local_watermark # combined_queue, combined_type process_batch = 0 # handled in CascadedConsumer copy_events = 0 # ok global_wm_event = 0 # ok local_wm_publish = 1 # ok process_events = 0 # ok send_tick_event = 0 # ok wait_behind = 0 # ok process_tick_event = 0 # ok target_queue = '' # ok keep_event_ids = 0 # ok create_tick = 0 # ok filtered_copy = 0 # ok process_global_wm = 0 # ok sync_watermark = 0 # ? wm_sync_nodes: Sequence[str] = [] def __init__(self, queue_name: str, nst: DictRow) -> None: self.node_type = nst['node_type'] self.node_name = nst['node_name'] self.local_watermark = nst['local_watermark'] self.global_watermark = nst['global_watermark'] self.node_attrs = {} attrs = nst.get('node_attrs', '') if attrs: self.node_attrs = skytools.db_urldecode(attrs) ntype = nst['node_type'] ctype = nst['combined_type'] if ntype == 'root': self.global_wm_event = 1 self.local_wm_publish = 0 elif ntype == 'branch': self.target_queue = queue_name self.process_batch = 1 self.process_events = 1 self.copy_events = 1 self.process_tick_event = 1 self.keep_event_ids = 1 self.create_tick = 1 if 'sync_watermark' in self.node_attrs: slist = self.node_attrs['sync_watermark'] self.sync_watermark = 1 self.wm_sync_nodes = slist.split(',') if slist else [] else: self.process_global_wm = 1 elif ntype == 'leaf' and not ctype: self.process_batch = 1 self.process_events = 1 elif ntype == 'leaf' and ctype: self.target_queue = nst['combined_queue'] if ctype == 'root': self.process_batch = 1 self.process_events = 1 self.copy_events = 1 self.filtered_copy = 1 self.send_tick_event = 1 elif ctype == 'branch': self.process_batch = 1 self.wait_behind = 1 else: raise Exception('invalid state 1') else: raise Exception('invalid state 2') if ctype and ntype != 'leaf': raise Exception('invalid state 3') class CascadedWorker(CascadedConsumer): """CascadedWorker base class. Config fragment:: ## Parameters for pgq.CascadedWorker ## # how often the root node should push wm downstream (seconds) #global_wm_publish_period = 300 # how often the nodes should report their wm upstream (seconds) #local_wm_publish_period = 300 """ global_wm_publish_time: float = 0 global_wm_publish_period: float = 5 * 60 local_wm_publish_time: float = 0 local_wm_publish_period: float = 5 * 60 max_evbuf: int = 500 cur_event_seq: int = 0 cur_max_id: int = 0 seq_buffer: int = 10000 main_worker: bool = True _worker_state: Optional[WorkerState] = None ev_buf: List[Event] = [] real_global_wm: Optional[int] = None was_wait_behind: bool = False def reload(self) -> None: super().reload() self.global_wm_publish_period = self.cf.getfloat('global_wm_publish_period', CascadedWorker.global_wm_publish_period) self.local_wm_publish_period = self.cf.getfloat('local_wm_publish_period', CascadedWorker.local_wm_publish_period) def _load_next_batch(self, curs: Cursor) -> Optional[int]: # handle wait_behind without blocking whole loop batch_id = super()._load_next_batch(curs) wst = self._worker_state cst = self._consumer_state info = self.batch_info if batch_id is not None and wst and cst and info and wst.wait_behind: tick_id = info['tick_id'] completed_tick_id = cst['completed_tick'] ahead = tick_id > completed_tick_id if ahead: # pretend no batch was available batch_id = None self.log.debug("Wait behind: tick_id=%r completed=%r", tick_id, completed_tick_id) return batch_id def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: # switch to alternative path for wait_behind wst = self._worker_state if wst and wst.wait_behind: dst_db = self.get_database(self.target_db) self.process_wait_behind(db, batch_id, event_list, dst_db) else: super().process_batch(db, batch_id, event_list) def process_wait_behind(self, src_db: Connection, batch_id: int, event_list: EventList, dst_db: Connection) -> None: # data events are already applied to target from main (target) queue, # need to process system events from events for leaf node. src_curs = src_db.cursor() dst_curs = dst_db.cursor() for ev in event_list: if ev.ev_type.split('.', 1)[0] in ("pgq", "londiste"): self.process_remote_event(src_curs, dst_curs, ev) st = self._worker_state assert st if st.local_wm_publish and self.main_worker: self.publish_local_wm(src_db, dst_db) def process_remote_batch(self, src_db: Connection, tick_id: int, event_list: EventList, dst_db: Connection) -> None: """Worker-specific event processing.""" self.ev_buf = [] max_id = 0 assert self._worker_state st = self._worker_state src_curs = src_db.cursor() dst_curs = dst_db.cursor() for ev in event_list: if st.copy_events: self.copy_event(dst_curs, ev, st.filtered_copy) if ev.ev_type.split('.', 1)[0] in ("pgq", "londiste"): # process cascade events even on waiting leaf node self.process_remote_event(src_curs, dst_curs, ev) else: if st.process_events: self.process_remote_event(src_curs, dst_curs, ev) if ev.ev_id > max_id: max_id = ev.ev_id if max_id > self.cur_max_id: self.cur_max_id = max_id def is_batch_done(self, state: DictRow, batch_info: BatchInfo, dst_db: Connection) -> bool: # handle combined_queue type change (branch->root) if self._worker_state and self.was_wait_behind: cur_tick = batch_info['tick_id'] dst_tick = state['completed_tick'] if cur_tick <= dst_tick: # current batch is already applied, skip it return True if not self._worker_state.wait_behind: # forget previous state self.was_wait_behind = False wst = self._worker_state assert wst # check if events have processed done = super().is_batch_done(state, batch_info, dst_db) if not wst.create_tick: return done if not done: return False # check if tick is done - it happens in separate tx # fetch last tick from target queue q = "select t.tick_id from pgq.tick t, pgq.queue q"\ " where t.tick_queue = q.queue_id and q.queue_name = %s"\ " order by t.tick_queue desc, t.tick_id desc"\ " limit 1" curs = dst_db.cursor() curs.execute(q, [self.queue_name]) last_tick = curs.fetchone()['tick_id'] dst_db.commit() # insert tick if missing cur_tick = batch_info['tick_id'] if last_tick != cur_tick: prev_tick = batch_info['prev_tick_id'] tick_time = batch_info['batch_end'] if last_tick != prev_tick: raise Exception('is_batch_done: last branch tick = %d, expected %d or %d' % ( last_tick, prev_tick, cur_tick)) self.create_branch_tick(dst_db, cur_tick, tick_time) return True def publish_local_wm(self, src_db: Connection, dst_db: Connection) -> None: """Send local watermark to provider. """ t = time.time() if t - self.local_wm_publish_time < self.local_wm_publish_period: return st = self._worker_state assert st assert self.batch_info wm = st.local_watermark if st.sync_watermark: # dont send local watermark upstream wm = self.batch_info['prev_tick_id'] elif wm > self.batch_info['cur_tick_id']: # in wait-behind-leaf case, the wm from target can be # ahead from source queue, use current batch then wm = self.batch_info['cur_tick_id'] self.log.debug("Publishing local watermark: %d", wm) src_curs = src_db.cursor() q = "select * from pgq_node.set_subscriber_watermark(%s, %s, %s)" src_curs.execute(q, [self.pgq_queue_name, st.node_name, wm]) src_db.commit() # if next part fails, dont repeat it immediately self.local_wm_publish_time = t if st.sync_watermark and self.real_global_wm is not None: # instead sync 'global-watermark' with specific nodes dst_curs = dst_db.cursor() nmap = self._get_node_map(dst_curs) dst_db.commit() # local lowest wm = st.local_watermark # the global-watermark in subtree can stay behind # upstream global-watermark, but must not go ahead if self.real_global_wm < wm: wm = self.real_global_wm for node in st.wm_sync_nodes: if node == st.node_name: continue if node not in nmap: # dont ignore missing nodes - cluster may be partially set up self.log.warning('Unknown node in sync_watermark list: %s', node) return n = nmap[node] if n['dead']: # ignore dead nodes continue wmdb = self.get_database('wmdb', connstr=n['node_location'], autocommit=1, profile='remote') wmcurs = wmdb.cursor() q = 'select local_watermark from pgq_node.get_node_info(%s)' wmcurs.execute(q, [self.queue_name]) row = wmcurs.fetchone() if not row: # partially set up node? self.log.warning('Node not working: %s', node) elif row['local_watermark'] < wm: # keep lowest wm wm = row['local_watermark'] self.close_database('wmdb') # now we have lowest wm, store it q = "select pgq_node.set_global_watermark(%s, %s)" dst_curs.execute(q, [self.queue_name, wm]) dst_db.commit() def _get_node_map(self, curs: Cursor) -> Dict[str, DictRow]: q = "select node_name, node_location, dead from pgq_node.get_queue_locations(%s)" curs.execute(q, [self.queue_name]) res = {} for row in curs.fetchall(): res[row['node_name']] = row return res def process_remote_event(self, src_curs: Cursor, dst_curs: Cursor, ev: Event) -> None: """Handle cascading events. """ if ev.retry: raise Exception('CascadedWorker must not get retry events') # non cascade events send to CascadedConsumer to error out if ev.ev_type[:4] != 'pgq.': super().process_remote_event(src_curs, dst_curs, ev) return # ignore cascade events if not main worker if not self.main_worker: return # check if for right queue t = ev.ev_type if ev.ev_extra1 != self.pgq_queue_name and t != "pgq.tick-id": raise Exception("bad event in queue: " + str(ev)) self.log.debug("got cascade event: %s(%s)", t, ev.ev_data) st = self._worker_state assert st if t == "pgq.location-info": node = ev.ev_data loc = ev.ev_extra2 dead = ev.ev_extra3 q = "select * from pgq_node.register_location(%s, %s, %s, %s)" dst_curs.execute(q, [self.pgq_queue_name, node, loc, dead]) elif t == "pgq.unregister-location": node = ev.ev_data q = "select * from pgq_node.unregister_location(%s, %s)" dst_curs.execute(q, [self.pgq_queue_name, node]) elif t == "pgq.global-watermark": if st.sync_watermark: tick_id = int(ev.ev_data) self.log.debug('Half-ignoring global watermark %d', tick_id) self.real_global_wm = tick_id elif st.process_global_wm: tick_id = int(ev.ev_data) q = "select * from pgq_node.set_global_watermark(%s, %s)" dst_curs.execute(q, [self.pgq_queue_name, tick_id]) elif t == "pgq.tick-id": tick_id = int(ev.ev_data) if ev.ev_extra1 == self.pgq_queue_name: raise Exception('tick-id event for own queue?') if st.process_tick_event: q = "select * from pgq_node.set_partition_watermark(%s, %s, %s)" dst_curs.execute(q, [self.pgq_queue_name, ev.ev_extra1, tick_id]) else: raise Exception("unknown cascade event: %s" % t) def finish_remote_batch(self, src_db: Connection, dst_db: Connection, tick_id: int) -> None: """Worker-specific cleanup on target node. """ # merge-leaf on branch should not update tick pos st = self._worker_state assert st if self.main_worker: dst_curs = dst_db.cursor() self.flush_events(dst_curs) # send tick event into queue if st.send_tick_event: q = "select pgq.insert_event(%s, 'pgq.tick-id', %s, %s, null, null, null)" dst_curs.execute(q, [st.target_queue, str(tick_id), self.pgq_queue_name]) super().finish_remote_batch(src_db, dst_db, tick_id) if self.main_worker: if st.create_tick and self.batch_info: # create actual tick tick_id = self.batch_info['tick_id'] tick_time = self.batch_info['batch_end'] self.create_branch_tick(dst_db, tick_id, tick_time) if st.local_wm_publish: self.publish_local_wm(src_db, dst_db) def create_branch_tick(self, dst_db: Connection, tick_id: int, tick_time: datetime.datetime) -> None: q = "select pgq.ticker(%s, %s, %s, %s)" # execute it in autocommit mode ilev = dst_db.isolation_level dst_db.set_isolation_level(0) dst_curs = dst_db.cursor() dst_curs.execute(q, [self.pgq_queue_name, tick_id, tick_time, self.cur_max_id]) dst_db.set_isolation_level(ilev) def copy_event(self, dst_curs: Cursor, ev: Event, filtered_copy: int) -> None: """Add event to copy buffer. """ if not self.main_worker: return if filtered_copy: if ev.type[:4] == "pgq.": return if len(self.ev_buf) >= self.max_evbuf: self.flush_events(dst_curs) if ev.type == 'pgq.global-watermark': st = self._worker_state if st and st.sync_watermark: # replace payload with synced global watermark row = dict(ev._event_row.items()) row['ev_data'] = str(st.global_watermark) ev = Event(self.queue_name, cast(DictRow, row)) self.ev_buf.append(ev) def flush_events(self, dst_curs: Cursor) -> None: """Send copy buffer to target queue. """ if len(self.ev_buf) == 0: return flds = ['ev_time', 'ev_type', 'ev_data', 'ev_extra1', 'ev_extra2', 'ev_extra3', 'ev_extra4'] st = self._worker_state assert st if st.keep_event_ids: flds.append('ev_id') bulk_insert_events(dst_curs, self.ev_buf, flds, st.target_queue) self.ev_buf = [] def refresh_state(self, dst_db: Connection, full_logic: bool = True) -> DictRow: """Load also node state from target node. """ queue_name = self.pgq_queue_name or '?' res = super().refresh_state(dst_db, full_logic) q = "select * from pgq_node.get_node_info(%s)" rows = self.exec_cmd(dst_db, q, [queue_name]) assert rows self._worker_state = WorkerState(queue_name, rows[0]) if self._worker_state.wait_behind: self.was_wait_behind = True return res def process_root_node(self, dst_db: Connection) -> None: """On root node send global watermark downstream. """ super().process_root_node(dst_db) t = time.time() if t - self.global_wm_publish_time < self.global_wm_publish_period: return self.log.debug("Publishing global watermark") dst_curs = dst_db.cursor() q = "select * from pgq_node.set_global_watermark(%s, NULL)" dst_curs.execute(q, [self.pgq_queue_name]) dst_db.commit() self.global_wm_publish_time = t python-pgq-3.8/pgq/consumer.py000066400000000000000000000105371447266245300165110ustar00rootroot00000000000000"""PgQ consumer framework for Python. """ from typing import Optional, Dict, Tuple, Iterator from skytools.basetypes import Cursor, DictRow from pgq.baseconsumer import BaseBatchWalker, BaseConsumer, EventList from pgq.event import Event __all__ = ['Consumer'] # Event status codes EV_UNTAGGED = -1 EV_RETRY = 0 EV_DONE = 1 class RetriableEvent(Event): """Event which can be retried Consumer is supposed to tag them after processing. """ __slots__ = ('_status', ) _status: int def __init__(self, queue_name: str, row: DictRow) -> None: super().__init__(queue_name, row) self._status = EV_DONE def tag_done(self) -> None: self._status = EV_DONE def get_status(self) -> int: return self._status def tag_retry(self, retry_time: int = 60) -> None: self._status = EV_RETRY self.retry_time = retry_time class RetriableWalkerEvent(RetriableEvent): """Redirects status flags to RetriableBatchWalker. That way event data can be gc'd immediately and tag_done() events don't need to be remembered. """ __slots__ = ('_walker', ) _walker: "RetriableBatchWalker" def __init__(self, walker: "RetriableBatchWalker", queue_name: str, row: DictRow) -> None: super().__init__(queue_name, row) self._walker = walker def tag_done(self) -> None: self._walker.tag_event_done(self) def get_status(self) -> int: return self._walker.get_status(self) def tag_retry(self, retry_time: int = 60) -> None: self._walker.tag_event_retry(self, retry_time) class RetriableBatchWalker(BaseBatchWalker): """BatchWalker that returns RetriableEvents """ status_map: Dict[int, Tuple[int,int]] def __init__(self, curs: Cursor, batch_id: int, queue_name: str, fetch_size: int = 300, consumer_filter: Optional[str] = None) -> None: super().__init__(curs, batch_id, queue_name, fetch_size, consumer_filter) self.status_map = {} def _make_event(self, queue_name: str, row: DictRow) -> RetriableWalkerEvent: return RetriableWalkerEvent(self, queue_name, row) def tag_event_done(self, event: Event) -> None: if event.id in self.status_map: del self.status_map[event.id] def tag_event_retry(self, event: Event, retry_time: int) -> None: self.status_map[event.id] = (EV_RETRY, retry_time) def get_status(self, event: Event) -> int: return self.status_map.get(event.id, (EV_DONE, 0))[0] def iter_status(self) -> Iterator[Tuple[int, Tuple[int, int]]]: for res in self.status_map.items(): yield res class Consumer(BaseConsumer): """Normal consumer base class. Can retry events """ _batch_walker_class = RetriableBatchWalker def _make_event(self, queue_name: str, row: DictRow) -> RetriableEvent: return RetriableEvent(queue_name, row) def _flush_retry(self, curs: Cursor, batch_id: int, ev_list: EventList) -> None: """Tag retry events.""" retry = 0 if self.pgq_lazy_fetch and isinstance(ev_list, RetriableBatchWalker): for ev_id, stat in ev_list.iter_status(): if stat[0] == EV_RETRY: self._tag_retry(curs, batch_id, ev_id, stat[1]) retry += 1 elif stat[0] != EV_DONE: raise Exception("Untagged event: id=%d" % ev_id) else: for ev in ev_list: if ev._status == EV_RETRY: self._tag_retry(curs, batch_id, ev.id, ev.retry_time) retry += 1 elif ev._status != EV_DONE: raise Exception("Untagged event: (id=%d, type=%s, data=%s, ex1=%s" % ( ev.id, ev.type, ev.data, ev.extra1)) # report weird events if retry: self.stat_increase('retry-events', retry) def _finish_batch(self, curs: Cursor, batch_id: int, ev_list: EventList) -> None: """Tag events and notify that the batch is done.""" self._flush_retry(curs, batch_id, ev_list) super()._finish_batch(curs, batch_id, ev_list) def _tag_retry(self, cx: Cursor, batch_id: int, ev_id: int, retry_time: int) -> None: """Tag event for retry. (internal)""" cx.execute("select pgq.event_retry(%s, %s, %s)", [batch_id, ev_id, retry_time]) python-pgq-3.8/pgq/coopconsumer.py000066400000000000000000000053451447266245300173730ustar00rootroot00000000000000"""PgQ cooperative consumer for Python. """ from typing import Sequence, Optional from skytools.basetypes import Cursor from pgq.baseconsumer import EventList from pgq.consumer import Consumer __all__ = ['CoopConsumer'] class CoopConsumer(Consumer): """Cooperative Consumer base class. There will be one dbscript process per subconsumer. Config params:: ## pgq.CoopConsumer # name for subconsumer subconsumer_name = # pgsql interval when to consider parallel subconsumers dead, # and take over their unfinished batch #subconsumer_timeout = 1 hour """ subconsumer_name: str subconsumer_timeout: str def __init__(self, service_name: str, db_name: str, args: Sequence[str]) -> None: """Initialize new subconsumer. @param service_name: service_name for DBScript @param db_name: name of database for get_database() @param args: cmdline args for DBScript """ super().__init__(service_name, db_name, args) self.subconsumer_name = self.cf.get("subconsumer_name") self.subconsumer_timeout = self.cf.get("subconsumer_timeout", "") def register_consumer(self) -> int: """Registration for subconsumer.""" self.log.info("Registering consumer on source queue") db = self.get_database(self.db_name) cx = db.cursor() cx.execute("select pgq_coop.register_subconsumer(%s, %s, %s)", [self.queue_name, self.consumer_name, self.subconsumer_name]) res = cx.fetchone()[0] db.commit() return res def unregister_consumer(self) -> None: """Unregistration for subconsumer.""" self.log.info("Unregistering consumer from source queue") db = self.get_database(self.db_name) cx = db.cursor() cx.execute("select pgq_coop.unregister_subconsumer(%s, %s, %s, 0)", [self.queue_name, self.consumer_name, self.subconsumer_name]) db.commit() def _load_next_batch(self, curs: Cursor) -> Optional[int]: """Allocate next batch. (internal)""" if self.subconsumer_timeout: q = "select pgq_coop.next_batch(%s, %s, %s, %s)" curs.execute(q, [self.queue_name, self.consumer_name, self.subconsumer_name, self.subconsumer_timeout]) else: q = "select pgq_coop.next_batch(%s, %s, %s)" curs.execute(q, [self.queue_name, self.consumer_name, self.subconsumer_name]) return curs.fetchone()[0] def _finish_batch(self, curs: Cursor, batch_id: int, ev_list: EventList) -> None: """Finish batch. (internal)""" self._flush_retry(curs, batch_id, ev_list) curs.execute("select pgq_coop.finish_batch(%s)", [batch_id]) python-pgq-3.8/pgq/event.py000066400000000000000000000045341447266245300157770ustar00rootroot00000000000000"""PgQ event container. """ from typing import Any, Optional, Mapping, KeysView, Iterator, ValuesView, ItemsView from skytools.basetypes import DictRow __all__ = ['Event'] _fldmap = { 'ev_id': 'ev_id', 'ev_txid': 'ev_txid', 'ev_time': 'ev_time', 'ev_type': 'ev_type', 'ev_data': 'ev_data', 'ev_extra1': 'ev_extra1', 'ev_extra2': 'ev_extra2', 'ev_extra3': 'ev_extra3', 'ev_extra4': 'ev_extra4', 'ev_retry': 'ev_retry', 'id': 'ev_id', 'txid': 'ev_txid', 'time': 'ev_time', 'type': 'ev_type', 'data': 'ev_data', 'extra1': 'ev_extra1', 'extra2': 'ev_extra2', 'extra3': 'ev_extra3', 'extra4': 'ev_extra4', 'retry': 'ev_retry', } class Event(Mapping[str, Any]): """Event data for consumers. Will be removed from the queue by default. """ __slots__ = ('_event_row', 'retry_time', 'queue_name') _event_row: DictRow retry_time: int queue_name: str def __init__(self, queue_name: str, row: DictRow) -> None: self._event_row = row self.retry_time = 60 self.queue_name = queue_name def __getattr__(self, key: str) -> Any: return self._event_row[_fldmap[key]] def __iter__(self) -> Iterator[str]: return iter(self._event_row) def __len__(self) -> int: return len(self._event_row) # would be better in RetriableEvent only since we don't care but # unfortunately it needs to be defined here due to compatibility concerns def tag_done(self) -> None: pass # be also dict-like def __getitem__(self, key: str) -> Any: return self._event_row.__getitem__(key) def __contains__(self, key: object) -> bool: return self._event_row.__contains__(key) def get(self, key: str, default: Optional[Any] = None) -> Any: return self._event_row.get(key, default) def has_key(self, key: str) -> bool: return key in self._event_row def keys(self) -> KeysView[str]: return self._event_row.keys() def values(self) -> ValuesView[Any]: return self._event_row.values() def items(self) -> ItemsView[str, Any]: return self._event_row.items() def __str__(self) -> str: return "" % ( self.id, self.type, self.data, self.extra1, self.extra2, self.extra3, self.extra4) python-pgq-3.8/pgq/localconsumer.py000066400000000000000000000167131447266245300175260ustar00rootroot00000000000000"""Consumer that stores last applied position in local file. For cases where the consumer cannot use single database for remote tracking. To be subclassed, then override .process_local_batch() or .process_local_event() methods. """ from typing import Optional import errno import os import sys import optparse import skytools from skytools.basetypes import Connection from pgq.event import Event from pgq.baseconsumer import BaseConsumer, EventList __all__ = ['LocalConsumer'] class LocalConsumer(BaseConsumer): """Consumer that applies batches sequentially in second database. Requirements: - Whole batch in one TX. - Must not use retry queue. Features: - Can detect if several batches are already applied to dest db. - If some ticks are lost, allows to seek back on queue. Whether it succeeds, depends on pgq configuration. Config options:: ## Parameters for LocalConsumer ## # file location where last applied tick is tracked local_tracking_file = ~/state/%(job_name)s.tick """ def reload(self) -> None: super().reload() self.local_tracking_file = self.cf.getfile('local_tracking_file') if not os.path.exists(os.path.dirname(self.local_tracking_file)): raise skytools.UsageError("path does not exist: %s" % self.local_tracking_file) def init_optparse(self, parser: Optional[optparse.OptionParser] = None) -> optparse.OptionParser: p = super().init_optparse(parser) p.add_option("--rewind", action="store_true", help="change queue position according to local tick") p.add_option("--reset", action="store_true", help="reset local tick based on queue position") return p def startup(self) -> None: if self.options.rewind: self.rewind() sys.exit(0) if self.options.reset: self.dst_reset() sys.exit(0) super().startup() self.check_queue() def check_queue(self) -> None: queue_tick = -1 local_tick = self.load_local_tick() db = self.get_database(self.db_name) curs = db.cursor() q = "select last_tick from pgq.get_consumer_info(%s, %s)" curs.execute(q, [self.queue_name, self.consumer_name]) rows = curs.fetchall() if len(rows) == 1: queue_tick = rows[0]['last_tick'] db.commit() if queue_tick < 0: if local_tick >= 0: self.log.info("Registering consumer at tick %d", local_tick) q = "select * from pgq.register_consumer_at(%s, %s, %s)" curs.execute(q, [self.queue_name, self.consumer_name, local_tick]) else: self.log.info("Registering consumer at queue top") q = "select * from pgq.register_consumer(%s, %s)" curs.execute(q, [self.queue_name, self.consumer_name]) elif local_tick < 0: self.log.info("Local tick missing, storing queue tick %d", queue_tick) self.save_local_tick(queue_tick) elif local_tick > queue_tick: self.log.warning("Tracking out of sync: queue=%d local=%d. Repositioning on queue. [Database failure?]", queue_tick, local_tick) q = "select * from pgq.register_consumer_at(%s, %s, %s)" curs.execute(q, [self.queue_name, self.consumer_name, local_tick]) elif local_tick < queue_tick: self.log.warning("Tracking out of sync: queue=%d local=%d. Rewinding queue. [Lost file data?]", queue_tick, local_tick) q = "select * from pgq.register_consumer_at(%s, %s, %s)" curs.execute(q, [self.queue_name, self.consumer_name, local_tick]) else: self.log.info("Ticks match: Queue=%d Local=%d", queue_tick, local_tick) def work(self) -> Optional[int]: if self.work_state < 0: self.check_queue() return super().work() def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: """Process all events in batch. """ # check if done if self.is_batch_done(): return # actual work self.process_local_batch(db, batch_id, event_list) # finish work self.set_batch_done() def process_local_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: """Overridable method to process whole batch.""" for ev in event_list: self.process_local_event(db, batch_id, ev) def process_local_event(self, db: Connection, batch_id: int, ev: Event) -> None: """Overridable method to process one event at a time.""" raise Exception('process_local_event not implemented') def is_batch_done(self) -> bool: """Helper function to keep track of last successful batch in external database. """ local_tick = self.load_local_tick() assert self.batch_info cur_tick = self.batch_info['tick_id'] prev_tick = self.batch_info['prev_tick_id'] if local_tick < 0: # seems this consumer has not run yet? return False if prev_tick == local_tick: # on track return False if cur_tick == local_tick: # current batch is already applied, skip it return True # anything else means problems raise Exception('Lost position: batch %d..%d, dst has %d' % ( prev_tick, cur_tick, local_tick)) def set_batch_done(self) -> None: """Helper function to set last successful batch in external database. """ if self.batch_info: tick_id = self.batch_info['tick_id'] self.save_local_tick(tick_id) def register_consumer(self) -> int: new = super().register_consumer() if new: # fixme self.dst_reset() return new def unregister_consumer(self) -> None: """If unregistering, also clean completed tick table on dest.""" super().unregister_consumer() self.dst_reset() def rewind(self) -> None: dst_tick = self.load_local_tick() if dst_tick >= 0: src_db = self.get_database(self.db_name) src_curs = src_db.cursor() self.log.info("Rewinding queue to local tick %d", dst_tick) q = "select pgq.register_consumer_at(%s, %s, %s)" src_curs.execute(q, [self.queue_name, self.consumer_name, dst_tick]) src_db.commit() else: self.log.error('Cannot rewind, no tick found in local file') def dst_reset(self) -> None: self.log.info("Removing local tracking file") try: os.remove(self.local_tracking_file) except BaseException: pass def load_local_tick(self) -> int: """Reads stored tick or -1.""" try: with open(self.local_tracking_file, "r", encoding="utf8") as f: buf = f.read() data = buf.strip() if data: tick_id = int(data) else: tick_id = -1 return tick_id except IOError as ex: if ex.errno == errno.ENOENT: return -1 raise def save_local_tick(self, tick_id: int) -> None: """Store tick in local file.""" data = str(tick_id) skytools.write_atomic(self.local_tracking_file, data) python-pgq-3.8/pgq/producer.py000066400000000000000000000031101447266245300164660ustar00rootroot00000000000000"""PgQ producer helpers for Python. """ from typing import Sequence, Optional, Any, Mapping, Union import skytools from skytools.basetypes import Cursor __all__ = ['bulk_insert_events', 'insert_event'] _fldmap = { 'id': 'ev_id', 'time': 'ev_time', 'type': 'ev_type', 'data': 'ev_data', 'extra1': 'ev_extra1', 'extra2': 'ev_extra2', 'extra3': 'ev_extra3', 'extra4': 'ev_extra4', 'ev_id': 'ev_id', 'ev_time': 'ev_time', 'ev_type': 'ev_type', 'ev_data': 'ev_data', 'ev_extra1': 'ev_extra1', 'ev_extra2': 'ev_extra2', 'ev_extra3': 'ev_extra3', 'ev_extra4': 'ev_extra4', } def bulk_insert_events(curs: Cursor, rows: Union[Sequence[Sequence[Any]], Sequence[Mapping[str, Any]]], fields: Sequence[str], queue_name: str) -> None: q = "select pgq.current_event_table(%s)" curs.execute(q, [queue_name]) tbl = curs.fetchone()[0] db_fields = [_fldmap[name] for name in fields] skytools.magic_insert(curs, tbl, rows, db_fields) def insert_event(curs: Cursor, queue: str, ev_type: Optional[str], ev_data: Optional[str], extra1: Optional[str] = None, extra2: Optional[str] = None, extra3: Optional[str] = None, extra4: Optional[str] = None) -> int: q = "select pgq.insert_event(%s, %s, %s, %s, %s, %s, %s)" curs.execute(q, [queue, ev_type, ev_data, extra1, extra2, extra3, extra4]) return curs.fetchone()[0] python-pgq-3.8/pgq/py.typed000066400000000000000000000000001447266245300157630ustar00rootroot00000000000000python-pgq-3.8/pgq/remoteconsumer.py000066400000000000000000000147251447266245300177300ustar00rootroot00000000000000"""Old RemoteConsumer / SerialConsumer classes. """ from typing import Optional, Sequence import sys import optparse from skytools.basetypes import Cursor, Connection from pgq.baseconsumer import EventList from pgq.consumer import Consumer __all__ = ['RemoteConsumer', 'SerialConsumer'] class RemoteConsumer(Consumer): """Helper for doing event processing in another database. Requires that whole batch is processed in one TX. """ remote_db: str def __init__(self, service_name: str, db_name: str, remote_db: str, args: Sequence[str]) -> None: super().__init__(service_name, db_name, args) self.remote_db = remote_db def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: """Process all events in batch. By default calls process_event for each. """ dst_db = self.get_database(self.remote_db) curs = dst_db.cursor() if self.is_last_batch(curs, batch_id): return self.process_remote_batch(db, batch_id, event_list, dst_db) self.set_last_batch(curs, batch_id) dst_db.commit() def is_last_batch(self, dst_curs: Cursor, batch_id: int) -> bool: """Helper function to keep track of last successful batch in external database. """ q = "select pgq_ext.is_batch_done(%s, %s)" dst_curs.execute(q, [self.consumer_name, batch_id]) return dst_curs.fetchone()[0] def set_last_batch(self, dst_curs: Cursor, batch_id: int) -> None: """Helper function to set last successful batch in external database. """ q = "select pgq_ext.set_batch_done(%s, %s)" dst_curs.execute(q, [self.consumer_name, batch_id]) def process_remote_batch(self, db: Connection, batch_id: int, event_list: EventList, dst_db: Connection) -> None: raise Exception('process_remote_batch not implemented') class SerialConsumer(Consumer): """Consumer that applies batches sequentially in second database. Requirements: - Whole batch in one TX. - Must not use retry queue. Features: - Can detect if several batches are already applied to dest db. - If some ticks are lost. allows to seek back on queue. Whether it succeeds, depends on pgq configuration. """ remote_db: str dst_schema: str def __init__(self, service_name: str, db_name: str, remote_db: str, args: Sequence[str]) -> None: super().__init__(service_name, db_name, args) self.remote_db = remote_db self.dst_schema = "pgq_ext" def startup(self) -> None: if self.options.rewind: self.rewind() sys.exit(0) if self.options.reset: self.dst_reset() sys.exit(0) return Consumer.startup(self) def init_optparse(self, parser: Optional[optparse.OptionParser] = None) -> optparse.OptionParser: p = super().init_optparse(parser) p.add_option("--rewind", action="store_true", help="change queue position according to destination") p.add_option("--reset", action="store_true", help="reset queue pos on destination side") return p def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> None: """Process all events in batch. """ dst_db = self.get_database(self.remote_db) curs = dst_db.cursor() # check if done if self.is_batch_done(curs): return # actual work self.process_remote_batch(db, batch_id, event_list, dst_db) # finish work self.set_batch_done(curs) dst_db.commit() def is_batch_done(self, dst_curs: Cursor) -> bool: """Helper function to keep track of last successful batch in external database. """ assert self.batch_info cur_tick = self.batch_info['tick_id'] prev_tick = self.batch_info['prev_tick_id'] dst_tick = self.get_last_tick(dst_curs) if not dst_tick: # seems this consumer has not run yet against dst_db return False if prev_tick == dst_tick: # on track return False if cur_tick == dst_tick: # current batch is already applied, skip it return True # anything else means problems raise Exception('Lost position: batch %d..%d, dst has %d' % ( prev_tick, cur_tick, dst_tick)) def set_batch_done(self, dst_curs: Cursor) -> None: """Helper function to set last successful batch in external database. """ if self.batch_info: tick_id = self.batch_info['tick_id'] self.set_last_tick(dst_curs, tick_id) def register_consumer(self) -> int: new = Consumer.register_consumer(self) if new: # fixme self.dst_reset() return new def unregister_consumer(self) -> None: """If unregistering, also clean completed tick table on dest.""" Consumer.unregister_consumer(self) self.dst_reset() def process_remote_batch(self, db: Connection, batch_id: int, event_list: EventList, dst_db: Connection) -> None: raise Exception('process_remote_batch not implemented') def rewind(self) -> None: self.log.info("Rewinding queue") src_db = self.get_database(self.db_name) dst_db = self.get_database(self.remote_db) src_curs = src_db.cursor() dst_curs = dst_db.cursor() dst_tick = self.get_last_tick(dst_curs) if dst_tick: q = "select pgq.register_consumer_at(%s, %s, %s)" src_curs.execute(q, [self.queue_name, self.consumer_name, dst_tick]) else: self.log.warning('No tick found on dst side') dst_db.commit() src_db.commit() def dst_reset(self) -> None: self.log.info("Resetting queue tracking on dst side") dst_db = self.get_database(self.remote_db) dst_curs = dst_db.cursor() self.set_last_tick(dst_curs, None) dst_db.commit() def get_last_tick(self, dst_curs: Cursor) -> int: q = "select %s.get_last_tick(%%s)" % self.dst_schema dst_curs.execute(q, [self.consumer_name]) res = dst_curs.fetchone() return res[0] def set_last_tick(self, dst_curs: Cursor, tick_id: Optional[int]) -> None: q = "select %s.set_last_tick(%%s, %%s)" % self.dst_schema dst_curs.execute(q, [self.consumer_name, tick_id]) python-pgq-3.8/pgq/status.py000066400000000000000000000067431447266245300162050ustar00rootroot00000000000000"""Status display. """ from typing import Optional, Sequence, List import sys from skytools.basetypes import DictRow import skytools __all__ = ['PGQStatus'] def ival(data: str, _as: Optional[str] = None) -> str: "Format interval for output" if not _as: _as = data.split('.')[-1] numfmt = 'FM9999999' expr = "coalesce(to_char(extract(epoch from %s), '%s') || 's', 'NULL') as %s" return expr % (data, numfmt, _as) class PGQStatus(skytools.DBScript): """Info gathering and display.""" def __init__(self, args: Sequence[str], check: int = 0) -> None: super().__init__('pgqadm', args) self.show_status() sys.exit(0) def show_status(self) -> None: db = self.get_database("db", autocommit=1) cx = db.cursor() cx.execute("show server_version") pgver = cx.fetchone()[0] cx.execute("select pgq.version()") qver = cx.fetchone()[0] print("Postgres version: %s PgQ version: %s" % (pgver, qver)) q = """select f.queue_name, f.queue_ntables, %s, %s, %s, %s, q.queue_ticker_max_count, f.ev_per_sec, f.ev_new from pgq.get_queue_info() f, pgq.queue q where q.queue_name = f.queue_name""" % ( ival('f.queue_rotation_period'), ival('f.ticker_lag'), ival('q.queue_ticker_max_lag'), ival('q.queue_ticker_idle_period'), ) cx.execute(q) event_rows = cx.fetchall() q = """select queue_name, consumer_name, %s, %s, pending_events from pgq.get_consumer_info()""" % ( ival('lag'), ival('last_seen'), ) cx.execute(q) consumer_rows = cx.fetchall() print("\n%-33s %9s %13s %6s %6s %5s" % ('Event queue', 'Rotation', 'Ticker', 'TLag', 'EPS', 'New')) print('-' * 78) for ev_row in event_rows: tck = "%s/%s/%s" % (ev_row['queue_ticker_max_count'], ev_row['queue_ticker_max_lag'], ev_row['queue_ticker_idle_period']) rot = "%s/%s" % (ev_row['queue_ntables'], ev_row['queue_rotation_period']) print("%-33s %9s %13s %6s %6.1f %5d" % ( ev_row['queue_name'], rot, tck, ev_row['ticker_lag'], ev_row['ev_per_sec'], ev_row['ev_new'], )) print('-' * 78) print("\n%-48s %9s %9s %8s" % ( 'Consumer', 'Lag', 'LastSeen', 'Pending')) print('-' * 78) for ev_row in event_rows: cons = self.pick_consumers(ev_row, consumer_rows) self.show_queue(ev_row, cons) print('-' * 78) db.commit() def show_consumer(self, cons: DictRow) -> None: print(" %-46s %9s %9s %8d" % ( cons['consumer_name'], cons['lag'], cons['last_seen'], cons['pending_events'])) def show_queue(self, ev_row: DictRow, consumer_rows: Sequence[DictRow]) -> None: print("%s:" % ev_row['queue_name']) for cons in consumer_rows: self.show_consumer(cons) def pick_consumers(self, ev_row: DictRow, consumer_rows: Sequence[DictRow]) -> List[DictRow]: res = [] for con in consumer_rows: if con['queue_name'] != ev_row['queue_name']: continue res.append(con) return res python-pgq-3.8/pyproject.toml000066400000000000000000000611661447266245300164350ustar00rootroot00000000000000[project] name = "pgq" description = "PgQ client library for Python" readme = "README.rst" keywords = ["database", "queue"] dynamic = ["version"] requires-python = ">= 3.7" maintainers = [{name = "Marko Kreen", email = "markokr@gmail.com"}] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: ISC License (ISCL)", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: 3", "Topic :: Database", "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = ["skytools"] [project.optional-dependencies] test = ["pytest", "pytest-cov", "coverage[toml]", "psycopg2-binary"] doc = ["sphinx"] [project.urls] homepage = "https://github.com/pgq/python-pgq" #documentation = "https://readthedocs.org" repository = "https://github.com/pgq/python-pgq" changelog = "https://github.com/pgq/python-pgq/blob/master/NEWS.rst" [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools] packages = ["pgq", "pgq.cascade"] package-data = {"pgq" = ["py.typed"]} zip-safe = false [tool.setuptools.dynamic.version] attr = "pgq.__version__" # # testing # [tool.pytest] testpaths = ["tests"] [tool.coverage.paths] source = ["pgq", "**/site-packages/pgq"] [tool.coverage.report] exclude_lines = [ "pragma: no cover", "def __repr__", "if self.debug:", "if settings.DEBUG", "raise AssertionError", "raise NotImplementedError", "if 0:", "if __name__ == .__main__.:", ] # # formatting # [tool.isort] atomic = true line_length = 100 multi_line_output = 5 known_first_party = ["pgq"] known_third_party = ["pytest", "yaml", "skytools"] include_trailing_comma = true balanced_wrapping = true [tool.autopep8] exclude = ".tox, git, tmp, build, cover, dist" ignore = ["E301", "E265", "W391"] max-line-length = 110 in-place = true recursive = true aggressive = 2 [tool.doc8] extensions = "rst" # # linters # [tool.mypy] python_version = "3.10" strict = true disallow_any_unimported = true disallow_any_expr = false disallow_any_decorated = false disallow_any_explicit = false disallow_any_generics = false warn_return_any = false warn_unreachable = false [[tool.mypy.overrides]] module = [] strict = false disallow_untyped_defs = false disallow_untyped_calls = false disallow_incomplete_defs = false [tool.ruff] line-length = 120 select = ["E", "F", "Q", "W", "UP", "YTT", "ANN"] ignore = [ "ANN101", # Missing type annotation for `self` in method "ANN102", # Missing type annotation for `cls` in classmethod "ANN401", # Dynamically typed expressions (typing.Any) are disallowed "UP006", # Use `dict` instead of `Dict` "UP007", # Use `X | Y` for type annotations "UP031", # Use format specifiers instead of percent format "UP032", # Use f-string instead of `format` call "UP035", # typing.List` is deprecated "UP037", # Remove quotes from type annotation "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` ] [tool.ruff.flake8-quotes] docstring-quotes = "double" # # reference links # # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ # https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html # [tool.pylint.main] # Analyse import fallback blocks. This can be used to support both Python 2 and 3 # compatible code, which means that the block might have code that exists only in # one or another interpreter, leading to false positives when analysed. # analyse-fallback-blocks = # Clear in-memory caches upon conclusion of linting. Useful if running pylint in # a server-like mode. # clear-cache-post-run = # Always return a 0 (non-error) status code, even if lint errors are found. This # is primarily useful in continuous integration scripts. # exit-zero = # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. # extension-pkg-allow-list = # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. (This is an alternative name to extension-pkg-allow-list # for backward compatibility.) # extension-pkg-whitelist = # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages # specified are enabled, while categories only check already-enabled messages. # fail-on = # Specify a score threshold under which the program will exit with error. fail-under = 10 # Interpret the stdin as a python script, whose filename needs to be passed as # the module_or_package argument. # from-stdin = # Files or directories to be skipped. They should be base names, not paths. ignore = ["CVS", "tmp", "dist"] # Add files or directories matching the regular expressions patterns to the # ignore-list. The regex matches against paths and can be in Posix or Windows # format. Because '\\' represents the directory delimiter on Windows systems, it # can't be used as an escape character. # ignore-paths = # Files or directories matching the regular expression patterns are skipped. The # regex matches against base names, not paths. The default value ignores Emacs # file locks # ignore-patterns = # List of module names for which member attributes should not be checked (useful # for modules/projects where namespaces are manipulated during runtime and thus # existing member attributes cannot be deduced by static analysis). It supports # qualified module names, as well as Unix pattern matching. # ignored-modules = # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). # init-hook = # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use, and will cap the count on Windows to # avoid hangs. jobs = 1 # Control the amount of potential inferred values when inferring a single object. # This can help the performance when dealing with large functions or complex, # nested conditions. limit-inference-results = 100 # List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. # load-plugins = # Pickle collected data for later comparisons. persistent = true # Minimum Python version to use for version dependent checks. Will default to the # version used to run pylint. py-version = "3.10" # Discover python modules and packages in the file system subtree. # recursive = # Add paths to the list of the source roots. Supports globbing patterns. The # source root is an absolute path or a path relative to the current working # directory used to determine a package namespace for modules located under the # source root. # source-roots = # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode = true # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. # unsafe-load-any-extension = [tool.pylint.basic] # Naming style matching correct argument names. argument-naming-style = "snake_case" # Regular expression matching correct argument names. Overrides argument-naming- # style. If left empty, argument names will be checked with the set naming style. # argument-rgx = # Naming style matching correct attribute names. attr-naming-style = "snake_case" # Regular expression matching correct attribute names. Overrides attr-naming- # style. If left empty, attribute names will be checked with the set naming # style. # attr-rgx = # Bad variable names which should always be refused, separated by a comma. bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] # Bad variable names regexes, separated by a comma. If names match any regex, # they will always be refused # bad-names-rgxs = # Naming style matching correct class attribute names. class-attribute-naming-style = "any" # Regular expression matching correct class attribute names. Overrides class- # attribute-naming-style. If left empty, class attribute names will be checked # with the set naming style. # class-attribute-rgx = # Naming style matching correct class constant names. class-const-naming-style = "UPPER_CASE" # Regular expression matching correct class constant names. Overrides class- # const-naming-style. If left empty, class constant names will be checked with # the set naming style. # class-const-rgx = # Naming style matching correct class names. class-naming-style = "PascalCase" # Regular expression matching correct class names. Overrides class-naming-style. # If left empty, class names will be checked with the set naming style. # class-rgx = # Naming style matching correct constant names. const-naming-style = "UPPER_CASE" # Regular expression matching correct constant names. Overrides const-naming- # style. If left empty, constant names will be checked with the set naming style. # const-rgx = # Minimum line length for functions/classes that require docstrings, shorter ones # are exempt. docstring-min-length = -1 # Naming style matching correct function names. function-naming-style = "snake_case" # Regular expression matching correct function names. Overrides function-naming- # style. If left empty, function names will be checked with the set naming style. # function-rgx = # Good variable names which should always be accepted, separated by a comma. good-names = ["i", "j", "k", "ex", "Run", "_"] # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted # good-names-rgxs = # Include a hint for the correct naming format with invalid-name. # include-naming-hint = # Naming style matching correct inline iteration names. inlinevar-naming-style = "any" # Regular expression matching correct inline iteration names. Overrides # inlinevar-naming-style. If left empty, inline iteration names will be checked # with the set naming style. # inlinevar-rgx = # Naming style matching correct method names. method-naming-style = "snake_case" # Regular expression matching correct method names. Overrides method-naming- # style. If left empty, method names will be checked with the set naming style. # method-rgx = # Naming style matching correct module names. module-naming-style = "snake_case" # Regular expression matching correct module names. Overrides module-naming- # style. If left empty, module names will be checked with the set naming style. # module-rgx = # Colon-delimited sets of names that determine each other's naming style when the # name regexes allow several styles. # name-group = # Regular expression which should only match function or class names that do not # require a docstring. no-docstring-rgx = "^_" # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. These # decorators are taken in consideration only for invalid-name. property-classes = ["abc.abstractproperty"] # Regular expression matching correct type alias names. If left empty, type alias # names will be checked with the set naming style. # typealias-rgx = # Regular expression matching correct type variable names. If left empty, type # variable names will be checked with the set naming style. # typevar-rgx = # Naming style matching correct variable names. variable-naming-style = "snake_case" # Regular expression matching correct variable names. Overrides variable-naming- # style. If left empty, variable names will be checked with the set naming style. # variable-rgx = [tool.pylint.classes] # Warn about protected attribute access inside special methods # check-protected-access-in-special-methods = # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods = ["__init__", "__new__", "setUp"] # List of member names, which should be excluded from the protected access # warning. exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make"] # List of valid names for the first argument in a class method. valid-classmethod-first-arg = ["cls"] # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg = ["cls"] [tool.pylint.design] # List of regular expressions of class ancestor names to ignore when counting # public methods (see R0903) # exclude-too-few-public-methods = # List of qualified class names to ignore when counting class parents (see R0901) # ignored-parents = # Maximum number of arguments for function / method. max-args = 15 # Maximum number of attributes for a class (see R0902). max-attributes = 37 # Maximum number of boolean expressions in an if statement (see R0916). max-bool-expr = 5 # Maximum number of branch for function / method body. max-branches = 50 # Maximum number of locals for function / method body. max-locals = 45 # Maximum number of parents for a class (see R0901). max-parents = 7 # Maximum number of public methods for a class (see R0904). max-public-methods = 420 # Maximum number of return / yield for function / method body. max-returns = 16 # Maximum number of statements in function / method body. max-statements = 150 # Minimum number of public methods for a class (see R0903). min-public-methods = 0 [tool.pylint.exceptions] # Exceptions that will emit a warning when caught. overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"] [tool.pylint.format] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format = "LF" # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines = "^\\s*(# )??$" # Number of spaces of indent required inside a hanging or continued line. indent-after-paren = 4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string = " " # Maximum number of characters on a single line. max-line-length = 190 # Maximum number of lines in a module. max-module-lines = 10000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. # single-line-class-stmt = # Allow the body of an if to be on the same line as the test if there is no else. # single-line-if-stmt = [tool.pylint.imports] # List of modules that can be imported at any level, not just the top level one. # allow-any-import-level = # Allow explicit reexports by alias from a package __init__. # allow-reexport-from-package = # Allow wildcard imports from modules that define __all__. # allow-wildcard-with-all = # Deprecated modules which should not be used, separated by a comma. deprecated-modules = ["optparse", "tkinter.tix"] # Output a graph (.gv or any supported image format) of external dependencies to # the given file (report RP0402 must not be disabled). # ext-import-graph = # Output a graph (.gv or any supported image format) of all (i.e. internal and # external) dependencies to the given file (report RP0402 must not be disabled). # import-graph = # Output a graph (.gv or any supported image format) of internal dependencies to # the given file (report RP0402 must not be disabled). # int-import-graph = # Force import order to recognize a module as part of the standard compatibility # libraries. # known-standard-library = # Force import order to recognize a module as part of a third party library. known-third-party = ["enchant"] # Couples of modules and preferred modules, separated by a comma. # preferred-modules = [tool.pylint.logging] # The type of string formatting that logging methods do. `old` means using % # formatting, `new` is for `{}` formatting. logging-format-style = "old" # Logging modules to check that the string format arguments are in logging # function parameter format. logging-modules = ["logging"] [tool.pylint."messages control"] # Only show warnings with the listed confidence levels. Leave empty to show all. # Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] # Disable the message, report, category or checker with the given id(s). You can # either give multiple identifiers separated by comma (,) or put this option # multiple times (only on the command line, not in the configuration file where # it should appear only once). You can also use "--disable=all" to disable # everything first and then re-enable specific checks. For example, if you want # to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". disable = [ "raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead", "bare-except", "broad-exception-caught", "useless-return", "consider-using-in", "consider-using-ternary", "fixme", "global-statement", "invalid-name", "missing-module-docstring", "missing-class-docstring", "missing-function-docstring", "no-else-raise", "no-else-return", "trailing-newlines", "unused-argument", "unused-variable", "using-constant-test", "useless-object-inheritance", "duplicate-code", "singleton-comparison", "consider-using-f-string", "broad-exception-raised", "arguments-differ", "multiple-statements", "use-implicit-booleaness-not-len", "chained-comparison", "unnecessary-pass", "cyclic-import", "too-many-ancestors", "import-outside-toplevel", "protected-access", "try-except-raise", "deprecated-module", "no-else-break", "no-else-continue", # junk "trailing-newlines", "consider-using-f-string", # expected "cyclic-import", # issues "broad-exception-caught", "no-else-return", ] # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where it # should appear only once). See also the "--disable" option for examples. enable = ["c-extension-no-member"] [tool.pylint.method_args] # List of qualified names (i.e., library.method) which require a timeout # parameter e.g. 'requests.api.get,requests.api.post' timeout-methods = [ "requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request" ] [tool.pylint.miscellaneous] # List of note tags to take in consideration, separated by a comma. notes = ["FIXME", "XXX", "TODO"] # Regular expression of note tags to take in consideration. # notes-rgx = [tool.pylint.refactoring] # Maximum number of nested blocks for function / method body max-nested-blocks = 10 # Complete name of functions that never returns. When checking for inconsistent- # return-statements if a never returning function is called then it will be # considered as an explicit return statement and no message will be printed. never-returning-functions = ["sys.exit"] [tool.pylint.reports] # Python expression which should return a score less than or equal to 10. You # have access to the variables 'fatal', 'error', 'warning', 'refactor', # 'convention', and 'info' which contain the number of messages in each category, # as well as 'statement' which is the total number of statements analyzed. This # score is used by the global evaluation report (RP0004). evaluation = "10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)" # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details. # msg-template = # Set the output format. Available formats are text, parseable, colorized, json # and msvs (visual studio). You can also give a reporter class, e.g. # mypackage.mymodule.MyReporterClass. # output-format = # Tells whether to display a full report or only the messages. # reports = # Activate the evaluation score. # score = [tool.pylint.similarities] # Comments are removed from the similarity computation ignore-comments = true # Docstrings are removed from the similarity computation ignore-docstrings = true # Imports are removed from the similarity computation # ignore-imports = # Signatures are removed from the similarity computation ignore-signatures = true # Minimum lines number of a similarity. min-similarity-lines = 4 [tool.pylint.spelling] # Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions = 4 # Spelling dictionary name. No available dictionaries : You need to install both # the python package and the system dependency for enchant to work.. # spelling-dict = # List of comma separated words that should be considered directives if they # appear at the beginning of a comment and should not be checked. spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" # List of comma separated words that should not be checked. spelling-ignore-words = "usr,bin,env" # A path to a file that contains the private dictionary; one word per line. spelling-private-dict-file = ".local.dict" # Tells whether to store unknown words to the private dictionary (see the # --spelling-private-dict-file option) instead of raising a message. # spelling-store-unknown-words = [tool.pylint.typecheck] # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that # produce valid context managers. contextmanager-decorators = ["contextlib.contextmanager"] # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. # generated-members = # Tells whether missing members accessed in mixin class should be ignored. A # class is considered mixin if its name matches the mixin-class-rgx option. # Tells whether to warn about missing members when the owner of the attribute is # inferred to be None. ignore-none = true # This flag controls whether pylint should warn about no-member and similar # checks whenever an opaque object is returned when inferring. The inference can # return multiple potential results while evaluating a Python object, but some # branches might not be evaluated, which results in partial inference. In that # case, it might be useful to still emit no-member and other checks for the rest # of the inferred objects. ignore-on-opaque-inference = true # List of symbolic message names to ignore for Mixin members. ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes = ["optparse.Values", "thread._local", "_thread._local"] # Show a hint with possible names when a member name was not found. The aspect of # finding the hint is based on edit distance. missing-member-hint = true # The minimum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance = 1 # The total number of similar names that should be taken in consideration when # showing a hint for a missing member. missing-member-max-choices = 1 # Regex pattern to define which classes are considered mixins. mixin-class-rgx = ".*[Mm]ixin" # List of decorators that change the signature of a decorated function. # signature-mutators = [tool.pylint.variables] # List of additional names supposed to be defined in builtins. Remember that you # should avoid defining new builtins when possible. # additional-builtins = # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables = true # List of names allowed to shadow builtins # allowed-redefined-builtins = # List of strings which can identify a callback function by name. A callback name # must start or end with one of those strings. callbacks = ["cb_", "_cb"] # A regular expression matching the name of dummy variables (i.e. expected to not # be used). dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" # Argument names that match this expression will be ignored. ignored-argument-names = "_.*|^ignored_|^unused_" # Tells whether we should check for unused import in __init__ files. # init-import = # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] python-pgq-3.8/setup.py000066400000000000000000000001051447266245300152150ustar00rootroot00000000000000"""Setup for pgq module. """ from setuptools import setup setup() python-pgq-3.8/tests/000077500000000000000000000000001447266245300146515ustar00rootroot00000000000000python-pgq-3.8/tests/test_pgq.py000066400000000000000000000027141447266245300170550ustar00rootroot00000000000000 import os import secrets import pytest import skytools import datetime import pgq TEST_Q_NAME = os.environ.get("TEST_Q_NAME") @pytest.fixture(scope="session") def dbconn(): db = skytools.connect_database("") db.set_isolation_level(0) return db @pytest.mark.skipif(not TEST_Q_NAME, reason="no db setup") def test_insert_event(dbconn): with dbconn.cursor() as curs: ev_id = pgq.insert_event(curs, TEST_Q_NAME, "mytype", "payload") assert ev_id > 0 with dbconn.cursor() as curs: curs.execute("select * from pgq.event_template where ev_id = %s", (ev_id,)) rows = curs.fetchall() assert len(rows) == 1 assert rows[0]["ev_id"] == ev_id assert rows[0]["ev_type"] == "mytype" @pytest.mark.skipif(not TEST_Q_NAME, reason="no db setup") def test_bulk_insert_events(dbconn): fields = ['ev_type', 'ev_data', 'ev_time'] my_type = secrets.token_urlsafe(12) ev_time = datetime.datetime.now() rows1 = [ {'ev_type': my_type, 'ev_data': 'data1', 'ev_time': ev_time}, {'ev_type': my_type, 'ev_data': 'data2', 'ev_time': ev_time}, {'ev_type': my_type, 'ev_data': 'data3', 'ev_time': ev_time}, ] with dbconn.cursor() as curs: pgq.bulk_insert_events(curs, rows1, fields, TEST_Q_NAME) with dbconn.cursor() as curs: curs.execute("select * from pgq.event_template where ev_type = %s", (my_type,)) rows2 = curs.fetchall() assert len(rows1) == len(rows2) python-pgq-3.8/tox.ini000066400000000000000000000022031447266245300150170ustar00rootroot00000000000000 [tox] envlist = lint,py3 [package] name = pgq deps = psycopg2-binary==2.9.7 skytools==3.9.2 test_deps = coverage==7.2.7 pytest==7.4.0 pytest-cov==4.1.0 lint_deps = mypy==1.5.1 pyflakes==3.1.0 typing-extensions==4.7.1 types-setuptools==68.1.0.0 types-psycopg2==2.9.21.11 xlint_deps = pylint==2.17.5 pytype==2023.8.22 [testenv] changedir = {envsitepackagesdir} deps = {[package]deps} {[package]test_deps} passenv = TEST_Q_NAME PGPORT PGHOST PGUSER PGDATABASE commands = pytest \ --cov=pgq \ --cov-report=term \ --cov-report=xml:{toxinidir}/cover/coverage.xml \ --cov-report=html:{toxinidir}/cover/{envname} \ {toxinidir}/tests \ {posargs} [testenv:lint] basepython = python3 changedir = {toxinidir} deps = {[package]deps} {[package]lint_deps} commands = pyflakes {[package]name} mypy {[package]name} [testenv:xlint] basepython = python3.10 changedir = {toxinidir} deps = {[package]deps} {[package]lint_deps} {[package]xlint_deps} commands = pylint {[package]name} pytype {[package]name}