pax_global_header00006660000000000000000000000064147201035270014513gustar00rootroot0000000000000052 comment=fb0fcc859a5995b7ba3d93037878425caf5f8f8b patroni-4.0.4/000077500000000000000000000000001472010352700131745ustar00rootroot00000000000000patroni-4.0.4/.github/000077500000000000000000000000001472010352700145345ustar00rootroot00000000000000patroni-4.0.4/.github/ISSUE_TEMPLATE/000077500000000000000000000000001472010352700167175ustar00rootroot00000000000000patroni-4.0.4/.github/ISSUE_TEMPLATE/bug_report.yml000066400000000000000000000052521472010352700216160ustar00rootroot00000000000000name: Bug Report description: Create a report to help us improve labels: - bug body: - type: markdown attributes: value: | If you have a question please post it on channel [#patroni](https://postgresteam.slack.com/archives/C9XPYG92A) in the [PostgreSQL Slack](https://pgtreats.info/slack-invite). Before reporting a bug please make sure to **reproduce it with the latest Patroni version**! Please fill the form below and provide as much information as possible. Not doing so may result in your bug not being addressed in a timely manner. - type: textarea id: problem attributes: label: What happened? validations: required: true - type: textarea id: repro attributes: label: How can we reproduce it (as minimally and precisely as possible)? validations: required: true - type: textarea id: expected attributes: label: What did you expect to happen? validations: required: true - type: textarea id: environment attributes: label: Patroni/PostgreSQL/DCS version value: | - Patroni version: - PostgreSQL version: - DCS (and its version): validations: required: true - type: textarea id: patroniConfig attributes: label: Patroni configuration file description: Please copy and paste Patroni configuration file here. This will be automatically formatted into code, so no need for backticks. render: yaml validations: required: true - type: textarea id: globalConfig attributes: label: patronictl show-config description: Please copy and paste `patronictl show-config` output here. This will be automatically formatted into code, so no need for backticks. render: yaml validations: required: true - type: textarea id: patroniLogs attributes: label: Patroni log files description: Please copy and paste any relevant Patroni log output. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true - type: textarea id: postgresLogs attributes: label: PostgreSQL log files description: Please copy and paste any relevant PostgreSQL log output. This will be automatically formatted into code, so no need for backticks. render: shell validations: required: true - type: checkboxes id: issueSearch attributes: label: Have you tried to use GitHub issue search? description: Maybe there is already a similar issue solved. options: - label: 'Yes' required: true validations: required: true - type: textarea id: additional attributes: label: Anything else we need to know? description: Add any other context about the problem here. patroni-4.0.4/.github/ISSUE_TEMPLATE/config.yml000066400000000000000000000002621472010352700207070ustar00rootroot00000000000000blank_issues_enabled: false contact_links: - name: Question url: https://pgtreats.info/slack-invite about: "Please ask questions on channel #patroni in the PostgreSQL Slack" patroni-4.0.4/.github/workflows/000077500000000000000000000000001472010352700165715ustar00rootroot00000000000000patroni-4.0.4/.github/workflows/install_deps.py000066400000000000000000000113621472010352700216270ustar00rootroot00000000000000import inspect import os import shutil import subprocess import stat import sys import tarfile import zipfile def install_requirements(what): subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip']) s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel', 'setuptools']) if s != 0: return s old_path = sys.path[:] w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) sys.path.insert(0, os.path.dirname(os.path.dirname(w))) try: from setup import EXTRAS_REQUIRE, read finally: sys.path = old_path requirements = ['flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave'] requirements += ['coverage'] # try to split tests between psycopg2 and psycopg3 requirements += ['psycopg[binary]'] if sys.version_info >= (3, 8, 0) and\ (sys.platform != 'darwin' or what == 'etcd3') else ['psycopg2-binary==2.9.9' if sys.platform == 'darwin' else 'psycopg2-binary'] for r in read('requirements.txt').split('\n'): r = r.strip() if r != '': extras = {e for e, v in EXTRAS_REQUIRE.items() if v and any(r.startswith(x) for x in v)} if not extras or what == 'all' or what in extras: requirements.append(r) return subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements) def install_packages(what): from mapping import versions packages = { 'zookeeper': ['zookeeper', 'zookeeper-bin', 'zookeeperd'], 'consul': ['consul'], } packages['exhibitor'] = packages['zookeeper'] packages = packages.get(what, []) ver = versions.get(what) if 15 <= float(ver) < 17: packages += ['postgresql-{0}-citus-12.1'.format(ver)] subprocess.call(['sudo', 'apt-get', 'update', '-y']) return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages) def get_file(url, name): try: from urllib.request import urlretrieve except ImportError: from urllib import urlretrieve print('Downloading ' + url) urlretrieve(url, name) def untar(archive, name): with tarfile.open(archive) as tar: f = tar.extractfile(name) dest = os.path.basename(name) with open(dest, 'wb') as d: shutil.copyfileobj(f, d) return dest def unzip(archive, name): with zipfile.ZipFile(archive, 'r') as z: name = z.extract(name) dest = os.path.basename(name) shutil.move(name, dest) return dest def unzip_all(archive): print('Extracting ' + archive) with zipfile.ZipFile(archive, 'r') as z: z.extractall() def chmod_755(name): os.chmod(name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) def unpack(archive, name): print('Extracting {0} from {1}'.format(name, archive)) func = unzip if archive.endswith('.zip') else untar name = func(archive, name) chmod_755(name) return name def install_etcd(): version = os.environ.get('ETCDVERSION', '3.4.23') platform = {'linux2': 'linux', 'win32': 'windows', 'cygwin': 'windows'}.get(sys.platform, sys.platform) dirname = 'etcd-v{0}-{1}-amd64'.format(version, platform) ext = 'tar.gz' if platform == 'linux' else 'zip' name = '{0}.{1}'.format(dirname, ext) url = 'https://github.com/etcd-io/etcd/releases/download/v{0}/{1}'.format(version, name) get_file(url, name) ext = '.exe' if platform == 'windows' else '' return int(unpack(name, '{0}/etcd{1}'.format(dirname, ext)) is None) def install_postgres(): version = os.environ.get('PGVERSION', '16.1-1') platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform] if platform == 'osx': return subprocess.call(['brew', 'install', 'expect', 'postgresql@{0}'.format(version.split('.')[0])]) name = 'postgresql-{0}-{1}-binaries.zip'.format(version, platform) get_file('http://get.enterprisedb.com/postgresql/' + name, name) unzip_all(name) bin_dir = os.path.join('pgsql', 'bin') for f in os.listdir(bin_dir): chmod_755(os.path.join(bin_dir, f)) return subprocess.call(['pgsql/bin/postgres', '-V']) def main(): what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all') if what != 'all': if sys.platform.startswith('linux'): r = install_packages(what) else: r = install_postgres() if r == 0 and what.startswith('etcd'): r = install_etcd() if r != 0: return r return install_requirements(what) if __name__ == '__main__': sys.exit(main()) patroni-4.0.4/.github/workflows/mapping.py000066400000000000000000000001571472010352700206010ustar00rootroot00000000000000versions = {'etcd': '9.6', 'etcd3': '16', 'consul': '17', 'exhibitor': '12', 'raft': '14', 'kubernetes': '15'} patroni-4.0.4/.github/workflows/release.yaml000066400000000000000000000021601472010352700210740ustar00rootroot00000000000000name: Publish Patroni distributions to PyPI and TestPyPI on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' release: types: - published jobs: build-n-publish: name: Build and publish Patroni distributions to PyPI and TestPyPI runs-on: ubuntu-latest permissions: id-token: write steps: - uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run tests and flake8 run: python .github/workflows/run_tests.py - name: Install Python packaging build frontend run: python -m pip install build - name: Build a binary wheel and a source tarball run: python -m build - name: Publish distribution to Test PyPI if: github.event_name == 'push' uses: pypa/gh-action-pypi-publish@v1.9.0 with: repository_url: https://test.pypi.org/legacy/ - name: Publish distribution to PyPI if: github.event_name == 'release' uses: pypa/gh-action-pypi-publish@v1.9.0 patroni-4.0.4/.github/workflows/run_tests.py000066400000000000000000000032231472010352700211710ustar00rootroot00000000000000import os import shutil import subprocess import sys import tempfile def main(): what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all') if what == 'all': flake8 = subprocess.call([sys.executable, 'setup.py', 'flake8']) test = subprocess.call([sys.executable, 'setup.py', 'test']) version = '.'.join(map(str, sys.version_info[:2])) shutil.move('.coverage', os.path.join(tempfile.gettempdir(), '.coverage.' + version)) return flake8 | test elif what == 'combine': tmp = tempfile.gettempdir() for name in os.listdir(tmp): if name.startswith('.coverage.'): shutil.move(os.path.join(tmp, name), name) return subprocess.call([sys.executable, '-m', 'coverage', 'combine']) env = os.environ.copy() if sys.platform.startswith('linux'): from mapping import versions version = versions.get(what) path = '/usr/lib/postgresql/{0}/bin:.'.format(version) unbuffer = ['timeout', '900', 'unbuffer'] else: if sys.platform == 'darwin': version = os.environ.get('PGVERSION', '16.1-1') path = '/opt/homebrew/opt/postgresql@{0}/bin:.'.format(version.split('.')[0]) unbuffer = ['unbuffer'] else: path = os.path.abspath(os.path.join('pgsql', 'bin')) unbuffer = [] env['PATH'] = path + os.pathsep + env['PATH'] env['DCS'] = what if what == 'kubernetes': env['PATRONI_KUBERNETES_CONTEXT'] = 'k3d-k3s-default' return subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env) if __name__ == '__main__': sys.exit(main()) patroni-4.0.4/.github/workflows/tests.yaml000066400000000000000000000162621472010352700206260ustar00rootroot00000000000000name: Tests on: pull_request: push: branches: - master - 'REL_[0-9]+_[0-9]+' env: CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} SECRETS_AVAILABLE: ${{ secrets.CODACY_PROJECT_TOKEN != '' }} jobs: unit: runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: os: [ubuntu, windows, macos] steps: - uses: actions/checkout@v4 - name: Set up Python 3.7 uses: actions/setup-python@v5 with: python-version: 3.7 if: matrix.os != 'macos' - name: Install dependencies run: python .github/workflows/install_deps.py if: matrix.os != 'macos' - name: Run tests and flake8 run: python .github/workflows/run_tests.py if: matrix.os != 'macos' - name: Set up Python 3.8 uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run tests and flake8 run: python .github/workflows/run_tests.py - name: Set up Python 3.9 uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run tests and flake8 run: python .github/workflows/run_tests.py - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run tests and flake8 run: python .github/workflows/run_tests.py - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run tests and flake8 run: python .github/workflows/run_tests.py - name: Set up Python 3.12 uses: actions/setup-python@v5 with: python-version: 3.12 - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run tests and flake8 run: python .github/workflows/run_tests.py - name: Combine coverage run: python .github/workflows/run_tests.py combine - name: Install coveralls run: python -m pip install coveralls - name: Upload Coverage env: COVERALLS_FLAG_NAME: unit-${{ matrix.os }} COVERALLS_PARALLEL: 'true' GITHUB_TOKEN: ${{ secrets.github_token }} run: python -m coveralls --service=github behave: runs-on: ${{ matrix.os }}-latest env: DCS: ${{ matrix.dcs }} ETCDVERSION: 3.4.23 PGVERSION: 16.1-1 # for windows and macos strategy: fail-fast: false matrix: os: [ubuntu] python-version: [3.7, 3.12] dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft] include: - os: macos python-version: 3.8 dcs: raft - os: macos python-version: 3.9 dcs: etcd - os: macos python-version: 3.11 dcs: etcd3 steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - uses: nolar/setup-k3d-k3s@v1 if: matrix.dcs == 'kubernetes' - name: Add postgresql and citus apt repo run: | sudo apt-get update -y sudo apt-get install -y wget ca-certificates gnupg debian-archive-keyring apt-transport-https sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg' sudo sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/ubuntu/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list' sudo sh -c 'wget -qO - https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg' if: matrix.os == 'ubuntu' - name: Install dependencies run: python .github/workflows/install_deps.py - name: Run behave tests run: python .github/workflows/run_tests.py - name: Upload logs if behave failed uses: actions/upload-artifact@v4 if: failure() with: name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs path: | features/output/*_failed/*postgres?.* features/output/*.log if-no-files-found: error retention-days: 5 - name: Generate coverage xml report run: python -m coverage xml -o cobertura.xml - name: Upload coverage to Codacy run: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r cobertura.xml -l Python --partial if: ${{ env.SECRETS_AVAILABLE == 'true' }} coveralls-finish: name: Finalize coveralls.io needs: unit runs-on: ubuntu-latest steps: - uses: actions/setup-python@v5 - run: python -m pip install coveralls - run: python -m coveralls --service=github --finish env: GITHUB_TOKEN: ${{ secrets.github_token }} codacy-final: name: Finalize Codacy needs: behave runs-on: ubuntu-latest steps: - run: bash <(curl -Ls https://coverage.codacy.com/get.sh) final if: ${{ env.SECRETS_AVAILABLE == 'true' }} pyright: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v5 with: python-version: 3.12 - name: Install dependencies run: python -m pip install -r requirements.txt psycopg2-binary psycopg - uses: jakebailey/pyright-action@v2 with: version: 1.1.389 ydiff: name: Test compatibility with the latest version of ydiff runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v5 with: python-version: 3.12 - name: Install dependencies run: python .github/workflows/install_deps.py - name: Update ydiff run: python -m pip install -U ydiff - name: Run tests run: python -m pytest tests/test_ctl.py -v docs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: 3.11 cache: pip - name: Install dependencies run: pip install tox - name: Install package dependencies run: | sudo apt update \ && sudo apt install -y \ latexmk texlive-latex-extra tex-gyre \ --no-install-recommends - name: Generate documentation run: tox -m docs isort: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v5 with: python-version: 3.12 cache: pip - name: isort uses: isort/isort-action@master with: requirementsFiles: "requirements.txt requirements.dev.txt requirements.docs.txt" sort-paths: "patroni tests features setup.py" patroni-4.0.4/.gitignore000066400000000000000000000012711472010352700151650ustar00rootroot00000000000000*.py[cod] # vi(m) swap files: *.sw? # C extensions *.so # Packages .cache/ *.egg *.eggs *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg lib lib64 # Installer logs pip-log.txt # Unit test / coverage reports .coverage* .tox nosetests.xml coverage.xml htmlcov junit.xml features/output* dummy result.json # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject pgpass scm-source.json # Sphinx-generated documentation docs/_build/ docs/build/ docs/source/_static/ docs/source/_templates/ docs/modules/ docs/pdf/ # Pycharm IDE .idea/ #VSCode IDE .vscode/ # Virtual environment venv*/ # Default test data directory data/ # macOS **/.DS_Store patroni-4.0.4/.readthedocs.yaml000066400000000000000000000007661472010352700164340ustar00rootroot00000000000000# .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # Set the version of Python and other tools you might need build: os: ubuntu-22.04 tools: python: "3.11" # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py formats: - epub - pdf - htmlzip python: install: - requirements: requirements.docs.txt - requirements: requirements.txt patroni-4.0.4/CODEOWNERS000066400000000000000000000000601472010352700145630ustar00rootroot00000000000000# global owners * @CyberDem0n @hughcapet patroni-4.0.4/Dockerfile000066400000000000000000000200451472010352700151670ustar00rootroot00000000000000## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine ## It has all the necessary components to play/debug with a single node appliance, running etcd ARG PG_MAJOR=16 ARG COMPRESS=false ARG PGHOME=/home/postgres ARG PGDATA=$PGHOME/data ARG LC_ALL=C.UTF-8 ARG LANG=C.UTF-8 FROM postgres:$PG_MAJOR as builder ARG PGHOME ARG PGDATA ARG LC_ALL ARG LANG ENV ETCDVERSION=3.3.13 CONFDVERSION=0.16.0 RUN set -ex \ && export DEBIAN_FRONTEND=noninteractive \ && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \ && apt-get update -y \ # postgres:10 is based on debian, which has the patroni package. We will install all required dependencies && apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ | xargs apt-get install -y vim curl less jq locales haproxy sudo \ python3-etcd python3-kazoo python3-pip busybox \ net-tools iputils-ping dumb-init --fix-missing \ \ # Cleanup all locales but en_US.UTF-8 && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \ && find /usr/share/i18n/locales/ -type f ! -name en_US ! -name en_GB ! -name i18n* ! -name iso14651_t1 ! -name iso14651_t1_common ! -name 'translit_*' -delete \ && echo 'en_US.UTF-8 UTF-8' > /usr/share/i18n/SUPPORTED \ \ # Make sure we have a en_US.UTF-8 locale available && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ \ # haproxy dummy config && echo 'global\n stats socket /run/haproxy/admin.sock mode 660 level admin' > /etc/haproxy/haproxy.cfg \ \ # vim config && echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \ \ # Prepare postgres/patroni/haproxy environment && mkdir -p "$PGHOME/.config/patroni" /patroni /run/haproxy \ && ln -s ../../postgres0.yml "$PGHOME/.config/patroni/patronictl.yaml" \ && ln -s /patronictl.py /usr/local/bin/patronictl \ && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \ && chown -R postgres:postgres /var/log \ \ # Download etcd && curl -sL "https://github.com/coreos/etcd/releases/download/v$ETCDVERSION/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \ | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \ \ && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ # Build confd apt-get install -y git make \ && curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \ && export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \ && git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \ && make -C confd \ && cp confd/bin/confd /usr/local/bin/confd \ && rm -rf /confd /usr/local/go; \ else \ # Download confd curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \ > /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \ fi \ \ # Clean up all useless packages and some files && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \ libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \ exim4-config gnupg-agent dirmngr \ git make \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ /root/.cache \ /var/cache/debconf/* \ /etc/rc?.d \ /etc/systemd \ /docker-entrypoint* \ /sbin/pam* \ /sbin/swap* \ /sbin/unix* \ /usr/local/bin/gosu \ /usr/sbin/[acgipr]* \ /usr/sbin/*user* \ /usr/share/doc* \ /usr/share/man \ /usr/share/info \ /usr/share/i18n/locales/translit_hangul \ /usr/share/locale/?? \ /usr/share/locale/??_?? \ /usr/share/postgresql/*/man \ /usr/share/postgresql-common/pg_wrapper \ /usr/share/vim/vim*/doc \ /usr/share/vim/vim*/lang \ /usr/share/vim/vim*/tutor \ # /var/lib/dpkg/info/* \ && find /usr/bin -xtype l -delete \ && find /var/log -type f -exec truncate --size 0 {} \; \ && find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \ && find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete # perform compression if it is necessary ARG COMPRESS RUN if [ "$COMPRESS" = "true" ]; then \ set -ex \ # Allow certain sudo commands from postgres && echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \ && ln -snf busybox /bin/sh \ && arch=$(uname -m) \ && darch=$(uname -m | sed 's/_/-/') \ && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \ && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \ && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \ && find /etc/alternatives -xtype l -delete \ && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \ && XZ_OPT=-e9v tar -X /exclude -cpJf a.tar.xz $save_dirs \ # we call "cat /exclude" to avoid including files from the $save_dirs that are also among # the exceptions listed in the /exclude, as "uniq -u" eliminates all non-unique lines. # By calling "cat /exclude" a second time we guarantee that there will be at least two lines # for each exception and therefore they will be excluded from the output passed to 'rm'. && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \ && /bin/busybox --install -s \ && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \ else \ /bin/busybox --install -s; \ fi FROM scratch COPY --from=builder / / LABEL maintainer="Alexander Kukushkin " ARG PG_MAJOR ARG COMPRESS ARG PGHOME ARG PGDATA ARG LC_ALL ARG LANG ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN ENV ETCDCTL_API=3 COPY patroni /patroni/ COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/ COPY extras/confd/templates/haproxy.tmpl /etc/confd/templates/ COPY patroni*.py docker/entrypoint.sh / COPY postgres?.yml $PGHOME/ WORKDIR $PGHOME RUN sed -i 's/env python/&3/' /patroni*.py \ # "fix" patroni configs && sed -i 's/^ listen: 127.0.0.1/ listen: 0.0.0.0/' postgres?.yml \ && sed -i "s|^\( data_dir: \).*|\1$PGDATA|" postgres?.yml \ && sed -i "s|^#\( bin_dir: \).*|\1$PGBIN|" postgres?.yml \ && sed -i 's/^ - encoding: UTF8/ - locale: en_US.UTF-8\n&/' postgres?.yml \ && sed -i 's/^\(scope\|name\|etcd\| host\| authentication\| connect_address\| parameters\):/#&/' postgres?.yml \ && sed -i 's/^ \(replication\|superuser\|rewind\|unix_socket_directories\|\(\( \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \ && sed -i 's/^ parameters:/&\n max_connections: 100/' postgres?.yml \ && sed -i 's/^ pg_hba:/&\n - local all all trust/' postgres?.yml \ && sed -i 's/^\(.*\) \(.*\) md5/\1 all md5/' postgres?.yml \ && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \ && chmod +s /bin/ping \ && chown -R postgres:postgres "$PGHOME" /run /etc/haproxy USER postgres ENTRYPOINT ["/bin/sh", "/entrypoint.sh"] patroni-4.0.4/Dockerfile.citus000066400000000000000000000251011472010352700163130ustar00rootroot00000000000000## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine ## It has all the necessary components to play/debug with a single node appliance, running etcd ARG PG_MAJOR=16 ARG COMPRESS=false ARG PGHOME=/home/postgres ARG PGDATA=$PGHOME/data ARG LC_ALL=C.UTF-8 ARG LANG=C.UTF-8 FROM postgres:$PG_MAJOR as builder ARG PGHOME ARG PGDATA ARG LC_ALL ARG LANG ENV ETCDVERSION=3.3.13 CONFDVERSION=0.16.0 RUN set -ex \ && export DEBIAN_FRONTEND=noninteractive \ && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \ && apt-get update -y \ # postgres:PG_MAJOR is based on debian, which has the patroni package. We will install all required dependencies && apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ | xargs apt-get install -y vim curl less jq locales haproxy sudo \ python3-etcd python3-kazoo python3-pip busybox \ net-tools iputils-ping lsb-release dumb-init --fix-missing \ && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ apt-get install -y postgresql-server-dev-$PG_MAJOR \ git gcc make autoconf \ libc6-dev flex libcurl4-gnutls-dev \ libicu-dev libkrb5-dev liblz4-dev \ libpam0g-dev libreadline-dev libselinux1-dev\ libssl-dev libxslt1-dev libzstd-dev uuid-dev \ && git clone -b "main" https://github.com/citusdata/citus.git \ && MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \ && cd citus && ./configure && make install && cd ../ && rm -rf /citus; \ else \ echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \ && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \ && apt-get update -y \ && apt-get -y install postgresql-$PG_MAJOR-citus-12.1; \ fi \ \ # Cleanup all locales but en_US.UTF-8 && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \ && find /usr/share/i18n/locales/ -type f ! -name en_US ! -name en_GB ! -name i18n* ! -name iso14651_t1 ! -name iso14651_t1_common ! -name 'translit_*' -delete \ && echo 'en_US.UTF-8 UTF-8' > /usr/share/i18n/SUPPORTED \ \ # Make sure we have a en_US.UTF-8 locale available && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ \ # haproxy dummy config && echo 'global\n stats socket /run/haproxy/admin.sock mode 660 level admin' > /etc/haproxy/haproxy.cfg \ \ # vim config && echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \ \ # Prepare postgres/patroni/haproxy environment && mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \ && ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \ && ln -s /patronictl.py /usr/local/bin/patronictl \ && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \ && chown -R postgres:postgres /var/log \ \ # Download etcd && curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \ | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \ \ && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ # Build confd curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \ && export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \ && git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \ && make -C confd \ && cp confd/bin/confd /usr/local/bin/confd \ && rm -rf /confd /usr/local/go; \ else \ # Download confd curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \ > /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \ fi \ # Prepare client cert for HAProxy && cat /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/certs/ssl-cert-snakeoil.pem > /etc/ssl/private/ssl-cert-snakeoil.crt \ \ # Clean up all useless packages and some files && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \ libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \ exim4-config gnupg-agent dirmngr \ postgresql-server-dev-$PG_MAJOR git gcc make autoconf \ libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \ libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ /root/.cache \ /var/cache/debconf/* \ /etc/rc?.d \ /etc/systemd \ /docker-entrypoint* \ /sbin/pam* \ /sbin/swap* \ /sbin/unix* \ /usr/local/bin/gosu \ /usr/sbin/[acgipr]* \ /usr/sbin/*user* \ /usr/share/doc* \ /usr/share/man \ /usr/share/info \ /usr/share/i18n/locales/translit_hangul \ /usr/share/locale/?? \ /usr/share/locale/??_?? \ /usr/share/postgresql/*/man \ /usr/share/postgresql-common/pg_wrapper \ /usr/share/vim/vim*/doc \ /usr/share/vim/vim*/lang \ /usr/share/vim/vim*/tutor \ # /var/lib/dpkg/info/* \ && find /usr/bin -xtype l -delete \ && find /var/log -type f -exec truncate --size 0 {} \; \ && find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \ && find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete # perform compression if it is necessary ARG COMPRESS RUN if [ "$COMPRESS" = "true" ]; then \ set -ex \ # Allow certain sudo commands from postgres && echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \ && ln -snf busybox /bin/sh \ && arch=$(uname -m) \ && darch=$(uname -m | sed 's/_/-/') \ && files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \ && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \ && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \ && find /etc/alternatives -xtype l -delete \ && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \ && XZ_OPT=-e9v tar -X /exclude -cpJf a.tar.xz $save_dirs \ # we call "cat /exclude" to avoid including files from the $save_dirs that are also among # the exceptions listed in the /exclude, as "uniq -u" eliminates all non-unique lines. # By calling "cat /exclude" a second time we guarantee that there will be at least two lines # for each exception and therefore they will be excluded from the output passed to 'rm'. && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \ && /bin/busybox --install -s \ && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \ else \ /bin/busybox --install -s; \ fi FROM scratch COPY --from=builder / / LABEL maintainer="Alexander Kukushkin " ARG PG_MAJOR ARG COMPRESS ARG PGHOME ARG PGDATA ARG LC_ALL ARG LANG ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN ENV ETCDCTL_API=3 COPY patroni /patroni/ COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/ COPY extras/confd/templates/haproxy-citus.tmpl /etc/confd/templates/haproxy.tmpl COPY patroni*.py docker/entrypoint.sh / COPY postgres?.yml $PGHOME/ WORKDIR $PGHOME RUN sed -i 's/env python/&3/' /patroni*.py \ # "fix" patroni configs && sed -i 's/^ listen: 127.0.0.1/ listen: 0.0.0.0/' postgres?.yml \ && sed -i "s|^\( data_dir: \).*|\1$PGDATA|" postgres?.yml \ && sed -i "s|^#\( bin_dir: \).*|\1$PGBIN|" postgres?.yml \ && sed -i 's/^ - encoding: UTF8/ - locale: en_US.UTF-8\n&/' postgres?.yml \ && sed -i 's/^scope:/log:\n loggers:\n patroni.postgresql.mpp.citus: DEBUG\n#&/' postgres?.yml \ && sed -i 's/^\(name\|etcd\| host\| authentication\| connect_address\| parameters\):/#&/' postgres?.yml \ && sed -i 's/^ \(replication\|superuser\|rewind\|unix_socket_directories\|\(\( \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \ && sed -i 's/^postgresql:/&\n basebackup:\n checkpoint: fast/' postgres?.yml \ && sed -i 's|^ parameters:|&\n max_connections: 100\n shared_buffers: 16MB\n ssl: "on"\n ssl_ca_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n ssl_cert_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n ssl_key_file: /etc/ssl/private/ssl-cert-snakeoil.key\n citus.node_conninfo: "sslrootcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslkey=/etc/ssl/private/ssl-cert-snakeoil.key sslcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslmode=verify-ca"|' postgres?.yml \ && sed -i 's/^ pg_hba:/&\n - local all all trust/' postgres?.yml \ && sed -i 's/^\(.*\) \(.*\) \(.*\) \(.*\) \(.*\) md5.*$/\1 hostssl \3 \4 all md5 clientcert=verify-ca/' postgres?.yml \ && sed -i 's/^#\(ctl\| certfile\| keyfile\)/\1/' postgres?.yml \ && sed -i 's|^# cafile: .*$| verify_client: required\n cafile: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \ && sed -i 's|^# cacert: .*$| cacert: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \ && sed -i 's/^# insecure: .*/ insecure: on/' postgres?.yml \ # client cert for HAProxy to access Patroni REST API && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \ && chmod +s /bin/ping \ && chown -R postgres:postgres $PGHOME /run /etc/haproxy USER postgres ENTRYPOINT ["/bin/sh", "/entrypoint.sh"] patroni-4.0.4/LICENSE000066400000000000000000000021241472010352700142000ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2024 Compose, Zalando SE, Patroni Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. patroni-4.0.4/MAINTAINERS000066400000000000000000000001321472010352700146650ustar00rootroot00000000000000Alexander Kukushkin Polina Bungina patroni-4.0.4/MANIFEST.in000066400000000000000000000001031472010352700147240ustar00rootroot00000000000000include requirements* include *.rst recursive-include patroni *.py patroni-4.0.4/README.rst000066400000000000000000000206511472010352700146670ustar00rootroot00000000000000|Tests Status| |Coverage Status| Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul -------------------------------------------------------------------- You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io `__. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation `__. Patroni is a template for high availability (HA) PostgreSQL solutions using Python. For maximum accessibility, Patroni supports a variety of distributed configuration stores like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in datacenters - or anywhere else - will hopefully find it useful. We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. Currently supported PostgreSQL versions: 9.3 to 17. **Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus `__ database extension to Postgres. Please check the `Citus support page `__ in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster. **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes `__ chapter of the Patroni documentation. .. contents:: :local: :depth: 1 :backlinks: none ================= How Patroni Works ================= Patroni (formerly known as Zalando's Patroni) originated as a fork of `Governor `__, the project from Compose. It includes plenty of new features. For additional background info, see: * `Elephants on Automatic: HA Clustered PostgreSQL with Helm `_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017 * `PostgreSQL HA with Kubernetes and Patroni `__, talk by Josh Berkus at KubeCon 2016 (video) * `Feb. 2016 Zalando Tech blog post `__ ================== Development Status ================== Patroni is in active development and accepts contributions. See our `Contributing `__ section below for more details. We report new releases information `here `__. ========= Community ========= There are two places to connect with the Patroni community: `on github `__, via Issues and PRs, and on channel `#patroni `__ in the `PostgreSQL Slack `__. If you're using Patroni, or just interested, please join us. =================================== Technical Requirements/Installation =================================== **Pre-requirements for Mac OS** To install requirements on a Mac, run the following: :: brew install postgresql etcd haproxy libyaml python **Psycopg** Starting from `psycopg2-2.8 `__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages. Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it. There are a few options available: 1. Use the package manager from your distro :: sudo apt-get install python3-psycopg2 # install psycopg2 module on Debian/Ubuntu sudo yum install python3-psycopg2 # install psycopg2 on RedHat/Fedora/CentOS 2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the list of dependencies when installing Patroni with pip (see below). **General installation for pip** Patroni can be installed with pip: :: pip install patroni[dependencies] where dependencies can be either empty, or consist of one or more of the following: etcd or etcd3 `python-etcd` module in order to use Etcd as DCS consul `py-consul` module in order to use Consul as DCS zookeeper `kazoo` module in order to use Zookeeper as DCS exhibitor `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper) kubernetes `kubernetes` module in order to use Kubernetes as DCS in Patroni raft `pysyncobj` module in order to use python Raft implementation as DCS aws `boto3` in order to use AWS callbacks all all of the above (except psycopg family) psycopg3 `psycopg[binary]>=3.0.0` module psycopg2 `psycopg2>=2.5.4` module psycopg2-binary `psycopg2-binary` module For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is: :: pip install patroni[psycopg3,etcd3,aws] Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed independently of Patroni. ======================= Running and Configuring ======================= To get started, do the following from different terminals: :: > etcd --data-dir=data/etcd --enable-v2=true > ./patroni.py postgres0.yml > ./patroni.py postgres1.yml You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves. Add more ``postgres*.yml`` files to create an even larger cluster. Patroni provides an `HAProxy `__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure, run: :: > haproxy -f haproxy.cfg :: > psql --host 127.0.0.1 --port 5000 postgres ================== YAML Configuration ================== Go `here `__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml `__. ========================= Environment Configuration ========================= Go `here `__ for comprehensive information about configuring(overriding) settings via environment variables. =================== Replication Choices =================== Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation `__ for details. ====================================== Applications Should Not Use Superusers ====================================== When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable. .. |Tests Status| image:: https://github.com/patroni/patroni/actions/workflows/tests.yaml/badge.svg :target: https://github.com/patroni/patroni/actions/workflows/tests.yaml?query=branch%3Amaster .. |Coverage Status| image:: https://coveralls.io/repos/patroni/patroni/badge.svg?branch=master :target: https://coveralls.io/github/patroni/patroni?branch=master patroni-4.0.4/docker-compose-citus.yml000066400000000000000000000105601472010352700177600ustar00rootroot00000000000000# docker compose file for running a Citus cluster # with 3-node etcd v3 cluster as the DCS and one haproxy node. # The Citus cluster has a coordinator (3 nodes) # and two worker clusters (2 nodes). # # Before starting it up you need to build the docker image: # $ docker build -f Dockerfile.citus -t patroni-citus . # The cluster could be started as: # $ docker-compose -f docker-compose-citus.yml up -d # You can read more about it in the: # https://github.com/patroni/patroni/blob/master/docker/README.md#citus-cluster version: "2" networks: demo: services: etcd1: &etcd image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] environment: ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380 ETCD_INITIAL_CLUSTER_STATE: new ETCD_INITIAL_CLUSTER_TOKEN: tutorial ETCD_UNSUPPORTED_ARCH: arm64 container_name: demo-etcd1 hostname: etcd1 command: etcd --name etcd1 --initial-advertise-peer-urls http://etcd1:2380 etcd2: <<: *etcd container_name: demo-etcd2 hostname: etcd2 command: etcd --name etcd2 --initial-advertise-peer-urls http://etcd2:2380 etcd3: <<: *etcd container_name: demo-etcd3 hostname: etcd3 command: etcd --name etcd3 --initial-advertise-peer-urls http://etcd3:2380 haproxy: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: haproxy container_name: demo-haproxy ports: - "5000:5000" # Access to the coorinator primary - "5001:5001" # Load-balancing across workers primaries command: haproxy environment: &haproxy_env ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379 PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'" PATRONI_SCOPE: demo PATRONI_CITUS_GROUP: 0 PATRONI_CITUS_DATABASE: citus PGSSLMODE: verify-ca PGSSLKEY: /etc/ssl/private/ssl-cert-snakeoil.key PGSSLCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem PGSSLROOTCERT: /etc/ssl/certs/ssl-cert-snakeoil.pem coord1: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: coord1 container_name: demo-coord1 environment: &coord_env <<: *haproxy_env PATRONI_NAME: coord1 PATRONI_CITUS_GROUP: 0 coord2: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: coord2 container_name: demo-coord2 environment: <<: *coord_env PATRONI_NAME: coord2 coord3: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: coord3 container_name: demo-coord3 environment: <<: *coord_env PATRONI_NAME: coord3 work1-1: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work1-1 container_name: demo-work1-1 environment: &work1_env <<: *haproxy_env PATRONI_NAME: work1-1 PATRONI_CITUS_GROUP: 1 work1-2: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work1-2 container_name: demo-work1-2 environment: <<: *work1_env PATRONI_NAME: work1-2 work2-1: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work2-1 container_name: demo-work2-1 environment: &work2_env <<: *haproxy_env PATRONI_NAME: work2-1 PATRONI_CITUS_GROUP: 2 work2-2: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] env_file: docker/patroni.env hostname: work2-2 container_name: demo-work2-2 environment: <<: *work2_env PATRONI_NAME: work2-2 patroni-4.0.4/docker-compose.yml000066400000000000000000000052051472010352700166330ustar00rootroot00000000000000# docker compose file for running a 3-node PostgreSQL cluster # with 3-node etcd cluster as the DCS and one haproxy node # # requires a patroni image build from the Dockerfile: # $ docker build -t patroni . # The cluster could be started as: # $ docker-compose up -d # You can read more about it in the: # https://github.com/patroni/patroni/blob/master/docker/README.md version: "2" networks: demo: services: etcd1: &etcd image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] environment: ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380 ETCD_INITIAL_CLUSTER_STATE: new ETCD_INITIAL_CLUSTER_TOKEN: tutorial ETCD_UNSUPPORTED_ARCH: arm64 container_name: demo-etcd1 hostname: etcd1 command: etcd --name etcd1 --initial-advertise-peer-urls http://etcd1:2380 etcd2: <<: *etcd container_name: demo-etcd2 hostname: etcd2 command: etcd --name etcd2 --initial-advertise-peer-urls http://etcd2:2380 etcd3: <<: *etcd container_name: demo-etcd3 hostname: etcd3 command: etcd --name etcd3 --initial-advertise-peer-urls http://etcd3:2380 haproxy: image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: haproxy container_name: demo-haproxy ports: - "5000:5000" - "5001:5001" command: haproxy environment: &haproxy_env ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379 PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'" PATRONI_SCOPE: demo patroni1: image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: patroni1 container_name: demo-patroni1 environment: <<: *haproxy_env PATRONI_NAME: patroni1 patroni2: image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: patroni2 container_name: demo-patroni2 environment: <<: *haproxy_env PATRONI_NAME: patroni2 patroni3: image: ${PATRONI_TEST_IMAGE:-patroni} networks: [ demo ] env_file: docker/patroni.env hostname: patroni3 container_name: demo-patroni3 environment: <<: *haproxy_env PATRONI_NAME: patroni3 patroni-4.0.4/docker/000077500000000000000000000000001472010352700144435ustar00rootroot00000000000000patroni-4.0.4/docker/README.md000066400000000000000000000523131472010352700157260ustar00rootroot00000000000000# Dockerfile and Dockerfile.citus You can run Patroni in a docker container using these Dockerfiles They are meant in aiding development of Patroni and quick testing of features and not a production-worthy! docker build -t patroni . docker build -f Dockerfile.citus -t patroni-citus . # Examples ## Standalone Patroni docker run -d patroni ## Three-node Patroni cluster In addition to three Patroni containers the stack starts three containers with etcd (forming a three-node cluster), and one container with haproxy. The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load-balancing between healthy standbys). Example session: $ docker compose up -d ✔ Network patroni_demo Created ✔ Container demo-etcd1 Started ✔ Container demo-haproxy Started ✔ Container demo-patroni1 Started ✔ Container demo-patroni2 Started ✔ Container demo-patroni3 Started ✔ Container demo-etcd2 Started ✔ Container demo-etcd3 Started $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES a37bcec56726 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd3 034ab73868a8 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni2 03837736f710 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni3 22815c3d85b3 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd2 814b4304d132 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes 0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp demo-haproxy 6375b0ba2d0a patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni1 aef8bf3ee91f patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd1 $ docker logs demo-patroni1 2024-08-26 09:04:33,547 INFO: Selected new etcd server http://172.29.0.3:2379 2024-08-26 09:04:33,605 INFO: Lock owner: None; I am patroni1 2024-08-26 09:04:33,693 INFO: trying to bootstrap a new cluster ... 2024-08-26 09:04:34.920 UTC [43] LOG: starting PostgreSQL 16.4 (Debian 16.4-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit 2024-08-26 09:04:34.921 UTC [43] LOG: listening on IPv4 address "0.0.0.0", port 5432 2024-08-26 09:04:34,922 INFO: postmaster pid=43 2024-08-26 09:04:34.922 UTC [43] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" 2024-08-26 09:04:34.925 UTC [47] LOG: database system was shut down at 2024-08-26 09:04:34 UTC 2024-08-26 09:04:34.928 UTC [43] LOG: database system is ready to accept connections localhost:5432 - accepting connections localhost:5432 - accepting connections 2024-08-26 09:04:34,938 INFO: establishing a new patroni heartbeat connection to postgres 2024-08-26 09:04:34,992 INFO: running post_bootstrap 2024-08-26 09:04:35,004 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0 2024-08-26 09:04:35,009 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog' 2024-08-26 09:04:35,189 INFO: initialized a new cluster 2024-08-26 09:04:35,328 INFO: no action. I am (patroni1), the leader with the lock 2024-08-26 09:04:43,824 INFO: establishing a new patroni restapi connection to postgres 2024-08-26 09:04:45,322 INFO: no action. I am (patroni1), the leader with the lock 2024-08-26 09:04:55,320 INFO: no action. I am (patroni1), the leader with the lock ... $ docker exec -ti demo-patroni1 bash postgres@patroni1:~$ patronictl list + Cluster: demo (7303838734793224214) --------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +----------+------------+---------+-----------+----+-----------+ | patroni1 | 172.29.0.2 | Leader | running | 1 | | | patroni2 | 172.29.0.6 | Replica | streaming | 1 | 0 | | patroni3 | 172.29.0.5 | Replica | streaming | 1 | 0 | +----------+------------+---------+-----------+----+-----------+ postgres@patroni1:~$ etcdctl get --keys-only --prefix /service/demo /service/demo/config /service/demo/initialize /service/demo/leader /service/demo/members/patroni1 /service/demo/members/patroni2 /service/demo/members/patroni3 /service/demo/status postgres@patroni1:~$ etcdctl member list 2bf3e2ceda5d5960, started, etcd2, http://etcd2:2380, http://172.29.0.3:2379 55b3264e129c7005, started, etcd3, http://etcd3:2380, http://172.29.0.7:2379 acce7233f8ec127e, started, etcd1, http://etcd1:2380, http://172.29.0.8:2379 postgres@patroni1:~$ exit $ docker exec -ti demo-haproxy bash postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W Password: postgres psql (16.4 (Debian 16.4-1.pgdg120+1)) Type "help" for help. postgres=# SELECT pg_is_in_recovery(); pg_is_in_recovery ─────────────────── f (1 row) postgres=# \q postgres@haproxy:~$ psql -h localhost -p 5001 -U postgres -W Password: postgres psql (16.4 (Debian 16.4-1.pgdg120+1)) Type "help" for help. postgres=# SELECT pg_is_in_recovery(); pg_is_in_recovery ─────────────────── t (1 row) ## Citus cluster The stack starts three containers with etcd (forming a three-node etcd cluster), seven containers with Patroni+PostgreSQL+Citus (three coordinator nodes, and two worker clusters with two nodes each), and one container with haproxy. The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001 (does load-balancing between worker primary nodes). Example session: $ docker-compose -f docker-compose-citus.yml up -d ✔ Network patroni_demo Created ✔ Container demo-coord2 Started ✔ Container demo-work2-2 Started ✔ Container demo-etcd1 Started ✔ Container demo-haproxy Started ✔ Container demo-work1-1 Started ✔ Container demo-work2-1 Started ✔ Container demo-work1-2 Started ✔ Container demo-coord1 Started ✔ Container demo-etcd3 Started ✔ Container demo-coord3 Started ✔ Container demo-etcd2 Started $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 79c95492fac9 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd3 77eb82d0f0c1 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work2-1 03dacd7267ef patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd1 db9206c66f85 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd2 9a0fef7b7dd4 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work1-2 f06b031d99dc patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work2-2 f7c58545f314 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord2 383f9e7e188a patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work1-1 f02e96dcc9d6 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord3 6945834b7056 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord1 b96ca42f785d patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes 0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp demo-haproxy $ docker logs demo-coord1 2024-08-26 08:21:05,323 INFO: Selected new etcd server http://172.19.0.5:2379 2024-08-26 08:21:05,339 INFO: No PostgreSQL configuration items changed, nothing to reload. 2024-08-26 08:21:05,388 INFO: Lock owner: None; I am coord1 2024-08-26 08:21:05,480 INFO: trying to bootstrap a new cluster ... 2024-08-26 08:21:17,115 INFO: postmaster pid=35 localhost:5432 - no response 2024-08-26 08:21:17.127 UTC [35] LOG: starting PostgreSQL 16.4 (Debian 16.4-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit 2024-08-26 08:21:17.127 UTC [35] LOG: listening on IPv4 address "0.0.0.0", port 5432 2024-08-26 08:21:17.141 UTC [35] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" 2024-08-26 08:21:17.155 UTC [39] LOG: database system was shut down at 2024-08-26 08:21:05 UTC 2024-08-26 08:21:17.182 UTC [35] LOG: database system is ready to accept connections 2024-08-26 08:21:17,683 INFO: establishing a new patroni heartbeat connection to postgres 2024-08-26 08:21:17,704 INFO: establishing a new patroni restapi connection to postgres localhost:5432 - accepting connections localhost:5432 - accepting connections 2024-08-26 08:21:18,202 INFO: running post_bootstrap 2024-08-26 08:21:19.048 UTC [53] LOG: starting maintenance daemon on database 16385 user 10 2024-08-26 08:21:19.048 UTC [53] CONTEXT: Citus maintenance daemon for database 16385 user 10 2024-08-26 08:21:19,058 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog' 2024-08-26 08:21:19.250 UTC [37] LOG: checkpoint starting: immediate force wait 2024-08-26 08:21:19,275 INFO: initialized a new cluster 2024-08-26 08:21:22.946 UTC [37] LOG: checkpoint starting: immediate force wait 2024-08-26 08:21:29,059 INFO: Lock owner: coord1; I am coord1 2024-08-26 08:21:29,205 INFO: Enabled synchronous replication 2024-08-26 08:21:29,206 DEBUG: query(SELECT groupid, nodename, nodeport, noderole, nodeid FROM pg_catalog.pg_dist_node, ()) 2024-08-26 08:21:29,206 INFO: establishing a new patroni citus connection to postgres 2024-08-26 08:21:29,206 DEBUG: Adding the new task: PgDistTask({PgDistNode(nodeid=None,host=172.19.0.8,port=5432,role=primary)}) 2024-08-26 08:21:29,206 DEBUG: Adding the new task: PgDistTask({PgDistNode(nodeid=None,host=172.19.0.2,port=5432,role=primary)}) 2024-08-26 08:21:29,206 DEBUG: Adding the new task: PgDistTask({PgDistNode(nodeid=None,host=172.19.0.9,port=5432,role=primary)}) 2024-08-26 08:21:29,219 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default'), ('172.19.0.2', 5432, 1, 'primary')) 2024-08-26 08:21:29,256 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default'), ('172.19.0.9', 5432, 2, 'primary')) 2024-08-26 08:21:29,474 INFO: no action. I am (coord1), the leader with the lock 2024-08-26 08:21:39,060 INFO: Lock owner: coord1; I am coord1 2024-08-26 08:21:39,159 DEBUG: Adding the new task: PgDistTask({PgDistNode(nodeid=None,host=172.19.0.8,port=5432,role=primary), PgDistNode(nodeid=None,host=172.19.0.11,port=5432,role=secondary), PgDistNode(nodeid=None,host=172.19.0.7,port=5432,role=secondary)}) 2024-08-26 08:21:39,159 DEBUG: Adding the new task: PgDistTask({PgDistNode(nodeid=None,host=172.19.0.2,port=5432,role=primary), PgDistNode(nodeid=None,host=172.19.0.12,port=5432,role=secondary)}) 2024-08-26 08:21:39,159 DEBUG: Adding the new task: PgDistTask({PgDistNode(nodeid=None,host=172.19.0.6,port=5432,role=secondary), PgDistNode(nodeid=None,host=172.19.0.9,port=5432,role=primary)}) 2024-08-26 08:21:39,160 DEBUG: query(BEGIN, ()) 2024-08-26 08:21:39,160 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default'), ('172.19.0.11', 5432, 0, 'secondary')) 2024-08-26 08:21:39,164 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default'), ('172.19.0.7', 5432, 0, 'secondary')) 2024-08-26 08:21:39,166 DEBUG: query(COMMIT, ()) 2024-08-26 08:21:39,176 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default'), ('172.19.0.12', 5432, 1, 'secondary')) 2024-08-26 08:21:39,191 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default'), ('172.19.0.6', 5432, 2, 'secondary')) 2024-08-26 08:21:39,211 INFO: no action. I am (coord1), the leader with the lock 2024-08-26 08:21:49,060 INFO: Lock owner: coord1; I am coord1 2024-08-26 08:21:49,166 INFO: Setting synchronous replication to 1 of 2 (coord2, coord3) server signaled 2024-08-26 08:21:49.170 UTC [35] LOG: received SIGHUP, reloading configuration files 2024-08-26 08:21:49.171 UTC [35] LOG: parameter "synchronous_standby_names" changed to "ANY 1 (coord2,coord3)" 2024-08-26 08:21:49.377 UTC [68] LOG: standby "coord2" is now a candidate for quorum synchronous standby 2024-08-26 08:21:49.377 UTC [68] STATEMENT: START_REPLICATION SLOT "coord2" 0/3000000 TIMELINE 1 2024-08-26 08:21:49.377 UTC [69] LOG: standby "coord3" is now a candidate for quorum synchronous standby 2024-08-26 08:21:49.377 UTC [69] STATEMENT: START_REPLICATION SLOT "coord3" 0/4000000 TIMELINE 1 2024-08-26 08:21:50,278 INFO: Setting leader to coord1, quorum to 1 of 2 (coord2, coord3) 2024-08-26 08:21:50,390 INFO: no action. I am (coord1), the leader with the lock 2024-08-26 08:21:59,159 INFO: no action. I am (coord1), the leader with the lock ... $ docker exec -ti demo-haproxy bash postgres@haproxy:~$ etcdctl member list 2b28411e74c0c281, started, etcd3, http://etcd3:2380, http://172.30.0.4:2379 6c70137d27cfa6c1, started, etcd2, http://etcd2:2380, http://172.30.0.5:2379 a28f9a70ebf21304, started, etcd1, http://etcd1:2380, http://172.30.0.6:2379 postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo /service/demo/0/config /service/demo/0/initialize /service/demo/0/leader /service/demo/0/members/coord1 /service/demo/0/members/coord2 /service/demo/0/members/coord3 /service/demo/0/status /service/demo/0/sync /service/demo/1/config /service/demo/1/initialize /service/demo/1/leader /service/demo/1/members/work1-1 /service/demo/1/members/work1-2 /service/demo/1/status /service/demo/1/sync /service/demo/2/config /service/demo/2/initialize /service/demo/2/leader /service/demo/2/members/work2-1 /service/demo/2/members/work2-2 /service/demo/2/status /service/demo/2/sync postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus Password for user postgres: postgres psql (16.4 (Debian 16.4-1.pgdg120+1)) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. citus=# select pg_is_in_recovery(); pg_is_in_recovery ------------------- f (1 row) citus=# table pg_dist_node; nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+-------------+----------+----------+-------------+----------+-----------+-------------+----------------+------------------ 1 | 0 | 172.19.0.8 | 5432 | default | t | t | primary | default | t | f 2 | 1 | 172.19.0.2 | 5432 | default | t | t | primary | default | t | t 3 | 2 | 172.19.0.9 | 5432 | default | t | t | primary | default | t | t 4 | 0 | 172.19.0.11 | 5432 | default | t | t | secondary | default | t | f 5 | 0 | 172.19.0.7 | 5432 | default | t | t | secondary | default | t | f 6 | 1 | 172.19.0.12 | 5432 | default | f | t | secondary | default | f | t 7 | 2 | 172.19.0.6 | 5432 | default | f | t | secondary | default | f | t (7 rows) citus=# \q postgres@haproxy:~$ patronictl list + Citus cluster: demo ----------+----------------+-----------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------+-------------+----------------+-----------+----+-----------+ | 0 | coord1 | 172.19.0.8 | Leader | running | 1 | | | 0 | coord2 | 172.19.0.7 | Quorum Standby | streaming | 1 | 0 | | 0 | coord3 | 172.19.0.11 | Quorum Standby | streaming | 1 | 0 | | 1 | work1-1 | 172.19.0.12 | Quorum Standby | streaming | 1 | 0 | | 1 | work1-2 | 172.19.0.2 | Leader | running | 1 | | | 2 | work2-1 | 172.19.0.6 | Quorum Standby | streaming | 1 | 0 | | 2 | work2-2 | 172.19.0.9 | Leader | running | 1 | | +-------+---------+-------------+----------------+-----------+----+-----------+ postgres@haproxy:~$ patronictl switchover --group 2 --force Current cluster topology + Citus cluster: demo (group: 2, 7407360296219029527) ---+-----------+ | Member | Host | Role | State | TL | Lag in MB | +---------+------------+----------------+-----------+----+-----------+ | work2-1 | 172.19.0.6 | Quorum Standby | streaming | 1 | 0 | | work2-2 | 172.19.0.9 | Leader | running | 1 | | +---------+------------+----------------+-----------+----+-----------+ 2024-08-26 08:31:45.92277 Successfully switched over to "work2-1" + Citus cluster: demo (group: 2, 7407360296219029527) ------+ | Member | Host | Role | State | TL | Lag in MB | +---------+------------+---------+---------+----+-----------+ | work2-1 | 172.19.0.6 | Leader | running | 1 | | | work2-2 | 172.19.0.9 | Replica | stopped | | unknown | +---------+------------+---------+---------+----+-----------+ postgres@haproxy:~$ patronictl list + Citus cluster: demo ----------+----------------+-----------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------+-------------+----------------+-----------+----+-----------+ | 0 | coord1 | 172.19.0.8 | Leader | running | 1 | | | 0 | coord2 | 172.19.0.7 | Quorum Standby | streaming | 1 | 0 | | 0 | coord3 | 172.19.0.11 | Quorum Standby | streaming | 1 | 0 | | 1 | work1-1 | 172.19.0.12 | Quorum Standby | streaming | 1 | 0 | | 1 | work1-2 | 172.19.0.2 | Leader | running | 1 | | | 2 | work2-1 | 172.19.0.6 | Leader | running | 2 | | | 2 | work2-2 | 172.19.0.9 | Quorum Standby | streaming | 2 | 0 | +-------+---------+-------------+----------------+-----------+----+-----------+ postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus Password for user postgres: postgres psql (16.4 (Debian 16.4-1.pgdg120+1)) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. citus=# table pg_dist_node; nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+-------------+----------+----------+-------------+----------+-----------+-------------+----------------+------------------ 1 | 0 | 172.19.0.8 | 5432 | default | t | t | primary | default | t | f 4 | 0 | 172.19.0.11 | 5432 | default | t | t | secondary | default | t | f 5 | 0 | 172.19.0.7 | 5432 | default | t | t | secondary | default | t | f 6 | 1 | 172.19.0.12 | 5432 | default | f | t | secondary | default | f | t 3 | 2 | 172.19.0.6 | 5432 | default | t | t | primary | default | t | t 2 | 1 | 172.19.0.2 | 5432 | default | t | t | primary | default | t | t 8 | 2 | 172.19.0.9 | 5432 | default | f | t | secondary | default | f | t (7 rows) patroni-4.0.4/docker/entrypoint.sh000077500000000000000000000063001472010352700172140ustar00rootroot00000000000000#!/bin/sh if [ -f /a.tar.xz ]; then echo "decompressing image..." sudo tar xpJf /a.tar.xz -C / > /dev/null 2>&1 sudo rm /a.tar.xz sudo ln -snf dash /bin/sh fi readonly PATRONI_SCOPE="${PATRONI_SCOPE:-batman}" PATRONI_NAMESPACE="${PATRONI_NAMESPACE:-/service}" readonly PATRONI_NAMESPACE="${PATRONI_NAMESPACE%/}" DOCKER_IP=$(hostname --ip-address) readonly DOCKER_IP export DUMB_INIT_SETSID=0 case "$1" in haproxy) haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D set -- confd "-prefix=$PATRONI_NAMESPACE/$PATRONI_SCOPE" -interval=10 -backend if [ -n "$PATRONI_ZOOKEEPER_HOSTS" ]; then while ! /usr/share/zookeeper/bin/zkCli.sh -server "$PATRONI_ZOOKEEPER_HOSTS" ls /; do sleep 1 done set -- "$@" zookeeper -node "$PATRONI_ZOOKEEPER_HOSTS" else while ! etcdctl member list 2> /dev/null; do sleep 1 done set -- "$@" etcdv3 while IFS='' read -r line; do set -- "$@" -node "$line" done <<-EOT $(echo "$ETCDCTL_ENDPOINTS" | sed 's/,/\n/g') EOT fi exec dumb-init "$@" ;; etcd) exec "$@" --auto-compaction-retention=1 -advertise-client-urls "http://$DOCKER_IP:2379" ;; zookeeper) exec /usr/share/zookeeper/bin/zkServer.sh start-foreground ;; esac ## We start an etcd if [ -z "$PATRONI_ETCD3_HOSTS" ] && [ -z "$PATRONI_ZOOKEEPER_HOSTS" ]; then export PATRONI_ETCD_URL="http://127.0.0.1:2379" etcd --data-dir /tmp/etcd.data -advertise-client-urls=$PATRONI_ETCD_URL -listen-client-urls=http://0.0.0.0:2379 > /var/log/etcd.log 2> /var/log/etcd.err & fi export PATRONI_SCOPE export PATRONI_NAMESPACE export PATRONI_NAME="${PATRONI_NAME:-$(hostname)}" export PATRONI_RESTAPI_CONNECT_ADDRESS="$DOCKER_IP:8008" export PATRONI_RESTAPI_LISTEN="0.0.0.0:8008" export PATRONI_admin_PASSWORD="${PATRONI_admin_PASSWORD:-admin}" export PATRONI_admin_OPTIONS="${PATRONI_admin_OPTIONS:-createdb, createrole}" export PATRONI_POSTGRESQL_CONNECT_ADDRESS="$DOCKER_IP:5432" export PATRONI_POSTGRESQL_LISTEN="0.0.0.0:5432" export PATRONI_POSTGRESQL_DATA_DIR="${PATRONI_POSTGRESQL_DATA_DIR:-$PGDATA}" export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}" export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-replicate}" export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}" export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}" export PATRONI_REPLICATION_SSLMODE="${PATRONI_REPLICATION_SSLMODE:-$PGSSLMODE}" export PATRONI_REPLICATION_SSLKEY="${PATRONI_REPLICATION_SSLKEY:-$PGSSLKEY}" export PATRONI_REPLICATION_SSLCERT="${PATRONI_REPLICATION_SSLCERT:-$PGSSLCERT}" export PATRONI_REPLICATION_SSLROOTCERT="${PATRONI_REPLICATION_SSLROOTCERT:-$PGSSLROOTCERT}" export PATRONI_SUPERUSER_SSLMODE="${PATRONI_SUPERUSER_SSLMODE:-$PGSSLMODE}" export PATRONI_SUPERUSER_SSLKEY="${PATRONI_SUPERUSER_SSLKEY:-$PGSSLKEY}" export PATRONI_SUPERUSER_SSLCERT="${PATRONI_SUPERUSER_SSLCERT:-$PGSSLCERT}" export PATRONI_SUPERUSER_SSLROOTCERT="${PATRONI_SUPERUSER_SSLROOTCERT:-$PGSSLROOTCERT}" exec dumb-init python3 /patroni.py postgres0.yml patroni-4.0.4/docker/patroni.env000066400000000000000000000004341472010352700166320ustar00rootroot00000000000000PATRONI_RESTAPI_USERNAME=admin PATRONI_RESTAPI_PASSWORD=admin PATRONI_SUPERUSER_USERNAME=postgres PATRONI_SUPERUSER_PASSWORD=postgres PATRONI_REPLICATION_USERNAME=replicator PATRONI_REPLICATION_PASSWORD=replicate PATRONI_admin_PASSWORD=admin PATRONI_admin_OPTIONS=createdb,createrole patroni-4.0.4/docs/000077500000000000000000000000001472010352700141245ustar00rootroot00000000000000patroni-4.0.4/docs/CONTRIBUTING.rst000066400000000000000000000003271472010352700165670ustar00rootroot00000000000000.. _contributing: Contributing ============ Resources and information for developers can be found in the pages below. .. toctree:: :maxdepth: 2 contributing_guidelines Patroni API docs patroni-4.0.4/docs/ENVIRONMENT.rst000066400000000000000000000774521472010352700165010ustar00rootroot00000000000000.. _environment: Environment Configuration Settings ================================== It is possible to override some of the configuration parameters defined in the Patroni configuration file using the system environment variables. This document lists all environment variables handled by Patroni. The values set via those variables always take precedence over the ones set in the Patroni configuration file. Global/Universal ---------------- - **PATRONI\_CONFIGURATION**: it is possible to set the entire configuration for the Patroni via ``PATRONI_CONFIGURATION`` environment variable. In this case any other environment variables will not be considered! - **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster. - **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service" - **PATRONI\_SCOPE**: cluster name Log --- - **PATRONI\_LOG\_TYPE**: sets the format of logs. Can be either **plain** or **json**. To use **json** format, you must have the :ref:`jsonlogger ` installed. The default value is **plain**. - **PATRONI\_LOG\_LEVEL**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging `_) - **PATRONI\_LOG\_TRACEBACK\_LEVEL**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **PATRONI\_LOG\_LEVEL=DEBUG**. - **PATRONI\_LOG\_FORMAT**: sets the log formatting string. If the log type is **plain**, the log format should be a string. Refer to `the LogRecord attributes `_ for available attributes. If the log type is **json**, the log format can be a list in addition to a string. Each list item should correspond to LogRecord attributes. Be cautious that only the field name is required, and the **%(** and **)** should be omitted. If you wish to print a log field with a different key name, use a dictionary where the dictionary key is the log field, and the value is the name of the field you want to be printed in the log. Default value is **%(asctime)s %(levelname)s: %(message)s** - **PATRONI\_LOG\_DATEFORMAT**: sets the datetime formatting string. (see the `formatTime() documentation `_) - **PATRONI\_LOG\_STATIC\_FIELDS**: add additional fields to the log. This option is only available when the log type is set to **json**. Example ``PATRONI_LOG_STATIC_FIELDS="{app: patroni}"`` - **PATRONI\_LOG\_MAX\_QUEUE\_SIZE**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m. - **PATRONI\_LOG\_DIR**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this env variable, the application will retain 4 25MB logs by default. You can tune those retention values with `PATRONI_LOG_FILE_NUM` and `PATRONI_LOG_FILE_SIZE` (see below). - **PATRONI\_LOG\_MODE**: Permissions for log files (for example, ``0644``). If not specified, permissions will be set based on the current umask value. - **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain. - **PATRONI\_LOG\_FILE\_SIZE**: Size of patroni.log file (in bytes) that triggers a log rolling. - **PATRONI\_LOG\_LOGGERS**: Redefine logging level per python module. Example ``PATRONI_LOG_LOGGERS="{patroni.postmaster: WARNING, urllib3: DEBUG}"`` Citus ----- Enables integration Patroni with `Citus `__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here `. - **PATRONI\_CITUS\_GROUP**: the Citus group id, integer. Use ``0`` for coordinator and ``1``, ``2``, etc... for workers - **PATRONI\_CITUS\_DATABASE**: the database where ``citus`` extension should be created. Must be the same on the coordinator and all workers. Currently only one database is supported. Consul ------ - **PATRONI\_CONSUL\_HOST**: the host:port for the Consul local agent. - **PATRONI\_CONSUL\_URL**: url for the Consul local agent, in format: http(s)://host:port - **PATRONI\_CONSUL\_PORT**: (optional) Consul port - **PATRONI\_CONSUL\_SCHEME**: (optional) **http** or **https**, defaults to **http** - **PATRONI\_CONSUL\_TOKEN**: (optional) ACL token - **PATRONI\_CONSUL\_VERIFY**: (optional) whether to verify the SSL certificate for HTTPS requests - **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If present it will enable validation. - **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate - **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate. - **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used. - **PATRONI\_CONSUL\_CONSISTENCY**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference `__) - **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. By default an empty list is used. - **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, primary, replica, or standby-leader depending on the node's role. Defaults to **false** - **PATRONI\_CONSUL\_SERVICE\_TAGS**: (optional) additional static tags to add to the Consul service apart from the role (``primary``/``replica``/``standby-leader``). By default an empty list is used. - **PATRONI\_CONSUL\_SERVICE\_CHECK\_INTERVAL**: (optional) how often to perform health check against registered url - **PATRONI\_CONSUL\_SERVICE\_CHECK\_TLS\_SERVER\_NAME**: (optional) override SNI host when connecting via TLS, see also `consul agent check API reference `__. Etcd ---- - **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL** - **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port - **PATRONI\_ETCD\_HOSTS**: list of etcd endpoints in format 'host1:port1','host2:port2',etc... - **PATRONI\_ETCD\_USE\_PROXIES**: If this parameter is set to true, Patroni will consider **hosts** as a list of proxies and will not perform a topology discovery of etcd cluster but stick to a fixed list of **hosts**. - **PATRONI\_ETCD\_PROTOCOL**: http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them. - **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint. - **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery. Patroni will try to query these SRV service names for specified domain (in that order until first success): ``_etcd-client-ssl``, ``_etcd-client``, ``_etcd-ssl``, ``_etcd``, ``_etcd-server-ssl``, ``_etcd-server``. If SRV records for ``_etcd-server-ssl`` or ``_etcd-server`` are retrieved then ETCD peer protocol is used do query ETCD for available members. Otherwise hosts from SRV records will be used. - **PATRONI\_ETCD\_SRV\_SUFFIX**: Configures a suffix to the SRV name that is queried during discovery. Use this flag to differentiate between multiple etcd clusters under the same domain. Works only with conjunction with **PATRONI\_ETCD\_SRV**. For example, if ``PATRONI_ETCD_SRV_SUFFIX=foo`` and ``PATRONI_ETCD_SRV=example.org`` are set, the following DNS SRV query is made:``_etcd-client-ssl-foo._tcp.example.com`` (and so on for every possible ETCD SRV service name). - **PATRONI\_ETCD\_USERNAME**: username for etcd authentication. - **PATRONI\_ETCD\_PASSWORD**: password for etcd authentication. - **PATRONI\_ETCD\_CACERT**: The ca certificate. If present it will enable validation. - **PATRONI\_ETCD\_CERT**: File with the client certificate. - **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate. Etcdv3 ------ Environment names for Etcdv3 are similar as for Etcd, you just need to use ``ETCD3`` instead of ``ETCD`` in the variable name. Example: ``PATRONI_ETCD3_HOST``, ``PATRONI_ETCD3_CACERT``, and so on. .. warning:: Keys created with protocol version 2 are not visible with protocol version 3 and the other way around, therefore it is not possible to switch from Etcd to Etcdv3 just by updating Patroni configuration. In addition, Patroni uses Etcd's gRPC-gateway (proxy) to communicate with the V3 API, which means that TLS common name authentication is not possible. ZooKeeper --------- - **PATRONI\_ZOOKEEPER\_HOSTS**: Comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity! - **PATRONI\_ZOOKEEPER\_USE\_SSL**: (optional) Whether SSL is used or not. Defaults to ``false``. If set to ``false``, all SSL specific parameters are ignored. - **PATRONI\_ZOOKEEPER\_CACERT**: (optional) The CA certificate. If present it will enable validation. - **PATRONI\_ZOOKEEPER\_CERT**: (optional) File with the client certificate. - **PATRONI\_ZOOKEEPER\_KEY**: (optional) File with the client key. - **PATRONI\_ZOOKEEPER\_KEY\_PASSWORD**: (optional) The client key password. - **PATRONI\_ZOOKEEPER\_VERIFY**: (optional) Whether to verify certificate or not. Defaults to ``true``. - **PATRONI\_ZOOKEEPER\_SET\_ACLS**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value. Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``. For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``. - **PATRONI\_ZOOKEEPER\_AUTH\_DATA**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary. .. note:: It is required to install ``kazoo>=2.6.0`` to support SSL. Exhibitor --------- - **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes. - **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port. .. _kubernetes_environment: Kubernetes ---------- - **PATRONI\_KUBERNETES\_BYPASS\_API\_SERVICE**: (optional) When communicating with the Kubernetes API, Patroni is usually relying on the `kubernetes` service, the address of which is exposed in the pods via the `KUBERNETES_SERVICE_HOST` environment variable. If `PATRONI_KUBERNETES_BYPASS_API_SERVICE` is set to ``true``, Patroni will resolve the list of API nodes behind the service and connect directly to them. - **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`. - **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates. - **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`. - **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing role (`primary`, `replica` or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``. - **PATRONI\_KUBERNETES\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `primary`. Default value is `primary`. - **PATRONI\_KUBERNETES\_FOLLOWER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `replica`. Default value is `replica`. - **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``primary``. - **PATRONI\_KUBERNETES\_TMP\_ROLE\_LABEL**: (optional) name of the temporary label containing role (`primary` or `replica`). Value of this label will always use the default of corresponding role. Set only when necessary. - **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state. - **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted. - **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set. - **PATRONI\_KUBERNETES\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret. - **PATRONI\_RETRIABLE\_HTTP\_CODES**: (optional) list of HTTP status codes from K8s API to retry on. By default Patroni is retrying on ``500``, ``503``, and ``504``, or if K8s API response has ``retry-after`` HTTP header. Raft (deprecated) ----------------- - **PATRONI\_RAFT\_SELF\_ADDR**: ``ip:port`` to listen on for Raft connections. The ``self_addr`` must be accessible from other nodes of the cluster. If not set, the node will not participate in consensus. - **PATRONI\_RAFT\_BIND\_ADDR**: (optional) ``ip:port`` to listen on for Raft connections. If not specified the ``self_addr`` will be used. - **PATRONI\_RAFT\_PARTNER\_ADDRS**: list of other Patroni nodes in the cluster in format ``"'ip1:port1','ip2:port2'"``. It is important to quote every single entity! - **PATRONI\_RAFT\_DATA\_DIR**: directory where to store Raft log and snapshot. If not specified the current working directory is used. - **PATRONI\_RAFT\_PASSWORD**: (optional) Encrypt Raft traffic with a specified password, requires ``cryptography`` python module. PostgreSQL ---------- - **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. - **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications. - **PATRONI\_POSTGRESQL\_PROXY\_ADDRESS**: IP address + port through which a connection pool (e.g. pgbouncer) running next to Postgres is accessible. The value is written to the member key in DCS as ``proxy_url`` and could be used/useful for service discovery. - **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni. - **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni. - **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, initdb, pg_controldata, pg_basebackup, postgres, pg_isready, pg_rewind) The default value is an empty string meaning that PATH environment variable will be used to find the executables. - **PATRONI\_POSTGRESQL\_BIN\_PG\_CTL**: (optional) Custom name for ``pg_ctl`` binary. - **PATRONI\_POSTGRESQL\_BIN\_INITDB**: (optional) Custom name for ``initdb`` binary. - **PATRONI\_POSTGRESQL\_BIN\_PG\_CONTROLDATA**: (optional) Custom name for ``pg_controldata`` binary. - **PATRONI\_POSTGRESQL\_BIN\_PG\_BASEBACKUP**: (optional) Custom name for ``pg_basebackup`` binary. - **PATRONI\_POSTGRESQL\_BIN\_POSTGRES**: (optional) Custom name for ``postgres`` binary. - **PATRONI\_POSTGRESQL\_BIN\_IS\_READY**: (optional) Custom name for ``pg_isready`` binary. - **PATRONI\_POSTGRESQL\_BIN\_PG\_REWIND**: (optional) Custom name for ``pg_rewind`` binary. - **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication - **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization. - **PATRONI\_REPLICATION\_SSLMODE**: (optional) maps to the `sslmode `__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation `__. The default mode is ``prefer``. - **PATRONI\_REPLICATION\_SSLKEY**: (optional) maps to the `sslkey `__ connection parameter, which specifies the location of the secret key used with the client's certificate. - **PATRONI\_REPLICATION\_SSLPASSWORD**: (optional) maps to the `sslpassword `__ connection parameter, which specifies the password for the secret key specified in ``PATRONI_REPLICATION_SSLKEY``. - **PATRONI\_REPLICATION\_SSLCERT**: (optional) maps to the `sslcert `__ connection parameter, which specifies the location of the client certificate. - **PATRONI\_REPLICATION\_SSLROOTCERT**: (optional) maps to the `sslrootcert `__ connection parameter, which specifies the location of a file containing one or more certificate authorities (CA) certificates that the client will use to verify a server's certificate. - **PATRONI\_REPLICATION\_SSLCRL**: (optional) maps to the `sslcrl `__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **PATRONI\_REPLICATION\_SSLCRLDIR**: (optional) maps to the `sslcrldir `__ connection parameter, which specifies the location of a directory with files containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **PATRONI\_REPLICATION\_SSLNEGOTIATION**: (optional) maps to the `sslnegotiation `__ connection parameter, which controls how SSL encryption is negotiated with the server, if SSL is used. - **PATRONI\_REPLICATION\_GSSENCMODE**: (optional) maps to the `gssencmode `__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server - **PATRONI\_REPLICATION\_CHANNEL\_BINDING**: (optional) maps to the `channel_binding `__ connection parameter, which controls the client's use of channel binding. - **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind. - **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb). - **PATRONI\_SUPERUSER\_SSLMODE**: (optional) maps to the `sslmode `__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation `__. The default mode is ``prefer``. - **PATRONI\_SUPERUSER\_SSLKEY**: (optional) maps to the `sslkey `__ connection parameter, which specifies the location of the secret key used with the client's certificate. - **PATRONI\_SUPERUSER\_SSLPASSWORD**: (optional) maps to the `sslpassword `__ connection parameter, which specifies the password for the secret key specified in ``PATRONI_SUPERUSER_SSLKEY``. - **PATRONI\_SUPERUSER\_SSLCERT**: (optional) maps to the `sslcert `__ connection parameter, which specifies the location of the client certificate. - **PATRONI\_SUPERUSER\_SSLROOTCERT**: (optional) maps to the `sslrootcert `__ connection parameter, which specifies the location of a file containing one or more certificate authorities (CA) certificates that the client will use to verify a server's certificate. - **PATRONI\_SUPERUSER\_SSLCRL**: (optional) maps to the `sslcrl `__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **PATRONI\_SUPERUSER\_SSLCRLDIR**: (optional) maps to the `sslcrldir `__ connection parameter, which specifies the location of a directory with files containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **PATRONI\_SUPERUSER\_SSLNEGOTIATION**: (optional) maps to the `sslnegotiation `__ connection parameter, which controls how SSL encryption is negotiated with the server, if SSL is used. - **PATRONI\_SUPERUSER\_GSSENCMODE**: (optional) maps to the `gssencmode `__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server - **PATRONI\_SUPERUSER\_CHANNEL\_BINDING**: (optional) maps to the `channel_binding `__ connection parameter, which controls the client's use of channel binding. - **PATRONI\_REWIND\_USERNAME**: (optional) name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions `__ will be granted. - **PATRONI\_REWIND\_PASSWORD**: (optional) password for the user for ``pg_rewind``; the user will be created during initialization. - **PATRONI\_REWIND\_SSLMODE**: (optional) maps to the `sslmode `__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation `__. The default mode is ``prefer``. - **PATRONI\_REWIND\_SSLKEY**: (optional) maps to the `sslkey `__ connection parameter, which specifies the location of the secret key used with the client's certificate. - **PATRONI\_REWIND\_SSLPASSWORD**: (optional) maps to the `sslpassword `__ connection parameter, which specifies the password for the secret key specified in ``PATRONI_REWIND_SSLKEY``. - **PATRONI\_REWIND\_SSLCERT**: (optional) maps to the `sslcert `__ connection parameter, which specifies the location of the client certificate. - **PATRONI\_REWIND\_SSLROOTCERT**: (optional) maps to the `sslrootcert `__ connection parameter, which specifies the location of a file containing one or more certificate authorities (CA) certificates that the client will use to verify a server's certificate. - **PATRONI\_REWIND\_SSLCRL**: (optional) maps to the `sslcrl `__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **PATRONI\_REWIND\_SSLCRLDIR**: (optional) maps to the `sslcrldir `__ connection parameter, which specifies the location of a directory with files containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **PATRONI\_REWIND\_SSLNEGOTIATION**: (optional) maps to the `sslnegotiation `__ connection parameter, which controls how SSL encryption is negotiated with the server, if SSL is used. - **PATRONI\_REWIND\_GSSENCMODE**: (optional) maps to the `gssencmode `__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server - **PATRONI\_REWIND\_CHANNEL\_BINDING**: (optional) maps to the `channel_binding `__ connection parameter, which controls the client's use of channel binding. REST API -------- - **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port to access the REST API. - **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy. - **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect unsafe REST API endpoints. - **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints. - **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. - **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format. - **PATRONI\_RESTAPI\_KEYFILE\_PASSWORD**: Specifies a password for decrypting the keyfile. - **PATRONI\_RESTAPI\_CAFILE**: Specifies the file with the CA_BUNDLE with certificates of trusted CAs to use while verifying client certs. - **PATRONI\_RESTAPI\_CIPHERS**: (optional) Specifies the permitted cipher suites (e.g. "ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:!SSLv1:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1") - **PATRONI\_RESTAPI\_VERIFY\_CLIENT**: ``none`` (default), ``optional`` or ``required``. When ``none`` REST API will not check client certificates. When ``required`` client certificates are required for all REST API calls. When ``optional`` client certificates are required for all unsafe REST API endpoints. When ``required`` is used, then client authentication succeeds, if the certificate signature verification succeeds. For ``optional`` the client cert will only be checked for ``PUT``, ``POST``, ``PATCH``, and ``DELETE`` requests. - **PATRONI\_RESTAPI\_ALLOWLIST**: (optional): Specifies the set of hosts that are allowed to call unsafe REST API endpoints. The single element could be a host name, an IP address or a network address using CIDR notation. By default ``allow all`` is used. In case if ``allowlist`` or ``allowlist_include_members`` are set, anything that is not included is rejected. - **PATRONI\_RESTAPI\_ALLOWLIST\_INCLUDE\_MEMBERS**: (optional): If set to ``true`` it allows accessing unsafe REST API endpoints from other cluster members registered in DCS (IP address or hostname is taken from the members ``api_url``). Be careful, it might happen that OS will use a different IP for outgoing connections. - **PATRONI\_RESTAPI\_HTTP\_EXTRA\_HEADERS**: (optional) HTTP headers let the REST API server pass additional information with an HTTP response. - **PATRONI\_RESTAPI\_HTTPS\_EXTRA\_HEADERS**: (optional) HTTPS headers let the REST API server pass additional information with an HTTP response when TLS is enabled. This will also pass additional information set in ``http_extra_headers``. - **PATRONI\_RESTAPI\_REQUEST\_QUEUE\_SIZE**: (optional): Sets request queue size for TCP socket used by Patroni REST API. Once the queue is full, further requests get a "Connection denied" error. The default value is 5. .. warning:: - The ``PATRONI_RESTAPI_CONNECT_ADDRESS`` must be accessible from all nodes of a given Patroni cluster. Internally Patroni is using it during the leader race to find nodes with minimal replication lag. - If you enabled client certificates validation (``PATRONI_RESTAPI_VERIFY_CLIENT`` is set to ``required``), you also **must** provide **valid client certificates** in the ``PATRONI_CTL_CERTFILE``, ``PATRONI_CTL_KEYFILE``, ``PATRONI_CTL_KEYFILE_PASSWORD``. If not provided, Patroni will not work correctly. CTL --- - **PATRONICTL\_CONFIG\_FILE**: (optional) location of the configuration file. - **PATRONI\_CTL\_USERNAME**: (optional) Basic-auth username for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "username" parameter. - **PATRONI\_CTL\_PASSWORD**: (optional) Basic-auth password for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "password" parameter. - **PATRONI\_CTL\_INSECURE**: (optional) Allow connections to REST API without verifying SSL certs. - **PATRONI\_CTL\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided :ref:`patronictl` will use the value provided for REST API "cafile" parameter. - **PATRONI\_CTL\_CERTFILE**: (optional) Specifies the file with the client certificate in the PEM format. - **PATRONI\_CTL\_KEYFILE**: (optional) Specifies the file with the client secret key in the PEM format. - **PATRONI\_CTL\_KEYFILE\_PASSWORD**: (optional) Specifies a password for decrypting the client keyfile. patroni-4.0.4/docs/Makefile000066400000000000000000000011341472010352700155630ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = Patroni SOURCEDIR = . BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) patroni-4.0.4/docs/README.rst000066400000000000000000000122761472010352700156230ustar00rootroot00000000000000.. _readme: ============ Introduction ============ Patroni is a template for high availability (HA) PostgreSQL solutions using Python. Patroni originated as a fork of `Governor `__, the project from Compose. It includes plenty of new features. For additional background info, see: * `PostgreSQL HA with Kubernetes and Patroni `__, talk by Josh Berkus at KubeCon 2016 (video) * `Feb. 2016 Zalando Tech blog post `__ Development Status ------------------ Patroni is in active development and accepts contributions. See our :ref:`Contributing ` section below for more details. We report new releases information :ref:`here `. Technical Requirements/Installation ----------------------------------- Go :ref:`here ` for guidance on installing and upgrading Patroni on various platforms. .. _running_configuring: Planning the Number of PostgreSQL Nodes --------------------------------------- Patroni/PostgreSQL nodes are decoupled from DCS nodes (except when Patroni implements RAFT on its own) and therefore there is no requirement on the minimal number of nodes. Running a cluster consisting of one primary and one standby is perfectly fine. You can add more standby nodes later. Running and Configuring ----------------------- The following section assumes Patroni repository as being cloned from https://github.com/patroni/patroni. Namely, you will need example configuration files `postgres0.yml` and `postgres1.yml`. If you installed Patroni with pip, you can obtain those files from the git repository and replace `./patroni.py` below with `patroni` command. To get started, do the following from different terminals: :: > etcd --data-dir=data/etcd --enable-v2=true > ./patroni.py postgres0.yml > ./patroni.py postgres1.yml You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves. Add more ``postgres*.yml`` files to create an even larger cluster. Patroni provides an `HAProxy `__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure, run: :: > haproxy -f haproxy.cfg :: > psql --host 127.0.0.1 --port 5000 postgres YAML Configuration ------------------ Go :ref:`here ` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml `__. Environment Configuration ------------------------- Go :ref:`here ` for comprehensive information about configuring(overriding) settings via environment variables. Replication Choices ------------------- Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation ` for details. Applications Should Not Use Superusers -------------------------------------- When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable. Testing Your HA Solution -------------------------------------- Testing an HA solution is a time consuming process, with many variables. This is particularly true considering a cross-platform application. You need a trained system administrator or a consultant to do this work. It is not something we can cover in depth in the documentation. That said, here are some pieces of your infrastructure you should be sure to test: * Network (the network in front of your system as well as the NICs [physical or virtual] themselves) * Disk IO * file limits (nofile in Linux) * RAM. Even if you have oomkiller turned off, the unavailability of RAM could cause issues. * CPU * Virtualization Contention (overcommitting the hypervisor) * Any cgroup limitation (likely to be related to the above) * ``kill -9`` of any postgres process (except postmaster!). This is a decent simulation of a segfault. One thing that you should not do is run ``kill -9`` on a postmaster process. This is because doing so does not mimic any real life scenario. If you are concerned your infrastructure is insecure and an attacker could run ``kill -9``, no amount of HA process is going to fix that. The attacker will simply kill the process again, or cause chaos in another way. patroni-4.0.4/docs/_static/000077500000000000000000000000001472010352700155525ustar00rootroot00000000000000patroni-4.0.4/docs/_static/custom.css000066400000000000000000000000371472010352700175760ustar00rootroot00000000000000li { margin-bottom: 0.5em } patroni-4.0.4/docs/_static/multi-dc-asynchronous-replication.drawio000066400000000000000000000047151472010352700255460ustar00rootroot000000000000007Vxtb9s2EP41BrYPNfTmt4+Jk2bFOixrihXYF4O2aEkNLaoUZTv99SMlUhZF+i2RE8dVEsDiiTxKd88deXeMO+54sb4jIAn/wj5EHcfy1x33puM4tud67INTngrKkLc4ISCRLzptCA/RTyiIlqBmkQ9TpSPFGNEoUYkzHMdwRhUaIASv1G5zjNRZExBAjfAwA0infot8Ggqq3R9tbvwBoyCk8v0GxY0FkJ3Fm6Qh8PGqQnJvO+6YYEyLq8V6DBEXnpRLMe7jlrvlgxEY00MG/L3474v99d/Hb+jnn596d99vsjj7ILgsAcrEC9+MhYJS+iSFkOAoprkge9fsj80ztjo9dmfMW12nVyPU2wOVYOstzkMl1NsDlWDX2du1+e36A1YIWkthb9XmtyoPyP7ca5xRFMVwXELOYsSAAD9iqhhjhAmjxThm0rsO6QKxls0uV2FE4UMCZlyqK2YujDbHMRWgtx3ZFoLnXBmsKWBzEcEj1wQkt0tYKKTogxBI0mhajiJwlpE0WsIvMC2YcyoDYMKvF+uA22oXrFKvGxCcJfnjf2JzGe9O2OVkhnDmcyaU4EcoX7LjuOz3Iwfc9TxCqPbyS0hoxGzpCkUB500xnwqIFoJzyjkyiURx8Dlv3biWkIJpCh+kIfTF6+j4l2Bms8J1hSTs4Q7iBaTkiXURd3vSNoVz8kRztbF0T9LCipF7chwQ3iUoWW8MkF0IGzzCHh3NHu8Bk3gcaTZpELemm95VfzzsVwVnb9VKHXk1HZSsTCiugFzXyk6/c7CqbHvAzXq3shyrpyur7Ni4slxdWXd8TJxSEDP5OH3EAT4l7CrIoc7o/vRJ02X6COksFII3epdtFrHF6xxmpYw+z3/qpiUR8hlMIbrHaUSj3DdMMaV4sdewZ5D7KBUX+xwdSJPibefRGvrbvBWBKc7IDBa+ivm51OS1/OlE6mAiRX5CZI5UJzLQcdk3+JD+qVDpHYtKAoHPOhCYIKbTFpyvB04u+YmU+wkR2lMRar81RHstRFuIKhB13DODaF+D6O3X8Q0PNFGWcu04lh4nKTisKE8BR76BSmthgIoqK/8x4bDEWz0k61pWHmR1+24t+BLxVY06MlKLOK3Wc7SF8SAfze4bmNg1mjOs9c0Dqb12olmE2XDqWH/MppDEkIm5GxVIT2R4wxTkn8zPDlUQu44O4qEpnBieCMQDDcQtZFvIViHbPzPEDjWAQj+AcqHDhIY4wDFAtxvqNcFZ7JdY3fT5jLmkczR/h5Q+ieUTZBS/JGYtVtAd/URmkAISwF38xBLDX3CnqghEgEZLNSFpkrwYes/trLK01r2S56ksigcVo2r6Kx/j+SodtU6odUI7nRDT23l5IVl8eNduaPBWbuhlorcvQPT9A0U/Oi/R6475ckU/OC/R66nkG74WFHkQtujFil76PzJeNSyWxA9iTbziYiQwF6nsIPMn7JMtgqPiChWUjwVb2aEt+bUlv03Jb4ZJggmgcOIDCiblPmJ7iel05T+9itVQ+c9Ttx1vX/2zDbn7yyz/ucfqyrbPrfpnt1nsS89iH49Sr16lfvNEtq0nAVuY/uIwdZzzg+lQg6lWcNFDwzZxdFGJo+P97blVXOw229mC9p3VXJyzznYK8e5N/JSnw/dlfuRKc+l1lzIebl1R64reS+XFgNF36IvkLuANfNHLpO9ehPSHB0pfyvFcpO/9UtKXRvL60n+kftr/Z/jtGi7DW39l/0juoeEwv8yM+NGyUkepJUsq9RRDv5xkKNzwhMKHIk/Pyzb9ZN3ZUrY5fjqVxGc65BFsd88zHMzIa4pRrylG+8R7MKNBU4yGTTEaNcSIeYCGGNlNMXKaYtQUsp1tyL6vWGXBTDPWX5qsuKV6BJIHDPJfax11bapvk+cIr2YhILTLq5JTkPIV0FSROtmG2altmAc9bb/slrnV6o6510Di1Lhu6QfVj1x87KYMbSsjY73hQLic2as0xigh0QJw9e+UhnHhf4aVMXRT1bT2BqqLyPeLHSY/UAA2Jw3UQJ5HxXxTKQ4d5Far5ABEkcdQr65UWdjW95RVuUGt2OHqUe7IFOWeymbPZKNvOFO1a2u8d0fvntWGXi/PS2MJ3WearjVDIE2VQTSiCL6Cw9l6BixntBKo5axiTBYAGZk9UALBIooD1u1LUWbME1iO9RtIn+JZSHCMs/T3ildRD4nt80FcsltcEEdFnjY7nR/SEb7L+jT/UX6JiJikU/2eDpNfsbqW5wwV1yJt4Lm5Y9kFz+cpPDItzJqbbxMpum++k8W9/R8=patroni-4.0.4/docs/_static/multi-dc-asynchronous-replication.png000066400000000000000000001172011472010352700250400ustar00rootroot00000000000000‰PNG  IHDR™¥pq2Ù ÐtEXtmxfile%3Cmxfile%20host%3D%22app.diagrams.net%22%20modified%3D%222023-03-13T14%3A29%3A14.439Z%22%20agent%3D%225.0%20(X11%3B%20Ubuntu)%22%20etag%3D%22OufOFJhmd-sLSoEhEUfs%22%20version%3D%2221.0.6%22%20type%3D%22device%22%3E%3Cdiagram%20name%3D%22Page-1%22%20id%3D%22Xu3tU9JEMeQEUPilRV_D%22%3E7Vxtb9s2EP41BrYPNfTmt4%2BJk2bFOixrihXYF4O2aEkNLaoUZTv99SMlUhZF%2Bi2RE8dVEsDiiTxKd88deXeMO%2B54sb4jIAn%2Fwj5EHcfy1x33puM4tud67INTngrKkLc4ISCRLzptCA%2FRTyiIlqBmkQ9TpSPFGNEoUYkzHMdwRhUaIASv1G5zjNRZExBAjfAwA0infot8Ggqq3R9tbvwBoyCk8v0GxY0FkJ3Fm6Qh8PGqQnJvO%2B6YYEyLq8V6DBEXnpRLMe7jlrvlgxEY00MG%2FL3474v99d%2FHb%2Bjnn596d99vsjj7ILgsAcrEC9%2BMhYJS%2BiSFkOAoprkge9fsj80ztjo9dmfMW12nVyPU2wOVYOstzkMl1NsDlWDX2du1%2Be36A1YIWkthb9XmtyoPyP7ca5xRFMVwXELOYsSAAD9iqhhjhAmjxThm0rsO6QKxls0uV2FE4UMCZlyqK2YujDbHMRWgtx3ZFoLnXBmsKWBzEcEj1wQkt0tYKKTogxBI0mhajiJwlpE0WsIvMC2YcyoDYMKvF%2BuA22oXrFKvGxCcJfnjf2JzGe9O2OVkhnDmcyaU4EcoX7LjuOz3Iwfc9TxCqPbyS0hoxGzpCkUB500xnwqIFoJzyjkyiURx8Dlv3biWkIJpCh%2BkIfTF6%2Bj4l2Bms8J1hSTs4Q7iBaTkiXURd3vSNoVz8kRztbF0T9LCipF7chwQ3iUoWW8MkF0IGzzCHh3NHu8Bk3gcaTZpELemm95VfzzsVwVnb9VKHXk1HZSsTCiugFzXyk6%2Fc7CqbHvAzXq3shyrpyur7Ni4slxdWXd8TJxSEDP5OH3EAT4l7CrIoc7o%2FvRJ02X6COksFII3epdtFrHF6xxmpYw%2Bz3%2FqpiUR8hlMIbrHaUSj3DdMMaV4sdewZ5D7KBUX%2BxwdSJPibefRGvrbvBWBKc7IDBa%2Bivm51OS1%2FOlE6mAiRX5CZI5UJzLQcdk3%2BJD%2BqVDpHYtKAoHPOhCYIKbTFpyvB04u%2BYmU%2BwkR2lMRar81RHstRFuIKhB13DODaF%2BD6O3X8Q0PNFGWcu04lh4nKTisKE8BR76BSmthgIoqK%2F8x4bDEWz0k61pWHmR1%2B24t%2BBLxVY06MlKLOK3Wc7SF8SAfze4bmNg1mjOs9c0Dqb12olmE2XDqWH%2FMppDEkIm5GxVIT2R4wxTkn8zPDlUQu44O4qEpnBieCMQDDcQtZFvIViHbPzPEDjWAQj%2BAcqHDhIY4wDFAtxvqNcFZ7JdY3fT5jLmkczR%2Fh5Q%2BieUTZBS%2FJGYtVtAd%2FURmkAISwF38xBLDX3CnqghEgEZLNSFpkrwYes%2FtrLK01r2S56ksigcVo2r6Kx%2Fj%2BSodtU6odUI7nRDT23l5IVl8eNduaPBWbuhlorcvQPT9A0U%2FOi%2FR6475ckU%2FOC%2FR66nkG74WFHkQtujFil76PzJeNSyWxA9iTbziYiQwF6nsIPMn7JMtgqPiChWUjwVb2aEt%2BbUlv03Jb4ZJggmgcOIDCiblPmJ7iel05T%2B9itVQ%2Bc9Ttx1vX%2F2zDbn7yyz%2FucfqyrbPrfpnt1nsS89iH49Sr16lfvNEtq0nAVuY%2FuIwdZzzg%2BlQg6lWcNFDwzZxdFGJo%2BP97blVXOw229mC9p3VXJyzznYK8e5N%2FJSnw%2FdlfuRKc%2Bl1lzIebl1R64reS%2BXFgNF36IvkLuANfNHLpO9ehPSHB0pfyvFcpO%2F9UtKXRvL60n%2Bkftr%2FZ%2FjtGi7DW39l%2F0juoeEwv8yM%2BNGyUkepJUsq9RRDv5xkKNzwhMKHIk%2FPyzb9ZN3ZUrY5fjqVxGc65BFsd88zHMzIa4pRrylG%2B8R7MKNBU4yGTTEaNcSIeYCGGNlNMXKaYtQUsp1tyL6vWGXBTDPWX5qsuKV6BJIHDPJfax11bapvk%2BcIr2YhILTLq5JTkPIV0FSROtmG2altmAc9bb%2FslrnV6o6510Di1Lhu6QfVj1x87KYMbSsjY73hQLic2as0xigh0QJw9e%2BUhnHhf4aVMXRT1bT2BqqLyPeLHSY%2FUAA2Jw3UQJ5HxXxTKQ4d5Far5ABEkcdQr65UWdjW95RVuUGt2OHqUe7IFOWeymbPZKNvOFO1a2u8d0fvntWGXi%2FPS2MJ3WearjVDIE2VQTSiCL6Cw9l6BixntBKo5axiTBYAGZk9UALBIooD1u1LUWbME1iO9RtIn%2BJZSHCMs%2FT3ildRD4nt80FcsltcEEdFnjY7nR%2FSEb7L%2BjT%2FUX6JiJikU%2F2eDpNfsbqW5wwV1yJt4Lm5Y9kFz%2BcpPDItzJqbbxMpum%2B%2Bk8W9%2FR8%3D%3C%2Fdiagram%3E%3C%2Fmxfile%3E°c& IDATx^ì ¼MÕûÆ_ÒhJ“1”1õ/*Q‘!* E(”B3fžçy.…È ¡Ì… †P”)M¦ÒÏ?Kû:®ëžsöÙûìµ÷~VŸûéºgßw­³÷³ßw­ê¹Æ½Ï¦gÓY£z§Ûpv¬H€HÀ‹¦ŽxCÛnç¹ë¡8\ÏJªT¼k; Ø1 ð)T5 ‘éôتT(!U/ét3¬ŸH€H€ ÔjÒGô™¥ è¬Áš4¨+ÍÕs¶ÖN$@$@!ò6®o™œ$@$@~$@‘)B‘éÇ™Í1‘ €Þ(2õ¶{G$@$ŠLŠÌ¦‹’ X$•ÈÚõUùå÷c4uC¦ŒÒ¼ë›a›g¸lXDÌ@$@$`3ŠLŠL›§«# ˆ€@T"³CÓZÒkøÔ ªMîoɵK‘5˜…H€HÀV™™¶N(VF$@$K"ó†Lä†ë¯•»ŠÌˆ83 € (2)2]˜vl’H€OÀ’È,˜7§Ê—Sæ,ZC‘ø)D$@$ /?ˆÌ‚ùóHù2çNg?~â¤,]¾Zú)bè<ø'bTÌH$@$`K"󚫯”k®¾J~ùí˜m"óå6ƒå¯¿O«£Ü/»ì2É•ý&©^±Œ³fúçŸ32óÃU²ñËoå豓’õ¦ë¤Ú“I‘Ûó^€ãë{Õ¿ï(Û&L¬†H€HÀ‹¼.2;´j,Ò§“9 +üø¢qÒÔ™Æß>ŠÈ$áDf‘’OÈü©Þ¥™&Íer[þ¼Òªé+R¼h‘ÄúO:-CF—¥+ÖÊ‘#¿È­¹o–毾$eº_åùaÿAiß­¿|³c—\ݵҺiyôa¼”‰H€H ˆ,‰L'Âe!2»´xArd½Aþ6.f›¾Ü% ï}$›Õ’Ü92+Û zk–¤1ès•ÊȵÓÉÖm{dÜÔ…†Ð­)¹þËsð§_eøĹR¾ä=òp‰óÈ —c&  ð²È¬S³š2ßäi³.2ãèÁ=%Áš6m kâHDæ{“FJ¾<·Èþ%Ë–¯‘.}†ÈÔñÃäö‚ùTý šµ—Ë/O#mš7”›n¸^V}º^Úué'SÆ •BFžGŸ~A*>^^ê¿P]>^¹VZuì-ÏŸ*Y³Ü¶Ì@$@$à?aEf色Ûwý ³®–ÐpÙª—”Ûþó6¦tÒl¸ƒBE¦‰ù½VÊO?ÿ.MêUx'ÇMýPwidx:S'Zbîâµr…qá{âábòÑÊòÉÚÍòëï'”¥Èôß„åˆH€H ^™ó§—Š5ê';Üô†GsÁŒñrüøIõùìù‹“£ø,‘i66xäxÙ÷ã~Ö¯«¬]·QÞ0¼”/˜&—§I“ØŸQo½-W\y…T~â)ýDuùòÓÅ* ©’ÑïƯԑòeÏ…ù2‘ ‹@X‘™ÜÁ>¡á²¡¸R:ÈŠÈܱçGy ²sCàüÛ§}¡Zù°™ð¾!|sQd†%Å $@$ào^™÷Ýs—»·ˆŒ;é’*voaÙ¶s·œ0öi³9lL‚ìøvÏEù­ˆÌÏ¿øRÚuí§¼‘œ'pÚÎm›&Û—3gþ•ß~?*7Ýx½úûFK=^]ÎL 'ÓßË‹£# KˆXdÖ®RN¦ÌY¦*Êiì—Ì•=³¬Þð•ú·ù™Ý"óúÚ¡‚LÔJFMž'7g»I*–?·ÿ#¥D‘Ž?' `ð¢ÈL—.­4kXO‰µ”Df¨Ë•~PªV¬l­‘ùÝÞ¤òs¯È—Ÿ-–oôùn•†/Ö ;ivíù^š·ë.¥K—ÖÍ„ÍÏ $@$@þ$±È ¡á²Àb~f·ÈÜix2ÇþçÉœ>…œ>ýÏEžÌÃ?ÿ&?8"÷.˜h!ŠLNVŽŠH€¢%à5‘ 9fH/%.×oÜÕpBÛÑ8(hýÆÍ dEdnÜü¥´5ö\“9`ØXã`¾Sy2÷îÛ/;ví‘ÇÊ•’ÿýï2fâT#dw¶¼Þ¸¾T¯òdT}gf  ˆXdF2l»Eæ,ã$ÙCG~S{27½[& 2Bgqú™Þž½Tþüó”4|þü"3k1 øŸ€×Df•§UF‰ôäؤ„ÐDèlí—›'~dEdys‚ìÝ÷£Ú“ùɪO¥kŸ¡²ÌœW\~yb½=ú0BuOHÿí¥÷ Q²õ«í2jPwã=Ú×ùbq„$@$@)ˆXdÆ3\ö´ñªˆÊñïâäØó§Ëö5]®2¨ùtYɘ>­:vÜ´Õ ´yreK(E&g= xMdBÂÉ©±—²0Df£×;Z™KˆÊ=ª“cÍÓeë6je¼ºìjy£å«†ˆÌ¤N Åa@86[–ÌRöÉç”Ítm†ÄvÓ‡¥N}þ >ÎH  àˆXdÆ#\ö”›ÚxOÞÕ…}ŸIß“‰÷hΘ¿\¾0èÉ?þ’l™¯—ªO”¼è=™™Á™À) ¤DÀk"¯-9pð,3ÞGi5AFëÉüû¯¿%•!S§N%óå¹è=™8øgàð·ÔëIŽ;¡Þ“٬ыê=™«>Ý /¾Úú¢îŽ6Â~Ë—)au,G$@$àa‹ÌHÆK¸l$õ3 DCÀk"3{¶,Ò¯[;iØ¢ƒœ<ùG4CUyn{[|ÒkàÈIJáÂe£n„H€H€H ˆE¦ÓžLZŠH€H€ì&à5‘‰ñßV ¯š8]6Ú„W˜ô0â‚b™ÑRd~  X PdÆJåI€H€´%àE‘i7LŠL»‰²>  p"™á*Âç —„ó Ä‹E¦Ef¼fÛ! 0 „™C»¾*?ÿv,"b7^—Qšw}3Ù¼U*”ª—Œ¨f"  ;PdRdÚ1X @tŠÌ誻tnŠL»H²  H PdRdF:W˜H€HÀ>™ö±dM$@$@š È¤ÈÔlJ²;$@$™03I$@Á$@‘I‘Ì™ÏQ“ €»”È|®qï³U=“N¦Ûòå’Bùr:Ùë&  è/2’¦ ë9jµb÷–âE‹8Ú+'  P‰žÌ©#Þ   ðÝE&.»·¬ôs†H€H€(29H€H€|K€"Ó·¦åÀH€H€4&@‘©±qØ5  ØPdÆÆ¥I€H€HÀ ŠL+ÔX†H€HÀ(2=a&v’H€HÀg(2}fP‡H€Hà<ŠLÎ  ˆ?ŠÌø3g‹$@$@q"@‘'Ðl†H€H€BPdr: ø–E¦oMË‘ hL€"Scã°k$@$@± ÈŒK“ €™V¨± €'PdzÂLì$ €ÏPdúÌ  Ày™œ $@$@$™ñgÎI€H€âD€"3N Ù „ Èät  ð-ŠLßš–# ИE¦ÆÆa×H€H€b#@‘?–&  +(2­Pc  O Èô„™ØI  Ÿ Èô™A{˜&۾ݧFUõñ’R¥B õûœEkdöÂÕü;9p>$YÛwÿ šÔôÙ7‡c Èä\ˆ7|§ô6URý×ðÔo$vóÑLüû9ä@5uѱi-)”/g¼—+ÛsE¦ƒpݨZ÷*7˜°MH‰×Œ¿ç‡îöÅEx÷–•þ6BÀF‡‡ºgÏžUz™H€Âàš ÏÈ‹9(2½hµúŒ…jz/}64‡!À5ãVm*¥ÈÔÆé<™†Æ¤W&0ç@c%À5+A=ËSdêiöŠH€HÀ™6@d$@$@$%ŠÌ(1; €wPdzÇVì) €PdúÇ–j$ ýó™A9Ç pÍ8ŽØÕ(2]ÅÈÆúH³sÐ1àš‰žÆE)256Ž•®é~CeeL,CNàšq’®ûuën_üãþ±»<ÄÄn¢¬Ïï¸füiaŠLŸÙU÷*Ÿáæp|@€kÆFLaºÛ—"Óó7Ìþ³)Gä,®gùºU;E¦[äj—¡ö‚ÝèiÛ{œ¤N}îg©S§–Ynª•—ynN±±þ£ß“6žµ­CýÞœ!”ºGŠÜž×¶:YCÌý>(2ýnaýÆÇÐ?ûmÂk±ýLuª‘kF'kØ׊LûX²&À…­óÀI2qP+5º“ü%s¯•u_l“‘=›J*ómÛÉŒýùf}åaí.øäßÿýOR…R¥Tð÷íÿI®»6½¤OwIsH$à ŠLg¸²Vˆ'^‹ãI›m‘€=(2íáÈZ|J é… Ãüå÷ãÒ¬ó(Õ«©üóÏ™0}‘|·ï¤¾,µ<òÐ=ê=¥ÇΔÍ_ï–Yo^m^”Fí‡É+µž‰3Kß7êËñªrÿ"Yoº^^¬ñ˜äΑY–¬Ú$?8"{ AùËoÇ$ï-Ù¥Þ³*q9pÌLy¸dz2}:×8,gPd:Õµ’@< ðZOÚl‹ì!@‘iGmja¸¬½¦Hzaûó¯SòþGkeåº/etïf2nÚ‡rÙe—©ðYˆÂöý&ÊЮ¯ÊµÒJ¨'óå6ƒåîÿË'uŸyT®¸"´îñ–^»YÞžµTz´®«Âr‡ŒŸ-·ÜœUžyò!ŠL{M›X׌C`5©–"SC¨ ý³ßؼÛÏT§¹ft²†}}¡È´¥5é~C¥¤(:aî¹<ÍeªÂ]³ÜxÔ«þ˜Ê—SNœüS®ºò %48xDz›"=ZÕ•¬™¯¿HdvlZKrÞÊï8,ÃæÊCTš©uÏ·”§óûËÚÏ¿‘n-_P-]ý…4¼užy„"3 »E“•k&ZÞË«»}yð÷æT¸ó“p„¢ÿœ×âè™y©׌—¬y_)2#g剜ºßPybH'“ Ñ Ã#LvòÌä¸!6oÎv“ìÜó£t{ý…dEæ Î %ƒ±Ÿró7»eáÇ륃!:Í„C}ÊQ{2—º…"Ó‡óˆCrE¦{ìÙ2 ØE€×b»H²ˆŠÌø±fK$î¶fÃ×2íýOäš«¯Tûüõ÷iÙkì«ÄÁ=ƒÇÍ6¼–‡”¸Ä鲦ȆŒd'L_,‡ŽüjìñÌdœ û˜Ü’3‹:]–žLNvY[™Úš†#ˆ ðZ1*f$mPdjc {:ÂÐ?{8²–ààšñ·­)2ým_GÇÐ?­Â>éL€kFgëXïE¦uvZ–Ôý†JKhìT  pÍøÛüºÛ—ÿøoþñÿÙ”#r–׌³|ݪ"Ó-òµ«û •CÃfµ$`™×Œetž(¨»})2=1¢ê$o˜£ÂÅÌ$ \3þœ™>³+Cÿ|fPÇq\3Ž#vµŠLWñ²q†þÒìt ¸fb€§qQŠLî‘ ÄF€"36~,M$@$@VPdZ¡Æ2$@$@ž @‘é 3±“$@$@>#@‘é3ƒ2ôÏgåp'À5ã8bW Èt gè_ ÍÎAÇ@€k&x¥ÈÔØ8Vº¦û ••1± 8I€kÆIºî×­»}yðûsÄî𻉲>¿àšñ§…)2}fWÝo¨|†›Ãñ®1…!èn_ŠLÿÍ?Þ0ûϦ‘³¸fœåëVí™n‘w¨]†þ9–Õú–׌oM«F‘éoûê8:†þéhöIg\3:[Çzß(2­³cI  Í Pdjn vH€HÀ—(2}iVŠH€H€žLÎ  p‡E¦;Ük•¡Ž¡eÅ>%À5ãSÃþ7,z2ým_GÇÐ?­Â>éL€kFgëXïE¦uvZ–Ôý†JKhìT  pÍøÛüºÛ—ÿøoþñÿÙ”#r–׌³|ݪ"Ó-òµ«û •CÃfµ$`™×Œetž(¨»})2=1¢ê$o˜£ÂÅÌ$ \3þœ™>³+Cÿ|fPÇq\3Ž#vµŠLWñ²q†þÒìt ¸fb€§qQŠLî‘ ÄF€"36~,M$@$@VPdZ¡Æ2$£GÊÖ­[UNü¾eËõûµ×^+… V¿g̘1ñ÷ªd ( PdF ŒÙI€H€HÀ™6@Ô© †þ¹c½{÷ʼyóäý÷ß—+VXêDîܹ¥téÒR¹re©T©’¥:X(z\3Ñ3óR ŠL/YË}eèŸ?ìÈQÄ×LüXdz%ŠÌxÒŽC[ºßPÅAÜšX¹r¥•øÈLšB½”¡ÞËP¯f¨·3iyˆMü”*UJ @™œ!À5ã W]jÕݾ<øG—™b_?xˆ‰},c©iÛŽ]râärüÄIÙ¾sw²UÝV ¯dHŸNÌÿÇÒËZ'À5cÎ%)2u¶Ž…¾é~CeaHÚ™ŸA´®ž"ÓŸvå¨l&ïb‹-Ô>¹råRžLˆËx$„â¢=xP‘ lçΫb"H™E&g ø‹Äe#äÄ"1[ÖÌRÜð:Âû/d¡‚ù,x¿!8·ïØ­Äë:Ã3zðÐOª.x9ëÕzFš6ŒÏußòX4"@‘©‘1Ø= À“X¦LÕ¹fÍšÉСC]é(<©ÅÝ·oŸšË—/w¥l”¼D€"ÓKÖb_IàÒày¬U¿YâþÊ*O=jx+Hñ¢EöîóÍ2gþ"™³à#ÕDl¿níb²Žu–“€f(253H¬Ýaè_¬/.Ë-·¨Y쉄GÑÍO*BølBBBܼ©nŽÙ鶹fœ&ìný™îòbë ýsÆê˜Øwyß=wIÿo¡°çÃ`iñ|­ðpöì?B–­X«„æ‚œn2PõsÍøÓÜ™>³«î7T^Ãmz1±ÿÒ|Ï¥ÛcÀ^Mì Å©³V_—âötjŸkF'kØßÝí˃ì·¹Û5òg,€µ‚ÐØU‹Þs¦jmؼ½šSÆ uÔƒAW|•…kÆWæL E¦Ïìªû •×pwîÜYzôè!õêÕ“‰'jÑ}x33eÊ$éÒ¥3ö¤œÐ¢O^î׌—­¾ïºÛ—"3¼ ½–ƒ7Ìö[ ¯#©X£¾qˆOåEtãÔW„ë>Uý%ur-E¦½6æš±—§.µQdêb ›úÁÐ?›@þW ö@Λ7Otòd"t!¼HgÏžµwÀ¬kÆßF§Èô·}uCÿì· öFÖ~¹¹ª᪣‡ôŒk¸ìÒOVKÏ#_B‘i¯¹fìå©Km™ºX‚ýÐ’@Æ eìرrÕUWÉ¡C‡´8ÑûB»uëF‘©åŒa§H€H€ì&*2ͺñZ¼^¯q"aæ†[eؘ„Dq‰p]œ8K‘éqÖé7™~³(Çc+PA‡waâDW7_‚W© t×LôdÚjnVF$@$ !SdâПªÆ‰²x…Iè;-‹…¼Â$»!­¼ÆÄ•ÛvîR á]šfB»MÖS1vE¦†s„]ÒE¦~6‰©G ý‹ ßE…M‘yå•WÊ©S§B§ºâÿñNð^&=Ý–"3v+pÍÄÎ5 œ'ÀÐ?ûgC¨Èœ6a¸j!¬xŸå’åk.œ¡­#´6ÜþÍm†˜4ß¹Z¶`þ<êÝ›õjWK Í6:"Ó~ó ׌P5¨’"S#ØÙÝ÷Ù9ÖxÔeŠÌ¶mÛÊâÅ‹eëÖ­ªÙºuëªWšàu"N§É“'+q‰½˜H¹¦7“"3vú\3±3d $@ç ðûgCr"3´ Ïã~ãPžõ7«6l:w½Ž$APBŒBT2„émó&»ç“"3šÑçᚉž™JPdzÁJQô‘7ÌQÀŠ «)2!(›7o.C‡U?xO%D&Â^)bG‚˜\¹r¥àU%ø1êGJ—.-©R¥R¦ÈŒ8×Lì Y Ày¼a¶6„™‘´ˆ:BSñ¢E")vAŠÌ¨‘ET€k&"LžËD‘é9“¥Üa†þÙkÐP‘i†ªâ"œ€¦ØD«Ø« ±‰PZ3œ6cÆŒ— ­E=¦g¿ã=œ¨3éû8q²-ÚFÝf¢È´ÏÎ\3ö±dM$@ÂÐ?&"ÓŽnQdÚAñâ:.ë W·k¥ÈtÛl_kɉÌÐCâ0ž+V$ ÆK "žÏ¤"2i~Sx+Mir Qdj=mØ9   PdÚ“U‘@œPdÆ 4›ñ&p"3tTs…'ÿ7…d¨·291iz–ZÔÄÐ?{Í@‘i/OkãšÑÑ*ì x—CÿD–.]*åË—·ÍˆVEæñ'eÙò52{þ"Ùf¼Gó„ño¤ôÆ;1ñ>L¼³\é¥PÁ|õ•§ËF„)êL\3Q#óDŠLO˜‰t‹E¦[äÙ. €W à0»S§NIÏž=¥eË–1ÊÈܶc—4z½£8x8lûSÆ •HÞ›I‘%3@"ŠLNHE&§‡np€T«V­dáÂ…röìYÁ{T{ôè!E‹U]}ã7¤fÍšòÿ÷qíz¿~ýä—_~‘ØÒîÍ7ß,‡ì?Æ8¯¹æ)Y²¤Œ?^²fÍj©aƩ÷† "*TÆËO<u]ß|ó¼ýöÛ‚1¯_¿^ÕóùçŸG]Ï¥ üõ×_R¯^=™>}ºmuZ©è™gž‘5jHÕªU­k™-Z(;äÉ“'®í²±ä àÕ^M›6•ÿýW­_|GÅ"6£™û’§ª×Oô\š½¼ïž»;¼aÓÖÄß)29“IÀ~™ö3uµF†þًߪÈÄ þ×_lg®¸â É›7¯Ü~ûíÒ®];¹ûî4þ‰Ù% xmÍ<ÿüój,ð@ˆ}ôÑGòÒK/ɧŸ~ªÞʵŽ;J©R¥.óéÓ§sÏ©´oß>AùòEv®Û¬Y³¤X±b*ëï¿ÿ.?þ¸*TH&L˜®x²Ÿ‡ŠL¼f(GŽrà 7D]×ÚµkãåË—ËñãÇeÇŽrß}÷E]Ï¥ @¼(P@½+×͉ÈxŸk–,Yäõ×_S´>\åûõ×_•›|Xy2qsÙ¾}{G=ðdþïÿ“.]ºHBB‚NíÚµ¥W¯^rÙe— ÛwÞ©>Cß! GŒ¡> M'OžTPÁóŠ¾áßø.G´~Ÿ={¶*Þ¨Q#%²^|ñEÕNóæÍÕß!†á½õÖ[åË/¿”Í›7 /^\ÙBû·ß~“W^yE‰fŒœÐôÌÀñÇ0CŸÑL™2)±ÿꫯ*~°a‡T]=ôŒ;V x´q©vñàýþøãU}˜£˜ñxgÎ x‹á½Ç¿áÝ®S§Žò _wÝuÊ&UªTQcÅüîÓ§z€ÇïH=¾ï0¿1WÍtÕUW©ïœ·Þz+ªF+2k¾ÔTLOe¿ní¤j¥ Qµw©Ì™¶`¼¨üã W·k¥ÈtÛ6·O‘i/ÐXEæ˜1cÔ bhúóÏ?Õ^Û¶mÕͼK›6m’;î¸#âÎÓ“1ª°½¶fˆ›~„žá¦¢ 4…z2Ÿ~úiùᇔGð–[nQânÛ¶mJèAŒA Ì;W ±‚ *! ‰ù ‘€~”ƒ¸€þá‡VåF­ê†PL*2K—.­Ú‡ù!ž<(<ð€¬\¹R Qü=mÚ´‰Ìï¿ÿ^^xáyöÙg¥I“&— D~ l!¸!œªU«&»wïVý1ÃeM‘ ^ùóçW‚!´­[·V"lÁ‚Š|÷Ýw²fÍ%ô!ÐŽ;¦D5쇇°Û’%KTyˆDüm7kÖL6lØ "@˜ÝsÏ=*Œñ¹çžS!È}ûö•÷Þ{O…9BDãïˆè3æ8͘1CÊ•+§D!DÝÈ‘#U^oŒý€}`oŒóçRíblHèßöíÛ °ÅƒŽK‰LÌxOÁ6‚!€‘ 0ñY§N®{'3ð†ùÝPO&Ö9Ä&ææW´QÑŠLÜÜš)ÒPØHæEf$”¢ÏÃ5=3/” Èô‚•¢è£×Bÿ¢š+Y™æ@f‡›=Üìcn#M™‘’ ŸÏkk7û¸yŸ6mšgÙ³gW7äðî`^$™ðH™^­œ9s*¡‡„ éÓ§Ws{=á…' óþçŸVB"í˜ûQ¶M›6ʱa˜Td¢<òˆjõBDÀ‹‘a‚!ö6^Ê“ ñoÆŒŒÍO— DG®\¹ÔxÒ¥K§Ú‚¨‚K*2‘ÞHˆE$Œ¢ù±6³e˦<ðB€Â“v)‘ áZ½zuåÁD‚`ƒ·â "Þ\çËÈ›t¿#<¨ÝºuSu Ü}„Єàƒx‡PD? ÒÀïÁL W=qâ„šøžex^1<€M ú‘Þ|óMeˆb„2¢ 2¨Ï0·`_Ì)Kxc‘À¤V­ZªÏ™ð… V¶G]Ø/‹»Á>ð ¢äÚ…Gíáá Á+Š‡&xøq)‘‰¹¹nÝ:ç} M°öB|º™ú'Ê“x bŠKx£­&Sd6iPWš5:÷p"¥ôP…gåà¡ŸT–òeJÈè!çæp¬‰"3V‚É—çšq†«ÛµRdºm¶¯5'E&^·n]–ˆ›`x0"M™‘’òw¾þùG…¾BDB¼@$™?„ÀPƒ'ËÐýs¿ÄM=D¼š55E&¼c)H8|/xÖ z’™_}õ•ª ^2ÔO 'ª áð"xŸ" —E(+Ä Â+ñs©±À«X¤H%tÌ>ðª`\IE&Ä(ÄÉœ9s.š,Å+B‡!ÐR™ðæApA$!!¤³~ýú°úƒtà%„‡5©È„ðG(«éåƒwÞ;ˆ~x%áÁD? °`ó‰'ªýª¶†‡âe°O\Ø Ðx ‘Î ›Àî¨â:iBŸ!†ñ¤Ð=š™° ¸B|ãa¼¾fºúê«eÿþýêáDrí‚+D&<æ÷<Ï(F¡"¢m`ž Ÿø 6à ¦.˜H_|ñ…úÜ™Ü%Ûàá Ö|,âÒE´"3tO&ê€Ðìк±äÈfíÐ0³™îÎ+¶î-™Þ²{gN‹LÜHâ‰/n áíˆ4QdFJÊ_ù êf‰pÂн¾D(*¼fIE&ć)d úp#ï<‘%¸y‡@—ðúë¯Wû9!c™0ªH¦ÈÄ>,ô^,$ôó?‘‰üðòAdÀ;v©±`mÀ“ ¯%Bô6üä“OªΤ"{FQ/„ ØB\Âc‡“Jáá¼í¶ÛTH&ök†ódB ¡,„ĸB ATÂ{‰t)‘‰r7„Ó#AÜB”›žjô ¶÷â‹úvà僇§àâÂ[!2C˜"¢œÀ u"áoh‚-ôtÙ¤"óùÑx¦M†˜cðdBà&×.æ„)ÊÁóŠ„0[Ìxßá-‡Ç ssóã‚}!bv‡$¨å°_aϦ·[fr…l`F2ØÑhE&N—}òÙ—Œ5ÿÇÍCl–/SRªT|ÌR·(2-ac¡€ Èô™á½ú§;~'E&nx.‹›W„ëáF-ÒD‘)©ðù¼¶fî¿ÿ~µ{› ⦈0Fú¯D&„Ê£>ªÄU¨ÈD8"öãÆb^IIxš°Ç7éø‚ Gˆ¤¤â$ROfr"ÞE„ׇCaÐWSä&µTrÿ@@`?3Dð¥Æ‚=£¿xËV­Z¥è'ìM*2Ë”)£ö™"Ô}ƒ— ýÀKˆ±â÷Œ3ª=°h"âõc¯bèžLˆSˆ'Loñc=¦¢ Ö"™S¨× ='x˜qØì¡Š°Y<”Â<—齄8DŸ1Oཆ(ƉK‰Lô|pXæ<ºè/ÊÁ ‰È„ÇÞWìÅÜD˜$8aÿfJí¢¯ʘ;wîT^q„*£ß8UƒàÆ\FØ%D&æ&ÚÀƒ|wÂ{Œù‹“•1ô"ÛÍÄÐ?ûéG+2ќ۰E‡Ä°ÙÐ^¥OŸNêÕzFêÖª&Œß#M™‘’Š.×Lt¼¼’›"Ó+–Š°Ÿ^;Ä$Âa¹–Í ‘‰×<à<©Ç“xÜ"üÍ|Ïa$ƒ¥ÈŒ„Rdy¼¶f°ç%pCoî}ƒàBx$Nw…›9s¦zp*2±ù Pps9A/Ä nÖ±o"¢ CˆQ+á²É‰Lx½Þ €“AÑ&Äm$"{ípÂ4!D’ öÞ{ï½Ê³Y0ÇëR§Ë"¼ áÙE¨-Ä"¼e8°û !ZÁ ‚ ¢§¨Â« ¯2¼‡¡§ËblG‚HƒXÅTdâ &;sï$„-ì¡‹°zØÞhض…(ÉðVC\™nx_!Ô B‘R{Ø»‰Ã‰Ð&\`Ï,DmÒW˜\Ê“‰ú!„1v„îb(<¥ˆ)µ ¯4Øa ¼®l O.쇹‰¿anBdb^!!»ðò¢$Œ\pà’›‰‡˜ØOߊÈD/ŽŸ8) SfJÂÔ™y5ñ9ÄfÇV#>}–"Ó~Û¢F®g¸º]+E¦Û°¹}¯Ý0Û<|Û«‹Ud"/éûÀà)ÂSy3an"£I™ÑÐJ9/׌},ÃÕ¡€ƒjðÚÌaˆ8xL!ÚìJ™ðª™'ŽÚUo<ëˆÂ>RœÔj5Á ‹}Œ¦AHßðÈâµ3n¿¾„7ÌöÏ8«"3´'K?Y-³ç/’e+Î…a‡¦Ñƒ{&ûͤù(2í·-jäšq†«ÛµRdºm›Û÷ZèŸÍ÷½ºXEæ¥:„}T¦û¢M™Ñ»t~®ûX†« §LBPBàÐ"xÚðÎOxâìJ~™Ø{‹]£}— â ˆyx"òj'[»läD=ðÞ#䡵n'†þÙo;D¦Ù+ì×>:Aæ,8<š+Î:K‘i¿mQ#׌3\Ý®•"Óm °}­ Ä*2“{O¦¦È´ƒ"ëð#6ƒ=‘^÷à!$ïÄŒ6á´a„þb.N¶ J²Ê+(|¼>N;E¦ÉÂŒæ¿#yŸ&E¦×gûO™ñ¤Í¶‘€w ðûmç„È —M—.­lY³0lÇ.‘¥ \3–°i_ˆ"S{E×AÞ0GÇ+\nŠÌp„¼ÿ9׌÷mÈ€NxÃl¿5¢™5_j*'Nþ!åË””Ù²HvãÇLV§Ì®ß¸%ñoujV“Nmš„í8EfXD–2pÍX¦}!ŠLíM]ú¯p¹)2Ãòþç\3Þ·!G@:`èŸýÖ°"27lÚQG æÏ#¼71¢¼™aŠ:×LÔÈ”z¼ºœ0þÿÅê%Cútqïƒ_äšñ§e)2}fWÞ0ÛoÐܹs˾}û¤K—.‚ðY7<—·Ür‹ò`&$$HݺuÝìŽ/Úæšñ…9Іo˜1…ªŠ“bûukg)ÜÕjÏàIí5p¤ ½uÛ›ju :—ãšÑÙ:ÖûF‘i–%úg¿YLo&jnÞ¼¹ 2ÄþF"¨qË–-Ê« I/fÀ"ÌÂ5!(f#ˆˆCÿ"Âd)“®ŠÂØWY×8¸§|™–ꊤÄ國eÎüÅ*{•§•þ=ÚGR”y¢ À5,e¥Èô±ØU÷ DóرcÏ&<šuêÔ‰K‡öîÝ+ݺuô óý÷ß—k¯½6.í³  Ð…„_›Î}äà¡ŸT—Òa«šÅï-"xI¡‚ù,wuÿÁC†·rñÍͲtùÁ;5‘²eÍ,Z7‘òeKZ®›I h(2ƒfqŽ×2x+W®¬Bg‘°O³téÒJlâw;¼•óæÍSbžTó€ ²s¼¬‹H€H€¢!°ô“Õ2{þ"Y¶bíEÅn+Wí™,fO3áßø;Òú[ÿ¾}ç.Á~Ëп™–+ý !`KJÕJ¢éó’ (2}6 úç¼AáQ„'Ó›h^EPüÀÓhÅË»råJå±Äï¡ Bm‹Êd/®{y²6:†þÅw@ âU%‰ð@âu%¦—3šžà½™… z[|†W´°3^qÂÃ}¢!h=/׌uv:—¤ÈÔÙ:úÆCL,@³XFxñ*8Íê 4MgèïðJš"2ô÷¤Ý¨T©’­ð–R\Z4RŸf"€äá,ºÛáÝ[Vz˜0»ž”1ÑcN ¬ §Án3ìIšB=›±¥ tÏn\3î±w²eŠL'éºP·î7T. ‰K“Ø7i Nx#­¤\¹r)AizD­ÔÁ2Ñàš‰ž™—Jèn_ŠL/ͦÈúÊæÈ81 ˜¸fü9(2}fW†þécÐKy,/åáÔ§çÁê ׌¿íM‘éoûê8:†þéhöIg\3:[Çzß(2­³cI  Í Pdjn vH€HÀ—(2}iVŠH€H(29H€H€H þ(2ãÏÜÑúç(^VîC\3>4jÈ(2ým_GÇÐ?­Â>éL€kFgëXïE¦uvZ–Ôý†JKhìT  pÍøÛüºÛ—ÿøoþñÿÙ”#r–׌³|ݪ"Ó-òµ«û •CÃfµ$`™×Œetž(¨»})2=1¢ê$o˜£ÂÅÌ$ \3þœ™>³+Cÿ|fPÇq\3Ž#vµŠLWñ²q†þÒìt ¸fb€§qQŠLî‘ ÄF€"36~,M$@$@VPdZ¡Æ2$@$@ž @‘é 3±“$@$@>#@‘é3ƒ2ôÏgåp'À5ã8bW Èt gè_ ÍÎAÇ@€k&x¥ÈÔØ8Vº¦û ••1± 8I€kÆIºî×­»}yðûsÄî𻉲>¿àšñ§…)2}fWÝo¨tÁ}ã7Êï¿ÿ.©R¥R]Ê‘#‡¼ñÆòÊ+¯¨ÿõ×_Ò¯_?™:uª8p@}þüóÏ«³+Cÿ"3(DæâÅ‹åž{î‘S§NÉìÙ³•ˆüꫯ¤P¡BòÜsÏÉñãÇ•ð»å–[ä³Ï>“îÝ»«ÏÆŒsQ#[¶lQBô†nˆ¬!¹Ö®]+;v”åË—G]–b'À5;Ck ÈÔÙ:þìCÿR¶kÑÒåØñ‰y³d¾Q½XKjT«¨ þý÷)›0Mæ/\*‡ü,Y2ß$O?ùˆ4|±¶ñ÷²‹*ñÕÖò|*Ræ¡û£žP»ö|/s|$m›7Œº, ØG€kÆ>–:ÕD‘©“5Ø—¸™f£·Þz«ôîÝ[®¼òJ屄à¼üòËû&þ>a„ þŽ O=õ”4lØPrçÎ- 4\¹rɲeËĬ³L™2rèÐ!©S§Ž¬_¿^®»î:4h”*UJŠ-*GŽ‘J•*)ÏéðáÃeÈ!ò믿*Q;yòd)P €Œ5J¾üòKÙ¼y³ìÛ·OŠ/®þq»uëVÕî·ß~+÷ÝwŸŒ7Nn¾ùfãbý·´hÑBÞÿ}É’%‹¼þúëJL3‘@PPdÅÒ§W@d&¼9@î(T@NŸ>-‹—­’–zÊÂY“$_žÜÒ¬m79ùÇŸ†ðk 9²g•Í[¿‘c'K^㳞[^4Ìm;v)!z]¦ŒQ#Ø´å+«T©¢<±:u’üùóËàÁƒa»­[·V"m´oß^¶mÛ&ï¾û® û}衇dîܹR¬X±ˆÇÆŒ$àe™^¶ûîG¡"Ó_éÇkH«¦/ËW\.†½% gO’ËC¶¦À£9Ðø{Ÿnm/ø;Ê¿Ü´Ô¬VɤY¤C’=kY»n£äÌ‘MÕY¼èÝräç_¥u§Þ²õ«m’1cißò5¹ïž»äéš ä×ß~—reJÈ>dò´Y2qÊL9zô˜ä½5· èÙ^nÍSÞ™>WvîÚ#ß‚öÀÁÃRäÎÛ¥kû’ÕðÂnß¹[:í~¿ïG¹ëŽÛ¤w—6’5ËMF”Ôié9`„,]¾Fn¼á:yñùg ì£~4)ÇDÉ ÈôÙÄ`è_d…ÈD8lêÔ©•ð»þúë¥C‡Ò¤I©\¹²ÀóA‡4~üxõw3A4–,Yò‚†BEæÝwß­¼éÒ¥Sâ^Ñ®]»Êºu딧¢ÐL¡"Ou=*7Ýt“üöÛoªÌÏ?ÿ¬D"D&<Ÿ~ú©*úæ›oÊöíÛ¥ZµjÊC¹iÓ&õw”[±b…¡9sæ”éÓ§Ë< >ƒ·3}úô2pàÀÈ@ ׌¿L‘éoûê8:†þ¥l•P‘ùqý]úÉjiÚ¦«ÌŸ>Þ]],×\}µ¼Þ¸~Ħ ™ªÖ•Ž­›HšUeÔ¸·eõ§ŸËŒI#¥kŸ¡ræߥs›&òñÊO¥×€‘²fÉ, õdB<–¯ü¼¼?m¬äΙCzô!g^À{ ‘Ù£ÿp™3u¬ä3Ä端w4<±ùåµWêH¹Šµ¤C«ÆRêÁbÒwÈh9pè°ŒÞWKv·W†öí"?"¹Æ‹Mdô^Røÿ E<¶ däšñ§¥)2}fWÝo¨tÁ\¸¬Ù76x7GŒqQwáìß¿¿ s M¡"³jÕª²cÇõñÎ;•„È„øk×®Úÿyíµ×J«V­¤Q£Fx2!x‘ç½÷ÞSá­ª™3gN™¨ÇÜ:vìXùúë¯UØ,êœ3gÎ}‚‡Á3 1m&ˆè=zèb ×ûÁ5㺠í€îöåÁ?Žšß•ÊyˆIx‘yòä’ʸ.ýk¿k Ïâk/?//”uò˜ARôî;/h TdV¬^_6®ú@Ò^sµìÚ³W‰×E†WtØèÙòå7Ò¹]3¹%×͉åCEæiãºâÄI¹þºLrôØqn”ùõ÷£2¬_%2±GtæÛoª²SfÌ•=ßï“ÇÊ•–ÞGʃ·3í5×È-_ue^êÜ(׌ÎÖ±Þ7ŠLëì´,©û •.ÐR™‹-’^xAöìÙ#2dHìòáÇÕK|ž’È„¨„‡)Td«‰½šWOiW®\©<¦£ß}÷]âÁ?ðXÂËo)¼«ï¼óŽ,\¸0QdBTŽ=ZÕmŠLìåÄÁAð’"Á‹š D,„*B€³e˦>C[W\q…ÚÇÉtŽ×Œ¿g‚îö¥Èôßüã sÊ6M.\Ö,ÑËlxØÚ¥]ó‹*©R»±O³‘»·ðŸ…ŠÌW_ï$KçM9w½Ûûƒ¼Ö²³™†5ö®” éÓIý:5¤Ö³•/ðdž9ó¯Ê³pÉ'Æ!~×+¡ ÁiŠÌ»¿KÜúî¬ùò­ñïÂÿw»,þx¥Œ|áö<äÍwÉsKN〣óy)[BZ¼¹—Ö«#ùqÍøÓÒ™>³+Cÿ"3hJ"5<ùä“ròäIµûñšì±ÄÁ:3fÌ°$2Q'ökb%öbâÀx&±W‚'ØŽ9Rí™ÄÉ·þù§êBgá©D¸lr"'àæÍ›W…õ>üðÃÊ <þ|yõÕWU=¦Ta¾²IErdÔü™‹kÆŸv5GE‘éoûê8:†þY™+׬“V{ËòÞ5"yÒ&Vôó/¿Ié'ªËÄQR™•KÞç"‘ ¯föl™å*ã`¿ ›¶ÓF¾)òドÿ,X´LÆOž.“ oi¦k3ÊÜ>’«×%ŠLˆÊÿN¦ pÍøsFPdúÓ®UáD&Âe»uë¦Ä<˜EŠQ'ÏâdXžƒ‡¦ÐpÙKy2!ëÕ«§¼›™2eR'ÕâDZ¼¯'ÂÞyç2qâÄÄðZüƒpÚ–-[ªý—‰É‰L„õ¢_”ðT¢oðdÂkŠ}§8pè£>R·6mÚ(±ÌDA!@‘Ksœ^!’'c¨ß¤­ñpô/éÔ¦©:Qv×îï¥{¿áê`áý»Z™¨3Þ[¥ñ+/È¡ÃGÔ? g'{%‘>ƒÞ”Y·§Ï‘%¯–„Ñäï¿þ6úÑNy2ßÜC…Ë&'2Û¯>)ûäsÒ§K[y ØÝÒßð„þðãAykxéÜk°:á½{‡–òÓÏ¿HzÕþ̤žX¯Øý$h PdFKŒùI€H€³2Cÿ|fPÇq\3Ž#vµŠLWñ²q†þÒìt ¸fb€§qQŠLc¥kºßPYË€“¸fœ¤ë~ݺۗÿ¸?Gìî1±›(ëó;®Z˜"ÓgvÕý†Êg¸9àšñS‚îö¥Èôßüã ³ÿlÊ9K€kÆY¾nÕN‘éy‡ÚeèŸC`Y­o pÍøÖ´j`™þ¶¯Ž£c蟎VaŸt&À5£³u¬÷"Ó:;–$‹à”Wœæúù矓 €(250»@$@$8™39ìS§N ~vìØ¡^IbgÂË©S§N­~˜H€"'@‘9+æ$¯À;-ÿ÷¿³Ré‰ò¶uù©ê/ÉàÞeÛŽÝƉ°ëeP¯Ž¶ÕÍŠH ˆ(2}fu†þÙoÐHùòåå®»î’E‹I–,YdРAòÄOÈÖ­[¥yóæòàƒÊâÅ‹eìرêÝ—ðdŽ5JV¬X¡ÞmùÓO?I¥J•¤lÙ²ê}›gÏžM¬ïÑ»+çÏŸ¯Þ©U®\9yçwäꫯVïÓœ0a‚z&ÚºÿþûeÎœ9R¸pa5Ð%JHÛ¶mïéd²F€kÆ7¯”¢ÈôŠ¥üÓO†þ9oËQãÞ‘ÿýWš6¬k[c™¶¡Œº"®™¨‘y¢E¦'Ìy'u¿¡Š|$úä„ÈÌ‘#‡´hÑBúõë'Ë—/—jÕªÉîÝ»åСCJ`véÒE‰Í/¿üò‘Ù®];Ù¶m›¤I“FòæÍ+?ü°Ì;WF-³fÍR"tòäÉ2tèPYºt©\vÙeJd¶nÝZjÔ¨¡DfÅŠeäÈ‘’>}z%F!rÛ·o/GŽ‘|ùò){ÕUWéÌc=ášñ˜Á¢ì®îöåÁ?QÔÙyˆ‰}F:òó¯ÒºSoã™Û$cÆ Ò¾åkÆõîJiݱ·jﻬWû™CåŒá±ìܦ‰|¼òSé5`¤¬Y2KB=™–ò•Ÿ—÷§•Ü9sHþ#ä¬Ñ…ž[*‘¹}çn™ùöh#‚èòxµz2¤O'¹£P)ýxuC<¶•‡K=(#Þš$£Þz[Íž¤DfË=eÔ R¶Ô2fÂY¸d¹,œ5IžxæEUïÝ…ï0„èJ™6sž¼=v°}hM\3þ4J*2* ¯%RÉ’%¥oß¾Êó 1Yºti%2Qï+¯¼¢ÊÂÛ ÁŠÃƒL‘ùí·ßª¶°ç3sæÌJ¤Ö©SGj×®-Õ«W·Ðª‘kÆßƦÈô·}uCÿì³Ê°Ñ ²åËo¤³á}„GÑL¡"óô?ÿkOÊõ×e’£ÇŽËp£Ì¯¿•aýº(‘ áÙªé+ªh“Ö]ä‰G˪ßß~w¶L›0\ý~úôi¹ç¡§dÎÔ1JdN~w–Ì™2V}væÌ¿r_™Š2wêX™ùþBõ@ùõÆõ•½û®;¤Ö³•íp@kâšñ§á)2ýiWŽÊF¦'óرc’6mZUóÓO?­æ½÷Þ«„ÞW_}¥þnEdBL" >\®¼òJy饗ÔÁA¦ÈDXîõ×_Ÿ8"xPüqiÕª•üøã*Œ–‰H y™œ$à]†U^à éÓIý:5”¨ ™ȳpÉ'rà ×a­W+ÁiŠÌÇÊ•’'{XAhÖ¶› ýù—ßä«m;¤÷7á<\±–ŒÒS‰Ìe+ÖȈÝ?{òÙ¥[û’Êø¯kß¡J€+[Éð|N–›n<}ö.iöœì'@‘i?SÖè3æžLˆ:x"W­Z¥ñÙ¾}»ò:Æ*2±¿óöÛo—nݺÉÎ;÷x6iÒD…Ë&™'N”:HÑ¢EÕaAL$@—&@‘ÉÙAÞ%°kÏ^Éž-³\e<€ÅžÉ†Í;È’÷§(¢yðNš?yºL3H2]›Qæ~ð‘¬X½.QdBTšÞKSd¦J•Jy2§Ž¦à7<¡÷–zJ>œ9Q‰L|6{ÊõÚ)nì½|Ú[’5ËM†¸¬,o{C§Ïž/ïMå]¸ì9 8L€"ÓaÀñ®ž¡ö‡È„Çò¹çž“wß}WíË0`€Ú[‰_c™›7oVû0ÿøã%N;pà@ùì³ÏÔ‰¶IEæáÇ%[¶l±Y·n]û°¹fümpŠLÛWÇÑ1ôÏ>«ÔoÒVòç½U¿ò‚:|Dž®Ù@ÎNy.“?ÿüSZ7k oOŸ#K>^- £Èßý-õ›´SžÌ7÷Pá²É‰Ì‹Ý#¥Œ=™ý{´—²Ý/£'L•!£Æ_°'sì°ÞR¦äý26aš,[¾&Qt¶êÐKÖ¬Û(/×­!/=Ïí*vX›kÆŠúÕA‘©ŸMbê‘î7T1 Υ™ŋW¡©:$ˆQœ0»wïÞ Âhuè›ûÀ5ãE«EÞgÝí˃"·¥Wròû,õíîï¤Mç¾òýÞ$C†ôê4ÙšÏTR§¾¶x£»¼ôB ©þôòš!&wîúNí¿D8mïA£¤{‡2á²dE&Bh?]¿Iº÷&¿=nœ:[Bp€PûV¯)Oæ‚ÅËŒ¿ <©ùóÞ"}»µ“œ9²©}øÑ'*ìvù‡ÓåæìYíl€kâšñ§ñ)2}fWÝo¨¼ˆ['‘ùqÀúàƒÔɲL±àš‰¡Î5èn_ŠLgµ¾ñ†Ù7¯”‚8í;d´ÌŸ>Þ+]Ö¾Ÿ\3Ú›ÈR)2-aÓ·Cÿì·Í_ý¥Þa‰×–¸°/ówÞQ¡µL±àš‰¡Î5Pdêlö¡þ´+Fõ÷ߧÔ{;o+W^­ÿ¼ç‘qÍÄxœš£ÈŒh6C$@$™ñgÎIÀ¯ªÔn ×\}µz‡fF#|—‰HàÒ(29;H€H€|K€"Ó·¦åÀH€H€4&@‘©±q¬t¡V¨±L pÍøÛú™þ¶¯Ž£c蟎VaŸt&À5£³u¬÷"Ó:;-Kê~C¥%4v*иfüm~Ýí˃ü7ÿxˆ‰ÿlÊ9K€kÆY¾nÕN‘éy‡ÚÕý†Ê¡a³Z°L€kÆ2:OÔݾ™ž˜FQu’7ÌQábf®NŠLŸÙ•¡>3(‡ã8®Ç»ÚE¦«øÙ8Cÿiv:\31ÀÓ¸(E¦ÆÆa×H€H€b#@‘?–&  +(2­Pc  O Èô„™ØI  Ÿ Èô™Aúç3ƒr8Žàšq±« PdºŠ?3ô/fç c À5<‹Rdjl+]Óý†ÊʘX†œ$À5ã$]÷ëÖݾ<øÇý9bwxˆ‰ÝDYŸß pÍøÓ™>³«î7T>ÃÍáø€×ŒŒ˜Ât·/E¦ÿæo˜ýgSŽÈY\3Îòu«vŠL·È;Ô.CÿËj}K€kÆ·¦U£Èô·}uCÿt´ û¤3®­c½o™ÖÙ±$ €æ(257»G$@$àK™¾4«¿uüÄIÙ¾s·$þ'MÒ§“Û ä•ôéÒJ¡‚ùü $ÊÑ‘_”À˜ÝÓ(2=m>v^cÛvì’'ÿP×`󚜴»¸›×cüŸé<òãlð;ŠLŸYØo¡û’ ·Êº›åÀÁò~ãKÃ….G¶,RìÞ"rß=wFx’_øéâ·5~ÄÁÊA‘,{ë0Z?†þáÚ{îg³ì7®Å¸G›Ò"³q-Æõ×âb÷V4‰üR¶²×Læu¸1Rd†#ä±Ïu¿¡Š'„Ѥ©³déò5]ÈÒÁ3i\ p‘ Mæß·ýçåÄg¸ â)ëŽo÷\7»!8Ë—)!ukU3ÄgÖHºå™<ä©ü°f¢q°rën_üã¿ùè—CLæÌ_l\‡W«kqÒT0%ñð6{È5ÔôZ†F0®é¦¸6Ÿ4<Ÿ¡ ×áòeJJ•Šùn"_ä&õËš‰|ÄÁÈI‘é3;ë~C•nˆ£^F^pAƒ×Ñ|≰×Xžz®û|³ éÁEsæ­‰]ÁÓÔ­{Þ»I~Ö³—׌µ«”îö¥Èôß|ôú 3ÄQ#äÄ[Q²eÍ,Åë$®ÅðBƲשí;Î]‡×žÑƒ‡~R^Îzµž‘¦ ëz~B_ô&ôúš‰~ÄÁ(A‘é3;{5ôoö¼EÒ¶K_e \Ðp±©Z©BL¢2%Ó»‰6¦ÎL¼È5mXϳ8ò³¾½ºf¬8X%)2ƒeoFëÕÐ?\kÕo–¸¿²ÊSÆ R¼è…QCv2ÆÃß9óÉœ©j!bûuk“µ³ÑÔE~Ñк0¯W׌õ£$Ef0ì¬õ(q‘©ýrsA(l3CèÕ«ýL\û †§¶ã=¸§”/[2®íÇÚùÅJåýL€"ÓÏÖåØì$‰½ƒˆ êßã¸n%‡³gÿ²lÅZ%4̘`çÐâRùÅ3ñŠL˯]6:AFŒ¤ž^Â{éFÂ)okÔW×i†»ÑËm’Ÿet,™02‡h Ü"’hÕ¢÷l©ÏJ% ›·WBsʸ¡ŽzP­ô-\ò GˆŸE¦Ï,îÅÐ?ó¢‚}‘ñöbšæO˜2Sz éI‘I~±-b/®™ØF¬Ò™Á²·£õbèŸù ‡âÁ‹ËùVm€pÓ§ª¿¤üóšÈ$?«V?W΋k&¶£4E¦Ïì¬û Ur¸k¾Ô4ñ žŽ­›¨_ã•pQëi„Êb£>’=™äÛlñ⚉mÄÁ*­»}yðÿæ£11·]ÀW=¤g\Ãe—~²ZzzÍW£xMd’_lëØ‹k&¶£4E¦Ïì¬û U8‘‰Ïñ$ÿ”+ó c9ì;™Ãþ—Â)w^™äý‚ö⚉~”Á-¡»})2ý77½xÃ*’L‹àµ"x½^3âD2ßã÷8œ˜5Gö,–örúQd’_x^\3áGÅ&ŠLÎ…xðbè_8‘ CÔš.õ@7¥ºü*2É/e^\3‘Ø4èy(2}6t¿¡Jw¬"Ó.UdŸ׌]6 B=ºÛ—ÿøozñ;D¦– ²È 2?/®;ìå÷:(2}faÝo¨(2íŸpé±1õ⚉mÄÁ*­»})2ý7½xÃL‘Û<$¿ØøyqÍÄ6â`”¦Èô™½úG‘Û$$¿ØøyqÍÄ6â`•¦È –½u­Cÿ(’b›9ä?/®™ØFŒÒ™Á°³Ö£¤HŠÍ<ä?–ö7ŠLÛ—£³‡ERlÉ/6~,íO™þ´«§FE‘›¹È/6~,ío™þ¶/GgŠ¤Ø8’_lüXÚŸ(2}fW/†þQ$Å6 É/6~^\3±8X¥)2ƒeoFëÅÐ?Š¤ØfùÅÆÏ‹k&¶£4E¦Ïì¬û Ur¸£Ifޔ̖Ýx'fù2%¤iÃzê½\‘¦ œ.K~Ï/®™Hç4ó‰èn_üã¿YêÅCL¢IfÞp–Ãu¸ÊSIù²%ÃeMü<§Ë’ßÅÓÁ‹k&âIàŒ™>3¾î7Tñ™fé 9uÜP)T0_DV¦È¼SPøyqÍD4¡™IÐݾ™þ›¨^¼avBdš–­Rñ1éßýˆ M‘y1¦ ðó⚉hB<E¦Ï&€Cÿ¬z2woYy‘õö<$³ç-–c'©Ï ”6¯þ0"+Md’ß¹iáÅ5Ñ„f&ŠLÎWx1ôϪÈlÒ ®4kTï"ÎK?Y-ÃÆ$ÈŽo÷¨Ï:´j,õj?ÖA™äwnJxqÍ„ÌÌ ™œ®°SdšƒiÓ©·ÌYð‘úg¿ní¤j¥ aÇI‘yQø…ÌàiôdzÚ|ì|œØ-2Ñm<ô-ýx 5lcY¹pFØÑPdžG$~a'3x’E¦'Íæ¯N;!2C÷ûÚrt8™ŽâeåÉðbèŸU‘‰º¡žÌõ7ˆM[•HÏCE4‘I~ç¬ïÅ5Ã/¿ð(2Ã3òTÝo¨’ƒiUd&=M5ô°šH÷„ö'h"“üÎYß‹kÆS_J.wVwûòà—'ˆÍ{ñ«"3éµ{ ªð¬œ<ù‡¤7ÂcW.œÕß ‰Lò;ÿ°÷ìÙ³Rõñ’¬HVéŠL·È;Ô®î7TNŠLÔ‹ÛÁC?©f"9."óB‹‘Ÿ׌C_¾¬VwûRdúoÚYdš³ç-’¶]ú*Ö/SBF鱑ƒ.2ƒÊÏ‹k&âIàŒ™>3¾Cÿìòd”¡'£bCýÔñÃ"¶pÐ=™AåçÅ5ñ¤fFí=Õ™þ›¤^ ý³Ë“iZ3ôÖSÆ ø|ŠÌsƒÆÏ‹kÆß\öˆ"Ó~¦¬1JvŠL4úòçhö„Pdž3\ÐøE9]™ÝcèÉô˜ÁØ]WØ-2·íØ%kÔWc‰æœŠÌsæ?W&=uœE¦ãˆÙ@8v‹ÌýɓϾõžŠÌs– ¿pó“Ÿ{›E¦·íÇÞLJ€Ý"½îÑ„Lž6K Òs(2ÏÛ;Hüâ3ËÙJ¼ PdÆ›¸Ãíy1ôÏn‘ Ä¡‡Eº'„"óüä ?/®‡¿F|U=E¦¯Ìé‰Áx1ôÏ ‘z É9 ™ç§xøyqÍxâËÈåNRdºl»›×ý†*¹ñ:!2ÑNè!6‘ì ¡È¼Ð:AáçÅ5c÷÷†ŸëÓݾܓé¿ÙçÅCLœ™°lè!@‘œ“@‘yáz ?/®ÿ}sÙ?"ŠLû™ºZ£î7T±ŠL„lß¹KU3mÂðYã¢9|L‚Ê“ÃxwfÿíSÌ‘I~O/®W¿d<Ö¸îö¥ÈôØ„Š »^¼aŽFdb¿`Ï#‰ª+HÕJR¤Ò¦SocÆa•§iÃz)‘I~O/®™¾ Ÿ…"ÓgSÀ‹¡Ñx24WD&ù%q«R¡„“hX·‹(2]„Ц½úÈtÒ¬A™äw1/®'íè—º)2ýbIÙ_¬þ0ª—6Û9dvSúñrß=w…õÚÙ®u‘ŸY‡_ PdúÕ²—L‘Y§f5éÔ¦‰UGU¼žs|$‘lq‰ªb‡3“ŸÃ€Y½' PdzÒlþê´ùäÒÍ‹›}°jUú®C¬òc9 Èô·}9:{˜‡Ì¤J•JÌol1ÉjOÅQÔ‚>”z¼ºœ0þïæCç(ºœ˜•ü¬Pc¿ Èô™…½.z‚ZÝZÕÔž éÓÅÅ2h{ÒÔYjïfºtiåƒ÷&¸rqe°ä =/®™ØF¬Ò™Á²·£õjèŸù°ïµì×­]Š{'íæ O`¯#3v‹›œcùY§çÕ5c}ÄÁ(I‘é3;ë~Cu)ÜØ_³~³Äw[Ö«õŒT©ø¨c‚ÂlÎüÅ’0u¦0$€Àœ6~˜*˜Ï“3‚ü¬›Í«kÆúˆƒURwûòàÿÍG/bb†«Â*8 ¶®q-ÆkÀœJ—s,V×c¤*O=ö>§úbG½äg¢—׌µ£E¦Ïì¬û UJ¸!üð$Ð|y3òÞV ¯q+©.vø=çú[ÔSÒõ7ËÒåk»‚‹ZÇ6Mcª[‡iD~Ö¬àå5cmÄÁ*¥»})2ý7½~à áצs9xè'eœôFd„fñ{‹HÁüybz‹ó¶ïÜ“xÆC^¤lY3K§ÖM¤|Ù’žŸä½ ½¾f¢q0JPdúÌÎ~ýÃEhö¼Å2{þ¢Ä‹œi&\ì bá<¡{FLŠ –yTúñ'”¨Ä¿Í ™Y.h¯õjWsÌ[êÖÔ"¿èÈûaÍD7â`å¦È –½u­_Bÿ–~²Z]‡—­X{Vóš[ÌžfÂC`ü ǘ׎á!hèßÌÏÊ•~P]‹Ã½E»FÛò‹œ˜_ÖLä#FNŠÌ`ØÙ³£D¨é}„XÜ°i«¥±àé+Þ•‰ "¼¢^ ‹vðä-1æ÷ŠL¿Y”ã‰7D\K  „k±é匦/Ø–‚‡Ä·ÈgxE K±¢E<AÉøÉ/JÌãG™~´ªÏÇOÝþçBl @ñž4áB¯'ž¬EPFjvò‹”óùE¦¬È1èHa¡H8 v›q-NšB=›¸DzÝEÇñÇÚ'ò‹• ËëN€"Sw EÙ?†þE ŒÙO€kÆßS€"ÓßöÕqt ýÓÑ*ì“θft¶Žõ¾QdZg§eIÝo¨´„ÆNš×Œ¿Í¯»}yðÿæ1ñŸM9"g pÍ8Ë×­Ú)2Ý"ïP»ºßP94lVK– pÍXF牂ºÛ—"ÓÓ(ªNò†9*\ÌLê}ÕgÏž•ª{ÿtašó<ŠLŸÍ†þùÌ Žã¸fGìj™®âdã ý ¤Ù9èpÍÄO㢙‡]# ˆEflüXšH€H€¬ È´BeH€H€0b CÐݾ™þ›¼aöŸM9"g pÍ8Ë×­Ú)2Ý"ïP» ýs,«õ-®ßšV Œ"ÓßöÕqt ýÓÑßMw IDAT*ì“θft¶Žõ¾QdZgÇ’$@$@š ÈÔÜ@ì €/ PdúÒ¬ =™œ$@$@$àŠLw¸;Ö*CÿCËŠ}J€kƧ†ýoXôdúÛ¾:ŽŽ¡:Z…}ҙ׌ÎÖ±Þ7ŠLëì´,éö UÑÒåØñ’*U*Å'Kæ¥Ñ‹µ¤FµŠêßÿ}JÆ&L“ù —Êá#?Ÿß$O?ùˆ4|±¶¤IsÙEL_|µµ<_£Š”yèþ¨yïÚó½ÌYð‘´mÞ0ê²n ¿ø“w{ÍÄÄÁjQwûòàÿÍG·1áu$¶9E~±ñ³RÚí5c¥Ï,žEfxFžÊáö ¾œÞ w* §OŸ–ÅËVIË=eá¬I’/OniÖ¶›œüãOCø5Ù³Êæ­ßȈ±“%¯ñYÏŽ-/b½mÇ.%D¯Ë”1j;lÚò• 9^¦ŽuY· _üÉ»½fâ?â`µ¨»})2ý7ݾaæu$¶9E~±ñ³RÚí5c¥Ï,žEfxFžÊávè_è—³ ®ôã5¤UÓ—åŠ+.—ÃÞ’…³'ÉåiÒ$r…Gs ñ÷>ÝÚ^ðwdx¹i;©Y­’!H³H‡%{Ö,²vÝFÉ™#›ª³xÑ»åÈÏ¿JëN½eëWÛ$cÆ Ò¾åkrß=wÉÓ5ȯ¿ý.åÊ”!}:Éäi³dâ”™rôè1É{knг½Üš;§¼3}®ìܵG¾1탇¥È·K×ö-$«á…ݾs·t4Úý~ßr×·Iï.m$k–›äÔ©ÓÒsÀYº|ÜxÃuòâóÏÙGcž+ä3¨+p{ÍDÝaˆŠEfT¸˜Ùn‡þñ:›É/6~VJ»½f¬ô™eÂ È ψ9¢ úåüÏ™3²ô“ÕÒ´MW™?}¼ººX®¹újy½qýˆk ™ªÖ•Ž­›HšUeÔ¸·eõ§ŸËŒI#¥kŸ¡ræߥs›&òñÊO¥×€‘²fÉ, õdB<–¯ü¼¼?m¬äΙCzô!g^À{ ‘Ù£ÿp™3u¬ä3Ä端w4<±ùåµWêH¹Šµ¤C«ÆRêÁbÒwÈh9pè°ŒÞWKv·W†öí"?"¹Æ‹Mdô^Røÿ E<¶ä2’_lüb‚Ͼ$@‘éK³rP)àu$¶ëùÅÆ‹“L™œ ¶À—óÉ“HªÔ©å_Cø]kx_{ùyyṪҰy{åy¬[«šjsÆœ¤[ßó¡¬“Ç ’¢wßyABEfÅêõeãª$í5WË®={•x]dxE‡N-_~#Û5“[rÝœX>Tdžþç9qâ¤\]&9zì¸ 7ÊüúûQÖ¯‹™Ø#:óí7UÙ)3æÊžï÷ÉcåJKï#ež!‘PnýÆ-òèÃIÉG«e»ÊÝ…ïPŸÁÛ™öškä–¯ÆÄ“übã|ö%ŠL_š•ƒ #2y¶>ExæuØúìaÉP™>›n‡þ%fb"îe¶3†w³K»æQ¯R»±O³‘»·ð%E櫯w’¥ó¦¨Ï¿Ûûƒ¼Ö²³™†5ö®” éÓIý:5¤Ö³•/ðdž9ó¯Ê³pÉ'rà ×+¡ ÁiŠÌ»¿KÜúî¬ùò­ñïÂÿw»,þx¥ŒÜó‚>={Vòß]FòÜ’Ó8à(uâg”-!-^‹ÜK›ÜÔ#¿ØøYYÎn¯+}f™È PdFΊ9í!àvè¯#±]GÈ/6~VV‘ÛkÆJŸY&<ŠÌðŒ<•Ãíª”¾œW®Y'­:ö–å¼+éÒ¥Mäúó/¿Ié'ªËÄQR™•KÞç"‘ ¯föl™åª+¯” ›¶ÓF¾)òドÿ,X´LÆOž.“ oi¦k3ÊÜ>’«×%ŠLˆÊÿûaÿA¹üòËÕ>ÎXùÅÆÏ {·×Œ•>³Lät·/þ‰Ü–^Ééö!&¼ŽÄv!¿ØøYY§n¯+}f™ð(2Ã3òT·o¨RúrÈúMÚÊŸþ%Ú4U'ÊîÚý½tï7\¬3¼WK"uæÏ{«4~å9tøˆ:ðgáìc¯ä/ÒgЛ2ëÑòöô9²äãÕ’0z€üý×ßF?Ú)O曃{¨pÙäDfãÕ'eŸ|Núti+»[úžÐ~<(o ï#{ 6^Çò·tïÐR~úù©Q¯±ÚŸ™Ôíä!¿ =ÙÑò³’ßí5c¥Ï,9ÝíK‘¹-½’Óíf^Gb»Ž_lü¬¬S·×Œ•>³Lx™áy*‡Û¡ᾜqЈ1“ThëÏ¿þ&·Ì'-›¼lœ »]Š'Ââß¡)tOæ¥<™ˆm:÷•ïÚ ÒK£—jKÍg*©÷uV©Õ@ äË#ýŒ“k_kÕÙ8Eö;uðÂi{eˆÄ†ý5Y‘‰°Þ-Ɖµ] AùÃCªoýºµ3¼¦YÔ¾S8´êÓ †3¼R÷9ã@¢s{McIä =keÝ^3ÖzÍR‘ ÈŒ”óÙEÀíÐ?^Gb³$ùÅÆÏJi·×Œ•>³Lx™á1 €G PdzÔpì6 €§ PdzÚ|ì< @J(29?H€H€H þ(2ãÏÜÑúç(^VîC\3>4jÈ(2ým_GÇÐ?­Â>éL€kFgëXïE¦uvZ–Ôý†JKhìT  pÍøÛüºÛ—ÿøoþñÿÙ”#r–׌³|ݪ"Ó-òµ«û •CÃfµ$`™×Œetž(¨»})2=1¢ê$o˜£ÂÅÌ$ \3þœ™>³«_Cÿz!¹nÎ!Ï×xÚv‹Mš:Kö<$[7‘B÷•—+çË5W_m{;nVH~—¦ï×5ãæ|Ó©mŠL¬Œ¾ø5ôבØæ/ù]šŸ_×Ll3Æû¥)2½oCmGpæÌ¿’:u*ã'uÌ}ŒäËùô?ÿÈ—_u[¡"sÝç_HÑ»ï’Ë.»,êzì.@~ve}A$@‘D«sÌ&^Gb› ä?–6ŠÌ`Ûß‘Ñ)ù„ôíÚV:ï—üpæDùçŸ3Ò¾[ùzû·R0iÝ´ÜyGA9uê´tï7L–­X+§NŸ–‹Ý#ƒzu”«®ºRúIÚvé+Ûvì’ÿ»½ d4Þyo‘;/òd6jÑAJ•(. SfJÓ†õäî»nO¶­y.•%Ÿ¬’ËÞöÿ÷îÌv’;W ™wÞÿ˜¬ûd®òd~°øc2j‚œüãO)_¶¤t2çNç?íé¿ö:Ÿý<œ³†ßú|×Ú{϶´½½‡Ì˜ÌË ?ü²Ím˜É6fÒ§'€ÉLÏ(V)Lx¡Ò›óô§GËiukË7ßý¯\óm²ì¯³J½š6o+÷ô¹KÎ;§ìß·O*T8™aìuÏ0iyírež‘» ÙòÑ{oäçi×¥·\~IãbM¦–Óñ¿[§¬ë矷ËÔç_–W¦MtbÐ%0ç]r½Ì™9I,\š¿'ÓÉœ4ufÞ æv'N½¾\»^~þù9½^çßU¬"[·ýŸ<þäÙ´e«Œq_`ý~¡Ì¨ ÆLF’ÈÓõåàO²É„CLxŽøë"ðóÇ/ÛÜ&Œ™lc&}z˜ÌôŒb•Â„*½9ÿuÎt9²Jeù`ùÇÒé®ÿ‘j8¡Ç>Ý:ËYgœ&‡Ž”ÏV­‘êÕNÍ[¶I»[Zå-‹­/·w ‹ßœ™ŸGÓÕ=åäbMæuW5•«›]’²®íyK^,Z"cGÉ/óÚ›:É{zɧŸqÉ| o†²N퓤}› Å­æt䘉òæüw娣ªJùr‡;†3h“ ¿è† c&ºÖ–¼šLדi_Ÿ4ᅙ簿~?ü²Ím˜É6fÒ§'€ÉLÏ(V)LXú§7çw_Ÿ!U*W’µë6ÈyKMÕ4¹×çyû,«Ÿx‚ ÿóxçG÷öï.eÊ”‘÷3O?U®Ê[–zA³òf2ç:{ õjqKg¹¡ùÕÅšÌæ×4sf?SÕ¥KcŸ{~–Ìš6Á)oïÞ½ò_—µ”—ÿò¤,Ì[Jëž.ëÎdN˜<]öä¥é×£‹“þ³U_È·ßÿ7ºG&M}Až0Êiß«oÌ“E[¸É„_tÃ΄1]kK^M˜Ì’§y®[lÂÒ?žÃþzüüñË6· c&Û˜IŸž&3=#RdI ñæ¼/o9ì5­;93”7µ¼FÞ[ú¡ôÎÛ¹dÞËÒoðCR»VMéÙµ“|½þ¹©ý]y‡÷tÈK{ƒ³òŒ<ÃÙ7oÆsùŠOåÖ¼ýšƒûuOi2SÕµ oÿ§îÉœ0ú!¹ô¢F2qÊ ç`Ÿ7^|&ï ŸY™Ì¯×+wt Ï?3V*æ-ç½³ç@ivYc9,ïôÚùý›Lyr¤ìڹ˙qÕ™Ìñ–%¥äÉáJ ‚ÿÐJ$ž#þd‡Ÿ?~䆀ÀdÒ'xsÖÂÕ@ö¨¬^³VŽ=æhgæ²Ñyg‹Îhö¾{˜ìعSÎÈ;AöœõeÒs/ÈËÏ=)û÷ïwN—Õ<ºLöôSkË Ç—Òd¦ªKO—7ëXú°Ò²â“Ï¥fõeøþR«fõ¤§Ë¾4{®Œ{ê/²}ǹ´Éù2,ïpŸÝ»vË]}ï•/¾üÚ9ø§íM-ä¡QãdèÀ^ÒìÒ‹a ¿@0RÌdÒJ"ž#þT‡Ÿ?~ä†&ÓÂ>ÀÒ¿âEU“ùÞû8ŸHáÊž€Íü3Ù÷‡8åÀdÆI-;beéÏá0z²ÍÏaÆL=&÷e2“™{ Àôª@›Ea6ßœ³Àà9©Íü3ž»E,2š®/ÿÄ¢e$‡˜`2³ê0&¶ù9̘ɰÄ,&3f‚¥ ×ôªtñ‡õ{ýDÊO7åh[/¬*¬.×f~Œ«».Ëeí–×ÈÖñÂ\¼,6?G¢èˆ6ócÌDу¢¯“=óPkdé_¨x)ÜBŒ EMh’éD`&Ó¾þÇÒ?û4¥Eá`Ì„Ë7W¥c2sEžz!@ t˜ÌÐS @à ˜L: XK“i­´4 € &€É4X/¡±ôÏ 5ò”dŒ»ÕÇdÚ­¯‰­c韉ª“É3&«ã=6L¦wvFæ4ý…ÊHhU¢ 0fì–ßt}Ù“i_ÿãû4¥Eá`Ì„Ë7W¥c2sE>¤zM¡ ©Ù Ï3žÑÅ"£éúb2cѲ ’æ¬p‘˜±³`2-Ó•¥– JsB'À˜ qN+Àd扬œ¥%RvíƒcÆ<ƒ³b2 ‡Ð @ÀL¦?~ä† x!€ÉôB<€  ˜ÌXÈD€ `L¦e‚²ôÏ2AiNè3¡#Îi˜Ìœâ/‘•³ô¯DÊN£}`Ìø€gpVL¦Áâx Íô*/m"Â$À˜ “nîË6]_þÉ} :1 š(åÙN€1c§Â˜LËt5ý…Ê2Ü4ÇŒ DLÑÓõÅdÚ×ÿxa¶OSZ.ÆL¸|sU:&3WäCª—¥!¥Xk 0f¬•Öi&Ón}MlKÿLT…˜L&À˜1Yï±a2½³#' N“i¸@„@VÀdZ)+‚ f2é€ ÜÀdæ†{hµ²ô/4´l)ÆŒ¥Âþ§YÌdÚ­¯‰­c韉ª“É3&«ã=6L¦wvFæ|pì YµfƒÜpuciuÕ…NŒú=ëÍ¿9ÿÍÏá@(<.N«]]vocäx&(ÿ0™þRBvô…yؘérH^¶écïÎϬ}ѽøùp€ƒ;.vo+§Õ®–Ý`#µÑ0™FËCp€ à‡&Ó=òB€¼ÀdzãF.@ˆLf D"D@°Ž&Ó:Ii ¸0™ô@€@ô0™Ñ3§F@ˆˆ&3"ÐT@H €É¤;@€€µ0™ÖJKà @À`˜LƒÅ!4@ðG“é¹!@^`2½P# Ä‚&32$ XF“i™ 4€ `2é € è `2£gN€ LfD ©€ @“Iw€ k `2­•–†A€€Á0™‹Ch€ à&Ó?rC€¼Àdz¡F@ˆLf,d"H@°Œ&Ó2Ai ÀdÒ @ÑÀdFÏœ!@ "˜Ìˆ@S  @ &“î@ÖÀdZ+- ƒ ƒ `2 ‡Ð @ÀL¦?~ä† x!€ÉôB<€  ˜ÌXÈD€ `L¦e‚Ò@( €É¤7@€¢'€ÉŒž95B€@D0™¦@€@L&Ý€¬%€É´VZ@Àd,¡A€€?˜LüÈ @ðB“é…y @ 0™±‰ !@À2˜LË¥9€ P@“Io€ DO“=sj„ ˆ`2#M5€ ˜Lº XK“i­´4 € &€É4XBƒ 0™þø‘€ à…&Ó 5ò@€@,`2c!AB€€e0™– Js @ €&“Þ@ˆž&3zæÔ@ÀdFšj @ 0™t@°–&ÓZii L“i°8„@þ`2ýñ#7 @À L¦jä XÀdÆB&‚„ Ë`2-”æ@€@L&½€ =LfôÌ©€""€ÉŒ4Õ@€`2é€ `-L¦µÒÒ0@0˜&Ó`q €üÀdúãGn@€€˜L/ÔÈ@± €ÉŒ…L @–ÀdZ&(Í ˜Lz @ z˜Ìè™S# DD“hª $ÀdÒ @ÀZ˜Lk¥¥a€ `0L¦Áâ ø#€ÉôÇÜ€ /0™^¨…gçÎ2qâDyôÑGåûï¿—5jH=äÎ;eˆP#EFI}£¤}]è=óLkÄdfJªä¦cüÚ­=ú¢¯ÝÌm&3ÇÚlÛ¶Mžxâ =z´lܸñ hŽ9æéÝ»·tíÚU*T¨ãh©>[è›-±x¥G_óõÂdš¯Q®"düæŠ|4õ¢o4œsU úæŠ|æõb23ghÊÍ›7˨Q£dܸq¢%ÝU¥JéÖ­›ôêÕK*W®œ.9¿Ï1ôͱ!W¾!°xLf€0-)Šñk‰Iš¾è›H€÷çÜõLfÄìøá1b„<ýôÓ²cÇŽ¬k?âˆ#œ%´}ûö•c=6ëüd—ú†Ë7×¥£o®Ⱦ~LföÌlÍÁøµUÙíB_ôME€÷çèû&3"æëÖ­“‡~X¦N*¿þú«ïZuŸfÇŽeÀ€R­Z5ßåQ€?èëŸé¹Ñ×t…’LJɌ¯vAEÎø Š¤™å ¯™ºúE2úr0™!3_µj•<øàƒ2sæLÙ»woൕ.]ZÚ¶m+ƒ ’“O>9ðò)05ôµ»‡ oüõÅdÆ_C¯-`üz%|è¼F‰¾^É™““’+V¬!C†Èœ9sdÿþý!ÕRPl©R¥ä†nÁƒKýúõC¯¯¤W€¾v÷ôµG_L¦=ZfÚÆo¦¤â™}ã©[¦Q£o¦¤ÌO‡É X£E‹ÉC=$ï¼óNÀ%g^Ü5×\#÷ß¿œ{î¹™g"eFÐ7#L±M„¾±•.ià˜Lû4MÖ"ƯÝZ£/ú†M€÷ç` c2â¹}ûv¹úê«å½÷Þó\¢ž«ßƼøâ‹Eo¦cÆŒ‘­[·z.Oã™;w®çüd, €¾v÷ôµW_L¦½Úº-cüÚ­1ú¢o:¼?§#”›ßc2â®'Æê!<^®5jHûöí¥gÏž…>O¢S¿ŸéÇlN˜0Aºtéâ%,ò$@_»»úÚ«/&Ó^mÝ–1~íÖ}Ñ7ÞŸÍî˜Ì€ôiØ°¡,_¾<«ÒtpÜwß}Ò¡C‡”ùÔlΞ=ÛÙã¹~ýú¬êhÔ¨‘¼ÿþûYå!ñÁÐ×î^¾öê‹É´W[·eŒ_»5F_ô-J€÷çxô Lf:íÚµKÊ—//ûöí˨´ÄÁ±mÛ6©T©RÚ|6lêի˳Ï>›•ÙÔÓgwîÜ)úo.oÐ×·¸äB߸(å-NL¦7nqÉÅø‹RÞâD_oÜâ’ }㢔·81™Þ¸Ê¥û'/¹ä’´%5—ºöØcÅîÕÔO­£è~ÍÄ=™Znº}™]ºt‘ &ølmfÙ,X M›6Í,±‡Tï¼óŽ\~ùårfŸ}f†¾…—¦Û6~³%fç`&Ól}üDÇý™û³»“÷+?#© /ïWsŒòý9Í)“éS‹:uêÈš5k’–¢&±yóæÎïõÓ$º—2Ñ0êÒX=”GžìÒJ}˜&ÎTê‹­Îržyæ™N¶d ¹eÖ¯__V®\é³µ©³ë)¶ƒ–råÊÉæÍ›C«KÛ¾{÷nyà¤OŸ>¡Õ££o^ôµ{ü†:rX8&3‡ðC®šû3÷gÞ¯‚d¼_Ì3Š÷ç`U4§4L¦-6mÚ$GuTÊÔPVªTÉI£Ë_ÝO‘èÿë`Ö¥j0õ[˜nºÄÝofj9º$6q¯fâ,©þ¼fÍšIc)Uª”cn+T¨à£ÅÅgUóqï½÷Ê¡‡*‡rˆŒ?^n½õÖÀëq ÔYÜîÝ»ËÞ½{E×ë6,³‰¾ˆ£¯Ýã7´jHÁ˜LC„8 îÏÜŸy¿ xPåÇûÕÁLÃ|^A³JÄdúÐcÎœ9ù³”Å£³”ëÖ­ËÿUQ“éÎrº³j:u¶²zõê…f=Ý=˜E—Ú]Š«&3ÕAo¿ý¶\qÅ>Z\õ·ß~s>›âšËíÛ·;¿<æ˜cäÇ ¤ŽT…}ôÑù'æêÌ©k6Õ|vØaÔ¾èkëø d€Ä¤LfL„Ê2LîÏÜŸm½?ó~%bóûs–·ºX'Çdú¯ÿþòÈ#$-¡C‡2eÊ”¤&ÓÝW˜/ÀÏNÊ'Š÷çpTÍ]©˜LìÛ´i#Ï?ÿ|ÊÜî^ÊTß°ÔY9 )îó%‰…§š M¶g³¸à:uê$“'OöØê’“ }íÖ}íÖ7±u˜Lû´füÚ§ib‹Ð}y¶£`2=êØ¢E ÑÈé.w_f¢ITc©{0Ý«gÏž¢ÿ·lV¿“©ƒMÿq¯ÄüúßzŒ÷âÅ‹EcJwa2Ó:ð{ôÍŒS\S¡o\•Ë>nLföÌLÏÁø5]!ñ¡¯?~¦çF_Ó .>L¦G–Ó¦MËèp÷Û—z¬ÎXꉬºœUñ)îÒ¥¯î¥µ¸K¿¯© éR½*Uª8†Hó%×dÍzë­·äÊ+¯ôØê’“ }íÖ}íÖ7±u˜Lû´füÚ§ib‹Ð}•ïÏñï˜L^{íµ2wîÜ´%èA;j—Ħû¦e²B÷_j=H–j¦[V«V­dÖ¬Yiã%ÁèkwO@_»õu[‡É´SgƯºº­B_ôU¼?Ç»`2}è÷Ë/¿HÆ eõêÕiK)úaYuÔ%´™Ì>º…'~GÓýY¦fµ~ýú¢'ò•+W.m¬$8@}íî èk·¾˜L»õeü¢¯K€÷«øõÆoü4ó1&Ó µ„úÏÔ¤Ñ×$5‚‹“K“Kbüš¬ŽÿØÐ×?C“K@_“Õñ&Ó;»bsf³Q=àªó‹ã Ÿ°ÈfwAXQ oXdÑ7<²¹+“™;öQ×Ìó7jâÑÖ‡¾ÑòŽº6ôšxøõa2C`œÍFæ «ç Ÿ ‰\ú†Ï8—5 o.é_7&3x¦&—Èø5Yÿ±¡¯†&—€¾&«“}l˜Ìì™e”#ÓÏ‹dTX‰†.ýû÷Ï"I½@_/Ôâ“}ã£UºH1™éÙ÷{Ư}š&¶}Ñ7 ¼?O“6LfzF¤˜7ožÜsÏ=òÑGRžR»vmyðÁ¥uëÖ•IAÞ ¯7nqÉ…¾qQêà81™ñÕ.¨È¿A‘4³ô5S— ¢Bß HF_&3zæÔ@ÀdFšj @ 0™t@°–&ÓZii L“i°8„@þ`2ýñ#7 @À L¦jä XÀdÆB&‚„ Ë`2-”æ@€@L&½€ =LfôÌ©€""€ÉŒ4Õ@€`2é€ `-L¦µÒÒ0@0˜&Ó`q €üÀdúãGn@€€˜L/ÔÈ@± €ÉŒ…L @–ÀdZ&(Í9@ uëÖòÊ+¯H©R¥ !™ú¨”-[6c Z¶l):tæÍ›gœÇMøùçŸËsÏ='#FŒÈ:oqüq¹ñÆåøã¤< €)0™¦(A%…ÏßÌ”~ë­·äî»ï–uëÖIýúõe„ rúé§g–™TˆLf D"Äì èCî /”=z¤ÌܱcG9ùä“eàÀùé{ì1Çd>óÌ3ræ™gÊÊ•+¥}ûöò§?ýIºvíê˜Ì%K–ÈK/½ääùꫯ¤{÷îrøá‡Ë¬Y³2ÖÉ\ºt© 4H.\˜q}»wï–ßýîw…ÒïÙ³G/^,-Z´åË—K:u2.„ˆLfT"F›ðü=XÍ¢Ïßï¾ûNêÖ­ëü1üüóÏwþð=wî\Ñ? sAÀ˜L[”¤…x}ÈmÞ¼YN:é$Y¶l™óp¯yóæ9¦sæÌ™™LM³eË©^½º|ðÁrꩧŠE˼ãŽ;Cxâ‰':ù›4i"‰&ó°Ã“]»v9³£z©aýå—_䧟~r ®–{ä‘GʨQ£œ¼ 6t~§³ Ó§OwŒ¢êüQ.ºè"yòÉ'ôZ×?ü Ÿ}ö™Ô¨QÙM¼Úµk'+V¬Õ«W;i0™ $Û`2mS”ö˜N€çoúçï /¼ S¦L}·ÐkÛ¶mR¥JÑ÷…Ê•+›.1ñA #˜ÌŒ0‘(n¼>ä,X }úô‘O>ù$i“‹Îdº /»ì2¹í¶Ûœå¶‰×-·Ü"Çwœ >\^|ñEgÖS ¢Æè.—Mf2Õ8êl£Öùúë¯;Ët¿ýö[IœÉÜ´i“3«¿oÔ¨‘ 0@¾øâ ™3gŽ“oÈ!òꫯ:æS—wi|‹-ÂdÆ­£oZ˜Ì´ˆH@ ðüMÿüݾ}»èì¦þ1X/}þêûÃÚµkÕ‚Â K˜Ì\Ò§îÐ$Û¢3”çœsN~½E—Ë>ýôÓÎ’×·ß~;k“©fòÜsÏuLª{©A,W®œlܸQ*V¬èüxÆŒÎòÔ¶mÛ¦5™Ã† sfUuÏä)§œ’_n¢ÉÔVý«èüùóßÿý÷R­Z5Ùºu«3ûª3¨³gÏNÉ“ZW¤àÀdæXª/qxþf÷üÕ?빺'ÓË %®ƒÑàØÀdÆF*͆€×¿¤êÒ=à§èL¦þÕqÚ´iÒ©S'?~|¡=™n\M›6u~Ÿ8“©†O7ô똢Wªå²z€Ö©KhtfR¯.¡éÛ·¯üñ,4“©3•º ö÷¿ÿ}¡*ÔtêÌéš5kdܸq˜Ìl:i­!€É´FJ<Åù£oºç¯>ßï¼óNùøãeÒ¤IrÁÄDa„@f0™™q"UÌx}Èýûßÿvöd~øᇅöVêL .]Õ“i‹[.« =ÔYÇÄ=™¿þú«”/_Þ1‹:£©—î«Ô}–;w.4“¹cÇÑe³ëׯ—š5k:ËduÙ«îõÔ=šî=ºò믿Î?øGÿúùÑGÉSO=唯ù4~=L@cÕƒ‰ÆŽ‹ÉŒY&Ü``2ƒáH)È”ÏßôÏßß~ûM7n쬬=z´óìç‚€m0™¶)J{^rš÷á‡vf-uS~ƒ œƒqn¾ùféÖ­›³¶¨ÉTã©¿Ó‡Dq§Ëêò—zõê‰.}Õ“ät&Rã¹é¦›òMfÕªUeâĉÎgDú÷ï/<òˆc5¯i>xð`g/¦øó駟:KbuVóïÿ»ós]¦ûÚk¯ÉÙgŸíÔ£Ëiß}÷]L&ã¡ÄÀd–ø.€ˆ ðüMÿü}ùå—sôYx=>bé¨ÀdŠ“ÂL! 9},]ºt¡n¿ýöB³zÅ}ÂD¿}©3:C¨R¿ƒ©³Žj0õ»›j2{÷î-eÊ”‘}ûö9ßѼþúëeäÈ‘ÎŒcÑë_ÿú—“ÿý÷ßwÒêÒÖK/½´Ðé²úÍK5–j/¾øbg&ò›o¾‘þóŸ¢1ꌦž<§ßÔÒå5zšíyç'gœq†clßxã g™¯šW5œºôFg@™É4¥GG®`2sEžzK*ž¿éŸ¿ú,W“YôÒ³*UªTR»í¶Œ&Ó2Ai ÀdÒ @ÑÀdFÏœ!@ "˜Ìˆ@S  @ &“î@ÖÀdZ+- ƒ ƒ `2 ‡Ð @ÀL¦?~ä† x!€ÉôB<€  ˜ÌXÈD€ `L¦e‚Ò@( €É¤7@€¢'€ÉŒž95B€@D0™¦@€@L&Ý€¬%€É´VZ@Àd,¡A€€?˜LüÈ @ðB“é…y @ 0™±‰ !@À2˜LË¥9€ P@“Io€ DO“=sj„ ˆ`2#M5€ ˜Lº XK“i­´4 € &€É4XBƒ 0™þø‘€ à…&Ó 5ò@€@,`2c!AB€€e0™– Js @ €&“Þ@ˆž&3zæÔ@ÀdFšj @ 0™t@°–&ÓZii L“i°8„@þ`2ýñ#7 @À L¦jä XÀdÆB&‚„ Ë`2-”æ@€@L&½€ =LfôÌ©€""€ÉŒ4Õ@€`2é€ `-L¦µÒÒ0@0˜&Ó`q €üÀdúãGn@€€˜L/ÔÈ@± €ÉŒ…L @–ÀdZ&(Í ˜Lz @ z˜Ìè™S# DD“hª $ÀdÒ @ÀZ˜Lk¥¥a€ `0L¦Áâ ø#€ÉôÇÜ€ /0™^¨‘€bA“ ™€,#€É´LPš@0™ô@€@ô0™Ñ3§F@ˆˆ&3"ÐT@H €É¤;@€€µ0™ÖJKà @À`˜LƒÅ!4@ðG“é¹!@^`2½P# Ä‚&32$ XF ßdZÖ.š@pL{·±$ô!Ì@°‘Àÿ‹C÷ ǤßIEND®B`‚patroni-4.0.4/docs/_static/multi-dc-synchronous-replication.drawio000066400000000000000000000046211472010352700254010ustar00rootroot000000000000007Vtbc5s4FP41frQHgY3tx9hO2u5kZ9Km7c70xSODDGxlxAoRO/31K4FkgwS2k+DGTZxmpuggjsS5fOci0nGmq80HCpPwb+Ij3LEtf9NxZh3bBn2nz/8TlMeCMhyOCkJAI19O2hHuo19IEi1JzSIfpZWJjBDMoqRK9EgcI49VaJBSsq5OWxJcXTWBATII9x7EJvWfyGehpAJ3vLvxEUVBKJce2cPixgqqyfJN0hD6ZF0iOdcdZ0oJYcXVajNFWAhPyaV47qbh7nZjFMXsmAf8b7fdj9cAuzc/wOTz92/W3Y+vXafg8gBxJl94NrXlftmjEkJCopjlghxM+C9fZ2p1BvzOVIx69kAj6ONhlQDMkeBRJejjYZUAdPZAWx/oGywRjFGFvaWtb5U2yH+dCckYjmI03ZqcxYkBhX7EVTElmFBOi0nMpTcJ2QrzEeCX6zBi6D6BnpDqmrsLpy1JzKTRA1uNpeAFV27WDPK1qOSRawLR6wdUKKSYgzFM0mixfYoiL6Np9IC+oLRgLqjcABNxvdoEwld7cJ32ewElWZJv/xNfq/bunF/OPUwyXzBhlPxE6iU7tsP/3QiDmywjjLWXf0CURdyXrnAUCN6MiKWgHGG0ZIIjl0gUB7f5aOZYUgp1S/gwDZEvX8e0f+kSYlW0KZGkP3xAZIUYfeRT5N2+JX3zUY2L4Xrn6Y5y37Dk5I56Dkp0Cbasdw7IL6QPPsEf3b7hkHeQizyODKeskbehnMGVOx25ZcmBRrXopqcpYcuqzoxLVm6qZS/wHK0roOvKNnQF6nQFHPtEunKAoavrr9OZCEY4S7mX8itgqC39iZgXShmX5Fax7VzGqQYVJX13hAmKnzqlL/MfBRYl2O5ZVg7EPdfRAFpisEYd11ILLNdmjhsYD/On+f0aJkCj2SNtbg62ylhv4QLhO5JGLMpxakEYI6sSnHhIIGOjeevo9zNbIBojLuZelCPfJFEQyBXkn8yM7aoZ25aJOaMaM+6PXm7F6ch2HA96nxfjq1/hp+zT9C+vaxpxx3axAGY/euCXAcuduiAtqE7ha9bMy0klq3f/y0Sak2NKtwhJV3yCm2yKh+TtFy1XJYmVjtkCcA7s4WhG/bYYDdpidEi8RzMatsVo1BajcUuMuIu3xAi0xchui1Fblm03WfZdySsLZoazvmtyBZb0NCCP2qqktKu5gB6rlpisvRBS1vMhgwuYooY8rCEaHRvImqNWvxq1RMWlJ8rD3sAMW4MWEuXasGXvD1tHQEhbftbI6O6DeIk4ZTDmOnqatZzZq7TGKKHRCgr175VGbdx/hpNxL2BVzzqYLK4i3xeP8xqavw7c1dTVZFpkpjBjRJbXudNW8nBZkdUUaaWS3+6f0mfBoOKzwDEzzXFdpnmq2tYsbWeiBig0yYuduNaycpjrylpI2FZCUZON8uJnXLWOgm2DeVzaWu+6reURmhAKGZqLqDrfQkJzF+V0LS6zUdNai+vMOlxgbHa46gLlLgZQBHk5blGUYC7p/Q2VWhtuaic22PZxLc5yo6WitIMNi/0mszXDkosdcieYJsXbLqMN8pt8gkc0klEPFR4hAlydb/iLuVLFXEh+ruR+dGuv/1Qb3UYmaaSu2dpza2zUPZWJqu1cGnvvt7H3dCN+xcZe/VGCWSK9zaOEp6vq8LHPbz5KAGYXdjZ1LgnrJWF9F+ewQ/fsstThJUt921kqeLKVnl2a6hpWhvwAKZETykISkBji6x11woEk9rdWsJtzS4R+cz3+ixh7lIoUPa5nBWUl3kKZ+95CSlFsfa8WKMKQcSytYEydUOWjdwKid9rrAg1kOMxUeTBIA8TkY5putvt4gbrMfOxSVbyzquIZoHN2ZYX7TsqKZ+jq7D5RskfnESLUl7s5wu6T+fj3BIOBFsoNDRRh63SxwEwvVUYZOnlEgGmxE3XKwSKGUQsHc219pnDwqLDxQCdntJbmL1jFhK4grmV2/xh7IccWkqV84pcix8sRvemI55lH9ULqDadEwhbzaNI52UnikYikvLnmrFB+/i5X6ZS/MK9Dqi4P7mPHqfiA+hLsua6lppDlMkUn8Rp7/Ieh2fA3odn4ldFM2WUDmh0GE/MrrBNCYFuffLXH6ALKLZD/DAhXp58vhnCO4CN3WEVw59wR3DF72q+J4IebE9aRUK/+FqA9qH9Za6j/h8n52JCqFHIuch68VTm33pVrkDMf7v4EsoCZ3R+SOtf/Aw==patroni-4.0.4/docs/_static/multi-dc-synchronous-replication.png000066400000000000000000001011261472010352700246760ustar00rootroot00000000000000‰PNG  IHDR-^{ø ztEXtmxfile%3Cmxfile%20host%3D%22app.diagrams.net%22%20modified%3D%222023-03-13T14%3A25%3A38.635Z%22%20agent%3D%225.0%20(X11%3B%20Ubuntu)%22%20etag%3D%22qRkI2CaWlAzzNtbhExOR%22%20version%3D%2221.0.6%22%20type%3D%22device%22%3E%3Cdiagram%20id%3D%22SVgELWPNXIlR7V7eDs_m%22%20name%3D%22Page-1%22%3E7Vtbc5s4FP41frQHgY3tx9hO2u5kZ9Km7c70xSODDGxlxAoRO%2F31K4FkgwS2k%2BDGTZxmpuggjsS5fOci0nGmq80HCpPwb%2BIj3LEtf9NxZh3bBn2nz%2F8TlMeCMhyOCkJAI19O2hHuo19IEi1JzSIfpZWJjBDMoqRK9EgcI49VaJBSsq5OWxJcXTWBATII9x7EJvWfyGehpAJ3vLvxEUVBKJce2cPixgqqyfJN0hD6ZF0iOdcdZ0oJYcXVajNFWAhPyaV47qbh7nZjFMXsmAf8b7fdj9cAuzc%2FwOTz92%2FW3Y%2BvXafg8gBxJl94NrXlftmjEkJCopjlghxM%2BC9fZ2p1BvzOVIx69kAj6ONhlQDMkeBRJejjYZUAdPZAWx%2FoGywRjFGFvaWtb5U2yH%2BdCckYjmI03ZqcxYkBhX7EVTElmFBOi0nMpTcJ2QrzEeCX6zBi6D6BnpDqmrsLpy1JzKTRA1uNpeAFV27WDPK1qOSRawLR6wdUKKSYgzFM0mixfYoiL6Np9IC%2BoLRgLqjcABNxvdoEwld7cJ32ewElWZJv%2FxNfq%2FbunF%2FOPUwyXzBhlPxE6iU7tsP%2F3QiDmywjjLWXf0CURdyXrnAUCN6MiKWgHGG0ZIIjl0gUB7f5aOZYUgp1S%2FgwDZEvX8e0f%2BkSYlW0KZGkP3xAZIUYfeRT5N2%2BJX3zUY2L4Xrn6Y5y37Dk5I56Dkp0Cbasdw7IL6QPPsEf3b7hkHeQizyODKeskbehnMGVOx25ZcmBRrXopqcpYcuqzoxLVm6qZS%2FwHK0roOvKNnQF6nQFHPtEunKAoavrr9OZCEY4S7mX8itgqC39iZgXShmX5Fax7VzGqQYVJX13hAmKnzqlL%2FMfBRYl2O5ZVg7EPdfRAFpisEYd11ILLNdmjhsYD%2FOn%2Bf0aJkCj2SNtbg62ylhv4QLhO5JGLMpxakEYI6sSnHhIIGOjeevo9zNbIBojLuZelCPfJFEQyBXkn8yM7aoZ25aJOaMaM%2B6PXm7F6ch2HA96nxfjq1%2Fhp%2BzT9C%2Bvaxpxx3axAGY%2FeuCXAcuduiAtqE7ha9bMy0klq3f%2Fy0Sak2NKtwhJV3yCm2yKh%2BTtFy1XJYmVjtkCcA7s4WhG%2FbYYDdpidEi8RzMatsVo1BajcUuMuIu3xAi0xchui1Fblm03WfZdySsLZoazvmtyBZb0NCCP2qqktKu5gB6rlpisvRBS1vMhgwuYooY8rCEaHRvImqNWvxq1RMWlJ8rD3sAMW4MWEuXasGXvD1tHQEhbftbI6O6DeIk4ZTDmOnqatZzZq7TGKKHRCgr175VGbdx%2FhpNxL2BVzzqYLK4i3xeP8xqavw7c1dTVZFpkpjBjRJbXudNW8nBZkdUUaaWS3%2B6f0mfBoOKzwDEzzXFdpnmq2tYsbWeiBig0yYuduNaycpjrylpI2FZCUZON8uJnXLWOgm2DeVzaWu%2B6reURmhAKGZqLqDrfQkJzF%2BV0LS6zUdNai%2BvMOlxgbHa46gLlLgZQBHk5blGUYC7p%2FQ2VWhtuaic22PZxLc5yo6WitIMNi%2F0mszXDkosdcieYJsXbLqMN8pt8gkc0klEPFR4hAlydb%2FiLuVLFXEh%2BruR%2BdGuv%2F1Qb3UYmaaSu2dpza2zUPZWJqu1cGnvvt7H3dCN%2BxcZe%2FVGCWSK9zaOEp6vq8LHPbz5KAGYXdjZ1LgnrJWF9F%2BewQ%2FfsstThJUt921kqeLKVnl2a6hpWhvwAKZETykISkBji6x11woEk9rdWsJtzS4R%2Bcz3%2Bixh7lIoUPa5nBWUl3kKZ%2B95CSlFsfa8WKMKQcSytYEydUOWjdwKid9rrAg1kOMxUeTBIA8TkY5putvt4gbrMfOxSVbyzquIZoHN2ZYX7TsqKZ%2Bjq7D5RskfnESLUl7s5wu6T%2Bfj3BIOBFsoNDRRh63SxwEwvVUYZOnlEgGmxE3XKwSKGUQsHc219pnDwqLDxQCdntJbmL1jFhK4grmV2%2Fxh7IccWkqV84pcix8sRvemI55lH9ULqDadEwhbzaNI52UnikYikvLnmrFB%2B%2Fi5X6ZS%2FMK9Dqi4P7mPHqfiA%2BhLsua6lppDlMkUn8Rp7%2FIeh2fA3odn4ldFM2WUDmh0GE%2FMrrBNCYFuffLXH6ALKLZD%2FDAhXp58vhnCO4CN3WEVw59wR3DF72q%2BJ4IebE9aRUK%2F%2BFqA9qH9Za6j%2Fh8n52JCqFHIuch68VTm33pVrkDMf7v4EsoCZ3R%2BSOtf%2FAw%3D%3D%3C%2Fdiagram%3E%3C%2Fmxfile%3E\4»" IDATx^ì ¼MÕÇ—©Á¬Q(Dš¨„"„ eîo*C’1B†ŒÉ˜9d®e® QKå*¥îùå×ýÔµ÷ ú~Ûºöš¬ôr›æôh…‡"²7“€î"‚.ñùëo¶PïÃi¯§oÎ{õêÚžŠÞY$˜ƒ½ˆˆ€%Á‰å ,"ôl߈n¼á::c¼Œnزƒ¦|°„º·mHùnÌ¡:3ô­Y”ÖêW/OY³d¤Í[wÑ„é !£åý÷žýÓÈÉs©R™{©Béÿ^p#¢›@G@g¡qƒ:Ê_ÓÞ›u™ßƾѦBÂW6‡õi$"ÂSGSÁü7Ó©¿NӲ嫩çëÃhúÄt{Ⴊþæm»Rºti©S»”ýºkiåë¨KÏôî„áTĸçÑš¨Z•Jô\£ºôYÜêؽ?}¶`:Ý3{Xûp€@0 è,"p\åÌ0éñ¹PÁ[¨ô£u¨×+í•è;ùèƒ9Óç¿ÌA‡^ƒDD ¬ˆz"Ã;~¡Ù WQèr†ÚUÊÐmÿf ¤tRC¸CEÓò>Ž£ƒ¿¥ÖMkgL˜þ ½Ñó#S!u|çæ.^CW/®U+” %qëéó5éðÑJh€ˆÑÀM h:‹ fL¤jõžKÒ™ŒŒ„fN¤?ÿ<©~?{Áâ$_fùwшfcoŒžH{öþJ#ö¢5k×Ó+F–Ág½GéÒ¦·gÌ[oÓW^A5ª>BåªÖ¥-_,V™f|U7ìnÕ¬1UzøŸe¸@@ 1E]âsµÊ©ý+}è½I#þ#GQ©ŠµèûuKUæ.HŠ@X!©C—3„VšÒ&‹VD„m»öÒ[,¼Ú‚XP8c,whT§RXOŽž2Ï6òBDK 7€è*"ÜïÝTâ¾b4jüÔdXâ¾¢´uûN:a|Æ™ #ÆM¡m?îºì~+"§¿vé5Pe° pÒXîðjç6IÚrþüõbšýúkÕïùÛ¹²UêÒ§  €@²ttŠÏ&ü¿ÏŸ§ï¶n§ —ÒÑcÒð¯bd‚€@Šñ9UƒVý/%w‡) _ùõz}8-3…+Ò¥‹¯·ï QÆRŠ4¨oWê?t mþö3´]wí5|  –€n"‚Žñ¹–‘¡öñâϨ_Žñþ(W¥ èÝÉ8çž°> Á$±ˆàær†sÆQŽ,L|ŸO^øït†cfÐUÆ&] j>LY2eP'8Lxïu‚Cþ¼¹â=!˜ƒ½+txÂÏ™‘œº^x©»%጑qÀ¢A·¾CÔÉ æé M^èhý{5½Ò¡¥!dS'8ðf‹|‚C®œ9èáÇë+‘![ÖÌñí¦56aLú¿r­øe@üK@7AÇøÌÙÕn¬ö@(eÛ»hiœ:}'náLÊ’9“z ˆE7–3œ5–+¤6Î!ç³Èyß…ºÕÊS‘O~à^ž66Vœ¹`9}c 'O¦\9®¥ÚUËP±Û $€!¦1 (º‰|¬ã¾ýhÙŠ5–ýÄ“ÿh3Îœ>C©Œ êÔ©¨pÁüÔ±M3ã[ªÿŽÑ凌|Kßxüø º%ßMÔö…gÔ‘a+¿øŠžiùòeöŽ5–eT*_Úr?P@ÀßttŒÏ<‚âÖ¬£¡£&Ð/¿î7ŽLÏeˆÁ/&ˆïþe耀‹‘TËr†HêÇ=  `7ÝD„ܹrÒÀÞ]¨…±ÖÉ“§¢ÆÁ鶷*H¯ _6Ür†¨A°€n"â³ NG ZˆXDp:A Z0@ÀwtØ·* „>!Ú‹xì7xT‚b¢¥ˆûAÜ  ›ˆ€øìƨ@ @DàØ àE»aAD°›(ê°ƒ€Ž"‚ý­ñÙn¢¨@À‹‘4†å ‘PÂ= ’@D ÂKª¤ [@LŸñ4€È$VDÞ«%ý~äxDÖ_Mj×ëÍ$ï­U¹4Õ®R&¢zp€¸E"^RÝkh@ :Ÿ£1¸@À-aE» ˆ`IÔ `'ˆxIµs<¡.û@D@|¶o4¡&; @D°“&êÐŽD¼¤j7ha0„DÄç€ ut´#A;—Á`; @DÀKªã uØG"â³}£ 5ØI@‰õ[õ¿TÛسÀÉ붂y©HÁI—n_@‡ º  àˆ.@F r @DëX lÒ'éÒí öèAïAœ$ÁIº¨@@<ˆâ]AJ@ú$]º}6è6€€  "¸M€È%A®o`€@° HŸ¤K·/Ø£½p’D'é¢nñ "ˆw (é“téötØ Û .€ˆàd4 —D¹¾e Á& }’.ݾ`ô@ÀIœ¤‹ºAÄ€ˆ ÞE0@  ¤OÒ¥ÛÐaƒnƒ¸@"‚ Ñ€€\äú–›€ôIºtû‚=zÐ{' @Dp’.êO"‚xÁ@€>I—n_@‡ º  àˆ.@F r @DëX lÒ'éÒí öèAïAœ$ÁIº¨@@<ˆâ]AJ@ú$]º}6è6€€  "¸M€È%A®o`€@° HŸ¤K·/Ø£½p’D'é¢nñ "ˆw (é“téötØ Û .€ˆàäh›øàãU4ÉjU¬v•2T«riõï9‹VÓì…«ðsp0}íØÀýö€ˆ`?SÔ v>I—nŸ>@  ˆ à2é“V—qxÞœtà%Õó!@<" =þI·Ï#·¡Y€ˆ'£‹²HŸ´Ê¢å¼5Òý—TçÇ@4- SÌÿ™qÈ‹æ‰pö^éñOº}ÎzG^íˆÏˆÏòF¥-‚ˆ з¼l/1c“Ið¯M mª"‚M Q ø€€ôxàÄQuAú$]º}QÁÆÍ œâ³,ADåe NI¾% ýyÃKªo‡:&€ôx ™£&IÒísÔ9¨\&€øì2ð0ÍADåˆý“üM@ú‡^Re?dÉò‡ÝÖÀ¿v­>éñOº}±Ñׯ4ž_ý|Åðo4´œ¿"‚óŒ£nIÔÈ´*ÿÊrDYþnôñ"ìhHŸ¤K·/Ö~¸ñÙ^Dt!AOÁNßÀ‡œ,WJ÷^R1^d€5 àéñOº}îyJFKÒ?ÏeP‚ `ˆöp d-#&Í¥¯7o£T©RÑ¥KD™2\M÷+LêT¢4©S'Ëä‹õ[û/уÅo·…ÛÎÝûiê‡K¨ßËMm©ÏéJð!ç4áèê—FçO§ïF&‘Ó„½­þõ–âÖ¥Ç?éöÉò¦óÖàùuž±—-À¿^Ò¿¼mˆ²ü¡¬Ñå!a¡@¾\TµB %"ìÚ³†¾5‹jW)CKß“,ÙyKÖÐÅ‹—.;âïó(]Ú4Q{äô™³´ÿàaÊŸ7WÔe½( ‹½`ãE›¼ Ž6A@&éñ@&5笒>I—nŸsžAÍ à>Äg÷™§Ô"DYþPÖèò„Š&ÆñÓ?¡´iÒгõ£%+ÖÓÂå_ÑÉS§)wÎëè…§§C‡Ó¸w>R·W{ä#‹èØñ“´÷Àïtý5Y¨ñ“ЬOVQÜÚÍêžÒÅï ÿ=QŽR§NEÍ» §>@ ?ÿŠÒbCåò÷Ó#ÝK?ÿòMš¹H›LC.Ð&IÞð’èái¹óȳ†Nz<°Ö+}KIÒíÓ×óþ¶ñÙšŸ­qsªD§ÈÆP¯.Ib'óCÆ@Õ*=@÷ÞU:öO}%9³_CïÌ^ª²X\ÍDX¼âkš³p5µoV› çÏC«¾ú–ÂC—ëQjCa8v&=ü`1zø¢JD(R0/µjZvüô+½6ê=š<ôeúuÿïboA/*ýyÃKª¬ªK&2Ŭ]ük­wú•’ÿ¤Û§ŸÇc³X—çñÙšŸuñ¯µÞéW "‚@Ÿéòp\¿e»Úÿࢡ¤K›–J{"<[¯²Úóà¯Óg)s¦ôtò¯Ó4{á*:qâ/C¨q™ˆ°õÇ=ôR³:ÊCÆ}H%ï½Me ðµö›hÅ—›•¨À"‹«Ó]·Ý¬~×â•4¤{3úÝÈnÐ)Aÿ |41 "‚#X}[©ôñb‚G¦˜o‡` :&}’.ݾ@ £³ˆÏÈä Ú˜÷²¿¼¤¯yÛI½¤š]ºpñ"͘¿\‰Y3g¤«®LGY2eHRD8pè5ýߣªh÷ÁSè©š¨p<êÿ¼iâ„÷>¡]ŸW"ÿ5sõ»–]GÒ nÏk'"èò!§ùðŒØ|éþÀKjÄ®tåFéã%9™b® 4b3éñOº}6»C|uˆÏÈä?H}d D9Óí®¤$"ð Ÿ|¶–^iUŸ2§6¬þê;ÚôýÎ$E„ƒ¿U{!ðÅ™¥î-rÃW›¶Ñg«7ªzXD`рňn{Û¿íIéÀKª¬±§K&2Ŭ]ük­wú•’ÿ¤Û§ŸÇc³X—çñÙšŸuñ¯µÞéW "‚@Ÿéò¤$"|·¾6–:tiYΞû›û@MþÛ=WK-g8söoªW­ñž¡"BÜÚ-´$n=u5D>:rà›3è¡wQÅ2÷øFDÐÅ¿ GL‚ˆàVTê1dŠYs€ôx`­Wú–’>I—nŸ¾ž÷·åˆÏÖü‹øl›S¥ "8E6†zuyHR ‚¼ˆIsè—ý‡(çõ×(`úœÏ¨iÝG)Cú«iÌÔyTåá”&Mê"ï¥ðáÇ+iåº-Šà÷ÝNu ±÷]ðK&B CE ýyÃKªN@•È³ædéñÀZ¯ô-%=þI·O_ÏûÛrÄgkþE|¶ÆÍ©Rœ"C½xHb€‡¢ %éÏ^R£t¨Ã·ë’I„L1kAÿZë~¥¤Ç?éöéçñØ,ÖåùE|¶æg]ük­wú•‚ˆ ÐgxH:ÅF“à_aÚPD ¨ éãÅt2Å4(}ÜUé“téöùxh$Ù5ÄgdòmÌ{Ù_ˆ^ÒGÛ$ Ë‡\Pœ#ÝxI•5¥Y´` ÄF@zü“n_lôõ+ø¬ŸÏ`±¾ "èë;X®)|ÈÉrœtà%UÖxA&‘,Øm ük7ÑØê“ÿ¤Û}ýJãùÕÏgÑX ÿFCËù{!"8Ï8êðDL«ð¯,wADåX^¼dãEÛÒ'éÒíóÂghœ"€øìYkõBD°ÆÍÑRxHÅ‹ÊA éÏ^R1`AÀ=Òã{$d´$=þI·O†aØCñÙŽvÕÁ.’6Öƒ‡ÄF˜¨ Âþ¼á%UÖF&‘,Øm ük7ÑØê“ÿ¤Û}ýJãùÕÏgÑX ÿFCËù{!"8Ï8êðDL«ð¯,wADåéÖH/ÒùÁ>ˆ†€ôIºtû¢aí‡{ŸýàEôAtñìô |ÈÉr¥tà%ãEXîÿ¤Û瞧d´$ýó\%Xö€ˆ`GÔÀ‡\Ĩ\¹Qº?ð’êÊ0ˆ¸dEŒJËá_Yn“ÿ¤Û'Ë›Î[ƒç×yÆ^¶ÿzIÿò¶!"Èò‡²‰@§Øhük#Lª‚ˆ`DT>! =øsÄÝ>I—n_Ä q#h@ñY–“ "Èò‡²ÆÏÉÖm;èÄÉSô牓ôÃöIÒ¿­PÊœ)#™ tLòéÏ^R}4ØÐñ¤Çñm6Pzü“nŸÍî@uÀû³¿çG-ËMCD°ŒÎ¹‚~y‰Y·~ýóg#ýºÿ7Úgü‰öÊdˆ E Q…÷3þU.°‹€ôç /©vyÚžzIdG©µÀ¿²<#=þI·O–7·Æ/Ï/ÞŸ“+~ñ¯óO‚;-@Dp‡sT­èüÌY°˜–._eüY}YŸ ßš_ 7æÊI¹sÝÿ{3ë€38C¯}û(áa«ñ³“FæBèU©|iªT¾ ÕªöXT\¥Ü¬³¥0´ÓˆvÒô]ÒÇ‹ÿ=€‰€ôIºtû‚4V¸¯:Çg¼?m´êß_ˆúûPD8øõ<ŠNü+äº!•4²8{€³Š.hÙÎ_ Aá‡m;•8±ÖÈlØà ª‹³š6|’Ú´hb¹n/ êü!ç/§Û”:=¢«_úx‰®7¸dÿ¤Û'Û»ö[§c|Æû³ýã5ºC"‚;œ}Û g4|®müþµžxÔȨL%‹s¬Ïk¿ÞHs,¢9-Qm°H1°w—˜„ ÇŒM¢b?äÜ䃶@@2dIöNì¶Á¿±3´³é“téöÙé êÒéùÅûsô#J'ÿFß;ýJ@Dè3xíÖý÷ÞMƒú¾b,Uøo™‚Óh9C¡ß Q´lÅ%$|4s’ÓMÚR¿Nþµ¥Ã¨@4!‘W–£¤OÒ¥Û'Ë›°&”ÞŸ£ˆÏÑ3s²D'éZ¬[§‡„/]X¹è‹½½X‹v]•ðî„áŽf@Än)j™Ž;F›7oVÆñ¿7mÚ¤þ5kV*Z´¨úw–,Yâÿ-³±[¥Óço콕_ƒôIºtûä{8¸âý9zß#>GÏÌÉœ¤k±n]>n¦Z½çŒMsª,/NMàt°'ê>«N~€ˆ`qÀ¡€@ÄüI´{÷nš?>Í›7V¬XqßCoÌ—/•+WŽjÔ¨AÕ«W·T‡ÄB~ð¯D®Vm’>I—nŸUÓåùÅû³µ¦‹­õN¿RúL—‡„÷&xêùvŠ /';¬Ÿ«Ë–~¾Šú t¤."‚.þøhÀ$ðœ€."obPqqqJ4à?,"$¾B³ B³B³B³—g1ÿ”-[–X`Àv>I—nŸ>Щ]â3ÞŸuU°5906, ‚f%|ì"¿ÈÇ0:qñ>_­ßL#ÆM‰x9ŸØ ‹ˆ Ë‡œþC  ;ÝžßiÓ¦Q¯^½,˜“~Î&`Ñ Ú‹—;pÃÔ©Sã—A˜u4iÒ„zöì 1!Z¨¸ÿ2Ò'éÒí ÚÒ%>ãý9h#ÓŸý…ˆàO¿ºÒ+3ò¦ŠµøˆÇ“'OÅ·]"äˆÇÜÆDßÊ1¦h°uûµãÛwÆ×Ïí¶iÑTý|Ôø©\ñ:`Ð%“ˆ'ù5kÖŒî¾ûnµü€'øævy’3Ì,Ž?®ªn×® 6Ì®f\©GÿºC@#Ò'éÒíàBWMÐåùÅû³µa¡‹­õN¿RúL—‡$4¾7i¤"ÉK –._EŸ._@PÅÌKÂퟰÕ Nû$¾ ßšŸJÜWŒš>U'~éĈ±S´tñ¯ÀG&D@€³Ú·o¯6HÌ›7¯ÊD`ñÀ‹—Jp{œÁ sçε”íà†½‰ÛÐå›L/Ø M{àýÙGÄgkÜœ*Á)²1Ô«ËC’T í6oÙ¿›®[¿Qýê« ÿìþÉÅ‚‹ ,1„‡Û HrÏÝD„HúŽ{@@À Î(_¾¼*Ú¶m[>|¸•jb.Ù¼dbÏž=JHX¾|yÌuºQ.Ÿ¿n°@ ÎÀû³5®ˆÏÖ¸9U "‚Sdc¨W—‡$\Œ×z•,^,’b îˆ52°H@z&ÑÍ7߬–0ðžœàåÅ™¼É"/o˜2eŠkÙ±ôYºcéÊ‚€ß èòüâýÙÚHÔÅ¿Öz§_)ˆ}¦ËCbG´¿n"‚.þµÃ7¨üF@²Èkf!ðþœ áâ½xo>µÁêq’ú@äŸCéáýYþX‚…á @DÏw$CAÐÚÐÐåCÎZïP üM@òóûꫯRß¾}©iÓ¦4yòdŽàl„lÙ²QÆŒ}nNˆ° F€ø“€äø ÁŸc.Ƚ‚ˆdïÇØwˆÖêò!g­w(þ& 9“ˆ÷ ˜?>IÊDॼĂ¯K—.‰’ý+  èòüâýÙÚ@ÑÅ¿Öz§_)ˆ}¦ËC‚ hmðèâ_k½C)¯´hÑ‚ÆOW]u8p@ĉ¼/CïÞ½µ òz5zÑ.‡ÞŸ­ùñÙ7§JADpŠl õêò Æàd› „NØ‹-ªNDÈš5«Í­D^5ÉK+ÌK‡L]>#÷îFïÏÖ<‚øl›S¥ "8E6†zuyHcp2Š&I`À€4qâD:räÝvÛmÔ¼ysjÔ¨‘­´¾ÿþ{ªW¯}ûí·¶ÖëFe“&M¢çŸžÒ¤IßÜ7ÞH]»vU?·zñšuN;߶mµjÕŠ¾þúkKU½òÊ+Ô AºóÎ;©råʪ®ªU«ZªKj!É™D¦ˆpå•WÒÙ³g‰…>ÿvûâìƒÄ§Cè "Hö¯Û>D{  ]ž_¼?[Yºø×Zïô+A ^[+ IDATÏtyH¢ ‚æ½ápW*_šj=ñUz¸L¸[ãïÔé Ë–-£Š+FlG¤7êâßHûc×}ø€:D™3g¦9sæÇ <¶µ\¹rôÞ{ï©1z%øw<1Ë’%‹|ðAŠ‹‹£¶mÛÒÁƒ顇¢±cÇÒ5×\£2¸>æø×_©¶¹MÞ5ßvîÜI¼®ž3.\¸ ·ß~;žkëÖ­éÌ™3Ô¦MZ°`ú7 mï¼ó 2„HÙ³gWÿgñ‡ëâL„ï¾ûNe•lݺ• *¤l*V¬± Ã?Ï›7¯òé-·ÜBýû÷§òåË»=ü"nOòókŠ<þÚµk§Æ;ÿ9~ü¸ê‹üðzì¸X,à1ÇϦù|r½\?ÛÃc™Å)s¬ÚÑ&ê¤8ŸùÝ„³»úõëG:tˆ¾_ߟcƒ ´"A+wÉ2Öjlݼ µ}á¿u²f¯–~¾ŠFŒ›BÛ~Ü¥~Ô­c+júÔ“a;m—ˆÀ“W>"'{üÒûæ›oÒÓO?¶ýhopêC.Z;¤ÝÏߨ?ûì³ÄÕºuëÒwÜ?ùøý÷ß©@Äsvß“3gNêÖ­›šÌóD…/¾øBMÈOžï(o~ƒÊ“í~øAÕ½fÍ*\¸p²í²pÁ"Ä’%KÔXâÉz³fÍÔ$>9Óÿ¹=Erçέläo/QH,"ðKÍÌ™3•ˆÂÀ©S§§>úˆJ•*¥ Îèà ?×Å;÷ó„鍊nRŒyÉOþ“xƒ¾wß}—>þøcúõ×_U}¼Æžž˜ò73Ü?^~ùeµD$4á‰'žP"£>ªÚéÔ©=óÌ34}útõœíرƒ~üñGå ®… ~AûôÓOiÕªUÒ†g¼=’3‰BEs)±È‚OðM1;Ã/Å,&ðRs¹‹QÉ-}àzÌÌþ7gšpüwèÅ'CpÛ¡ÙC:‰’ýëåC¥f)ÓÇR3/Gçm;õüòþ.üÅâ: ÷|”n,b‚ßÞŸÝò¾SþuË~¿µA GuyHì‚ìŠ_÷ rUê)¯äΕ“âÎ ë¡XD„¿ÿþ[M`Lñ€'a|ñ·©üM®—.þu¢ïáêä ú˜1cèóÏ?WÔÏ=÷ñ:{þ¶ü >}ú¨Ikž|Xù‚Yó½:îWnl»ñû¤D„ÐvyÂÏ/Ã,¬™‚@rv±ÈÀþN,$¾ŸÇ‹xf†CR9ê$"@ä½|D`©Yø§KÍÂ3Òýþìûã?T78«ÏX\H—.]TÝóÃûsT¶éfÄg›@ÚT D›@ÚY.‰A96x¶ }µáŸµ¼+ΠsÝ"ÞXDþ¦™']œ–m^üí*«Í¸Ü#ÔZiÎ*ào¸o½õVµ4€ÓÜùœ'­©¾ÕçIîùóçã3"xÎßÊsCrír ¼”€¯uëÖ)1„mL,"ð½ÜGÎpàeümßÇßÞs,X„^Ie"°è2hÐ Ú¿¿úv„Å“…Y–EÊXÎ.`ö¼¿oŠÈë÷yÒÏ“V󔞈ó)æñq)‰ü­;ïÀß ð‰<©å yò©ˆÀû&ðæ€,ðDŸ'Ëæ^ɉä‚0ðxrΓWž¨âŠŽ‹¼#/?Àe€äL"ˆÖýj–”ìßØ{g­,5ÃR3–šñèvêùååˆææÍÖž¢„¥üþþl£¤êpÊ¿NÙë÷z!"ô°.‰Õ Xâ¾¢T⾄ûü°}­]¿Ie ðUøÖüôÞ¤‘”9SÆ°rJDÛ°Åtñ¯Åî9RŒ³Ž=ª¾½æL^ã+:¢ã¥ãÝb÷DÞËb©–šé""ÄÜ©ïÏÖ8#>[ãæT)ˆN‘¡^]«A0%4¼„¡iÃ'©éSOF$ p]º‰1 Àåã6yN}çý pEO€—RðÞe—? @DˆÝ¯º|þÆÞÓÈkÀR3,5ƒˆùóÉxŽ„Òå÷ >[ãæT)ˆN‘¡^]«Aðþ{ïN‰°nýÆø#»ul¥„h.ˆÑн ±œIdUDàÓ2ø„“¤.Þ¬“÷±àÍ4ùisCÓhê´±¢dÿFÃÜí{±ÔÌ:qd‰Yg—¸¤.Ï/ÞŸ­ù\ÿZë~¥ "ô™.‰Õ ˜x¯Þ á¡Êÿ£“'OGêe¤¸…3#ÎB`÷é&"èâ_LÏ HycxãMþz={V©ÊŸ6›ŸvèÐ!*?è$"DÕ1ܬÆ–šÅ6 "ÄÆ/´´äøj'ÞŸíó9jòŽDïØkß²]AAÌž¿ˆ:÷ ˜T*_šÆ{-b>º‰º|ÈEìÜ" ùùUDàÓKš7ožÀ›|Ô(ŸÈÁÇrò1œ™°aõ¼(Ò "B¤¤ô»KÍb÷–šÅÎЬAr|vBDÚû³}#5ÙA"‚Z‡"#lðl›øe ïNN%‹'Ü|19Ì:Ñmð€€äL"'DñŸþIEŠ¡}ûö©cIùØÒH/DÉþ”7î ÐåùÅû³µª‹­õN¿RúL—‡Äî ¸uÛªVï9å‘ܹrÒG3'E´¬A7Aÿ |4`€@ œ¸Ù&MšÐ´iÓ¨Zµj4þüˆ}¡“ˆ Ë7™ÃÇ âàýÙšKŸ­qsªD§ÈÆP¯.‰ÝA‘õ4Š¦½7KÑK¼wBrHub(  ÉpZDhÚ´)M:U—¾dÉ’ˆ=!bT¸@ ðþlÍɺ̬õN¿RúL—‡Ä‰ ºÉ"»fÁŒ‰T¤pÁ½Aà †I àS’3‰œNž<©–3ìÝ»—^zé%:thÄÖIDì߈ãF(]ž_¼?[ ºø×Zïô+A ÏtyHœ‚ìŽÐMKÜW”¦Oá+Aÿ |4`xN@²È너pîÜ9Ú´iuêÔ‰âââÔé _|ñ/^[ãæT)ˆN‘¡^]sD~AühæDº1× 1ôÚZQ¶¡l•ºtÂøû›UŸDt$¤µ–P @d€ˆ»tùü½§¨@À+x¶FñÙ7§JADpŠl õêô˜K rçÊI{wIqï‚$Y”•Ü׆Œ¦¶ï$¯³!ìîêI@r&D„ØÇŒdÿÆÞ;Ôþ& Óó‹÷çèÇ¢Nþ¾wú•€ˆ Ðgº=$ærFɧ)4iø$U*_Ú1²,Ìùh1ÍY°XµQë‰GiPß®Žµgwźù×îþ£>Й€d‘"‚Î# ¶ƒÄJ@r|NªoxŽÕã(ï%ˆ^Ò÷QÛ<±ïôêë´ÿÀAÕ«L™2*!¡ä}Ũð­ù©Há‚–{ûëþF¶Á.Z·~#-]¾šöý{jC®rP—[S¥‡ËX®Û‹‚º}ÈyÁÈÍ6¥ûƒƒôÎMqn"A[)<^ "`è‚™€äøœœ_ðþä«wß!"èí?qÖ/ý|Í^°ˆ–­Xs™m·* ö,(a æÅÿçŸóµný¦øŸÿ°}ñš±ÐŸ™¿¬XîAC (ö˜Hqpþ5HÇ9©,í°Kº? "ØáeûêœI!v?Köoì½Ó¯éñOº}úy<6‹u~~ñþÞ÷:û7|ïô»"‚@Ÿùá!a€Ï¶e€3~5²Ì,…hg̘Š"Ãm… Y E©DñbÚožèÿFãCé÷BDî!Ø)ˆ‘’Jþ>éñ öêUƒôIºtûôò6¬exF|ÖåI€ˆ ÐS~~‰á´-¾ø4…­Æ†ˆ‰¯ÐÌ^ÁÿÇNþ¼á%ÕIïû«nˆ±ûSz<ˆ½‡zÕ =þI·O/oÃÚ”àý™ñYÖ3A–?”5xH:&ù–€ôç /©²†žäL¢áÇSûöí©mÛ¶Äÿ–p;vŒ²eËFY²d!þ·ôK²¥³sÂ>éñOº}NøDrx~%{'vÛàßØÚYD;iÚT›@ ­þ•åˆ²ü!ÝÉãeÅŠT¾|y*Z´(mÜøOÖ—××Ô©S©iÓ¦T¶lYbûp@4¤OÒ¥Û k?Ü+9>û/ú¡ "`<€€Ëð!ç2ð0ÍI÷^R1^¢!/_>Ú³gõìÙ“xyƒ—gÜ|óÍ*aÊ”)Ô¤I/ÍAÛÿ¤Û§¡Ëc2YúçyLCaF"‚0‡ÀÿÀ‡œ,K÷^Reé™Df6Sk×® 6Ì€›6mRY, è”… Ý¿ž8ÓÃF¥Ç?éöyè:OšÆóë v×…]CQC"ÂäîMxHÜåívkð¯ÛÄSn"‚,ÀšØ ðŽ?Nœ™À 7Ž½âjؽ{7õîݛؾX@˜7oeÍš5‚ÒÞß"=xOÈ] ¤OÒ¥Ûç®·Ð8KñÙY¾ÑÖ!Zb.܇ÄÈhþ% ýyÃK*†ªœ P£F µ´/Þ'¡\¹rJLàÛyq¶Áüùó•XÀ™æŠ’6xŒ´¿ÒãA¤ýðË}ÒãŸtûü2Ð`ˆÏ²ÆDYþPÖà!è˜ä[ÒŸ7¼¤ÊzºeqFg"˜bÓä¬øg XÉ`‘"..Neð¿C/*¸M΂ÐíÒÍ¿ºñÖ^éñOº}ÑòÖý~<¿º{0eûá_Yþ…ˆ ËÊ<$b£Ið¯0m¨ "‚ T…ôñ’œ+8C€3øO¨ `ÞÏB‚™¡úoÎ*0E‚Ð'n§zõêJ”àlŃ a­º*}’.Ý>­œmƒ±ºÆgºŽ*@Àu\GŽƒNr²F€tà%ãÅn¼o)(p6•+oÞ¼J003¬Ô2 Ž€ôø'ݾp|ýö{éŸç~ãþ›D„`û½÷€>ä<€žB“Òý—TYãů™DÉe$—¡ Ë+öYãWÿÚGÈÝš¤Ç?éö¹ë-ï[Ãó뽜´þu’nôuCDˆž™ã%ð8ŽØÓà_Oñ_Ö8DYþ€5 à%éñÀK6^´-}’.Ý>/|†6AÀ)ˆÏN‘µV/DkÜ-…‡ÄQ¼¨þ¼á%Ü# =¸GBFKÒãŸtûdxV€€=ŸíáhW-ì"ic=xHl„‰ª@ éÏ^Re adÉò‡ÝÖÀ¿v­>éñOº}±Ñׯ4ž_ý|Åðo4´œ¿"‚óŒ£nÁ‹‡¤x¹jtüÏ”*U*eoÎ×Ó Ï4¤zuª©ÿŸ9s–ÆOy,\J¿úÝø}vªùø#Ô♧(mÚ4—õñ™–/ÓÓõjQù‡JEÝÿ»~¦9-¡ÎíZD]V‡^øW.^ÙÁ+òz¶+}¼èIVƒ@Ò¤OÒ¥Û´qåE|ÆûsÐFúk€ˆ€± pœòæ`º£H!:wî-^¶’:tëG gM¥‚ùóQÛνéä©¿Œ‰}sº1÷ ´qó÷4jü4*`ü®_÷—Qܺm‡®É–%jÂ6}KoŒžHÓ'Žˆº¬¼øÓ‹W6J÷^R½I·+}¼È¢k@ 6ÒãŸtûb£¯_i/â3ÞŸõ'°Øìá¨}-¡AÐìL¹*õ¨c›çéŠ+ÒÑàoÑÂÙS)]Ú´ñ}匄!ÆÏ_ïÝ9ÁÏù†çÛt¡uª‚CNêÖwå¾!'­Y»žòܘKÕY²ø=tè÷Ãôrþ´ùÛ­”%KfêÚáEºÿÞ»©fƒætøÈQªX¾4 {½M{oM~÷C:vì8¸% îוnÉ—‡Þ™1—¶ïØEß‚žý¿Q±»n§^]ÛÓ FÅÛwRw£ÝŸ÷쥻ï¸ú÷ìD7äÌNgÏž£~ƒGÑÒå«éú뮡gžþŸ‘Qñ¨«þóâCÎÕjÖ˜tà%UÖ€B&‘,Øm ük7ÑØê“ÿ¤Û}ýJ{ñüâýÙ½qâ…Ýë~-ADè3/’Ð ø÷ùó´ôóUÔ¦S/Z0c¢±´`1¥¿újz©ÕsÓ *×nBÝ_nMÔ¦1Þ¦U_|M3§Ž¦^¯§ó.ЫZÓgq_ÐkƒGÓêOgQh&‹•jÿ45ª_›Z´ëª2š4¬£ì›9çcê=à¿¥ÓÆ ¥â÷Ü•€d¨ˆP­îs´~åÇ”!ýÕ´c×n%N,2²FŒB›¶|O¯viK7ç½)¾|¨ˆpîï¿éĉ“tí5ÙèØñ?i¤QæðÑc4b`O%"ð ¾ý¦*ûî̹´ëç=ôXÅrÔÈhšo |q¹uë7Ñ£¢2Ö1Êö¢{ŠÞ¡~ÇÙ Ò§§W:´8`’¼xÞ¢é^R£¡…{A 6ÒãAl½Ó¯´ôø'Ý>ý<®ŸÅxvïýñYÖóA–?<Ì=#y͘Ÿ7²zviw­ZO57öIxJÜW4Y¡åK=héüwÕïÚý ½ØáU%"ðä~ðˆñÆþ q”9SFz®q=jø¿ 2Ο¿ îYøéçtÝu×*!‚SDؾó§ø=ÞŸµ€~4þ_ôÎÛiñgq4ö~ lºtéÝzOyÊscÉÔñ¿{äáÒÔþÅȳ,˜éJxIÁ¹E&‘PU ÿ r†aŠôø'Ý>YÞtÞ/žß¤–3˜=Åû³½>÷¿ööÀ_µADèO/’”‚`ÜêµÔ±{Zþñû”1c†xb¿ÿq„ÊU­K“Ç NQD`ÑàÓyï\&"pVBî\9èª+¯¤¯6l22º÷½K{÷íßXñ£EËhâ´4ÕÈvÈ–5 Íýx ­Xµ6^D`Ñ ï¿;š"BÅr¥UùÙïŽSm5öR˜5=oˆ%®¡–häÈ~úÝ/¿î§téÒ©}ܺ¼ð¯[}Ó±ˆ:zÍ;›¥ïÈ e°Ÿ€ôIºtûì÷ˆì½ˆÏxvïýYöè žu‚çó${œRäϵîLýušztj£Ndرógê3p¤Ú¸pä ^–D®óÖ·P«fèÀo‡Ô†Š gO1ö*øƒ^ú&Ízg,½=c}úÙ*š2v09}Æ°£‹ÊDxó¾j9CR"B'ãhȇ¯O¯÷ìL”¸‡™ ¿ìÝOo|^}í ã¸Ê3Ô§[:øûT¯i+µ?BâL '‡…rNöG÷º¥û/©²F˜ãåú믧£GÆÁ{ã7Ò+¯¼BÍš5SpNŸ>M¤éÓ§Ó¾}ûˆÿôÓO«{Ò†l†k’¬\¹2µjÕŠªV­5Üï¿ÿžÞ~ûmÕ.pš€ôø'Ý>§ý#­~/â3ÞŸf"K°Ç9œc«UÍá‚ o¶8jÜTµôà÷ÃGèö©Cëç“~ âƉ üÿÐ+tO„ä2XèôêúÙXâ9s&záÙ§¨Á“ÕéøŸ'¨VÃæT¨`~ú*d} IDAThœüðbÇWS~R+òr‡þCÇ"@{Cl8œ¤ˆÀË.6'>ô4ƒ_öP¶ ìÝÅÈzÈ©ö}à W~ñ•‘–š5©oløøÏ^n]^|ȹÕ7ۑÊU^d±ˆ°xñbº÷Þ{fÎÒìÙ³•Hðí·ßR‘"E¨~ýúô矪‰ýÍ7ßL_~ù%õéÓGýnܸ2²B¯M›6)¡áºëþÉÈŠæZ³f uïÞ–/_M1mîõ¿ÚÀñÀPéñOº}¸ÌÓ&½x~ñþìžË½ð¯{½Ó¯%ˆ}†‡D Sl4 þµ¦ UAD°"ªp”@¨ˆ`6tË-·PÿþýéJc9g° ÀK³Ì‹3øç“&MJðsþýO1ú &ˆ à2|ȹ n¥7^x¥_ø“å!éþÀKª¬ñâE&QJ"ÓyüñÇ“gNªý xÿ>†‘÷8à gÎœiIDà:y¿ÞÀ÷Bà 9³€÷*à ?Ÿ1zôhµgŸñ×_);xigðr†¤D>A¢@jÙE… T& , –-[ªzXxØ¿¿Z†ÁBEbÄÉá…ìîuKÒíÓÝÿÑÚï§çïÏ—{ßOþvlK¼"‚@¯xýœ?ÁH'M• ¥Ô*¦H‚à9#÷Š]Å#m+TDXûõ7Tüž»)Mš4‘÷ì>¯ýëYÇ…6 A¨c`Vþ(=fLÔ|¤}³òãø2š¿D•Ê—IRDàzš>õdŠm8qŠ¦½?‹æ¼;^ÙÀ©b÷—¯F fN¤eË×Äï‰`f"Lœ6ÓÈ@8¥ìäkÇ®ÝtâÄIºãöBêïk¯ÉFÇŽÿI#ÇN¡ÃGш=]^ù×õŽjÒ DM%ÄLéãE&˜¶>I—nŸ-NШ¯â3ÞŸ5$0Õ6lC©E?[0®É–•Ö­ßDϼø2å¹1w‚Žuhý<»«uë3˜¾Ûú#åÍ“›Ž=Nê×2–-ÜIϵîBq gÆ—áû ßZ Iá‰Ê©Ê#åSlë”±$aÙŠÕ4jpïø:ÿß3Ô»k{úöûí—‰ýŒ ƒBo¡Æ ê$°›Å‡Á#ÆÓÂO?§ë®»–2¤¿Z ^ˆ^}Èé?Bétà%Õ¿[­Uúx±Ú/”‰¤Ç?éöIô©“6yŸñþì¤WQ·T¤zÆ»8~þÑ{”-kÚõójf,`QÁ¼¾7ö9È{SnðÆ›êG¯vn£ŽßêÒk Ý}ÇmTÙX6ðà#µL„OÔ|Õ¨ÿ<Õ®^%I¡zÕGTöBJmñÒ…·ßŸM³ß§ê»pᕬP“f½3––KÌÓÌL„q“¦ÓyãžNm›«û¿Ûºöî;`d0œ§‰ÓfÐÔqCUÿæ~¼„V¬Z Áƒq&­I¯^:"倗ÔHI¹s2‰ÜáìU+ð¯Wä“nWzü“nŸ,o:oWÏ/ÞŸ÷-·à•Ýé~­@Dè3¯’Ð xÑX®PõÉgT†ÁÿjV¥•k¾¢—Œ½ V/™Ezô§‚ùo¦v-Ÿ¡ŸvÿBÿkü¢±9bãÞÚj?‚» A¡£‘±°~ã·ô´±_BNmRRjk™±ÿï‰0nxzø¡R4~Ê{jãÄ?˜ll”8û2á§Ý{©Y›.ôþäQ”ÙXnÑ¢]7z¤BµÓ÷§Ÿ­¢)cÓ™ÓgTÆg"¼ùF_×G€Wþu½£š4AGÁLÛ ð) |Â×_m{ݺV(=èÊÕªÝÒ'éÒí³Êå¢#€÷çèxY½ñÙ*9gÊADp†kLµzõ„Aî Ýû¡m?î¢Ù¯W™¥î¿‡8#á¥WúÒ_§OÓ]Æ ÷½“&¾=ƒf½=–.]º¤Ngà2¼ŒáŽÛ Rî\7¤("¤ÔŸÎ0ÏÈH›.-mÜü=Ýœ÷&л3å¿9o²§3|8ïóÖ;tÊ8oüá²P_cóijgÎÒ‹_¥í;~R+6ü_ ê?t õéÖžyø¡˜ü…Âzðêy‹”^R#%…û¢!pöìY㤳´mÛ6ud£g~¥NÚ–c‚í´+’º¤ÇƒHúà§{¤Ç?éöùi,Hî ÞŸÝñâ³;œ#m"B¤¤\¼É°YDXùÅ:u„$.p‚€ôç /©NxÝzÒ3‰öíÛG•*U¢»ï¾›-ZD9s椡C‡RÕªUióæÍÔ®];zðÁiñâÅ4~üxjÑ¢…ÊD3f ­X±‚¾ûî;:xð U¯^~øaêß¿¿‡Í:Îœ9CmÚ´¡ ÿ»bÅŠôÎ;ïÐÕÆñºÙ²e£I“&QË–-U[¥J•¢9sæPÑ¢EðÒ¥KSçÎé‰'ž°î‡KJ÷¯ÃÝW½ôø'Ý>quØ <¿þ~†~€¢¬"B”Àܸ ‚ ã müC"FB4¤n¼ñFjß¾= 8–/_NuêÔ¡;wÒ”€Ð³gO%&lÙ²%ˆÐ¥KÚºu+¥M›– (@*T ¹sçÒرciÖ¬YJd˜6m >œ–.]jñ›F‰/¿ü2Õ«WO‰ÕªU£Ñ£G§÷dRb‹]»v¥C‡QÁ‚•@qÕUWEƒ÷˜€ôIºtû‚6t¤Çg7ý/áܤ̶ "ÓïÚôš<ôÇaãDˆÛµ±9œ¡ø GÈÝßK÷^RÝáZ“>^XDÈ›7/;vŒ2f̨ºÃûZµjQ±bŨ|ùòôǨ¥6lH "¬ZµŠf̘¡Ê<ðÀÔ©S'ªQ£ýôÓOJPøù矣rOïc“%KÚ³g5lØž~úijÞ¼¹X´03>ýôSêÓ§­^½Ze(ðÿgÎüïôžp¬ñ{ÿ¤Û´$=>»é?¾?»Ém…'!<#ܶÀ‡œ­8c®Lº?𒳋m­@z¦‹,ð7ÿæÅY×_==þøãÔ AúþûïÕ¯‹¼”³ø*S¦ 0@e.°XP®\9%"p½Íš5Se9[ ÞœÑ~üñGÕ_¼çBŽ9”Ѹqczꩧ¨nݺ¶úÃîʤû×îþJ¯Ozü“nŸtÿÚmž_»‰Êªþ•åˆ²ü¡¬ÁC"Ð)6šÿÚÓ†ª "ØUˆ!`f"?~œ2dÈ ìªY³¦î»ï>5‘ÿöÛo-‹,ð5räHã(ß+éÙgŸU3š"/›¸öÚkãypD•*U¨cÇŽ´wï^µÌAò%=Hfç„mÒ'éÒísÂ'¨¼"€øìù¤Û…ˆ ËÊ<$“|K@úó†—Tß=G:fî‰À“vÎ$X¹r¥Ú$ñ‡~PY±Š¼¿Âí·ßN½{÷¦íÛ·Çï±Ðºukµœ!±ˆ0yòdêÖ­/^\mÆ(ý’¤ó³Û>éñOº}vûõ€—Ÿ½¤yÛdù"‚@À$þ¡„—TYãOz&‹œqP¿~}zÿý÷Õ¾ƒV{ð‰ ±Š7nTû œ:uJ ¼ÜaÈ!ôå—_ª!‹¿ýöåÊ•‹XLhÒ¤‰,g&atÿŠh³ÒãŸtûlv‡øêðüŠwQL¿1á³½0DÛ‘Æ^!’ØJ®þ•åˆ²ü!Ýéã…E„’%Kª¥.ø„†Ý»w'Xæ Á6Ø Ÿ€ôIºtûä{Ø^ ¥Çg{{‹Ú@À[¼åÖHr²œ.ÝxIÅx‰†€$áï¿ÿV5~üñÇêd\ -éñOº}ÑòÖý~éŸçºó…ý J"ƸLr.Óœtà%UÖx‘žItúôiZºt©:ÖÑë‹÷e`áwÞQKt¸¤ûW†vÚ(=þI·ÏN_èPž_¼dÝFø×:;'JBDp‚jŒuâ!‰ ðâð¯,ADåX^¼dãEÛÒ'éÒíóÂghœ"€øìYkõBD°ÆÍÑRxHÅ‹ÊA éÏ^R1`AÀ=Òã{$d´$=þI·O†aØCñÙŽvÕÁ.’6Öƒ‡ÄF˜¨ Âþ¼á%UÖF&‘,Øm ük7ÑØê“ÿ¤Û}ýJãùÕÏgÑX ÿFCËù{!"8Ï8êðDL«ð¯,wADåéÖH/ÒùÁ>ˆ†€ôIºtû¢aí‡{ŸýàEôAtñìô |ÈÉr¥tà%ãEXîÿ¤Û瞧d´$ýó\%Xö€ˆ`GÔÀ‡\Ĩ\¹Qº?ð’êÊ0ˆ¸dEŒJËá_Yn“ÿ¤Û'Ë›Î[ƒç×yÆ^¶ÿzIÿò¶!"Èò‡²‰@§Øhük#Lª‚ˆ`DT>! =øsÄÝ>I—n_Ä q#h@ñY–“ "Èò‡²‰@§À$ßþ¼á%Õ·CH@z<ˆÌQ“¤Ç?éö9êT.@|vx˜æ "ÈòDþ€Iþ& ýC /©²Æ2‰dùÃnkà_»‰ÆVŸôø'ݾØèëWϯ~>‹Æbø7ZÎß ÁyÆQ·úð¿g/\¥ê¨]¥ Õª\Zý?÷‡¨ ØN"‚íH}]¡ôñâkøè\àHŸ¤K·/hñ9hG½$ÁKúh@ÀsÒ_:ð’êùI`€ôñ"‹¬ØHÒ틾~¥Ÿõó,Ö—D}}ËAl ý¥/©68ÙÆ*)œÌ8‡ ª²H@zü“nŸEìÚC|F|Övðjh8D “Aì#Á>–¨ @ì$ }’.Ý>;}º@@ ”DŒ@€ˆh÷£ó ‚ HŸ¤K·O°ka€€æ "hî@˜ ˆ±ñCipŠ€ôIºtûœò êˆ &!ÐîGçA>I—nŸ`×Â4Í @DÐÜ0@ 6bã‡Ò  àé“téö9åÔ  0@M"B Ý΃& }’.Ý>Á®…i š€ˆ ¹a>€@l "ÄÆ¥A@À)Ò'éÒísÊ/¨@ "` €šD„@»L@ú$]º}‚] Ó@4'AsÂ|Ø@DˆJƒ€€S¤OÒ¥Ûç”_P/€@DÀ4ˆv?: ˜€ôIºtû»¦hN"‚æ„ù ±€ˆ?”§HŸ¤K·Ï)¿ ^€ˆ€1 hí~t@@0é“téö v-LÐœDÍóAb#!6~(  N>I—nŸS~A½  c@ Ð "Úýè<€€`Ò'éÒíìZ˜  9ˆš;æƒÄF"BlüP@œ" }’.Ý>§ü‚zA@"Æ€@  @D´ûÑyÁ¤OÒ¥Û'ص0 @@s4w ̈D„Øø¡4€8E@ú$]º}Nùõ‚€DŒ@€ˆh÷£ó ‚ HŸ¤K·O°ka€€æ "hî@˜ ˆ±ñCipŠ€ôIºtûœò êˆ &!ÐîGçA>I—nŸ`×Â4Í @DÐÜ0@ 6bã‡Ò  àé“téö9åÔ  0@M"B Ý΃& }’.Ý>Á®…i š€ˆ ¹a>€@l "ÄÆ¥A@À)Ò'éÒísÊ/¨@ "` €šD„@»L@ú$]º}‚] Ó@4'AsÂ|Ø@DˆJƒ€€S¤OÒ¥Ûç”_P/€@DÀ4ˆv?: ˜€ôIºtû»¦hN"‚æ„ù ±€ˆ?”§HŸ¤K·Ï)¿ ^€ˆ€1à Ó§OÓøñãiÈ!´oß>Ê—/µmÛ–Z´hAW]u•'6¡Ñ`€ˆL¿£×É@|ÆèB@ú$]º}Rü;ì#€ølKÔˆ±ñCé( ?~œFMǧ?þøã²ÒÙ³g§—^z‰Z¶lI™2eŠ²vÜÑ€ˆ=3”ð'ÄgúUç^IŸ¤K·OgßÃö„Ÿ1"¤€ˆ Í#>µçÈ‘#4tèP3f q weË–Z·nMíÛ·§¬Y³†»¿Ë "XF‡‚>!€øìGú°Ò'éÒíóá\—Ÿçrm: AWéièhàÀ4aÂú믿¢îDƌՇŽ;RŽ9¢. ŽD„p„ð{¿@|ö«gýÓ/é“téöùg$¯'ˆÏÁó¹n=†ˆ ›Ç4±÷矦×_¦M›Fç΋ÙjÞ'¡iÓ¦Ô¥KÊ“'OÌõ¡0 @DÀXÄç y\ßþJŸ¤K·O_Ï×rÄçàú^·žCDÐÍcÂíݺu+½öÚk4sæLºpá‚íÖ¦M›–6lHÝ»w§ Ø^?* ˆÁóyP{ŒøTÏëÛoé“téöéëùàYŽø<ŸëÞcˆº{Pˆý7n¤Þ½{Ó‚ èÒ¥KŽ[•:ujª]»6õèуî¼óNÇÛCþ%Á¿¾EÏþ!€øŒ‘ +é“téöéê÷ Ùø$oû«¯üåO×{³bÅ êß¿?-]ºÔõ¶Í«V­J½zõ¢ûî»Ï3а¾ "èë;Xž2ÄgŒÝ HŸ¤K·OwÿûÙ~Äg?{7}ƒˆ ?ÛÞËS§NQ•*UhåÊ•–ëæSÚ¶mKåÊ•#¦#FŒ cÇŽY®íùä“O,—GÁ`€ˆL¿û¹×ˆÏ~ön°ú&}’.ݾ`=z‹ø¬‡Ÿ`exÂ3ÂIàx“C+W¾|ù¨qãÆÔ®]»Ç7²€0|øð˜Ä„qãÆQóæÍ­˜…2%! Ž÷q·Ÿ}ìÜ€uMú$]º}.ZtñY 7ÁÈ@Dˆn¹œ@ñâÅiýúõQ¡añ gÏžÔ¤I“˱˜0oÞ<µÇÂîÝ»£j£T©RôÅ_DU7›D„`ûß½G|ö£WƒÙ'é“téösÔÈî5â³lÿÀºÈ @Dˆœîü—À™3g(C† tñâň˜„ŠÇ§,Y²„-·gÏÊ›7/M:5*1Oo8}ú4ñ߸@ "¡„{t!€ø¬‹§`g$¤OÒ¥Û cÜãÄg÷X£%ç @Dpž±ïZàý Ê—/¶_‰Å^¾À¢g°@`^¦``þŸ…Þ/3xÃÄhÅ„5kÖÐ<Ö>ÜL"ÆŸ >ûÉ›è‹ôIºtû0‚d@|–åXˆ±ñ di>¡[·nÉö^¶À¢_qqqT£FøMùç,ðþ¦¨À‚‹ü;þ9ÿž/®‹ÿÏ{(ðÅ÷·oß>Å L;v ¤oÐéè @DˆžJÈ%€ø,×7°,zÒ'éÒí‹ž8J8IñÙIº¨ÛmÜ&îƒöøHÅ… &ÛXDàkÚ´iIîÀâ@R'1$÷sL!E’»X°˜;w®H£ n€ˆàe´áÄg·H£7HŸ¤K·Ï ¡È >GÎ wÊ'A¾ÄY˜9sf:qâD²vqºVÙ²eÕïyÙÿ?Ö«hÑ¢´qãFUÍüùóUfCr×µ×^KüñG¬M¢|@@Dˆ£ÒMÄç€8: Ý”>I—n_@†‰6ÝD|ÖÆU04"€„[þ#°uëVºýöÛSDréÒ¥¿¯Y³¦:m!¹‹—2¤t C¹råhùòåñÅ9ƒ![¶l)Ú°sçNÊŸ??\a @D‹7hBñYGÁ̈ HŸ¤K·/bиÑqˆÏŽ#F.€ˆà2pÝ››8q"=ÿüóÉv#ñ„Ÿo1bDüþ¡ù^^öÀs¶é˜TÖ/]hÛ¶m‚6‹+F›6mJÖ^FѨQ#ÝqÃ~@Dp2šp…â³+˜Ñˆ‹¤OÒ¥Û碫ÐTˆÏ"~#Áou¸?M›6U›&w…î‡À÷$µô€÷=6lX’{%pÆ·‘x¿„Ð=¸Þpû"4oÞœÆç0ª_¶lU¬XÑ•¶Ðˆý "Øχ* sIDAT5þG`éÒ¥T©R%W >_ŽñÙ•¡çX#Ò'éÒísÌ1>©ñùrGºùþì“aØn@D¬ë­u¼P¡Bôã?&[˜E€êÕ««ßóѼ—A¨ ÀKxÓCþyrgðËph¦ œ¥p÷Ýw«bÉmØhÖyçwÒ–-[¬u2ÂR| D=(}úôtäÈ‘Ká6i "Hóˆ¿ìáØuöìYêׯuèÐÁÑÎ!>ÿ‡ñÙÑ¡æZåÒ'éÒísÍQš6„ø|¹ãÜxÖt¸ÀìD "`HDLàðáÃtÝu×¥x? Y²dQ÷ðòó¨Fþ?kÞÛ€„ãÇÇßZ! yóæUÂ/YÝ+!4Ë~óÍ7'kKêÔ©U™2eŠ¸‘ÞÈ/§¯¾ú*¥I“†R¥JEo¾ù&=ýôÓ‘Ç}Â@D柙ÃYTmÚ´¡ .ïÓ·o_GÄÄç⳿ é“téöùk4ØßÄçË™:ùþl¿Q£— "xI_³¶,XŸe”éœeðóÏ?Çÿ*±ˆ`f)˜Y,*p¶‹¡Y 扗B$^*Á"BJ2.^¼˜}ôQ[(ÿý÷ßêXIS<8uꔪ7{öìtðàA[Ú@%Þ€ˆà ÷ µzýõ×ÇŸÙK¦˜ÀâBºtélAøŒølË@V‰ôIºtû„¹S¤9ˆÏ—»ÅÎ÷g‘N‡Q¶€ˆ` Æ`TÒ¹sg4hP²mÒ¤ M™2%YÁÜ× ôØGS]ž`nÎNDwêOøYȈõJ,†ÄZʃ€€I ±8j• â³Ur( I@|ÆÈä @DÀ舘@™2ehõêÕÉÞŸø…äDxÏÎDظq#™G<òò^‚`Ö“xòžxsÅäN}0 äÍyÓ;.ÎDàÍ Y˜H›6-!Áª2ê@&‚ ?øÙŠÐoº2dÈ –6ð²ÎD¸âŠ+lé:â3â³-IX%Ò¿é—nŸ0wŠ4ñùr·Øùþ,Òé0ÊlÁèÿJΟ?OW_}5ñßÉ]¼ñaÙ²e‰3 X H|#ï…À¢_ü{_¼ÿ‡ñÙÖ¡åyeÒ'éÒíóÜ @|NÚAN½? 0/J¢ÔÛ×®]K¥J•J±û¼Î—¯”ö*`‘€÷@wñÆ‹I‰ \Ž~ôèQUol˜ÒÅ¢EJ'A„³#¥ßóË*ŸÎ1cFâMÍpéI"‚ž~ÓÅjŽWgÎœQ§3Ø-˜ Ÿ/ ˆÏºŒçzøœ<['ߟó(jv“D7ikÜÖСCþ›"BJg•Gº¾,¥¥ æž Œ“3g<„bæ,–-[:JÞÍs†íH@+‡ˆPÇ»ÔíO?ý”yäG[C|N/â³£CÏñÊ¥OÒ¥Û縃4oñyS²tãýYóáxó!"~D Aƒôþûï§x³¹—Aè&‰‰ °êËÙæ1ÉU˜R6Cr{&$U×3ÏI—n“÷Ÿ½÷,p†Dg¸ú®Ö5jo”î2÷EX8à=Ì«]»vÄ’ZÖÀG=²HÀÌ+´<ÿ›‘Œ‹‹#¶)Ü!!ü"Æ€îŸu÷ ìOŽ€ôIºtû0²¼'€øì½`3 "8ÃÕwµ¾ûî»mÈ“|^^À›ÕpÆŸhÀÿçM“ºxi‚y±‘Ô5oÞ<µa#/…È–-›¸\¨0‘ðE‹Ñc=æ; Cö€ˆ`KÔä Ägo¸£Uç HŸ¤K·Ïy¡…pŸÃÂïu%AWÏy`÷ã?NŸ|òIØ–y#CB—,Ô¬Y“X ˆö Ýÿ€Ëò†‹ü³”öA0Û¨U«Íž=;Ú&qÀ@D˜Ã}Ú]ÄgŸ:6àÝ’>I—n_À‡˜î#>‹q ±‘Daú½ª“'ORñâÅiÛ¶ma»Ú¤Iš2eJü}œ5ÀK"É0 qVïkùÈ?TŒ¸óÎ;‰w,OŸ>}X[qC° @D¶ÿýÒ{Äg¿xý% }’.Ý>Œ&ŸeøVØK"‚½<}_Û®]»Ô‘‰Ã]戡÷q6gp¦ïkøâºyég$Þó wïÞjIC¸‹3 ¶lÙByòä w+~ƒÀ/ŸýâIôÃ$ }’.Ý>Œ$9Ÿåø–ØC"‚=U /ixâ‰'È<Ò1¥Î›-&uOªT©.ûqr÷󦎑l¤˜:ujZ²d U¬X1P>Ag­€ˆ`JÊ#€ø,Ï'°È:é“téöY'’N@|v‚*êôŠD¯ÈkÞnŸ>}¨gÏža{an´˜ø$†Í›7«Œ†ÄWR"ßéFŠ¤N:…µ 7€€I"Æ‚ß >ûÍ£ÁíôIºtû‚;räöñY®o`Yt "DÇ wÿK€³8ÁêF‹¼¬—'„nÈBAãƉ÷S0/l¤ˆ!ç4ˆNFýn@|v›8ÚsŠ€ôIºtûœò êµNñÙ:;””E"‚,heM,ÅDÚQl¤))Üg•D«äPN2ÄgÉÞm‘>I—n_¤œqŸ»ŸÝåÖœ!Á®©5šbœ‚‚"Œz!"ÃÏAì%âs½î¯>KŸ¤K·Ï_£Á_½A|ö—?ƒØˆAôºÍ}Žf£››&l¤h7ÑàÕ!x>RŸƒämÿõUú$]º}þþê⳿ü´Þ@DšÇêo¤Ç/ÚÝü€¨sçÎvW‹úD"B€œЮ">Ôñ>è¶ôIºtû|0|ßÄg߻ط„ˆà[׺۱h6Š±Ë²ZµjÑìÙ³íªõ”D„€:>@ÝF|³}ÖUé“téöùl8ø²;ˆÏ¾tk :!nv§“¼Q ÛÈ뼜¾ .L6l ôéÓ;Ýê÷9ˆ>w0º§ >c èH@ú$]º}:ú<ˆ6#>Ñëú÷"‚þ>ÕƒmÛ¶QñâÅÕ «So¤ÈBþüùjõˆD„9;à]E|øаûÒ'éÒíÓÐå5ñ9°®×¶ã´u\ÃÜ()Êõ»®–ADÐÕs°Û Äg+ÔPÆ+Ò'éÒíóÊoh×ÄgkÜPʼáîûV—,YB]»v¥o¾ùƶ¾,X^{í5zòÉ'm«DŒ @|šÇõí¯ôIºtûôõ|p-G|®ïuë9DÝ<{Al%ÁVœ¨ @l# }’.Ý>ÛŠ@@ ˆ &!ÐîGçA>I—nŸ`×Â4Í @DÐÜ0@ 6bã‡Ò  àé“téö9åÔ  0@M"B Ý΃& }’.Ý>Á®…i š€ˆ ¹a>€@l "ÄÆ¥A@À)Ò'éÒísÊ/¨@ "` €šD„@»L@ú$]º}‚] Ó@4'AsÂ|Ø@DˆJƒ€€S¤OÒ¥Ûç”_P/€@DÀ4ˆv?: ˜€ôIºtû»¦hN"‚æ„ù ±€ˆ?”§HŸ¤K·Ï)¿ ^€ˆ1ðä“OÒœ9s(uêÔ zGÇKç»!"èì½(lç XºtijÛ¶mŠ¥š6mJ  nݺÅß7lØ0%"Lž<™î¾ûnÚ²e 5nܘZµjE-[¶T"ÂêÕ«éÃ?TevîÜImÚ´¡«¯¾šfÏž±•±ˆkÖ¬¡îÝ»ÓòåË#nïìÙ³tå•W&¸ÿüùóG5jÔ õë×S¡B…"®7êI"‚ž~ó“Ոϗ{ñÙO#Üz_¤OÒ¥Ûg] ,˜•]z5jÔˆ6nÜHÛ¶mS÷@DðÿÁÿ>–ÞCÄgÄgécÔ+û¤OÒ¥Ûç•ßüÔ.â3ⳟƳ}ˆ`'MÁuY ‚Ë–-£:ÐæÍ›“í]âLóÆ *гÏ>«–C„^õëקn¸ @|ðÊZ`€m4—3$'"°0ÀÙÜæG}¤–QìÝ»—B3>¬²)ø÷¥J•¢.]ºÐöíÛiÁ‚ª\ïÞ½iîܹJ\à¥I]lߊ+ "Óv™Á.’¨Ç*ÄgÄg«cÇïå¤OÒ¥Ûç÷ñáFÿŸŸÝg:¶AG¯Y°9¹5]œapï½÷Æטx9Äÿ·wÿ >wqÀ¿IJd ”²XüI’Á$™ 6Q )ÝÁdPŒ"ÿJÉ##eA= eÐS¥'I)ÅÀÀ`àé|ËÍ•8çþ9Ïç~î뎜ßïûù¼ÎéÔyßï÷{¯^I¸ÿ~sˆP‚íÛ·!Ä÷Ÿ,]ºtxÿþý°|ùòñŸoݺ5>>pàÀ?†§OŸïŠ(Ï\mØ°aò{ Ê·oß|ïž(w@ܹsç·ŠB„i,²yú!Â<¸DeÛŸíω–ó¬¶ý½¾YŒúeögûó]úl[ˆðG¢¦›¤–ÇÊ ¾áÓ§OÃÍ›7‡ƒ/^œòN„ïb{öìÿÿÇ;Ê~óæÍCy¤áçŸß=ÎP^ÐX®ùñãÇñ΂l¬X±b8qâÄpôèÑ)w"”; Êc kÖ¬™r‰*”;^¼x1\¸pAˆciϸ !ÂŒ }Á ìÏÃúÚŸg¸~<ú!=z} —D÷–ìÏöçî‹nž\Pˆ0O&j¦eNw|÷îÝøN„'OžLy·AùM~y´ üe‡_=ÎPûå·ÿå®߉ðåË—aÙ²ecPîH(?å½å=‡žr'ÂçÏŸ‡òXëW¯†õë×1”ÇÊ»Ê;¾¿±¼¿àåË—“/V¼téÒðôéÓáÊ•+ã÷—Ï•úwîÜ9ÖZ^üxþüy!ÂLU’Ï ’LäªðsˆP‚…ò%øÕ_g(/?ܸqãPM(v²ÜIP^v¸oß¾ÉaÕªUÃåË—Ç?³xòäÉáìÙ³cP>»iÓ¦áÔ©Sã»Ê Ÿ={6>²PîJxüøñøïå1Š»wïÛ¶m¯Swxøð¡a~.ß9­Zˆ0§¼¾¼BÀþl®X& rHôCzôú䢙å¦íÏöçY^Ri¾Nˆf*ßHÙËÝ‹/ž2ðСCS~+ÿ«?ñøíÛ·qLù  Ö®];Þ5P„E‹óãÇK–,¾~ý:¬^½zØ»wïpîܹñŽŸÞ¾};~þÑ£GãØòèÁîÝ»§üu†7nŒÁA víÚ5ÞIðúõëáùóçC©±Ü‘°råÊabbb8räÈø× vìØ1lÙ²e .îÝ»7>†Q‰(\»vm¼ƒÁ dÁ7´)DhÀ2tNìÏöç9YX ¾4ú!=z} –ÀÿÞ‚ýÙþü¿/ ‚NŒ²è# Dèãì*hˆ~H^_«·ñ¨"ÔJG€@J!BÊiÕ ¢Ò£×—` h B„ £,úú8» Z¢Ò£××êm<j„µRÆ R@ˆrZ5E€@è‡ôèõ%XZ @ ¨€!èÄ(‹>B„>ήB€Vè‡ôèõµzO€Z!B­”q¤"¤œVM @ ú!=z} –€* D:1Ê"@ €¡³« @ U ú!=z}­ÞÆ @ V@ˆP+e)„)§US$ˆ~H^_‚% ‚ ‚NŒ²è# Dèãì*hˆ~H^_«·ñ¨"ÔJG€@J!BÊiÕ ¢Ò£×—` h B„ £,úú8» Z¢Ò£××êm<j„µRÆ R@ˆrZ5E€@è‡ôèõ%XZ @ ¨€!èÄ(‹>B„>ήB€Vè‡ôèõµzO€Z!B­”q¤"¤œVM @ ú!=z} –€* D:1Ê"@ €¡³« @ U ú!=z}­ÞÆ @ V@ˆP+e)„)§US$ˆ~H^_‚% ‚ ‚NŒ²è# Dèãì*hˆ~H^_«·ñ¨"ÔJG€@J!BÊiÕ ¢Ò£×—` h B„ £,úú8» Z¢Ò£××êm<j„µRÆ R@ˆrZ5E€@è‡ôèõ%XZ @ ¨€!èÄ(‹>B„>ήB€Vè‡ôèõµzO€Z!B­”q¤"¤œVM @ ú!=z} –€* D:1Ê"@ €¡³« @ U ú!=z}­ÞÆ @ V@ˆP+e)„)§US$ˆ~H^_‚% ‚ ‚NŒ²è# Dèãì*hˆ~H^_«·ñ¨"ÔJG€@J!BÊiÕ ¢Ò£×—` h B„ £,úú8» Z¢Ò£××êm<j„µRÆ R@ˆrZ5E€@è‡ôèõ%XZ @ ¨€!èÄ(‹>B„>ήB€Vè‡ôèõµzO€Z!B­”q¤"¤œVM @ ú!=z} –€* D:1Ê"@ €¡³« @ U ú!=z}­ÞÆ @ V@ˆP+e)„)§US$ˆ~H^_‚% ‚ ‚NŒ²è# Dèãì*hˆ~H^_«·ñ¨"ÔJG€@J!BÊiÕ ¢Ò£×—` h B„ £,úú8» Z¢Ò£××êm<j&C„ÚG€lŸÛRÙ¤ý @`¡ üûÏßa[·?‡… ÐAà?QvÑ—»1¤ÈIEND®B`‚patroni-4.0.4/docs/citus.rst000066400000000000000000000421641472010352700160140ustar00rootroot00000000000000.. _citus: Citus support ============= Patroni makes it extremely simple to deploy `Multi-Node Citus`__ clusters. __ https://docs.citusdata.com/en/stable/installation/multi_node.html TL;DR ----- There are only a few simple rules you need to follow: 1. `Citus `__ database extension to PostgreSQL must be available on all nodes. Absolute minimum supported Citus version is 10.0, but, to take all benefits from transparent switchovers and restarts of workers we recommend using at least Citus 11.2. 2. Cluster name (``scope``) must be the same for all Citus nodes! 3. Superuser credentials must be the same on coordinator and all worker nodes, and ``pg_hba.conf`` should allow superuser access between all nodes. 4. :ref:`REST API ` access should be allowed from worker nodes to the coordinator. E.g., credentials should be the same and if configured, client certificates from worker nodes must be accepted by the coordinator. 5. Add the following section to the ``patroni.yaml``: .. code:: YAML citus: group: X # 0 for coordinator and 1, 2, 3, etc for workers database: citus # must be the same on all nodes After that you just need to start Patroni and it will handle the rest: 0. Patroni will set ``bootstrap.dcs.synchronous_mode`` to :ref:`quorum ` if it is not explicitly set to any other value. 1. ``citus`` extension will be automatically added to ``shared_preload_libraries``. 2. If ``max_prepared_transactions`` isn't explicitly set in the global :ref:`dynamic configuration ` Patroni will automatically set it to ``2*max_connections``. 3. The ``citus.local_hostname`` GUC value will be adjusted from ``localhost`` to the value that Patroni is using in order to connect to the local PostgreSQL instance. The value sometimes should be different from the ``localhost`` because PostgreSQL might be not listening on it. 4. The ``citus.database`` will be automatically created followed by ``CREATE EXTENSION citus``. 5. Current superuser :ref:`credentials ` will be added to the ``pg_dist_authinfo`` table to allow cross-node communication. Don't forget to update them if later you decide to change superuser username/password/sslcert/sslkey! 6. The coordinator primary node will automatically discover worker primary nodes and add them to the ``pg_dist_node`` table using the ``citus_add_node()`` function. 7. Patroni will also maintain ``pg_dist_node`` in case failover/switchover on the coordinator or worker clusters occurs. patronictl ---------- Coordinator and worker clusters are physically different PostgreSQL/Patroni clusters that are just logically grouped together using the `Citus `__ database extension to PostgreSQL. Therefore in most cases it is not possible to manage them as a single entity. It results in two major differences in :ref:`patronictl` behaviour when ``patroni.yaml`` has the ``citus`` section comparing with the usual: 1. The ``list`` and the ``topology`` by default output all members of the Citus formation (coordinators and workers). The new column ``Group`` indicates which Citus group they belong to. 2. For all ``patronictl`` commands the new option is introduced, named ``--group``. For some commands the default value for the group might be taken from the ``patroni.yaml``. For example, :ref:`patronictl_pause` will enable the maintenance mode by default for the ``group`` that is set in the ``citus`` section, but for example for :ref:`patronictl_switchover` or :ref:`patronictl_remove` the group must be explicitly specified. An example of :ref:`patronictl_list` output for the Citus cluster:: postgres@coord1:~$ patronictl list demo + Citus cluster: demo ----------+----------------+---------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------+-------------+----------------+---------+----+-----------+ | 0 | coord1 | 172.27.0.10 | Replica | running | 1 | 0 | | 0 | coord2 | 172.27.0.6 | Quorum Standby | running | 1 | 0 | | 0 | coord3 | 172.27.0.4 | Leader | running | 1 | | | 1 | work1-1 | 172.27.0.8 | Quorum Standby | running | 1 | 0 | | 1 | work1-2 | 172.27.0.2 | Leader | running | 1 | | | 2 | work2-1 | 172.27.0.5 | Quorum Standby | running | 1 | 0 | | 2 | work2-2 | 172.27.0.7 | Leader | running | 1 | | +-------+---------+-------------+----------------+---------+----+-----------+ If we add the ``--group`` option, the output will change to:: postgres@coord1:~$ patronictl list demo --group 0 + Citus cluster: demo (group: 0, 7179854923829112860) -+-----------+ | Member | Host | Role | State | TL | Lag in MB | +--------+-------------+----------------+---------+----+-----------+ | coord1 | 172.27.0.10 | Replica | running | 1 | 0 | | coord2 | 172.27.0.6 | Quorum Standby | running | 1 | 0 | | coord3 | 172.27.0.4 | Leader | running | 1 | | +--------+-------------+----------------+---------+----+-----------+ postgres@coord1:~$ patronictl list demo --group 1 + Citus cluster: demo (group: 1, 7179854923881963547) -+-----------+ | Member | Host | Role | State | TL | Lag in MB | +---------+------------+----------------+---------+----+-----------+ | work1-1 | 172.27.0.8 | Quorum Standby | running | 1 | 0 | | work1-2 | 172.27.0.2 | Leader | running | 1 | | +---------+------------+----------------+---------+----+-----------+ Citus worker switchover ----------------------- When a switchover is orchestrated for a Citus worker node, Citus offers the opportunity to make the switchover close to transparent for an application. Because the application connects to the coordinator, which in turn connects to the worker nodes, then it is possible with Citus to `pause` the SQL traffic on the coordinator for the shards hosted on a worker node. The switchover then happens while the traffic is kept on the coordinator, and resumes as soon as a new primary worker node is ready to accept read-write queries. An example of :ref:`patronictl_switchover` on the worker cluster:: postgres@coord1:~$ patronictl switchover demo + Citus cluster: demo ----------+----------------+---------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------+-------------+----------------+---------+----+-----------+ | 0 | coord1 | 172.27.0.10 | Replica | running | 1 | 0 | | 0 | coord2 | 172.27.0.6 | Quorum Standby | running | 1 | 0 | | 0 | coord3 | 172.27.0.4 | Leader | running | 1 | | | 1 | work1-1 | 172.27.0.8 | Leader | running | 1 | | | 1 | work1-2 | 172.27.0.2 | Quorum Standby | running | 1 | 0 | | 2 | work2-1 | 172.27.0.5 | Quorum Standby | running | 1 | 0 | | 2 | work2-2 | 172.27.0.7 | Leader | running | 1 | | +-------+---------+-------------+----------------+---------+----+-----------+ Citus group: 2 Primary [work2-2]: Candidate ['work2-1'] []: When should the switchover take place (e.g. 2024-08-26T08:02 ) [now]: Current cluster topology + Citus cluster: demo (group: 2, 7179854924063375386) -+-----------+ | Member | Host | Role | State | TL | Lag in MB | +---------+------------+----------------+---------+----+-----------+ | work2-1 | 172.27.0.5 | Quorum Standby | running | 1 | 0 | | work2-2 | 172.27.0.7 | Leader | running | 1 | | +---------+------------+----------------+---------+----+-----------+ Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y 2024-08-26 07:02:40.33003 Successfully switched over to "work2-1" + Citus cluster: demo (group: 2, 7179854924063375386) ------+ | Member | Host | Role | State | TL | Lag in MB | +---------+------------+---------+---------+----+-----------+ | work2-1 | 172.27.0.5 | Leader | running | 1 | | | work2-2 | 172.27.0.7 | Replica | stopped | | unknown | +---------+------------+---------+---------+----+-----------+ postgres@coord1:~$ patronictl list demo + Citus cluster: demo ----------+----------------+---------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------+-------------+----------------+---------+----+-----------+ | 0 | coord1 | 172.27.0.10 | Replica | running | 1 | 0 | | 0 | coord2 | 172.27.0.6 | Quorum Standby | running | 1 | 0 | | 0 | coord3 | 172.27.0.4 | Leader | running | 1 | | | 1 | work1-1 | 172.27.0.8 | Leader | running | 1 | | | 1 | work1-2 | 172.27.0.2 | Quorum Standby | running | 1 | 0 | | 2 | work2-1 | 172.27.0.5 | Leader | running | 2 | | | 2 | work2-2 | 172.27.0.7 | Quorum Standby | running | 2 | 0 | +-------+---------+-------------+----------------+---------+----+-----------+ And this is how it looks on the coordinator side:: # The worker primary notifies the coordinator that it is going to execute "pg_ctl stop". 2024-08-26 07:02:38,636 DEBUG: query(BEGIN, ()) 2024-08-26 07:02:38,636 DEBUG: query(SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s), (3, '172.19.0.7-demoted', 5432, 10000)) # From this moment all application traffic on the coordinator to the worker group 2 is paused. # The old worker primary is assigned as a secondary. 2024-08-26 07:02:40,084 DEBUG: query(SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s), (7, '172.19.0.7', 5432, 10000)) # The future worker primary notifies the coordinator that it acquired the leader lock in DCS and about to run "pg_ctl promote". 2024-08-26 07:02:40,085 DEBUG: query(SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s), (3, '172.19.0.5', 5432, 10000)) # The new worker primary just finished promote and notifies coordinator that it is ready to accept read-write traffic. 2024-08-26 07:02:41,485 DEBUG: query(COMMIT, ()) # From this moment the application traffic on the coordinator to the worker group 2 is unblocked. Secondary nodes --------------- Starting from Patroni v4.0.0 Citus secondary nodes without ``noloadbalance`` :ref:`tag ` are also registered in ``pg_dist_node``. However, to use secondary nodes for read-only queries applications need to change `citus.use_secondary_nodes `__ GUC. Peek into DCS ------------- The Citus cluster (coordinator and workers) are stored in DCS as a fleet of Patroni clusters logically grouped together:: /service/batman/ # scope=batman /service/batman/0/ # citus.group=0, coordinator /service/batman/0/initialize /service/batman/0/leader /service/batman/0/members/ /service/batman/0/members/m1 /service/batman/0/members/m2 /service/batman/1/ # citus.group=1, worker /service/batman/1/initialize /service/batman/1/leader /service/batman/1/members/ /service/batman/1/members/m3 /service/batman/1/members/m4 ... Such an approach was chosen because for most DCS it becomes possible to fetch the entire Citus cluster with a single recursive read request. Only Citus coordinator nodes are reading the whole tree, because they have to discover worker nodes. Worker nodes are reading only the subtree for their own group and in some cases they could read the subtree of the coordinator group. Citus on Kubernetes ------------------- Since Kubernetes doesn't support hierarchical structures we had to include the citus group to all K8s objects Patroni creates:: batman-0-leader # the leader config map for the coordinator batman-0-config # the config map holding initialize, config, and history "keys" ... batman-1-leader # the leader config map for worker group 1 batman-1-config ... I.e., the naming pattern is: ``${scope}-${citus.group}-${type}``. All Kubernetes objects are discovered by Patroni using the `label selector`__, therefore all Pods with Patroni&Citus and Endpoints/ConfigMaps must have similar labels, and Patroni must be configured to use them using Kubernetes :ref:`settings ` or :ref:`environment variables `. __ https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors A couple of examples of Patroni configuration using Pods environment variables: 1. for the coordinator cluster .. code:: YAML apiVersion: v1 kind: Pod metadata: labels: application: patroni citus-group: "0" citus-type: coordinator cluster-name: citusdemo name: citusdemo-0-0 namespace: default spec: containers: - env: - name: PATRONI_SCOPE value: citusdemo - name: PATRONI_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_LABELS value: '{application: patroni}' - name: PATRONI_CITUS_DATABASE value: citus - name: PATRONI_CITUS_GROUP value: "0" 2. for the worker cluster from the group 2 .. code:: YAML apiVersion: v1 kind: Pod metadata: labels: application: patroni citus-group: "2" citus-type: worker cluster-name: citusdemo name: citusdemo-2-0 namespace: default spec: containers: - env: - name: PATRONI_SCOPE value: citusdemo - name: PATRONI_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_LABELS value: '{application: patroni}' - name: PATRONI_CITUS_DATABASE value: citus - name: PATRONI_CITUS_GROUP value: "2" As you may noticed, both examples have ``citus-group`` label set. This label allows Patroni to identify object as belonging to a certain Citus group. In addition to that, there is also ``PATRONI_CITUS_GROUP`` environment variable, which has the same value as the ``citus-group`` label. When Patroni creates new Kubernetes objects ConfigMaps or Endpoints, it automatically puts the ``citus-group: ${env.PATRONI_CITUS_GROUP}`` label on them: .. code:: YAML apiVersion: v1 kind: ConfigMap metadata: name: citusdemo-0-leader # Is generated as ${env.PATRONI_SCOPE}-${env.PATRONI_CITUS_GROUP}-leader labels: application: patroni # Is set from the ${env.PATRONI_KUBERNETES_LABELS} cluster-name: citusdemo # Is automatically set from the ${env.PATRONI_SCOPE} citus-group: '0' # Is automatically set from the ${env.PATRONI_CITUS_GROUP} You can find a complete example of Patroni deployment on Kubernetes with Citus support in the `kubernetes`__ folder of the Patroni repository. __ https://github.com/patroni/patroni/tree/master/kubernetes There are two important files for you: 1. Dockerfile.citus 2. citus_k8s.yaml Citus upgrades and PostgreSQL major upgrades -------------------------------------------- First, please read about upgrading Citus version in the `documentation`__. There is one minor change in the process. When executing upgrade, you have to use :ref:`patronictl_restart` instead of ``systemctl restart`` to restart PostgreSQL. __ https://docs.citusdata.com/en/latest/admin_guide/upgrading_citus.html The PostgreSQL major upgrade with Citus is a bit more complex. You will have to combine techniques used in the Citus documentation about major upgrades and Patroni documentation about :ref:`PostgreSQL major upgrade`. Please keep in mind that Citus cluster consists of many Patroni clusters (coordinator and workers) and they all have to be upgraded independently. patroni-4.0.4/docs/conf.py000066400000000000000000000245601472010352700154320ustar00rootroot00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Patroni documentation build configuration file, created by # sphinx-quickstart on Mon Dec 19 16:54:09 2016. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys from sphinx.application import ENV_PICKLE_FILENAME sys.path.insert(0, os.path.abspath('..')) from patroni.version import __version__ project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) module_dir = os.path.abspath(os.path.join(project_root, 'patroni')) excludes = ['tests', 'setup.py', 'conf'] # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', # 'sphinx.ext.viewcode', 'sphinx_github_style', # Generate "View on GitHub" for source code 'sphinxcontrib.apidoc', # For generating module docs from code 'sphinx.ext.autodoc', # For generating module docs from docstrings 'sphinx.ext.napoleon', # For Google and Numpy formatted docstrings ] apidoc_module_dir = module_dir apidoc_output_dir = 'modules' apidoc_excluded_paths = excludes apidoc_separate_modules = True # Include autodoc for all members, including private ones and the ones that are missing a docstring. autodoc_default_options = { "members": True, "undoc-members": True, "private-members": True, } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = 'Patroni' copyright = '2024 Compose, Zalando SE, Patroni Contributors' author = 'Patroni Contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = __version__[:__version__.rfind('.')] # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' on_rtd = os.environ.get('READTHEDOCS', None) == 'True' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Replace "source" links with "edit on GitHub" when using rtd theme html_context = { 'display_github': True, 'github_user': 'patroni', 'github_repo': 'patroni', 'github_version': 'master', 'conf_py_path': '/docs/', } # sphinx-github-style options, https://sphinx-github-style.readthedocs.io/en/latest/index.html # The name of the top-level package. top_level = "patroni" # The blob to link to on GitHub - any of "head", "last_tag", or "{blob}" # linkcode_blob = 'head' # The link to your GitHub repository formatted as https://github.com/user/repo # If not provided, will attempt to create the link from the html_context dict # linkcode_url = f"https://github.com/{html_context['github_user']}/" \ # f"{html_context['github_repo']}/{html_context['github_version']}" # The text to use for the linkcode link # linkcode_link_text: str = "View on GitHub" # A linkcode_resolve() function to use for resolving the link target # linkcode_resolve: types.FunctionType # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'Patronidoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'Patroni.tex', 'Patroni Documentation', 'Patroni Contributors', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'patroni', 'Patroni Documentation', [author], 1) ] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'Patroni', 'Patroni Documentation', author, 'Patroni', 'One line description of project.', 'Miscellaneous'), ] # -- Options for Epub output ---------------------------------------------- # Bibliographic Dublin Core info. epub_title = project epub_author = author epub_publisher = author epub_copyright = copyright # The unique identifier of the text. This can be a ISBN number # or the project homepage. # # epub_identifier = '' # A unique identification for the text. # # epub_uid = '' # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'python': ('https://docs.python.org/', None)} # Remove these pages from index, references, toc trees, etc. # If the builder is not 'html' then add the API docs modules index to pages to be removed. exclude_from_builder = { 'latex': ['modules/'], 'epub': ['modules/'], } # Internal holding list, anything added here will always be excluded _docs_to_remove = [] def config_inited(app, config): """Run during Sphinx `config-inited` phase. rtd reuses the environment, and there is no way to customize this behavior. Thus we remove the saved env. """ pickle_file = os.path.join(app.doctreedir, ENV_PICKLE_FILENAME) if on_rtd and os.path.exists(pickle_file): os.remove(pickle_file) def builder_inited(app): """Run during Sphinx `builder-inited` phase. Set a config value to builder name and add module docs to `docs_to_remove`. """ print(f'The builder is: {app.builder.name}') app.add_config_value('builder', app.builder.name, 'env') # Remove pages when builder matches any referenced in exclude_from_builder if exclude_from_builder.get(app.builder.name): _docs_to_remove.extend(exclude_from_builder[app.builder.name]) def _to_be_removed(doc): for remove in _docs_to_remove: if doc.startswith(remove): return True return False def env_get_outdated(app, env, added, changed, removed): """Run during Sphinx `env-get-outdated` phase. Remove the items listed in `docs_to_remove` from known pages. """ to_remove = set() if hasattr(env, 'found_docs'): for doc in env.found_docs: if _to_be_removed(doc): to_remove.add(doc) added.difference_update(to_remove) changed.difference_update(to_remove) removed.update(to_remove) if hasattr(env, 'project'): env.project.docnames.difference_update(to_remove) return [] def doctree_read(app, doctree): """Run during Sphinx `doctree-read` phase. Remove the items listed in `docs_to_remove` from the table of contents. """ from sphinx import addnodes for toc_tree_node in doctree.traverse(addnodes.toctree): for e in toc_tree_node['entries']: if _to_be_removed(str(e[1])): toc_tree_node['entries'].remove(e) def autodoc_skip(app, what, name, obj, would_skip, options): """Include autodoc of ``__init__`` methods, which are skipped by default.""" if name == "__init__": return False return would_skip # A possibility to have an own stylesheet, to add new rules or override existing ones # For the latter case, the CSS specificity of the rules should be higher than the default ones def setup(app): if hasattr(app, 'add_css_file'): app.add_css_file('custom.css') else: app.add_stylesheet('custom.css') # Run extra steps to remove module docs when running with a non-html builder app.connect('config-inited', config_inited) app.connect('builder-inited', builder_inited) app.connect('env-get-outdated', env_get_outdated) app.connect('doctree-read', doctree_read) app.connect("autodoc-skip-member", autodoc_skip) patroni-4.0.4/docs/contributing_guidelines.rst000066400000000000000000000141411472010352700215760ustar00rootroot00000000000000.. _contributing_guidelines: Contributing guidelines ======================= .. _chatting: Chatting -------- If you have a question, looking for an interactive troubleshooting help or want to chat with other Patroni users, join us on channel `#patroni `__ in the `PostgreSQL Slack `__. .. _reporting_bugs: Reporting bugs -------------- Before reporting a bug please make sure to **reproduce it with the latest Patroni version**! Also please double check if the issue already exists in our `Issues Tracker `__. Running tests ------------- Requirements for running behave tests: #. PostgreSQL packages including `contrib `__ modules need to be installed. #. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`. #. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`. Install dependencies: .. code-block:: bash # You may want to use Virtualenv or specify pip3. pip install -r requirements.txt pip install -r requirements.dev.txt After you have all dependencies installed, you can run the various test suites: .. code-block:: bash # You may want to use Virtualenv or specify python3. # Run flake8 to check syntax and formatting: python setup.py flake8 # Run the pytest suite in tests/: python setup.py test # Moreover, you may want to run tests in different scopes for debugging purposes, # the -s option include print output during test execution. # Tests in pytest typically follow the pattern: FILEPATH::CLASSNAME::TESTNAME. pytest -s tests/test_api.py pytest -s tests/test_api.py::TestRestApiHandler pytest -s tests/test_api.py::TestRestApiHandler::test_do_GET # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/; # modify DCS as desired (raft has no dependencies so is the easiest to start with): DCS=raft python -m behave Testing with tox ---------------- To run tox tests you only need to install one dependency (other than Python) .. code-block:: bash pip install tox>=4 If you wish to run `behave` tests then you also need docker installed. Tox configuration in `tox.ini` has "environments" to run the following tasks: * lint: Python code lint with `flake8` * test: unit tests for all available python interpreters with `pytest`, generates XML reports or HTML reports if a TTY is detected * dep: detect package dependency conflicts using `pipdeptree` * type: static type checking with `pyright` * black: code formatting with `black` * docker-build: build docker image used for the `behave` env * docker-cmd: run arbitrary command with the above image * docker-behave-etcd: run tox for behave tests with above image * py*behave: run behave with available python interpreters (without docker, although this is what is called inside docker containers) * docs: build docs with `sphinx` Running tox ^^^^^^^^^^^ To run the default env list; dep, lint, test, and docs, just run: .. code-block:: bash tox The `test` envs can be run with the label `test`: .. code-block:: bash tox -m test The `behave` docker tests can be run with the label `behave`: .. code-block:: bash tox -m behave Similarly, docs has the label `docs`. All other envs can be run with their respective env names: .. code-block:: bash tox -e lint tox -e py39-test-lin It is also possible to select partial env lists using `factors`. For example, if you want to run all envs for python 3.10: .. code-block:: bash tox -f py310 This is equivalent to running all the envs listed below: .. code-block:: bash $ tox -l -f py310 py310-test-lin py310-test-mac py310-test-win py310-type-lin py310-type-mac py310-type-win py310-behave-etcd-lin py310-behave-etcd-win py310-behave-etcd-mac You can list all configured combinations of environments with tox (>=v4) like so .. code-block:: bash tox l The envs `test` and `docs` will attempt to open the HTML output files when the job completes, if tox is run with an active terminal. This is intended to be for benefit of the developer running this env locally. It will attempt to run `open` on a mac and `xdg-open` on Linux. To use a different command set the env var `OPEN_CMD` to the name or path of the command. If this step fails it will not fail the run overall. If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command. .. code-block:: bash OPEN_CMD=: tox -m docs Behave tests ^^^^^^^^^^^^ Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 16 and then run all behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of Postgres or to a specific feature set or steps. To specify the version of postgres include the full name of the dependent image build env that you want and then the behave env name. For instance if you want Postgres 14 use: .. code-block:: bash tox -e pg14-docker-build,pg14-docker-behave-etcd-lin If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run the watchdog behave feature test scenario with all versions of Postgres. .. code-block:: bash tox -m behave -- features/watchdog.feature Of course you can combine the two. Contributing a pull request --------------------------- #. Fork the repository, develop and test your code changes. #. Reflect changes in the user documentation. #. Submit a pull request with a clear description of the changes objective. Link an existing issue if necessary. You'll get feedback about your pull request as soon as possible. Happy Patroni hacking ;-) patroni-4.0.4/docs/dcs_failsafe_mode.rst000066400000000000000000000110561472010352700202700ustar00rootroot00000000000000.. _dcs_failsafe_mode: DCS Failsafe Mode ================= The problem ----------- Patroni is heavily relying on Distributed Configuration Store (DCS) to solve the task of leader elections and detect network partitioning. That is, the node is allowed to run Postgres as the primary only if it can update the leader lock in DCS. In case the update of the leader lock fails, Postgres is immediately demoted and started as read-only. Depending on which DCS is used, the chances of hitting the "problem" differ. For example, with Etcd which is only used for Patroni, chances are close to zero, while with K8s API (backed by Etcd) it could be observed more frequently. Reasons for the current implementation --------------------------------------- The leader lock update failure could be caused by two main reasons: 1. Network partitioning 2. DCS being down In general, it is impossible to distinguish between these two from a single node, and therefore Patroni assumes the worst case - network partitioning. In the case of a partitioned network, other nodes of the Patroni cluster may successfully grab the leader lock and promote Postgres to primary. In order to avoid a split-brain, the old primary is demoted before the leader lock expires. DCS Failsafe Mode ----------------- We introduce a new special option, the ``failsafe_mode``. It could be enabled only via global :ref:`dynamic configuration ` stored in the DCS ``/config`` key. If the failsafe mode is enabled and the leader lock update in DCS failed due to reasons different from the version/value/index mismatch, Postgres may continue to run as a primary if it can access all known members of the cluster via Patroni REST API. Low-level implementation details -------------------------------- - We introduce a new, permanent key in DCS, named ``/failsafe``. - The ``/failsafe`` key contains all known members of the given Patroni cluster at a given time. - The current leader maintains the ``/failsafe`` key. - The member is allowed to participate in the leader race and become the new leader only if it is present in the ``/failsafe`` key. - If the cluster consists of a single node the ``/failsafe`` key will contain a single member. - In the case of DCS "outage" the existing primary connects to all members presented in the ``/failsafe`` key via the ``POST /failsafe`` REST API and may continue to run as the primary if all replicas acknowledge it. - If one of the members doesn't respond, the primary is demoted. - Replicas are using incoming ``POST /failsafe`` REST API requests as an indicator that the primary is still alive. This information is cached for ``ttl`` seconds. F.A.Q. ------ - Why MUST the current primary see ALL other members? Can’t we rely on quorum here? This is a great question! The problem is that the view on the quorum might be different from the perspective of DCS and Patroni. While DCS nodes must be evenly distributed across availability zones, there is no such rule for Patroni, and more importantly, there is no mechanism for introducing and enforcing such a rule. If the majority of Patroni nodes ends up in the losing part of the partitioned network (including primary) while minority nodes are in the winning part, the primary must be demoted. Only checking ALL other members allows detecting such a situation. - What if node/pod gets terminated while DCS is down? If DCS isn’t accessible, the check “are ALL other cluster members accessible?†is executed every cycle of the heartbeat loop (every ``loop_wait`` seconds). If pod/node is terminated, the check will fail and Postgres will be demoted to a read-only and will not recover until DCS is restored. - What if all members of the Patroni cluster are lost while DCS is down? Patroni could be configured to create the new replica from the backup even when the cluster doesn't have a leader. But, if the new member isn't present in the ``/failsafe`` key, it will not be able to grab the leader lock and promote. - What will happen if the primary lost access to DCS while replicas didn't? The primary will execute the failsafe code and contact all known replicas. These replicas will use this information as an indicator that the primary is alive and will not start the leader race even if the leader lock in DCS has expired. - How to enable the Failsafe Mode? Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API ` or :ref:`patronictl edit-config -s failsafe_mode=true ` patroni-4.0.4/docs/dynamic_configuration.rst000066400000000000000000000303771472010352700212430ustar00rootroot00000000000000.. _dynamic_configuration: ============================== Dynamic Configuration Settings ============================== Dynamic configuration is stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes. In order to change the dynamic configuration you can use either :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API `. - **loop\_wait**: the number of seconds the loop will sleep. Default value: 10, minimum possible value: 1 - **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30, minimum possible value: 20 - **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10, minimum possible value: 3 .. warning:: when changing values of **loop_wait**, **retry_timeout**, or **ttl** you have to follow the rule: .. code-block:: python loop_wait + 2 * retry_timeout <= ttl - **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election. - **maximum\_lag\_on\_syncnode**: the maximum bytes a synchronous follower may lag before it is considered as an unhealthy candidate and swapped by healthy asynchronous follower. Patroni utilize the max replica lsn if there is more than one follower, otherwise it will use leader's current wal lsn. Default is -1, Patroni will not take action to swap synchronous unhealthy follower when the value is set to 0 or below. Please set the value high enough so Patroni won't swap synchrounous follower frequently during high transaction volume. - **max\_timelines\_history**: maximum number of timeline history items kept in DCS. Default value: 0. When set to 0, it keeps the full history in DCS. - **primary\_start\_timeout**: the amount of time a primary is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for primary failure is: loop\_wait + primary\_start\_timeout + loop\_wait, unless primary\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff. - **primary\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by primary\_stop\_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, primary\_stop\_timeout does not apply. - **synchronous\_mode**: turns on synchronous replication mode. Possible values: ``off``, ``on``, ``quorum``. In this mode the leader takes care of management of ``synchronous_standby_names``, and only the last known leader, or one of synchronous replicas, are allowed to participate in leader race. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation ` for details. - **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the primary. See :ref:`replication modes documentation ` for details. - **synchronous\_node\_count**: if ``synchronous_mode`` is enabled, this parameter is used by Patroni to manage the precise number of synchronous standby instances and adjusts the state in DCS and the ``synchronous_standby_names`` parameter in PostgreSQL as members join and leave. If the parameter is set to a value higher than the number of eligible nodes, it will be automatically adjusted. Defaults to ``1``. - **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode `. Defaults to `false`. - **postgresql**: - **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`. Note that either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``, or ``pg_rewind`` will not work. - **use\_slots**: whether or not to use replication slots. Defaults to `true` on PostgreSQL 9.4+. - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. There is no recovery.conf anymore in PostgreSQL 12, but you may continue using this section, because Patroni handles it transparently. - **parameters**: configuration parameters (GUCs) for Postgres in format ``{max_connections: 100, wal_level: "replica", max_wal_senders: 10, wal_log_hints: "on"}``. Many of these are required for replication to work. - **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. Patroni ignores this parameter if ``hba_file`` PostgreSQL parameter is set to a non-default value. - **- host all all 0.0.0.0/0 md5** - **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication. - **pg\_ident**: list of lines that Patroni will use to generate ``pg_ident.conf``. Patroni ignores this parameter if ``ident_file`` PostgreSQL parameter is set to a non-default value. - **- mapname1 systemname1 pguser1** - **- mapname1 systemname2 pguser2** - **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster. - **host**: an address of remote node - **port**: a port of remote node - **primary\_slot\_name**: which slot on the remote node to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`). - **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote primary, can be different from the list defined in :ref:`postgresql_settings` - **restore\_command**: command to restore WAL records from the remote primary to nodes in a standby cluster, can be different from the list defined in :ref:`postgresql_settings` - **archive\_cleanup\_command**: cleanup command for standby leader - **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader - **member_slots_ttl**: retention time of physical replication slots for replicas when they are shut down. Default value: `30min`. Set it to `0` if you want to keep the old behavior (when the member key expires from DCS, the slot is immediately removed). The feature works only starting from PostgreSQL 11. - **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Permanent slots that don't exist will be created by Patroni. With PostgreSQL 11 onwards permanent physical slots are created on all nodes and their position is advanced every **loop_wait** seconds. For PostgreSQL versions older than 11 permanent physical replication slots are maintained only on the current primary. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent replication slots requires **postgresql.use_slots** to be set to ``true``. If there are permanent logical replication slots defined Patroni will automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+. - **my\_slot\_name**: the name of the permanent replication slot. If the permanent slot name matches with the name of the current node it will not be created on this node. If you add a permanent physical replication slot which name matches the name of a Patroni member, Patroni will ensure that the slot that was created is not removed even if the corresponding member becomes unresponsive, situation which would normally result in the slot's removal by Patroni. Although this can be useful in some situations, such as when you want replication slots used by members to persist during temporary failures or when importing existing members to a new Patroni cluster (see :ref:`Convert a Standalone to a Patroni Cluster ` for details), caution should be exercised by the operator that these clashes in names are not persisted in the DCS, when the slot is no longer required, due to its effect on normal functioning of Patroni. - **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``. - **database**: the database name where logical slots should be created. - **plugin**: the plugin name for the logical slot. - **ignore\_slots**: list of sets of replication slot properties for which Patroni should ignore matching slots. This configuration/feature/etc. is useful when some replication slots are managed outside of Patroni. Any subset of matching properties will cause a slot to be ignored. - **name**: the name of the replication slot. - **type**: slot type. Can be ``physical`` or ``logical``. If the slot is logical, you may additionally define ``database`` and/or ``plugin``. - **database**: the database name (when matching a ``logical`` slot). - **plugin**: the logical decoding plugin (when matching a ``logical`` slot). Note: **slots** is a hashmap while **ignore_slots** is an array. For example: .. code:: YAML slots: permanent_logical_slot_name: type: logical database: my_db plugin: test_decoding permanent_physical_slot_name: type: physical ... ignore_slots: - name: ignored_logical_slot_name type: logical database: my_db plugin: test_decoding - name: ignored_physical_slot_name type: physical ... Note: When running PostgreSQL v11 or newer Patroni maintains physical replication slots on all nodes that could potentially become a leader, so that replica nodes keep WAL segments reserved if they are potentially required by other nodes. In case the node is absent and its member key in DCS gets expired, the corresponding replication slot is dropped after ``member_slots_ttl`` (default value is `30min`). You can increase or decrease retention based on your needs. Alternatively, if your cluster topology is static (fixed number of nodes that never change their names) you can configure permanent physical replication slots with names corresponding to the names of the nodes to avoid slots removal and recycling of WAL files while replica is temporarily down: .. code:: YAML slots: node_name1: type: physical node_name2: type: physical node_name3: type: physical ... .. warning:: Permanent replication slots are synchronized only from the ``primary``/``standby_leader`` to replica nodes. That means, applications are supposed to be using them only from the leader node. Using them on replica nodes will cause indefinite growth of ``pg_wal`` on all other nodes in the cluster. An exception to that rule are physical slots that match the Patroni member names (created and maintained by Patroni). Those will be synchronized among all nodes as they are used for replication among them. .. warning:: Setting ``nostream`` tag on standby disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas if any. patroni-4.0.4/docs/existing_data.rst000066400000000000000000000163421472010352700175070ustar00rootroot00000000000000.. _existing_data: Convert a Standalone to a Patroni Cluster ========================================= This section describes the process for converting a standalone PostgreSQL instance into a Patroni cluster. To deploy a Patroni cluster without using a pre-existing PostgreSQL instance, see :ref:`Running and Configuring ` instead. Procedure --------- You can find below an overview of steps for converting an existing Postgres cluster to a Patroni managed cluster. In the steps we assume all nodes that are part of the existing cluster are currently up and running, and that you *do not* intend to change Postgres configuration while the migration is ongoing. The steps: #. Create the Postgres users as explained for :ref:`authentication ` section of the Patroni configuration. You can find sample SQL commands to create the users in the code block below, in which you need to replace the usernames and passwords as per your environment. If you already have the relevant users, then you can skip this step. .. code-block:: sql -- Patroni superuser -- Replace PATRONI_SUPERUSER_USERNAME and PATRONI_SUPERUSER_PASSWORD accordingly CREATE USER PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD 'PATRONI_SUPERUSER_PASSWORD'; -- Patroni replication user -- Replace PATRONI_REPLICATION_USERNAME and PATRONI_REPLICATION_PASSWORD accordingly CREATE USER PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD 'PATRONI_REPLICATION_PASSWORD'; -- Patroni rewind user, if you intend to enable use_pg_rewind in your Patroni configuration -- Replace PATRONI_REWIND_USERNAME and PATRONI_REWIND_PASSWORD accordingly CREATE USER PATRONI_REWIND_USERNAME WITH ENCRYPTED PASSWORD 'PATRONI_REWIND_PASSWORD'; GRANT EXECUTE ON function pg_catalog.pg_ls_dir(text, boolean, boolean) TO PATRONI_REWIND_USERNAME; GRANT EXECUTE ON function pg_catalog.pg_stat_file(text, boolean) TO PATRONI_REWIND_USERNAME; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO PATRONI_REWIND_USERNAME; GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) TO PATRONI_REWIND_USERNAME; #. Perform the following steps on all Postgres nodes. Perform all steps on one node before proceeding with the next node. Start with the primary node, then proceed with each standby node: #. If you are running Postgres through systemd, then disable the Postgres systemd unit. This is performed as Patroni manages starting and stopping the Postgres daemon. #. Create a YAML configuration file for Patroni. You can use :ref:`Patroni configuration generation and validation tooling ` for that. * **Note (specific for the primary node):** If you have replication slots being used for replication between cluster members, then it is recommended that you enable ``use_slots`` and configure the existing replication slots as permanent via the ``slots`` configuration item. Be aware that Patroni automatically creates replication slots for replication between members, and drops replication slots that it does not recognize, when ``use_slots`` is enabled. The idea of using permanent slots here is to allow your existing slots to persist while the migration to Patroni is in progress. See :ref:`YAML Configuration Settings ` for details. #. Start Patroni using the ``patroni`` systemd service unit. It automatically detects that Postgres is already running and starts monitoring the instance. #. Hand over Postgres "start up procedure" to Patroni. In order to do that you need to restart the cluster members through :ref:`patronictl restart cluster-name member-name ` command. For minimal downtime you might want to split this step into: #. Immediate restart of the standby nodes. #. Scheduled restart of the primary node within a maintenance window. #. If you configured permanent slots in step ``1.2.``, then you should remove them from ``slots`` configuration through :ref:`patronictl edit-config cluster-name member-name ` command once the ``restart_lsn`` of the slots created by Patroni is able to catch up with the ``restart_lsn`` of the original slots for the corresponding members. By removing the slots from ``slots`` configuration you will allow Patroni to drop the original slots from your cluster once they are not needed anymore. You can find below an example query to check the ``restart_lsn`` of a couple slots, so you can compare them: .. code-block:: sql -- Assume original_slot_for_member_x is the name of the slot in your original -- cluster for replicating changes to member X, and slot_for_member_x is the -- slot created by Patroni for that purpose. You need restart_lsn of -- slot_for_member_x to be >= restart_lsn of original_slot_for_member_x SELECT slot_name, restart_lsn FROM pg_replication_slots WHERE slot_name IN ( 'original_slot_for_member_x', 'slot_for_member_x' ) .. _major_upgrade: Major Upgrade of PostgreSQL Version =================================== The only possible way to do a major upgrade currently is: #. Stop Patroni #. Upgrade PostgreSQL binaries and perform `pg_upgrade `_ on the primary node #. Update patroni.yml #. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running :ref:`patronictl remove cluster-name ` . It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier. #. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one. It will help you to retain some PostgreSQL parameters you had set before. #. Start Patroni on the primary node. #. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes. #. Start Patroni on the standby nodes and wait for the replication to complete. Running pg_upgrade on standby nodes is not supported by PostgreSQL. If you know what you are doing, you can try the rsync procedure described in https://www.postgresql.org/docs/current/pgupgrade.html instead of wiping data_dir on standby nodes. The safest way is however to let Patroni replicate the data for you. FAQ --- - During Patroni startup, Patroni complains that it cannot bind to the PostgreSQL port. You need to verify ``listen_addresses`` and ``port`` in ``postgresql.conf`` and ``postgresql.listen`` in ``patroni.yml``. Don't forget that ``pg_hba.conf`` should allow such access. - After asking Patroni to restart the node, PostgreSQL displays the error message ``could not open configuration file "/etc/postgresql/10/main/pg_hba.conf": No such file or directory`` It can mean various things depending on how you manage PostgreSQL configuration. If you specified `postgresql.config_dir`, Patroni generates the ``pg_hba.conf`` based on the settings in the :ref:`bootstrap ` section only when it bootstraps a new cluster. In this scenario the ``PGDATA`` was not empty, therefore no bootstrap happened. This file must exist beforehand. patroni-4.0.4/docs/faq.rst000066400000000000000000000522401472010352700154300ustar00rootroot00000000000000.. _faq: FAQ === In this section you will find answers for the most frequently asked questions about Patroni. Each sub-section attempts to focus on different kinds of questions. We hope that this helps you to clarify most of your questions. If you still have further concerns or find yourself facing an unexpected issue, please refer to :ref:`chatting` and :ref:`reporting_bugs` for instructions on how to get help or report issues. Comparison with other HA solutions ---------------------------------- Why does Patroni require a separate cluster of DCS nodes while other solutions like ``repmgr`` do not? There are different ways of implementing HA solutions, each of them with their pros and cons. Software like ``repmgr`` performs communication among the nodes to decide when actions should be taken. Patroni on the other hand relies on the state stored in the DCS. The DCS acts as a source of truth for Patroni to decide what it should do. While having a separate DCS cluster can make you bloat your architecture, this approach also makes it less likely for split-brain scenarios to happen in your Postgres cluster. What is the difference between Patroni and other HA solutions in regards to Postgres management? Patroni does not just manage the high availability of the Postgres cluster but also manages Postgres itself. If Postgres nodes do not exist yet, it takes care of bootstrapping the primary and the standby nodes, and also manages Postgres configuration of the nodes. If the Postgres nodes already exist, Patroni will take over management of the cluster. Besides the above, Patroni also has self-healing capabilities. In other words, if a primary node fails, Patroni will not only fail over to a replica, but also attempt to rejoin the former primary as a replica of the new primary. Similarly, if a replica fails, Patroni will attempt to rejoin that replica. That is way we call Patroni as a "template for HA solutions". It goes further than just managing physical replication: it manages Postgres as a whole. DCS --- Can I use the same ``etcd`` cluster to store data from two or more Patroni clusters? Yes, you can! Information about a Patroni cluster is stored in the DCS under a path prefixed with the ``namespace`` and ``scope`` Patroni settings. As long as you do not have conflicting namespace and scope across different Patroni clusters, you should be able to use the same DCS cluster to store information from multiple Patroni clusters. What occurs if I attempt to use the same combination of ``namespace`` and ``scope`` for different Patroni clusters that point to the same DCS cluster? The second Patroni cluster that attempts to use the same ``namespace`` and ``scope`` will not be able to manage Postgres because it will find information related with that same combination in the DCS, but with an incompatible Postgres system identifier. The mismatch on the system identifier causes Patroni to abort the management of the second cluster, as it assumes that refers to a different cluster and that the user has misconfigured Patroni. Make sure to use different ``namespace`` / ``scope`` when dealing with different Patroni clusters that share the same DCS cluster. What occurs if I lose my DCS cluster? The DCS is used to store basically status and the dynamic configuration of the Patroni cluster. They very first consequence is that all the Patroni clusters that rely on that DCS will go to read-only mode -- unless :ref:`dcs_failsafe_mode` is enabled. What should I do if I lose my DCS cluster? There are three possible outcomes upon losing your DCS cluster: 1. The DCS cluster is fully recovered: this requires no action from the Patroni side. Once the DCS cluster is recovered, Patroni should be able to recover too; 2. The DCS cluster is re-created in place, and the endpoints remain the same. No changes are required on the Patroni side; 3. A new DCS cluster is created with different endpoints. You will need to update the DCS endpoints in the Patroni configuration of each Patroni node. If you face scenario ``2.`` or ``3.`` Patroni will take care of creating the status information again based on the current status of the cluster, and recreate the dynamic configuration on the DCS based on a backup file named ``patroni.dynamic.json`` which is stored inside the Postgres data directory of each member of the Patroni cluster. What occurs if I lose majority in my DCS cluster? The DCS will become unresponsive, which will cause Patroni to demote the current read/write Postgres node. Remember: Patroni relies on the state of the DCS to take actions on the cluster. You can use the :ref:`dcs_failsafe_mode` to alleviate that situation. patronictl ---------- Do I need to run :ref:`patronictl` in the Patroni host? No, you do not need to do that. Running :ref:`patronictl` in the Patroni host is handy if you have access to the Patroni host because you can use the very same configuration file from the ``patroni`` agent for the :ref:`patronictl` application. However, :ref:`patronictl` is basically a client and it can be executed from remote machines. You just need to provide it with enough configuration so it can reach the DCS and the REST API of the Patroni member(s). Why did the information from one of my Patroni members disappear from the output of :ref:`patronictl_list` command? Information shown by :ref:`patronictl_list` is based on the contents of the DCS. If information about a member disappeared from the DCS it is very likely that the Patroni agent on that node is not running anymore, or it is not able to communicate with the DCS. As the member is not able to update the information, the information eventually expires from the DCS, and consequently the member is not shown anymore in the output of :ref:`patronictl_list`. Why is the information about one of my Patroni members not up-to-date in the output of :ref:`patronictl_list` command? Information shown by :ref:`patronictl_list` is based on the contents of the DCS. By default, that information is updated by Patroni roughly every ``loop_wait`` seconds. In other words, even if everything is normally functional you may still see a "delay" of up to ``loop_wait`` seconds in the information stored in the DCS. Be aware that that is not a rule, though. Some operations performed by Patroni cause it to immediately update the DCS information. Configuration ------------- What is the difference between dynamic configuration and local configuration? Dynamic configuration (or global configuration) is the configuration stored in the DCS, and which is applied to all members of the Patroni cluster. This is primarily where you should store your configuration. Settings that are specific to a node, or settings that you would like to overwrite the global configuration with, you should set only on the desired Patroni member as a local configuration. That local configuration can be specified either through the configuration file or through environment variables. See more in :ref:`patroni_configuration`. What are the types of configuration in Patroni, and what is the precedence? The types are: * Dynamic configuration: applied to all members; * Local configuration: applied to the local member, overrides dynamic configuration; * Environment configuration: applied to the local member, overrides both dynamic and local configuration. **Note:** some Postgres GUCs can only be set globally, i.e., through dynamic configuration. Besides that, there are GUCs which Patroni enforces a hard-coded value. See more in :ref:`patroni_configuration`. Is there any facility to help me create my Patroni configuration file? Yes, there is. You can use ``patroni --generate-sample-config`` or ``patroni --generate-config`` commands to generate a sample Patroni configuration or a Patroni configuration based on an existing Postgres instance, respectively. Please refer to :ref:`generate_sample_config` and :ref:`generate_config` for more details. I changed my parameters under ``bootstrap.dcs`` configuration but Patroni is not applying the changes to the cluster members. What is wrong? The values configured under ``bootstrap.dcs`` are only used when bootstrapping a fresh cluster. Those values will be written to the DCS during the bootstrap. After the bootstrap phase finishes, you will only be able to change the dynamic configuration through the DCS. Refer to the next question for more details. How can I change my dynamic configuration? You need to change the configuration in the DCS. That is accomplished either through: * :ref:`patronictl_edit_config`; or * A ``PATCH`` request to :ref:`config_endpoint`. How can I change my local configuration? You need to change the configuration file of the corresponding Patroni member and signal the Patroni agent with ``SIHGUP``. You can do that using either of these approaches: * Send a ``POST`` request to the REST API :ref:`reload_endpoint`; or * Run :ref:`patronictl_reload`; or * Locally signal the Patroni process with ``SIGHUP``: * If you started Patroni through systemd, you can use the command ``systemctl reload PATRONI_UNIT.service``, ``PATRONI_UNIT`` being the name of the Patroni service; or * If you started Patroni through other means, you will need to identify the ``patroni`` process and run ``kill -s HUP PID``, ``PID`` being the process ID of the ``patroni`` process. **Note:** there are cases where a reload through the :ref:`patronictl_reload` may not work: * Expired REST API certificates: you can mitigate that by using the ``-k`` option of the :ref:`patronictl`; * Wrong credentials: for example when changing ``restapi`` or ``ctl`` credentials in the configuration file, and using that same configuration file for Patroni and :ref:`patronictl`. How can I change my environment configuration? The environment configuration is only read by Patroni during startup. With that in mind, if you change the environment configuration you will need to restart the corresponding Patroni agent. Take care to not cause a failover in the cluster! You might be interested in checking :ref:`patronictl_pause`. What occurs if I change a Postgres GUC that requires a reload? When you change the dynamic or the local configuration as explained in the previous questions, Patroni will take care of reloading the Postgres configuration for you. What occurs if I change a Postgres GUC that requires a restart? Patroni will mark the affected members with a flag of ``pending restart``. It is up to you to determine when and how to restart the members. That can be accomplished either through: * :ref:`patronictl_restart`; or * A ``POST`` request to :ref:`restart_endpoint`. **Note:** some Postgres GUCs require a special management in terms of the order for restarting the Postgres nodes. Refer to :ref:`shared_memory_gucs` for more details. What is the difference between ``etcd`` and ``etcd3`` in Patroni configuration? ``etcd`` uses the API version 2 of ``etcd``, while ``etcd3`` uses the API version 3 of ``etcd``. Be aware that information stored by the API version 2 is not manageable by API version 3 and vice-versa. We recommend that you configure ``etcd3`` instead of ``etcd`` because: * API version 2 is disabled by default from Etcd v3.4 onward; * API version 2 will be completely removed on Etcd v3.6. I have ``use_slots`` enabled in my Patroni configuration, but when a cluster member goes offline for some time, the replication slot used by that member is dropped on the upstream node. What can I do to avoid that issue? There are two options: 1. You can tune ``member_slots_ttl`` (default value ``30min``, available since Patroni ``4.0.0`` and PostgreSQL 11 onwards) and replication slots for absent members will not be removed when the members downtime is shorter than the configured threshold. 2. You can configure permanent physical replication slots for the members. Since Patroni ``3.2.0`` it is now possible to have member slots as permanent slots managed by Patroni. Patroni will create the permanent physical slots on all nodes, and make sure to not remove the slots, as well as to advance the slots' LSN on all nodes according to the LSN that has been consumed by the member. Later, if you decide to remove the corresponding member, it's **your responsibility** to adjust the permanent slots configuration, otherwise Patroni will keep the slots around forever. **Note:** on Patroni older than ``3.2.0`` you could still have member slots configured as permanent physical slots, however they would be managed only on the current leader. That is, in case of failover/switchover these slots would be created on the new leader, but that wouldn't guarantee that it had all WAL segments for the absent node. **Note:** even with Patroni ``3.2.0`` there might be a small race condition. In the very beginning, when the slot is created on the replica it could be ahead of the same slot on the leader and in case if nobody is consuming the slot there is still a chance that some files could be missing after failover. With that in mind, it is recommended that you configure continuous archiving, which makes it possible to restore required WALs or perform PITR. What is the difference between ``loop_wait``, ``retry_timeout`` and ``ttl``? Patroni performs what we call a HA cycle from time to time. On each HA cycle it takes care of performing a series of checks on the cluster to determine its healthiness, and depending on the status it may take actions, like failing over to a standby. ``loop_wait`` determines for how long, in seconds, Patroni should sleep before performing a new cycle of HA checks. ``retry_timeout`` sets the timeout for retry operations on the DCS and on Postgres. For example: if the DCS is unresponsive for more than ``retry_timeout`` seconds, Patroni might demote the primary node as a security action. ``ttl`` sets the lease time on the ``leader`` lock in the DCS. If the current leader of the cluster is not able to renew the lease during its HA cycles for longer than ``ttl``, then the lease will expire and that will trigger a ``leader race`` in the cluster. **Note:** when modifying these settings, please keep in mind that Patroni enforces the rule and minimal values described in :ref:`dynamic_configuration` section of the docs. Postgres management ------------------- Can I change Postgres GUCs directly in Postgres configuration? You can, but you should avoid that. Postgres configuration is managed by Patroni, and attempts to edit the configuration files may end up being frustrated by Patroni as it may eventually overwrite them. There are a few options available to overcome the management performed by Patroni: * Change Postgres GUCs through ``$PGDATA/postgresql.base.conf``; or * Define a ``postgresql.custom_conf`` which will be used instead of ``postgresql.base.conf`` so you can manage that externally; or * Change GUCs using ``ALTER SYSTEM`` / ``ALTER DATABASE`` / ``ALTER USER``. You can find more information about that in the section :ref:`important_configuration_rules`. In any case we recommend that you manage all the Postgres configuration through Patroni. That will centralize the management and make it easier to debug Patroni when needed. Can I restart Postgres nodes directly? No, you should **not** attempt to manage Postgres directly! Any attempt of bouncing the Postgres server without Patroni can lead your cluster to face failovers. If you need to manage the Postgres server, do that through the ways exposed by Patroni. Is Patroni able to take over management of an already existing Postgres cluster? Yes, it can! Please refer to :ref:`existing_data` for detailed instructions. How does Patroni manage Postgres? Patroni takes care of bringing Postgres up and down by running the Postgres binaries, like ``pg_ctl`` and ``postgres``. With that in mind you **MUST** disable any other sources that could manage the Postgres clusters, like the systemd units, e.g. ``postgresql.service``. Only Patroni should be able to start, stop and promote Postgres instances in the cluster. Not doing so may result in split-brain scenarios. For example: if the node running as a primary failed and the unit ``postgresql.service`` is enabled, it may bring Postgres back up and cause a split-brain. Concepts and requirements ------------------------- Which are the applications that make part of Patroni? Patroni basically ships a couple applications: * ``patroni``: This is the Patroni agent, which takes care of managing a Postgres node; * ``patronictl``: This is a command-line utility used to interact with a Patroni cluster (perform switchovers, restarts, changes in the configuration, etc.). Please find more information in :ref:`patronictl`. What is a ``standby cluster`` in Patroni? It is a cluster that does not have any primary Postgres node running, i.e., there is no read/write member in the cluster. These kinds of clusters exist to replicate data from another cluster and are usually useful when you want to replicate data across data centers. There will be a leader in the cluster which will be a standby in charge of replicating changes from a remote Postgres node. Then, there will be a set of standbys configured with cascading replication from such leader member. **Note:** the standby cluster doesn't know anything about the source cluster which it is replicating from -- it can even use ``restore_command`` instead of WAL streaming, and may use an absolutely independent DCS cluster. Refer to :ref:`standby_cluster` for more details. What is a ``leader`` in Patroni? A ``leader`` in Patroni is like a coordinator of the cluster. In a regular Patroni cluster, the ``leader`` will be the read/write node. In a standby Patroni cluster, the ``leader`` (AKA ``standby leader``) will be in charge of replicating from a remote Postgres node, and cascading those changes to the other members of the standby cluster. Does Patroni require a minimum number of Postgres nodes in the cluster? No, you can run Patroni with any number of Postgres nodes. Remember: Patroni is decoupled from the DCS. What does ``pause`` mean in Patroni? Pause is an operation exposed by Patroni so the user can ask Patroni to step back in regards to Postgres management. That is mainly useful when you want to perform maintenance on the cluster, and would like to avoid that Patroni takes decisions related with HA, like failing over to a standby when you stop the primary. You can find more information about that in :ref:`pause`. Automatic failover ------------------ How does the automatic failover mechanism of Patroni work? Patroni automatic failover is based on what we call ``leader race``. Patroni stores the cluster's status in the DCS, among them a ``leader`` lock which holds the name of the Patroni member which is the current ``leader`` of the cluster. That ``leader`` lock has a time-to-live associated with it. If the leader node fails to update the lease of the ``leader`` lock in time, the key will eventually expire from the DCS. When the ``leader`` lock expires, it triggers what Patroni calls a ``leader race``: all nodes start performing checks to determine if they are the best candidates for taking over the ``leader`` role. Some of these checks include calls to the REST API of all other Patroni members. All Patroni members that find themselves as the best candidate for taking over the ``leader`` lock will attempt to do so. The first Patroni member that is able to take the ``leader`` lock will promote itself to a read/write node (or ``standby leader``), and the others will be configured to follow it. Can I temporarily disable automatic failover in the Patroni cluster? Yes, you can! You can achieve that by temporarily pausing the cluster. This is typically useful for performing maintenance. When you want to resume the automatic failover of the cluster, you just need to unpause it. You can find more information about that in :ref:`pause`. Bootstrapping and standbys creation ----------------------------------- How does Patroni create a primary Postgres node? What about a standby Postgres node? By default Patroni will use ``initdb`` to bootstrap a fresh cluster, and ``pg_basebackup`` to create standby nodes from a copy of the ``leader`` member. You can customize that behavior by writing your custom bootstrap methods, and your custom replica creation methods. Custom methods are usually useful when you want to restore backups created by backup tools like pgBackRest or Barman, for example. For detailed information please refer to :ref:`custom_bootstrap` and :ref:`custom_replica_creation`. Monitoring ---------- How can I monitor my Patroni cluster? Patroni exposes a couple handy endpoints in its :ref:`rest_api`: * ``/metrics``: exposes monitoring metrics in a format that can be consumed by Prometheus; * ``/patroni``: exposes the status of the cluster in a JSON format. The information shown here is very similar to what is shown by the ``/metrics`` endpoint. You can use those endpoints to implement monitoring checks. patroni-4.0.4/docs/ha_loop_diagram.dot000066400000000000000000000177401472010352700177520ustar00rootroot00000000000000// Graphviz source for ha_loop_diagram.png // recompile with: // dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png digraph G { rankdir=TB; fontname="sans-serif"; penwidth="0.3"; layout="dot"; newrank=true; edge [fontname="sans-serif", fontsize=12, color=black, fontcolor=black]; node [fontname=serif, fontsize=12, fillcolor=white, color=black, fontcolor=black, style=filled]; "start" [label=Start, shape="rectangle", fillcolor="green"] "start" -> "load_cluster_from_dcs"; "update_member" [label="Persist node state in DCS"] "update_member" -> "start" subgraph cluster_run_cycle { label="run_cycle" "load_cluster_from_dcs" [label="Load cluster from DCS"]; "touch_member" [label="Persist node in DCS"]; "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"] "cluster.has_member" -> "touch_member" [label="no" color="red"] "long_action_in_progress?" [shape="diamond" label="Is the PostgreSQL currently being\nstopping/starting/restarting/reinitializing?"] "load_cluster_from_dcs" -> "cluster.has_member"; "touch_member" -> "long_action_in_progress?"; "cluster.has_member" -> "long_action_in_progress?" [label="yes" color="green"]; "long_action_in_progress?" -> "recovering?" [label="no" color="red"] "recovering?" [label="Was cluster recovering and failed?", shape="diamond"]; "recovering?" -> "post_recover" [label="yes" color="green"]; "recovering?" -> "data_directory_empty" [label="no" color="red"]; "post_recover" [label="Remove leader key (if I was the leader)"]; "data_directory_empty" [label="Is data folder empty?", shape="diamond"]; "data_directory_empty" -> "cluster_initialize" [label="no" color="red"]; "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"]; "data_belongs_to_cluster" -> "exit" [label="no" color="red"]; "data_belongs_to_cluster" -> "is_healthy" [label="yes" color="green"] "exit" [label="Fail and exit", fillcolor=red]; "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"] "cluster_initialize" -> "cluster.has_leader" [label="no" color="red"] "cluster.has_leader" [label="Does the cluster has leader?", shape="diamond"] "cluster.has_leader" -> "dcs.initialize" [label="no", color="red"] "cluster.has_leader" -> "is_healthy" [label="yes", color="green"] "cluster_initialize" -> "data_belongs_to_cluster" [label="yes" color="green"] "dcs.initialize" [label="Initialize new cluster"]; "dcs.initialize" -> "is_healthy" "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"]; "recover" [label="Start as read-only\nand set Recover flag"] "is_healthy" -> "recover" [label="no" color="red"]; "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; "cluster.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"] } "post_recover" -> "update_member" "recover" -> "update_member" "long_action_in_progress?" -> "async_has_lock?" [label="yes" color="green"]; "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"] "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"] "data_directory_empty" -> "bootstrap.is_unlocked" [label="yes" color="green"] subgraph cluster_async { label = "Long action in progress\n(Start/Stop/Restart/Reinitialize)" "async_has_lock?" [label="Do I have the leader lock?", shape="diamond"] "async_update_lock" [label="Renew leader lock"] "async_has_lock?" -> "async_update_lock" [label="yes" color="green"] } "async_update_lock" -> "update_member" "async_has_lock?" -> "update_member" [label="no" color="red"] subgraph cluster_bootstrap { label = "Node bootstrap"; "bootstrap.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"] "bootstrap.is_initialized" [label="Does the cluster has an initialize key?", shape="diamond"] "bootstrap.is_unlocked" -> "bootstrap.is_initialized" [label="no" color="red"] "bootstrap.is_unlocked" -> "bootstrap.select_node" [label="yes" color="green"] "bootstrap.select_node" [label="Select a node to take a backup from"] "bootstrap.do_bootstrap" [label="Run pg_basebackup\n(async)"] "bootstrap.select_node" -> "bootstrap.do_bootstrap" "bootstrap.is_initialized" -> "bootstrap.initialization_race" [label="no" color="red"] "bootstrap.is_initialized" -> "bootstrap.wait_for_leader" [label="yes" color="green"] "bootstrap.initialization_race" [label="Race for initialize key"] "bootstrap.initialization_race" -> "bootstrap.won_initialize_race?" "bootstrap.won_initialize_race?" [label="Do I won initialize race?", shape="diamond"] "bootstrap.won_initialize_race?" -> "bootstrap.initdb_and_start" [label="yes" color="green"] "bootstrap.won_initialize_race?" -> "bootstrap.wait_for_leader" [label="no" color="red"] "bootstrap.wait_for_leader" [label="Need to wait for leader key"] "bootstrap.initdb_and_start" [label="Run initdb, start postgres and create roles"] "bootstrap.initdb_and_start" -> "bootstrap.success?" "bootstrap.success?" [label="Success", shape="diamond"] "bootstrap.success?" -> "bootstrap.take_leader_key" [label="yes" color="green"] "bootstrap.success?" -> "bootstrap.clean" [label="no" color="red"] "bootstrap.clean" [label="Remove initialize key from DCS\nand data directory from filesystem"] "bootstrap.take_leader_key" [label="Take a leader key in DCS"] } "bootstrap.do_bootstrap" -> "update_member" "bootstrap.wait_for_leader" -> "update_member" "bootstrap.clean" -> "update_member" "bootstrap.take_leader_key" -> "update_member" subgraph cluster_process_healthy_cluster { label = "process_healthy_cluster" "healthy_has_lock" [label="Am I the owner of the leader lock?", shape=diamond] "healthy_is_leader" [label="Is Postgres running as primary?", shape=diamond] "healthy_no_lock" [label="Follow the leader (async,\ncreate/update recovery.conf and restart if necessary)"] "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"] "healthy_has_lock" -> "healthy_update_leader_lock" [label="yes" color="green"] "healthy_update_leader_lock" [label="Try to update leader lock"] "healthy_update_leader_lock" -> "healthy_update_success" "healthy_update_success" [label="Success?", shape=diamond] "healthy_update_success" -> "healthy_is_leader" [label="yes" color="green"] "healthy_update_success" -> "healthy_demote" [label="no" color="red"] "healthy_demote" [label="Demote (async,\nrestart in read-only)"] "healthy_failover" [label="Promote Postgres to primary"] "healthy_is_leader" -> "healthy_failover" [label="no" color="red"] } "healthy_demote" -> "update_member" "healthy_is_leader" -> "update_member" [label="yes" color="green"] "healthy_failover" -> "update_member" "healthy_no_lock" -> "update_member" subgraph cluster_process_unhealthy_cluster { label = "process_unhealthy_cluster" "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"] "unhealthy_is_healthiest" -> "unhealthy_leader_race" [label="yes", color="green"] "unhealthy_leader_race" [label="Try to create leader key"] "unhealthy_leader_race" -> "unhealthy_acquire_lock" "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"] "unhealthy_is_leader" [label="Is Postgres running as primary?", shape=diamond] "unhealthy_acquire_lock" -> "unhealthy_is_leader" [label="yes" color="green"] "unhealthy_is_leader" -> "unhealthy_promote" [label="no" color="red"] "unhealthy_promote" [label="Promote to primary"] "unhealthy_is_healthiest" -> "unhealthy_follow" [label="no" color="red"] "unhealthy_follow" [label="try to follow somebody else()"] "unhealthy_acquire_lock" -> "unhealthy_follow" [label="no" color="red"] } "unhealthy_follow" -> "update_member" "unhealthy_promote" -> "update_member" "unhealthy_is_leader" -> "update_member" [label="yes" color="green"] } patroni-4.0.4/docs/ha_loop_diagram.png000066400000000000000000020305441472010352700177470ustar00rootroot00000000000000‰PNG  IHDR  :ÚS%ïbKGDÿÿÿ ½§“ IDATxœìÝ{”y/þïÜ’r‰˜ì$3Q8’!N(‰Q, ¥‡¨§”b‚-Ç Â±þ×¢-¨uÙÊe-TŽV-j#ä"š‚Z”ËL²“@’ÉL2™Ìüþ€ìf’ÉH²sù|ÖÚë}yÞyž¬x™Éû}Ÿª¾¾¾¾¼êJ7ì„ €$BÀËj+Ýû‡;ï¼³Ò-ìõ¦Nš &Tº €mªêëëë«tìûªªª*ÝÂ^ïŽ;îÈŒ3*ÝÀ6UWºö#w$éóð°2’/2’/2’/2’/2’/2’/2’/2`ï27ÉŸ%™dp’æ$3“Ü›¤o³ÏÍNR•ä‰ÝÔÇî>?À^HÈ€½Ç¼$S““<˜du’{’ KrR^ °ÛÔVº(ûF^Ú^pMþûvÚ‡$ùr^Ú,Àne“{®$’¬à½ß$™òò×—%yçË_ONR•—ÂI²4É%IÞ˜¤äè$ßßâ\×¼\Sõò×MÒðòóìàüû1!ö’d]’Ó’Ü—¤oŸ»*ɼüõã/îÙ—Ÿ_™—‚ ç¿ïO2³ú%Yýò×ÿ˜äíI:’|u'ΰ2`ïñ¡$g&¹7ÉÛ’ŒOrQ^ 쬯¼üdX’¿HrJþ;@°¥éIÎJ24/…n}ì'„ Ø{Ô%ùn’Ÿ'ùË$/&¹.ÉñIÞ•dÅ«ºÙçŠ/—déËÏ—&98É-IžL²./mA¸ûUô1Ðù缊óìC„ Ø{\–䟓|+ɱI†%iÍKúÿ8É)›}öˆ$&9?É’œ™¤-É’ŒLr\’Ó|7É;“<–¤*Éâ$·'þòy>òòë+¶èe óOÙUƒìj+Ýû¦Õ«WgáÂ…Y²dI.\¸kN:,ɇ^~ìŒk_~lîØ$÷í î/?^Íùw³¾¾¾<ûì³™?~ù±nݺÜyç{¶à€$d@?Ë–-ËâÅ‹³hÑ¢ò£T*eñâÅåãÂ… ³víÚrMm­ËR_ŽŽŽ<ùä“yâ‰'òä“OfÞ¼yùÝï~—®®®$ÉàÁƒÓÝÝ÷¾÷½î8Pøos€DgggJ¥R94ÐÙÙYþzÓ±££#«W¯.× 4(£GθqãR,sÄGäío{êëë˯7.MMM©«««àt{¹%/î¾ûîüìg?Ë£>šùóççÅ_LòR˜ ¯¯/ÝÝÝýÊÖ¯_ŸÁƒçÍo~óžî8@ ì㶠 tìèèȆ Ê5…B¡_P ¹¹9mmmý‚Åb1cÇŽMuuu§ÛOt¾tøæ7¿9àÛëׯßfiwwwZZZvCS[2Ø uwwgéÒ¥[¶Ü>ÐÞÞžžžžr]¡Pèhmm-?ßó™ÜvÛm©©©IOOOêêêrä‘GnU3nܸŒ7.§Ÿ~zùµ_|1óçÏÏc=–ÇúèqÄûÕ† à•2özëׯϲe˶Ø´}à¹çžËÆËu…Ba‡ÁMá^;!ƒØÉA’ôõõå»ßýn.»ì²Ü}÷Ý9ì°Ãvi+ÝÝÝyꩧ2wîÜòcÞ¼yY»vmêêêrØa‡¥µµµü8úè£3lØ°]Ú°w2*fݺuY¾|ùv·”J¥,Y²$½½½åº 466fĈœîÀ#d°¯ d°Iwwwjkk÷Èžžžüþ÷¿Ï“O>™ùóçgîܹyøá‡ó /$IŠÅb¿àÁ±Ç›±cÇîö¾€=KÈØ庺ºvØ´}`sõõõÛ Œ7.&LÈ Aƒ*4Û#d°¯"d°7xæ™gòè£fÞ¼yåã‚ ’$&LÈ”)S2eÊ”´¶¶fÊ”)yÝë^WáŽ×BÈØi;X¸paV®\Ù¯ngÂMMM©­­­Ðdì B;°† ²téÒ<ú裙;wnæÌ™“9sæäÙgŸM’rÈ!åÐÁ±Ç›ÖÖÖŒ5ª² ;MÈHggç·´··gÍš5åšÁƒ§¡¡a»Áb±˜1cƤ¦¦¦‚Ó±§ìÀ~2ÈÊ•+óøãgîܹåÇ“O>™$)‹imm-?¦NšÑ£GW¸c` â~°ÛQx`Ñ¢EiooOOOO¹¦P(¤¾¾¾hii0°`Á‚¬Zµª\3hРŒ=ºØ´}`Ëð@SSSjkk+8¼rUUUÉÿIrB¥;ÙK½_ÈàµX³fM}ôÑrèàÞ{ïÍóÏ?ŸÚÚÚ~øáåÀAkkk&MšôÒßGà2€-l loë@GGG6lØP®) Û l~;vlª««+8ì>.êÞ1!ƒ]«T*•C›¶¬[·.#GŽÌ±Ç[´µµ¥¡¡¡ÒíÂ>AÈ€Bwww–.]ºÝ­‹-J{{{zzzÊu…Ba‡[ŠÅbŠÅ¢ ¬*lݺuyä‘GòË_þ2?üpzè¡ttt¤¦¦&“&MÊ 'œiÓ¦eÚ´i9ôÐC+Ý.ì•„ ا­[·.Ë—/ßnp T*eÉ’%éíí-×mØÖqüøñ5jT§àµ*•Jyøá‡óË_þ2>ø`~ýë_gýúõ?~|9p0mÚ´Lž<9555•n*NÈ€½RWWW9 °½í‹/Îæ—ÂÕ××÷ ´} ©©)ǯàtTJOOO{ì±ÜÿýyàrÏ=÷dÙ²e6lXŽ?þø´µµeÚ´iikkË!C*Ý.ìqBìQ›‡¶u,•JY±bE¿º-ÃSWWW¡ÉØW=ýôÓ™={vî¿ÿþÜÿýyæ™gR[[›£Ž:ª:8ùä“óº×½®Ò­Ân'dÀ.ÑÙÙÙ/(0ÐöŽŽŽ¬^½º\3hРŒ=z«ÀÀ–ÛšššR[[[Áé8”J¥<ðÀåmóæÍKoooš››ûm:hii©t«°Ë °][†:vttdÆ åšB¡°UP` ãرcS]]]Áé`Ç–/_^Üÿý™3gNº»»3~üø¼õ­oM[[[N<ñÄLž<9UUU•n^!€Pwww–.]ºU``Ëííííééé)× …n(‹)‹.²`¿µaÆüæ7¿)o:øÙÏ~–åË—çu¯{]Ž?þøL›6-Ó§OÏ[Þòÿ{È>GÈ`?²nݺ,_¾|»[J¥R–,Y’ÞÞÞrÝ@á-&LÈÈ‘#+8ì6nܘÇ{,?ÿùÏsÏ=÷ä¿øEV¯^qãÆåä“OÎÛßþöœ|òÉ9ôÐC+Ý*ìÀ> ««k‡[J¥R/^œÍ/ «¯¯00°ùö‰'fØ°aœö/7nÌ£>šÙ³gçþûïÏ}÷Ý—U«V¥X,–·¼ë]ïÊĉ+Ý*lEÈ ‚ ly\¸paV®\Ù¯n[áÍ©«««ÐdÀ&===yì±Ç2{öìÌž=;¿øÅ/²~ýú477gúôéikkË;ÞñŽŒ?¾Ò­‚ÀîÐÙÙ9```óííííY³fM¹fðàÁihhØfp`Óö‰'¦¦¦¦‚Ó¯ÅÚµkóàƒæþûïÏ<ûî»/ÝÝÝåÐÁôéÓóŽw¼# •n•À+°­ðÀæÇŽŽŽlØ°¡\S(Êím;vlª««+8P «V­Ê½÷Þ›ŸÿüçùùÏžßüæ7©ªªÊ1Ç“·¿ýí9ùä“sâ‰'æ ƒªt«„ €Þúõë³lÙ²m6mxî¹ç²qãÆr]¡PØap`S¸`g­^½:?üpfÏžÙ³gç‘GIMMMŽ;~úé™>}zÞò–·¤ªªªÒ­²2ö[ëÖ­ËòåË·»u T*eÉ’%éíí-×íLx ±±1#FŒ¨àtÀ¢T*å?þã?òÓŸþ4ÿñÿ‘^x!cÇŽÍ;ßùÎœrÊ)yç;ß™1cÆTºMöBÀ>§««k‡ÁMÛ6W__¿Ã­‡rH†Z¡É¶¯··7>úh~úÓŸæ§?ýixàlØ°!GuTN9唜rÊ)™6mZ\éVÙG { ,\¸0+W®ìW·£ðÀ¸qãÒÔÔ”ÚÚÚ M°{¬]»6>ø`fÏžÙ³gç‘GI¡PH[[[N;í´¼ï}ïËĉ+Ý&û!`·ëììÜáÖööö¬Y³¦\3xðà444l78P,3f̘ÔÔÔTp:€½G{{{~üãçßÿýßó³Ÿý,kÖ¬IKKKÞóž÷äÔSOÍ´iÓRWWWé6Ù‹ ¯ÚŽÂ‹-J{{{zzzÊ5…B!õõõ; ‹ÅTUUUp:€}[OOO~ùË_æ‡?üafÏž¹sçfèС9ùä“súé§çÔSOMccc¥Ûd/#dô³~ýú,[¶l»ÁR©”çŸ>7n,× …Æ—úúú Npàúãÿ˜»ï¾;?úÑrï½÷fݺu9úè£sÚi§åŒ3ÎHkk«'Bp èêêJggçË/Îæ—Õ×קX,nwû@cccFŒQÁéx%Ö®]›ÿüÏÿÌøÃüð‡?LGGGÆ_¼ýíoO¡P¨t›T€ì㺺ºvX´hQ:;;ûÕm loë@cccêêê*4{ÊüùóóÃþ0?øÁòàƒ¦P(äïxGN?ýôœ~úé)‹•n‘=DÈöR)•JÛÝ>°`Á‚¬Zµª\3hРŒ=z»[Æ—¦¦¦ÔÖÖVp:öV ,ÈøÃüÛ¿ý[î¹çžôôôäøãÏgœ‘÷½ï}9üðÃ+Ý"»‘ìa›ÂÛÛ:ÐÑÑ‘ 6”k …ƒÅb1cÇŽMuuu§`²fÍšüä'?É~ðƒüèG?ÊÒ¥Ksä‘GæÌ3ÏÌ™gž™£Ž:ªÒ-²‹ À.ÐÝÝ¥K—nwëÀ¢E‹ÒÞÞžžžžr]¡P(¶"(‹©ªªªà„èz{{óàƒæÛßþvîºë®,X° ‡rHÎ8㌜uÖYikkóÿY÷B°ëÖ­ËòåË·(•JY²dIz{{Ëu›‡¶u?~|FUÁéàÕ›?~¾ýíoçöÛoÏïÿû|ðÁy÷»ß³Î:+ï~÷»SWWWéy„ 8 uuu•ÛÛ>°xñâl~‰M}}}¿ À@Ûššš2|øð N{Öc=–»îº+wÝuWžxâ‰|ðÁ9óÌ3óþ÷¿?'žxbjjj*Ý";IÈ€ýÊæámK¥RV¬XѯnËðÀ@ÇÆÆFwc€ø¯ÿú¯|÷»ßÍwÞ™G}4cÇŽÍYg•3fdêÔ©©®®®t‹l‡û„ÎÎÎ~A¶tttdõêÕåšAƒeôèÑ[¶Ü>ÐÔÔ”ÚÚÚ Nû§gŸ}6ßÿþ÷sóÍ7ç‘GÉ„ ræ™g欳ÎJ[[[ªªª*Ý"[2 ¢¶  tìèèȆ Ê5…Ba« À@DZcǺK*ì%üñÜyç¹ãŽ;òÔSOåÐCÍûßÿþ¼ÿýïÏÑG]éöx™»\www–.]ºU``Ëííííééé)× …n(‹)‹îx û°ùóççÛßþvn½õÖüáÈG‘3fäÜsÏÍ¡‡ZéöhBì´uëÖeùòåÛÝ:P*•²dÉ’ôöö–ë lyœ0aBFŽYÁé€=­¯¯/=ôP¾õ­oåŽ;îÈŠ+rÒI'åì³ÏΟÿùŸgøðá•nñ€#d@ºººv¸u T*eñâÅÙür“úúú›o˜8qb† VÁé€}Awww~ò“ŸäÛßþv¾ûÝ醴·7Ó§OÏ9眓÷½ï}©«««t‹!€ýØ@á- .ÌÊ•+ûÕm+<°ù±±±ÑE>Àn±|ùòÜyç¹å–[òÐCe̘1ùà?˜sÎ9'G}t¥ÛÛ¯ ìƒ:;; l¾} ½½=kÖ¬)× <8 Û lÚ>0qâÄÔÔÔTp:€ÿÖÞÞžý×Í7Þ˜§žz*“&MÊ9眓óÏ??£G®t{û!€½È¶Â›;::²aÆrM¡P(¶·}`ìر©®®®àt¯Íܹs3kÖ¬Üzë­Ù¸qcN?ýôÌœ93ïxÇ;RUUUéöö BPa—]vYš››+ÝTÜÓO?«®ºªÒmÀn±~ýú,[¶l›ÁMÛž{î¹lܸ±\W(vØ.8¬Zµ*·ß~{n¾ùæ<ðÀ9üðÃsÞyçåCúPÆŒSéööiBPa³fÍÊÌ™3+ÝTœŸ‘Ø­[·.Ë—/ßîÖR©”%K–¤···\·3áÆÆÆŒ1¢‚ÓìæÍ›—o¼1·Þzk^|ñÅœvÚiùÈG>’w½ë]¶¹½ µ•n`oÓÕÕµÃàÀ¦í›«¯¯ïhiiÙjëÀ!‡’¡C‡Vh2€ýÏ1Ç“k®¹&_üâóÝï~77ÞxcþôOÿ4ÍÍÍùèG?šóÎ;/õõõ•nsŸ!d0v&<°pá¬\¹²_Ý–áÖÖÖ­¶455¥¶Ö¥•2dÈüå_þeþò/ÿ2O=õTn¼ñÆüýßÿ}þöoÿ6gœqF>þñç„N¨t›{=?Ùû¼ÎÎÎnhooÏš5kÊ5ƒNCCC9(ÐÜÜœ¶¶¶~Áb±˜1cƤ¦¦¦‚ÓðJvØa¹êª«òÙÏ~6·Ýv[®½öÚL:5­­­™9sfÎ>ûì 2¤Òmî•„ €½ÖŽÂ‹-J{{{zzzÊ5…B!õõõå@KKËV[ŠÅbŠÅbªªª*8»Û°aÃ2sæÌÌœ93ÿùŸÿ™ë®».]tQ>÷¹ÏåüóÏÏG?úÑŒ?¾ÒmîU„ €=jýúõY¶lÙvƒ¥R)Ï?ÿ|6nÜX®+ ý‚­­­[Æ—úúú NÀÞꤓNÊI'”… fÖ¬Y™5kV¾øÅ/æø@>õ©OeòäÉ•nq¯ d0gÎœ{ì±¹üòËsÅWTº€]¢««+; ,^¼8}}}åºúúú‹Åòö¶¶¶­Â1bD§`1~üø|þóŸÏßþíßæöÛoÏ¿øżùÍoN[[[.½ôÒœvÚiôæ;!`»ºººvX´hQ:;;ûÕm l ´´´lµu ±±1uuuš €Ù AƒrÎ9çäœsÎÉý÷ߟ«¯¾:ï}ï{3yòä\tÑE9÷Üs3xðàJ·¹Ç Àª³³3¥Ri»Û,XU«V•k ”Ñ£G—·‹Å´¶¶nhjjJm­ËØ7L›6-Ó¦MË#<’üÇÌE]”+¯¼2_|q>ò‘PÛõü4û™MáímèèèȆ Ê5…B¡_p ¹¹9mmmý‚Åb1cÇŽMuuu§€Ýç-oyKn½õÖ|éK_Ê 7Ü+¯¼2Ÿÿüçsá…æÒK/M}}}¥[ÜíüÔÀnñãÿ8UUUùÊW¾’{ï½7o{ÛÛ2|øðL™2%Iòõ¯=UUUùÎw¾³Uí¦÷¾÷½ï x¾_þò—9餓2tèÐŒ=:çž{n–/_þªúìëëË7¿ùÍœxâ‰5jT†žc=6_ûÚ×ÒÓÓ“‡z(UUUùØÇ>6`ýwÞ™ªªª|éK_Úés›nº)S§NÍðáÃ3dÈuÔQ¹öÚkÓ××÷ªföÝÝÝ)•J™?~fÏž›o¾9W_}¶,¤ç IDATu.¾øâ̘1#Ó¦MËÞð†ÔÕÕ¥¡¡!GydÞùÎwæÿïÿY³feîܹéêêJKKKÎ>ûì\sÍ5ù·û·Ì™3' .ÌÚµkS*•2gÎœüà?È 7Ü+®¸"3gÎÌ駟žÖÖÖŒ7NÀ€B±XÌW\‘gžy&ŸúÔ§2kÖ¬477çŠ+®Hggg¥ÛÛ­l2`·zðÁsÉ%—”/®ïíí}Mçûõ¯Ë.»,ëׯO’¬]»67ß|sž}öÙÜ{ｯè\}}}ùà?˜;ßësæÌÉœ9sr衇fúôé9öØcsóÍ7窫®Ê°aÃú}öºë®ËСCsþùç¿¢sn«Ÿ³Ï>;·Þzk¿×ó›ßäcûX{ì±Ìš5ëÍìýÖ¯_ŸeË–mwë@©TÊ’%KúýLU(úmØØ|ëÀøñã3jÔ¨ Nû¶Q£FåsŸû\>ñ‰OäÚk¯Í¾ð…üã?þc.ºè¢|úÓŸNCCC¥[Üå„ Ø­¾ýíoç¼óÎË¥—^š7¼á ©©©yMç»í¶ÛrÁä“ŸüdÆŸGy$ÿëý¯Üwß}yì±ÇrÔQGíô¹nºé¦ÜqÇ=ztþîïþ.ïyÏ{ÒÐÐßþö·¹á†RWW—$ù›¿ù›œ}öÙ¹å–[òÑ~´\ÿä“OæÞ{ïÍG?úÑòE;;{Î|ë[ßÊ­·ÞšÉ“'çꫯÎqÇ—Áƒgîܹù›¿ù›|ík_ˇ?üáœp ¯òOØ“ºººÊÎÎÎm†/^ÜosY}}}¿ @ssóVᦦ¦ >¼‚ÓÀeØ°a¹ôÒKsÑEåÆoÌ?üÃ?äÚk¯Íyç—Ï|æ33fL¥[Üe„ Ø­Ž?þø|ýë_OUUÕ.9ß)§œ’믿¾ü¼­­-Ÿþô§Ëwú%!ƒù—I’Ü~ûíý¶ L™2%S¦L)?Ÿ1cF.¹ä’\wÝuýB×]w]’ä¯ÿú¯_ñ9òo|#555ùÉO~’b±X~ýÄOÌm·Ý––––|ÿûß2€ Û<<°­c©TÊŠ+úÕmhiié÷|ܸqillÜn8¨¬aÆåâ‹/Î_ýÕ_åšk®É—¿üå|ãßÈÇ?þñ\rÉ%[mFß °[MŸ>}— ’䤓NÚêµæææ$ÉêÕ«_ѹ~÷»ß¥¾¾¾_` ƒ Ê\+®¸"÷Ýw_N<ñĬY³&·ÜrKN9å”qįøœ™?~6nܘÆÆÆ$I___ùn¦›Žííí¯ø¼ÀÎéìììhû@GGG¿Ÿ= ”Ñ£G÷Û:ÐÖÖ–úúú~ᦦ¦ÔÖú'zØ_ 6,—]vY>ö±åšk®ÉÕW_n¸!—_~yÎ?ÿü}ú&~ƒÀn5zôè_¯®®N’ôöönõ^WW×6Ï7dÈ­^ÛbØt!þîpÁäïÿþïsÝu×åÄOÌ-·Ü’U«Våâ‹/ÞeßcÓŸÅÆ·ù™îîî]öýà@±ex` cGGG6lØP®) ý‚›Â›ŠÅbÆŽ[þù8ðl Ìœ93_øÂò‰O|"_ùÊWrå•WæþÏÿ¹Koȶ§P¯ýë“$Ï<óÌVïÝsÏ={¤‡7½éMùÅ/~‘ŸýìgyÇ;Þ±ÝÏŽ3&ïÿûsûí·gñâŹþúësØa‡åÔSO}Õ稟Gy$¥R)#GŽ|ÅóÀ¤»»;K—.Ý*0°åööööôôô”ë …B¿ @kkëV[ŠÅbŠÅâ>yP ¹êª«rÑEåÊ+¯Ì>ðL™2%_üâsâ‰'Vº½WDÈ€Š˜4iR’ä+_ùJŽ;î¸wÜqY²dI¾úÕ¯æ{ßûÞéáÜsÏÍ/~ñ‹|ðƒÌßýÝßåÔSOM}}}~÷»ßå†nÈ_üÅ_ämo{[ùó_|qn¹å–œwÞyyüñÇóÏÿüÏ[]tôJϹ¹¿ú«¿Êý÷ߟéÓ§çòË/ÏqÇ—‘#GfÑ¢Eyâ‰'rÓM7å /|ÅáØ—¬[·.Ë—/ßîÖR©”%K–ôÛŒ6Px`ËàÀ„ z€Ýª±±17ÜpC>ò‘äÓŸþtÞö¶·åÌ3ÏÌÕW_7¾ñ•no§PÍÍÍ9óÌ3s×]wåä“O.¿^[[›³Ï>;·ÜrËnïáÃþp~üãç;ßùNfÎœ¹Õû3fÌè÷¼µµ5S§NÍÝwß#FäCúÐk>çæÎ=÷ÜÜ{ï½ùæ7¿™ÓO?}ÀÏ|ä#ÙÁT°wêêêÚáÖR©”Å‹§¯¯¯\W__ß/(ÐÒÒ’b±ØoûÀĉ3lØ° NÐß”)SrÏ=÷äßÿýßóéO:“'OÎ¥—^šË.»,…B¡Òím—sã7fÔ¨Qùþ÷¿Ÿ_|1ÇsLþáþ!O=õÔ TWWçÎ;ïÌ׿þõÜtÓMyüñÇSSS“7½éM™9sfN:餭j.¸à‚<øàƒ9ï¼ó2|øð]rÎMªªªòo|#§žzj¾öµ¯eîܹyñÅ3a„¼ùÍoι瞛éÓ§ïÂ?xí ly\¸paV®\Ù¯n[áÍ·466¦®®®B“¼vïyÏ{rÊ)§äÚk¯Íç>÷¹ÜrË-ùêW¿š÷¼ç=•nm›ªú6¿°ÇÍš5kÀ»{§üãù§ú§<õÔSyÃÞPév`¿âgdØ»tvvØ|û@{{{Ö¬YS®0qâÄÔÔÔTp:€=oÑ¢E¹ôÒKsË-·ä´ÓNË5×\“‰'Vº­­Ød;aãÆ™={v®¿þúœxâ‰쳶ØüØÑÑ‘ 6”k …B9 P,ÓÜÜœ¶¶¶­BcÇŽMuuu§Ø{‹ÅÜ|óÍùð‡?œ‹.º(“&MÊ%—\’Ï|æ34hP¥Û+2`¿òè£æ˜cŽÙáçÞûÞ÷æ{ßûÞNóŠ+®Èç?ÿùòóK/½ôU÷»Ãúõë³lÙ²m6mxî¹ç²qãÆr]¡PèhmmÝæöv“O>9óæÍËÕW_«®º*wÝuWþå_þe§þsO2€4nܸ|êSŸÊ©§žZéV8@¬[·.Ë—/ßîÖR©”%K–¤···\·ex ¹¹y«ð@cccFŒQÁé\ƒÎç>÷¹üÅ_üEÎ?ÿüwÜqùÜç>—Ë.»,µµ•½Ì¿ª¯¯¯¯¢ÀnÖ¬Y™9sf¥Û€Šó32’®®®6mØ\}}}¿ À@[9ä :´B“ðJõõõåk_ûZ>þñçÈ#ÌÍ7ßœÿñ?þGÅú±É`Ù™ðÀÂ… ³råÊ~u[†Z[[· 455Uü.vìzUUU™9sfÞúÖ·æœsÎÉ1Ç“Ë/¿<—\rIª««÷x?~°;Ü:ÐÞÞž5kÖ”kœ†††rP ¹¹9mmm[m3fLjjj*8{ƒ#Ž8">ø`®¼òÊ|ö³ŸÍìÙ³sÓM7¥±±qö!d°vX´hQÚÛÛÓÓÓS®) ©¯¯/‡ZZZ¶Ú:P,S,SUUUÁéØ×ÔÕÕåóŸÿ|N;í´òVƒÛn»-§œrÊëAÈد¬_¿>Ë–-Ûnp T*åùçŸÏÆËu…B¡_P µµu«àÀ¸qãR___Áé8{ì±™7o^.¸à‚¼ç=ïÉg?ûÙ\~ùå{ä¦BÀ>¡««+; ,^¼8}}}åºúúú‹Åòö¶¶¶­Â1bD§€þ …B¾ùÍofêÔ©ùØÇ>–yóæåæ›oÎÈ‘#wë÷2*ª««k‡ÁE‹¥³³³_ݦðÀ¦ @KKËV[SWWW¡É൛9sfŽ8âˆÌ˜1#Çw\îºë®Lš4i·}?!`·èììL©TÚîö dÕªUåšAƒeôèÑå­Åb1­­­[…šššR[ëŸ;80¼õ­oͯ~õ«üùŸÿy¦Nšo}ë[9í´ÓvË÷ò[7àÙØÞÖŽŽŽlØ°¡\S(úš››ÓÖÖÖ/8P,3vìØTWWWp:Ø;566æ¾ûîË…^˜÷½ï}¹æškrÁìòï#d¤»»;K—.ÝîÖE‹¥½½====åºB¡PÔ×ק¥¥%Ó§Oß*°xñâôõõ•ëêëëûš››· 455eøðáœØ‘øÃ=ztf̘‘îîî|ýë_OMMÍk>¯ìE6lëX*•²bÅŠ~u[†ZZZú=7n\SWWW¡É€]íŒ3ÎÈÿûÿ/gžyfº»»sóÍ7¿æ ìý‚mèèèÈêÕ«Ë5ƒ ÊèÑ£ûmhkkK}}}¿ð@SSSjkýÓˆN=õÔüà?È駟ž¡C‡æ†nHUUÕ«>Ÿß4Àk°ex` cGGG6lØP®) ý‚›Â›ŠÅbÆŽ›êêê Nì ¦OŸž;ï¼3gžyfrÕUW½ês Àº»»³téÒ­[nhooOOOO¹®P(ô ´¶¶nµu X,¦X,¾¦;‰léôÓOÏM7Ý”sÏ=7ãÇÏ_ÿõ_¿ªóp@éééÉêÕ«³råʬ]»6/¾øbV®\™5kÖäÅ_Ì‹/¾˜+VäÅ_ÌúõëÓÕÕ•uëÖeãÆYµjU’dåÊ•éííð½$å×Ò××—+Vl³¿ƒ:(ƒ𽚚šŒ1¢ü¼P(dÈ!ý^1bDjjj¶z¯ºº:#GŽÌ°aÃ2tèÐ :4£FÊСCsÐAeĈ1bD†š!C†¼â?WØW¬[·.Ë—/ßîÖR©”%K–¤···\7Px`ËàÀ„ 2räÈ NèÎ>ûì,\¸0ŸøÄ'røá‡ç]ïz×+>‡ûœÞÞÞ<ÿüóY¾|y–/_žÎÎÎ[¾¶jÕªtwwoó¼uuu6lXFŽ™ƒ:(C† ÉàÁƒsÐA¥ªª*£FJ’Œ?>uuu¾—$µµµ>|ø6¿ÏðáÃS[;ð¯è7²nݺtuu•Ÿ¯]»6ëׯφ ²fÍš$ÉŠ+Ò××—eË–¥»»»üÞ¦ ĪU«²víÚ¬]»v›ýmšgĈihhH}}}ÊMÏ7?Ž=:|p …Â6Ï »SWW×·”J¥,^¼8}}}åºúúú~A–––‹Å~Û&Nœ˜aÆUp:€wÙe—åÉ'ŸÌ>ð<ôÐCyÓ›ÞôŠê… Ø+ôööfñâÅY²dI-Z”^x!K–,ÉâÅ‹ó /”ß{þùçó /lu!þ!C¶º~ÓÛ^1bD:è  >¼|×þƒ:¨|GÿAƒUhúÊèìì,osX½zu9€°i›ÃªU«ú…5ž~úéÌ™3§ü|åÊ•[sĈ)‹9øàƒ3f̘~_;6|pÆŽ›qãÆ $°S ly\¸páV·Ø|û@cccêêê*4Àîóµ¯}-'tRf̘‘‡~øm1®êÛüV-À7k̜֬9³ÒmÀn·lÙ²ttt¤££#íííéèèÈ‚ òÜsÏ¥££#¥R)6l(~È!yýë_¿ÕEê¯ýëË_=º(pÁúž·qãÆò¦ˆeË–  Ùüë+Vô«3fL3a„455¥©©)&LHccc&Nœ˜±cǦ¦¦¦BÓ±»uvvnøéOšñãÇ—_koo/oéH’Áƒ§¡¡¡_P`óã¦í'Nôw8àuttäè£ÎŒ3rýõ×ïtT˜û‹žžž<÷Üsùãÿ˜?üáåãþð‡<óÌ3éêê*öu¯{]¿‹É'L˜ &dâĉå`ÁðáÃ+8 »Ãúõëó /¤T*•/ ß<Ù8Y´hQ6nܘ$©­­ÍøñãóÆ7¾1oxö::´Â1Â[;::ú…Š …Bêëë3hРLžø`w8çœsrÏ=÷dþüù9rä?_»z`´jÕª<ùä“å@ÁüùóóÄOdñâÅI’úúú¼éMoÊa‡–©S§ö»Ë|CCC…»g_U[[[Þl1uëÖå駟.‡6m͸çž{òÜsÏ%I†šI“&åÈ#LKKK&OžœI“&móœ»Û’%KòÏÿüÏùêW¿š?ýÓ?Í¿þë¿V¤¬[·.Ë—/ßîÖR©”%K–¤···\·ex ¹¹y«ð@cccFŒQÁéH’/ùË™4iRþïÿý¿¹îºëvøy!²zõê<òÈ#™3gN~ýë_gÎœ9ùãÿ˜$6lXŽ8âˆLž<9ï~÷»3yòä´´´düøñîšQ¡PȤI“2iÒ¤­Þ(s÷Ýw—ƒ1 ™2eJŽ=öØòqwþ=þÃþ/}éKùÆ7¾‘¾¾¾lØ°!ííí»íûm®««k‡[6½¶¹úúú~A–––­¶Lœ8ñÿgïÎ㪬óþ¿ûŽdS<©€“‚Ž[†…¥ÜnYj·år;Š6cÙ2.Mõ˜¦»\Z4÷enµEš¹]¨T,q\rµ\!MEÖóûc~ž[K9¨¯çãq\纾×÷|¾×9øxÈù¾¯¯ÜÜÜêe€_¯I“&š5k–FŽ©ñãÇ«]»vµ¶7˜¯­) À*/^¬ØØXk—€ûHyy¹8 ={öXGUee¥|||,°Û·o¯°°0µlÙRƒÁÚe¿Ø¹sçtøða}ÿý÷– ÍñãÇUYY)__ß*¡ƒnݺÉÝÝýW½ÞÁƒõÁhõêÕ²±±QYY™åX‹-têÔ©_Üwmák?³²²táÂ…*çý<LàêêzGÇ ¸½üòËzòÉ'uäÈ‘oHEÈîR•••Ú¼y³þö·¿iÆ ÊËËSHHˆ  þýû«K—.²µµµv™À]ËÇÇGC† Ñ!C$IiiiZ¿~½V¬X¡?þ¸JÛëW"¸™3fèᇾ­µPýúõSpp°,X 9sæTÛƦžküJ§OŸÖ[o½¥   ÅÄÄèÈ‘#zõÕWuäÈ;vLï½÷žºwïNÀ¸Í‚ƒƒõòË/ëàÁƒÊÎÎÖ믿®ÐÐÐ*¿kvvvµþî åääÔG¹ÜÀÆÆFÇך5kj\—Ü***ôùçŸë‰'žÉdÒ¢E‹4lØ0=zT»víÒ”)SÔºukk— Ü7|}}õÎ;ïèðáÃ:wîœâããÕºuk•——ËÉÉIŽŽŽ’${{{ÙÙýßÂÒvvv„ Võì³Ï*77WÛ¶m«ö8!hÀJKKµxñbµnÝZÏ<óŒìííõÅ_觟~ÒŒ3bíûž§§§þð‡?èÈ‘#:pà€þë¿þKÎÎÎrqqQ÷îÝÕ«W/FIRYY!€U+<<\ Õ·«v/ÀªÌf³¾øâ ½öÚkÊÈÈЈ#ôÕW_©U«VÖ. @-zè!Í;WÓ§O×¢E‹4kÖ,]¸pA¯¾úªú÷ï¯}ûöÉÙÙÙÚeîs={öÔöíÛ«=ÆJÐÀdggkàÀ2dˆÚ·o¯#GŽhñâÅõ0ÈÌÌ”Á`¨òøè£îøëÖdéÒ¥–:.\ø«úš:uª¥¯¯¿þú6Uøe0tøðáÛÞwM’““Õ­[7¹ººÊ`0(  Þ^ûVT÷¹²µµ•———ºwï®÷Þ{O/^¬ñü””9RrttT@@€yäMŸ>]ǯÒ699YO>ù¤äèè(“ɤØØX%%%Él6ßé¡Z¸¹¹éÕW_Õ?þ¨wß}WsæÌQÿþýåëë«#FÔ[T'**Jßÿ½.\¸pÃ1BЀlØ°Amڴщ'´sçN­]»V-[¶¬·×ÙlÖÈ‘#åêê*³Ù¬—^z©Þ^ÿçÆŒ£âââÛÒ׌3täÈ‘ÛÒWCñì³ÏÊßß_gÏžUjjª\\\¬]Rµªû\•––êСC=z´æÍ›§víÚéСC7œ»zõjEDD¨Y³fÚ¾}».]º¤äädýþ÷¿×‚ Ô®];KÛ¨k×®òõõÕ®]»téÒ%}óÍ7rssSTT”’““ësØ’$Mœ8QÇŽÓÃ?¬è•W^QEEE½×À5:tPEE…RSSo8FȈyóæiàÀ:t¨öï߯.]ºX»$4`W¯^Õ±cÇ-777µiÓ憻ú7d¶¶¶ò÷÷×èÑ£µwï^IRLLŒ.]ºdisøða5J¯¼òŠfÎœ©–-[ÊÁÁA>>>:t¨¾üòKÙØüßWýë_åèè¨øøx5oÞ\ Ò¬Y³^ïc¼ž···V¯^­?þX .TÿþýURRbÕš÷¯ÀÀ@9;;WûÝ!hV­Z¥^xAï¾û®/^,'''k—Tg›6mRçÎåìì¬ÆkøðáÊÉÉ©Ò&??_“&MR«V­ää䤇zHëÖ­«¶¿yóæ©E‹rqqQÏž=uâĉ_TK`` ž~úiíÙ³§Æö/½ô’ ƒ ƒ>,IúüóÏ-û>þøcKÛ´´4 0@Mš4‘»»»¨ï¾ûN’4uêTõêÕK’.ƒÁ    ˹7nTdd¤œœœäãã£ñãÇëâÅ‹’¤øøxËëÅÇÇëù矗——— ƒžyæ™j뎗³³³$iܸq2 êÝ»wûºÙ{v}? .Ô‹/¾(jÙ²e*++Ó„ äéé©-ZhÙ²eu~ªãíí­iÓ¦)33S .´ìŸ6mß êÀ IDATš*++5iÒ¤jÏ Ó•+W,Ï‹‹‹UVVVeß5?üðƒ"##U·Ã³Ï>«¤¤$íÚµKÆ “Ùl¶vI€ûL&“ÒÒÒnûL&LÐرc•••¥Y³fiòäÉ·TK¯^½”™™©={öÈÑÑQ=öXç|ôÑGÚ²eK•}O?ý´òòònhûÔSOÉÙÙY©©©ÊÈÈPPP¢££%I3fÌ°ôsèÐ!™Íf:uJ’´nÝ:õïß_111ÊÉÉÑæÍ›•””¤'Ÿ|Rf³Y&L°\«?üP>ú¨2224wîÜë¾þœ%K–Èl6ë믿®S_uyÏ®ïgáÂ…êÓ§²²²ôÌ3ÏhìرŠÕO<¡ÌÌL=óÌ3?~¼NŸ>]ó›S}ûö•Á`ÐW_}eÙ÷õ×_+,,LF£±N}têÔIW¯^Uß¾}µ}ûö;¿cÇŽÚ°aƒÖ¯_¯E‹Y»À}Êh4ª°°ð†ý„ ÀÊfÎœ)½óÎ;Ö.å–ýñTÛ¶mõî»ïªI“&jݺµ/^¬“'Ojþüù–v}ô‘>úè#5nÜXnnnzöÙgõøãß0‘þ­·ÞÒC=¤7ÞxCF£QíÛ·Wlllk Õÿ÷«qãÆòóóÓ¢E‹äèèø«ÇyõêU}ÿý÷4h¼½½Õ¨Q#}ðÁruu½é¹“'OVhh¨Þ~ûmFýæ7¿Ñû￯o¾ùFß~ûm•¶ÑÑÑ}äîî®—^zIf³YeeeêׯŸe_yy¹víÚõ‹k•$yyyY /^TAAš5kVç>F¥Aƒ)))I<òˆüýýõ‡?ü¡ÖÕ,¬¥{÷îš8q¢Þ~ûm•••Y»À}ÈÝݽÊM£®!dVöÅ_hìرrpp°v)·$33SÇŽÓ#ßÞÞ^_|ñ…¾ýö[=÷Üsº|ù²æÏŸ¯Î;ë‰'ž¨önLÖô‡?üA999¿: À/áêêªË—/ß°ŸXÑ¥K—”ŸŸ¯¶mÛZ»”[–ŸŸ/éßaŸkܸ±å¸$¥¦¦jРAjÖ¬™llld0´bÅ XÚäääTÛ_uýßJ-·Ë¦M›4pà@½ôÒKjÔ¨‘þã?þã¦wÈ¿VW||¼ ƒåáíí-Iú駟ª´wvv¾mõV××­¼g׸¹¹Y¶mlljÜWYYù«ê½pá‚ Ô¢E Iÿ^ÙÀh4Z>·"**J«V­Rnn®Ö­[§ž={jóæÍ nµ–-[ÊÕÕUéééÖ.pªéF?„ ÀŠ\]]åäät[î_ßš4i"éß+üܹsç,ÇËÊÊ­ŒŒ mÛ¶Meee2›Í9rd•;×ûúúVÛß… ~U-7sm’|YY™eßÅ‹ohg4õá‡*++KIIIºzõªzôè¡´´´›Ö5yòd™Íæ«V­ºåzº¾gÖ°~ýz™ÍfÅÄÄXöÅÄÄèðáÿxGGGõïß_›7oV`` þõ¯Ý®ro‹¢¢"]¹rŪ×€Ÿ#dVdcc£=zhÍš5Ö.¥Îz÷î­mÛ¶) @>ø ¶mÛVåøÁƒuñâEEGGK’ÒÓÓ•““£¡C‡ªuëÖ²µµ•$•””T9ÏËËK!!!Ú±cG•ýû÷ï¿iM×jIJJª²?;;[NNN:wî\ç6mÚT’tæÌ˾~ø¡J›3gÎ(<<Üò¼sçÎZ²d‰JKKµoß>IÿVøy]­[·ÖÞ½{o8ö›ßü¦Þß÷º¾gõíìÙ³zýõרqãÆYö¿þúë²µµÕ|Píy³gÏ–§§§%4ñÆoèÍ7ß¼¡ìííÕ¸qã;3€_híÚµrppP—.]¬] „ ÀÊ^yå}ùå—Ú°aƒµK¹e|ðŽ9¢×_]çÎÓ±cÇ4nÜ8µjÕJÏ?ÿ¼$)((HM›6ÕªU«”ššª«W¯jóæÍúꫯnèï­·ÞÒÁƒõÎ;蘒 @?üðƒÞ{ï½:×’ššª7ß|SçÎÓéÓ§5zôh9²ÖÉå!!!òööÖüùóuþüy;vL+V¬¸¡ÝáÇ5{öl]¼xQ………Z´h‘œœœÔ±cGIÿ·ÃÑ£G•ŸŸ/___íÛ·O~ø¡¶oß®3f(??_ùùùzå•WT^^®Ôil·S]Þ³úPQQ¡¬¬,-_¾\;v”äîîniÓ¦M}üñÇš5k–^{í5:uJeee:}ú´Þ}÷]ýéOÒ¼yóäååe9'..NŸ|ò‰òòòTZZªüQ/¾ø¢N:U¯ã»™sçÎé7ÞЈ#XÉР2+{â‰'4zôh >ÜrW|kÉÌÌ”Á`Њ+tùòe †›6m²´ïÛ·¯”˜˜(uéÒE<ð€¶oß.I’£££äéé©ßþö· Ñ_|¡^½zéûï¿—Á`°¬"ðŸÿùŸš7ož–,Y"___7NÓ¦M“$=ÿüóêܹsµ_«eóæÍò÷÷W×®]ª¸¸8IÒÔ©SÕ¦MIRŸ>}ôÜsÏYêûä“O”žž®€€7No¼ñ†$iøðáêÝ»·š5k¦7ꫯ¾’ÉdR‹-´k×.%$$Èd2Iú÷„øßÿþ÷3fŒ‚ƒƒ5hÐ EFF*&&F_~ù¥Ö­[§€€………éìÙ³Ú´i“œœœ´zõjËÄú±cÇÊ`0¨°°°Ö÷)>>þ†svìØQ§¾êòžý¼Ÿçž{N‰‰‰²··—$½ð zú駵mÛ6 IÒË/¿\ãJÕ}®ìíí®eË–i„ úᇪ¬qÍSO=¥}ûö)++K]»v•›››ºuë¦ýû÷+11Ñò>^{çÌ™£?þX;v”›››"""tôèQ}ýõ×züñÇk½®õ¥¨¨Hýúõ“ƒƒƒÞÿ}k—@³Ùl¶vÀýlñâÅŠµv°²ÒÒR 4HÛ·o×Ê•+5pà@k—àÈÊÊÒ“O>©S§N)))É|Á¿ñd¨?C† ‘$­]»¶Ê~V2€ÀÁÁA_|ñ…ž~úi=õÔSúãÿ¨ââbk—à6Z»v­Ú·o¯ââbíÞ½›€ A"d „£££–/_®%K–héÒ¥j×®þþ÷¿[»,¿Ò‘#GÔ¯_?=óÌ34hvïÞ­àà`k—@µ@3zôh¥¤¤¨cÇŽzúé§Õ½{wmÚ´ÉÚe¸Eiii;v¬Úµk§ŒŒ }óÍ7Z¸p¡ÜÜܬ]5"d ¿¿¿>ýôSíÙ³GnnnêÝ»·"##µbÅ ]¹rÅÚå¨ÙlVRR’† ¦Ö­[ëÛo¿ÕÒ¥Kµÿ~EEEY»<nŠ4`;vÔ×_­}ûö©U«VŠ•¯¯¯Æ¯½{÷Z»<ÿ_NNŽf̘¡|PQQQ:yò¤V¬X¡cÇŽiäÈ‘²±á+ÀÝÁÎÚn.""B«W¯V~~¾V­Z¥eË–iÑ¢Ej×®†®ªU«VÖ.¸¯êË/¿ÔêÕ«õÕW_ÉÃÃCÏ=÷œ>ÿüsµk×ÎÚåð‹pÛ¸‹4iÒD/¿ü²>¬Ý»w«S§Nš>}ºxà………éOú“öìÙ£ÊÊJk— Ü“~úé'ÅÇÇ«W¯^òööÖ¨Q£TZZªU«V);;[qqq w5V2€»TçÎÕ¹sg-X°@;vìкuë´fÍMŸ>]¾¾¾êׯŸ}ôQEEEÉÇÇÇÚåw¥ââbíÞ½[ß~û­tàÀyzzªwïÞZ±b…úôé£FY»LnBp—³³³STT”¢¢¢4{öl:tHëׯWBB‚–/_®òòrµiÓÆÒæ‘G!tÔàZ¨ ))Iß~û­þõ¯©¤¤D­ZµÒã?®3f(**JÖ.€;‚ÜcÂÃî×_]EEEÚ±c‡¶mÛ¦¤¤$-Y²DåååjÛ¶­ºt颎;ªcÇŽ —½½½µKêÝ?þ¨½{÷jïÞ½Ú³g%T¬¨¨(;V={öT@@€µK ^2€{˜›››z÷î­Þ½{K’ŠŠŠôÏþSIIIÚ³gÖ®]«K—.ÉÉÉI¿ùÍoÔ±cGEFF*22R­[·–­­­•GÜ>999Ú»w¯öíÛg œ;wNvvv U§N4vìXEEE)00ÐÚå`„ à>âææ¦>}ú¨OŸ>–}ÙÙÙÚ¹s§vìØ¡ääd-[¶LÅÅÅrppP«V­ª¶mÛZ~¶iÓF666VP» .èäÉ“JIIQjjªRRR”œœ¬œœI’¯¯¯"""4iÒ$uëÖM:t‹‹‹•« a d÷9??? ¬ÔÔT%$$Xîo0¨àà`µjÕÊò–Éd’»»»5†ƒ»Xiii• ÁÉ“'-ôôt•””Húw˜àPXX˜zõꥰ°0µmÛV-Z´°ò¸û2ÔI£FÔ½{wuïÞ½Êþ’’eee)==]éééJIIQjjª¶nݪӧO«¢¢B’äää$???™L&ùúúÞ°Bá>SPP ôôtegg+''ç†íë??F£Q&“I&“Iýû÷·l›L&µlÙRƒÁÊ£àÞ@Èð«8::Z&{ÿ\II‰ÒÓÓõã?*##C™™™ú駟”‘‘¡ï¾ûNºzõª¥½···¼½½Õ¬Y35kÖLM›6•¯¯¯|||Ô´iSùûû«iÓ¦òöö–­­m}uTPP 3gÎ(//OÙÙÙÊÍÍUnn®²³³•——§³gÏ*''GgÎœQyy¹$ÉÖÖVÍš5SóæͨvíÚ©oß¾ TóæÍ,£Ñhå‘p d¸cÕ¦MµiÓ¦Æ6¹¹¹ÊÌÌTff¦NŸ>­ÜÜ\åää(77W'Nœ°LN¿>Œ`0äíí-///yyyÉÎÎNÍ›7————ŒFc?ÝÝÝåìì\C¿«•——ëÒ¥K*((PAAΟ?_íÏë·Ï;§¼¼<•””Xú±µµUÓ¦MÕ´iSùùùÉÛÛ[!!!òóó“ŸŸŸ%Dàçç';;¾² !à/ö«º¶zA‡jmwáÂåääX­íÛ·kûöíÊÏÏWtt´ÒÓÓ«L€¿~Âû5ƒA5’›››\\\äææ¦FÉÅÅE...òôô”»»»\\\äêê*[[[yxxH’<==ecc#ggg999ÉÎÎNîîîUŽ]ãááQãj µ»xñ¢***j¼•••7´½zõªŠ‹‹UYY© .T{¬¬¬LEEE*,,Ô•+Wtùòe]¼xQ—.]²B€»‚§§§<== eË–iöìÙÊÈÈPLLŒ^ýuuîÜù†s®Mœ¿<¸6¡¾°°PEEEº|ù²._¾¬ÂÂBËvzzº.^¼¨Ë—/«¸¸X¥¥¥º|ù²$©   ¾‡ý‹¹ººÊÁÁAUÂruu•«««L&“ÜÜÜ,ÏF£%\áîî^%PШQ#k ÔB€»ÂÙ³gµ`ÁÍ;W¥¥¥6l˜^}õU…„„ÔxεÉó·µ–Ë—/«´´´ÆÂõ+ ü\mÇ$ÉÅÅEŽŽŽu:æææ&{{{K@’ŒFã-àB€íĉŠ×âÅ‹åáá¡^xA/¾ø¢¼¼¼¬VÓµðÀ½† AÚ±c‡fÎœ©„„kÆŒŠ•³³³µK¸gÙX»®©¬¬Ô† Ô¹sg=üðÃ*((К5ktôèQMœ8‘€ÀÆJ«+**Ò§Ÿ~ª>ø@iiiŠ‰‰Ñîݻչsgk—p_!d°šÜÜ\ÍŸ?_sçÎUII‰ž}öYmܸQ!!!Ö. à¾DÈPïNž<©¹sçjÉ’%rww× /¼ ^xA7¶vi÷5B€z³cÇÍœ9S ÖôéÓ+gggk—I6Ö.po«¬¬Ô† Ô¥K=üðÃ*((К5ktôèQMœ8‘€@ÂJ€;âòåËúä“Oôá‡êäÉ“Š‰‰Ñ®]»Ô¥Kk—€2ÜV¹¹¹š?¾âããUTT¤!C†hýúõzðÁ­]n‚à¶8yò¤æΫ%K–ÈÍÍM&LÐ /¼ Æ[»4Ô!À¯²cÇÍ™3GÿûߤéÓ§+66VÎÎÎÖ. ·ÈÆÚî>•••Ú°aƒºvíª‡~XéééZ¾|¹Ž;¦‰'0¸K±’ ÎJJJ´fÍM›6M'NœPLLŒvîÜ©®]»Z»4Ü„ 7•››«ùóç+>>^EEE2dˆþ÷ÿW­[·¶vi¸j”––¦9sæhÉ’%rssÓ„ 4aÂ5iÒÄÚ¥à d¸Arr²âââô駟*((HÓ§O×رcåââbíÒpÙX»@ÃPYY© 6¨[·nŠŒŒTjjª–/_®cÇŽiâĉ î¬d÷¹’’­Y³FÓ§O×ñãÇ£;v¨[·nÖ. õŒܧòòò4oÞ<Í›7O—.]Ò!CôüC­[·¶vi°BpŸIKKÓœ9s´téR¹¸¸èw¿û&Nœ(___k—+#d÷‰äädÅÅÅéÓO?U‹-4mÚ4;V...Ö. „µ Ü9•••Ú°aƒºwï®ÈÈH¥¦¦jùòå:~ü¸&NœHÀU2€{PII‰V®\©ÐÐP 8PF£Q[¶lѾ}û4bÄÙÚÚZ»D4@vÖ.pûäååiùò劋‹Óùóç5dÈýýïW›6m¬]î„ àžž®¸¸8-]ºTööö9r¤¦L™"???k—€»!¸‹%''+..NŸ~ú©š7o®iÓ¦i̘1ruuµvi¸ 2€»Lee¥4gÎ%&&ªC‡Z¾|¹† &;;þì €_ÎÆÚꦤ¤D+W®TXX˜  '''mÙ²EÉÉÉ1büj|ã \~~¾–-[¦9sæ(??_C‡Õ矮¶mÛZ»4Üc@•žž®¸¸8-]ºTööö9r¤&Ož,k—€{!h`öï߯>úHŸ}ö™5mÚ43F®®®Ö. ÷8BÐTVV*!!AsæÌQbb¢Ú·o¯eË–iØ°a²³ãO¹¨6Ö.îg¥¥¥Z¹r¥ÂÃÃ5`ÀIÒúõëµÿ~1‚€êßN€\¸pAÿó?ÿ£÷Þ{Oùùù:t¨Ö®]«ÐÐPk—€û!¨G?þø£>úè#-[¶L¶¶¶5j”&Ož,k—2€úpàÀÍž=[Ÿ}ö™ôæ›ojüøñòôô´vi€!¸CÌf³¶nݪ¸¸8mܸQí۷ײeË4lØ0ÙÙñçY4<6Ö.î5¥¥¥Z¹r¥ÂÂÂôøãëêÕ«Z¿~½öï߯#F0@ƒÅ7Yp›\¼xQýë_õþûï+//OC‡ÕÚµkjíÒ€:!d¿Ò©S§´páB-\¸Pf³Y£FÒ¤I“`íÒ€[BÈþ¿³gÏÊËËKöööujàÀÍž=[Ÿ}ö™ôÚk¯iܸqjԨѮ¸3l¬]4™™™zøᇵvíÚZÛ™Íf%&&ª_¿~êСƒ:¤eË–éĉš2e Üո不§«K—.:qΩOŸ^m›ÒÒR­\¹RáááêÕ«— ´~ýzíß¿_#FŒ ÇàîGÈÀ}íرcêÚµ«Îž=+IJIIQbb¢åøÅ‹§àà`;Ö²zÁŽ;Ô¯_? k•ÜvÜZ À}+55U<òˆ U^^.I²³³ÓŒ3ÔªU+-\¸P .”ÙlÖ¨Q£4iÒ$X¹jàÎ!dྔœœ¬Ç{LEEEª¨¨°ì///×Ö­[ÕªU+ùûûëÏþ³ÆŒ#www+V ÔBnInn®233•‘‘¡Ÿ~úIÊÎΖ——— æÍ›+00P~~~²···vÉ7رc‡z÷î­’’’*ƒkìííÕµkW%&&ÊÎŽ?£àþÁ·c, •™™©Ó§O+33S™™™– Áµ`ÁÕ«W-í›6mª€€ùûûëÔ©SZ·n²³³U^^.I²±±Q³fÍ,¡ƒë ”¯¯¯ C½qÛ¶mŠ‰‰QiiiµI*++ÓŽ;”­æÍ›×[m€µ2î%%%ÊÊÊRvv¶rrr”žž^e;--M………–öNNNòóó“ÉdR@@€:uê$“É$___ùùù)$$DîîîÕ¾VAAÁ ýgggëСCZ·nNŸ>m™àooo¯&MšX^ëZÿ×o·lÙò¶4hÐ •——«²²²Ö¶666Š×{ï½÷«_¸[2îeeeÊËË«6|¸eò¹Éd’Ñh´ò¨P£Ñ¨ˆˆEDDT{¼´´TùùùÕ~vîÜyÓÏÃÏÃ|ÐP2À}©º;×_¿]Ûë»uëvÃÄñ   ÙØØXyT¸Säçç'??¿ƒµ­l‘˜˜xÓ•-~¾ÍÊ°B¸çÔHOOWFF†ÊÊÊ,íF£er·ÉdRttt•Iß-Z´­­­G„»£££e•‚š×ø¹LNNVFF†.]ºdio4k] !00Pöööõ1<Ü'à®RÓ$íkÛǯu’öÏW!hÙ²¥\\\¬8"ÜOœoD¨-$“˜˜XkH¦º0!Ü Bh0JJJ”••Uãë´´4ZÚ;99U™TªØØXËë¹»»[qDÀ­32 ­±MAAAµ«!¤¤¤(11Q§OŸVEE…$ÉÞÞ^Mš4©6€pm»eË–2 õ5D4`„ P/ÊÊÊ”——WmxàÚó3gÎÈl6K’åïïo™­áÇ[&G›L&F+ °£Ñ¨ˆˆEDDT{¼´´TùùùÕþ¾íܹó¦¿o?#ðûpÿ d€Û¢º;«_¿]ÛÕ»uëvÃÄæ   ÙØØXyTÀÝÉÁÁA~~~òóó«1ˆPÛÊ!‰‰‰7]9äçÛ¬po d€›*((¨1<žž®ŒŒ •••YÚFËäc“ɤèèè*“’[´h![[[+Ž€£££e•‚š×ø{Ÿœœ¬ŒŒ ]ºtÉÒÞh4ÖºB`` ìííëcxø…ÜçjšD|mûøñãµN"þù*-[¶”‹‹‹G„ºÚ¹s§ú÷ï¯þóŸjÛ¶­µËAäìì|Ó Bm!¤ÄÄÄZCHÕ…!X!€{XII‰²²²jœœ––¦ÂÂBK{''§*“~CCCk™"ww÷;Zsff¦«ì›={¶^zé¥;úº¿ÔÒ¥K5vìXIÒ‚ 4~üx+WTw•••2›Í2›Íu>'11Q½zõÒ¡C‡v««›†tý«ûìÚØØÈÓÓSmÛ¶Uÿþý5~üxyxxT{~JJŠÞ{ï=}óÍ7ÊÍÍUÓ¦M¬Þ½{멧žRHHˆ¥mrr²ÞyçíÝ»Wyyyò÷÷Wtt´ž}öYõèÑCƒA’tàÀ½ÿþûJJJÒåË—Õ¶m[ýéORß¾}oÛ¸F£ŒF£BCCklSPPPíj)))JLLÔéÓ§UQQ!I²··W“&Mj\ Á××W¾¾¾–1àö"dp—*++S^^^µákÏÏœ9c™@îèè(Ë„Ýèèh >¼Êä]£ÑhåQI2›Í5j”>ÿüsY»¤Z3FÏ=÷œœ­]Ê-{øá‡uþüyk—ñ«4¤ë_Ýg·¢¢BgΜѦM›ô—¿üEóçÏ׆ ^åÜÕ«WkÔ¨Qš8q¢¶oß.hÛ¶mš4i’þò—¿èêÕ«’þèÚµ«~÷»ßi×®]jÖ¬™²³³5gÎEEEiïÞ½ŠŒŒ”$õéÓG={öTrr²ìììôæ›oªÿþJHHPŸ>}êíÚFEDD(""¢Ú㥥¥ÊÏϯöß³;wÞôß³Ÿ‡‚ƒƒÕ¨Q£zÀ½„@UÝ¿¯ß®íÎßݺu»aâmPPlll¬<*àþbkk+=Z}ûöU§N£ÔÔT˪ ‡Ö¨Q£ôÊ+¯hÚ´i–s}||4tèP…††ªS§N–ýýë_åèè¨øøxËïtPPfÍš¥ÄÄÄ*¯ïâ⢥K—ÊÕÕU’4wî\}òÉ'Z²dI½† nÆÁÁA~~~òóó«1ˆPÛÊ,‰‰‰7]™åçÛõ±2 À݈o•¬   @)))JLLÔâÅ‹õÖ[oiܸqêÕ«—‚ƒƒåàà ///EFFªÿþš:uªUPP “ɤØØX-_¾\[¶lQZZšŠ‹‹•­}ûöiíÚµŠ‹‹Ó”)S4xð`uïÞ]&“鞤¥¥iÀ€jÒ¤‰ÜÝÝ5pà@}÷Ýw5¶—Á`Á`ÐÂ… õâ‹/ÊÓÓSþþþzûí·oh¿iÓ&uîÜYÎÎÎjܸ±†®œœœÚÍ›7O-Z´‹‹‹zöì©'NTûú7nTdd¤œœœäãã£ñãÇëâÅ‹uª7>>^Ï?ÿ¼¼¼¼d0ôÌ3ÏÜR¿×jtrrR—.]´{÷nKß;w®òZK—.­Ó5ž:uªzõê%I —Á`PPPPêºc»Ùõ¯ÎÍÞã[ýÌÜ oooM›6M™™™Z¸p¡eÿ´iÓTYY©I“&U{^XX˜®\¹by^\\¬²²²*û®ùá‡,«HRzzº%` IvvvrvvÖ¹sç~ÕX¬ÁÑÑQ&“IÝ»w×àÁƒ5eÊÅÅÅiíÚµÚ·oŸ tåÊ¥¥¥iË–-Z´h‘bcce2™”““£ 6èÕW_Uÿþý)yyy)44T½zõÒ¸qãôÖ[oiñâÅJLLTzzºÊÊʬ=l€zw÷³ ÐÀ+==]‰‰‰Z¹r¥fÎœ©qãÆ©_¿~U&¶†……©W¯^š:uªþö·¿)==]&“IÇW||¼¶l٢ÇëòåË:þ¼öíÛ§ 6hÑ¢Eš2eŠFŒ¡èèh™L&ÙÚÚZ{Øõ⩧ž’³³³RSS•‘‘¡   EGG×Ø~„ ºté’$iÁ‚ŠŽŽVff¦¦L™¢?ÿùÏJJJ²´Ý¸q£bbbôØc)##C;wîÔ±cÇÔ£GK’ôÙgŸi„ ;v¬²²²4kÖ,Mž<ù†×^·nú÷ﯘ˜åäähóæÍJJJÒ“O>)³Ù|Óz?üðC=úè£ÊÈÈÐܹso©ßëkÌÉÉÑÂ… 5eÊËuøî»ïª¼V]¯ñŒ3´eËIÒ¡C‡d6›uêÔ©:Õu'ÆVÛõ¯N]Þã[ùÌü}ûö•Á`ÐW_}eÙ÷õ×_+,,LF£±N}têÔIW¯^Uß¾}µ}ûö?OÕÙ¶m›rssÕ¡C‡[®ýnàìì,“ɤèèh1BS¦LÑ¢E‹´eË¥¤¤èÒ¥K:þ¼>¬-[¶hÆŒ_\®ÆûzòÉ'1bÄnãììlÈܹs9âˆ#~V]µ5¶ÊŽev÷ïî{¦*?}ïþTbb" ,Y²„ 6À)§œÂÛo¿]m¿;•––rî¹çò¿ÿû¿$%%qúé§sÑEÑ»wïJ·)..æÅ_äOú5böìÙ´lÙr·öw0***ªðùþã×999‘pAll,‰‰‰•~Öï|””D( xTÒþÁ#K’$I’$IRÝÙùà+¯¼RnyLÅH’$I’$í‹JKKY³fM¥á¯ "w ‹‹#%%%rAiVVǯp¡©jFýúõéÙ³'7Ýt¡PˆSO=• T0ø±Î;GžGGG“˜˜ÈêÕ«ÈËËcÑ¢EŒ5ªÜ6ݺu£I“&dggsà 7°nÝ:/^ÌUW]U®ÝO/ìÎËË«¶]u!ƒºtéRaÙîôÛ­[·JÛôèÑ£ÚýÁžãŸ;Þš[UÖÿ´ÿÝ9Ç?VÝ{foTvß—Ÿszll,¯¾ú*|ðO?ý4¯¿þ:>ú(>ú('žx"&L iӦ嶹æškxóÍ79ÿüó=z4-Z´ØëqÈÂá0™™™dffVº¾¤¤„ÂÂÂJÿ^LŸ>}—/~Fhß¾}…s&I’$I’$I’C’$I’$é °}ûv ÈÉÉ!//¼¼¼rÏ—/_^î‚ÐØØX’““IKK£M›6dee‘ššJëÖ­IKK#55•C9$àQ|&MšÄ˜1c¸æšk¸ð ÉÊÊâÖ[oÝ­‹Ì7n\îull,Û·o °°€fÍšUØ®yóæ‘õ+W®¬´ÝO_ïl?~üxÆ_¡ÏåË—ï²Þ TX¶;ýVUãîΘ±'Çø玷¦ÇVÙy«ªÿ]ã«î=³§Ö¯_OQQ]»vvœ—p8ÛÏ1`À À–-[˜4i>ø “'Of̘1Œ;¶BûñãÇsî¹çîUýÚ¡^½z$''“œœ\eá‡~¨ðw&77—¼¼<Þxã –/_^n¶‹„„ÒÒÒhݺ5©©©¤¦¦Ò¦M›Èßœ´´4êׯ_WC”$I’$I’$I±¨  $I’$I’vW8fܸq¬X±‚©S§²yófŽ=öX–,Y²Wý&&&°nݺ ëÖ®]YŸ””Ti»õë×WÚßõ×_OYYY…ÇóÏ?¿WuV×oU5V6¶ÊìÉ1®‰ñîÍØ~zü«ëW縶½þú딕•1hРȲAƒ1oÞ<Š‹‹÷¨Ï¸¸8†ÊäÉ“IKKãÓO?­©r%I’$I’$I’tr&I’$I’tPˆŠŠŠÜuº*¥¥¥¬Y³†•+W²téRòóó#ϳ³³ÉÏÏ/7ÛA\\)))$%%‘œœLzzz¥ÏU3 8p sçΠOŸ><ùä“´oßž™3gÒ¾}û=î;55•N:ñÁ”[>gÎ6lØ@VV°ã.ø;vdÚ´iåÚÍž=»B‡vŸ}öY…}uíÚ•?ýéOœsÎ9{Tçîô[Y•móS»sŒ£¢*Þ·¤&Æ»7cûéñ¯ªÿÝ9ǵiÕªUÜ|óͤ¥¥1räÈÈò›o¾™W_}•±cÇ2f̘ Û=ðÀŒ=šo¿ý–fÍšqË-· …¸óÎ;˵‹‰‰!66–æÍ›WèãÙgŸ­ññÌJJJ(,,¬ôïÅÎ×Õý½2dH¹¿íÛ·§iÓ¦J’$I’$I’$iC’$I’$Iÿ_lll$ˆ™™Yi›-[¶°bÅŠr”î|žÍ’%KÊݼ~ýúÕ=ôPš4iRWCÜïÍ›7x€Ë.»ŒíÛ·ó׿þ•úõëÓ³gϽî{ìر 6Œ›o¾™k¯½–ÂÂBFŽI‡¸òÊ+#íFÍùçŸÏ˜1c5j¹¹¹Üwß}ú7nC‡åž{îaĈÜu×]lݺ•aÆíq»ÓïOkÌÉÉá¯ýënõ¿«c¼s6… ÒªU+:wîÌo¼Q#ãÝ“±Uuü+³»ç¸&mÛ¶‚‚&MšÄèÑ£‰ŠŠâ7Þ >>>ÒæðÃç…^`øðálÛ¶‘#G’’’B~~>/¼ðcÆŒáÉ'Ÿ¤Y³f‘mzè!;ì0N<ñDX±b<ðË–-ã±Ç+WCQQÝ»w§wïÞL˜0¡VÆy )**ª2<°råJrrrضm°ãoGbbbäó½_¿~‘ç;?ï“’’…BJ’$I’$I’$i÷„ÊvÞJI’$IR žxâ ®¸âŠ Ë$Õ M›6UzwëÏ/^ÌÆ#íÃápµ³!´k׎† 8¢º•——GZZZ¹e<ð×\s o½õ=ô³gϦ´´”.]ºpûí·süñÇWÚ×Ë/¿Ìyçy}ÁpÏ=÷”ë¿}ûö|óÍ7üûßÿæ¶Ûnã‹/¾ aÆ 4ˆ¿üå/‘ ëwzôÑG¹÷Þ{YµjÝ»wçþûï稣Ž wïÞ|üñÇLž<™Ûn»Ï?ÿœ¦M›r 'pï½÷’ššº[õÂŽ‹z‡óÝéwg«W¯¦gÏž<øàƒdffòÔSOqÙe—1~üx~ûÛßFÚ÷ïߟ>ø`·Žñ¨Q£øÇ?þAYY^x!<òÈ.몱íêøWfWçøç¾g~¬²÷n(¢iÓ¦~øá 6Œ_ÿú×U‹,XÀ=÷ÜCvv6k×®åC¡wïÞ\{íµôë×/Òî»ï¾ãþç˜0a_}õùùù4nܘ=zpÝu×1pàÀ ǹk×®ôéÓ‡W^y¥Êcs°(**ªô³yg 77—ÒÒÒHûp8\eP,==6mÚàˆ¤‹ÿF–$I’$I’¤ºsöÙgTøÉ$I’0/ ¤ƒ“¹ª®Í™3‡îÝ»3uêTŽ=öØ Ë‘jEU!¯¯—/_Îwß}i_]È+==´´4bcc‘tðñßÈ’$I’$I’Twª ÄQŒ$I’$IÒÁ.‡ÉÈȨ²MQQQ¥³!,]º”ììlrrrضm±±±$&&V9Brr2mÛ¶%**ª®†¨=õÔSLœ8‘G}”–-[²dÉ~÷»ßÑ­[7úöítyÒÙ²e +V¬¨2 µdÉŠ‹‹#íëׯ_îs033³ÜgbÇŽ‰pD’$I’$I’$Iû&C’$I’$Iû¨p8Lff&™™™•®/--eÍš5•^l;}útòóó)((`çD–qqq¤¤¤T{×îp8\—CT-9ÿüóY»v-ƒfÉ’%$&&’••Å˜1c¼+»öI%%%V; AuŸgYYY ><òyÖ¾}{š6mð¨$I’$I’$I’öO† $I’$I’öS±±±$''“œœ\e¡º;gggïòÎß?}î¿÷ 6ä†nà†nº ¨|f–¿®nf–~ýú• C%%%‘””D( xT’$I’$I’$I&C’$I’$I°¸¸¸È…¹UÙ´iS¥wÏÏÏgÁ‚,^¼˜7FÚ‡Ãá*gCHJJ¢]»v4lØ°.†'iPTTTiig 77—ÒÒÒHûp8ù¼ÈÈÈ`È!å>GÚ´iCttt€#’$I’$I’$I:¸2$I’$I:È5hÐ`—A„ê."ÎÎήö"âÊÂ^D,íª !í|››[m駳¤¥¥àˆ$I’$I’$I’´+† $I’$I’´Káp˜p8LFFF•mŠŠŠ* ag!''‡mÛ¶Kbbb•³!$''Ó¶m[¢¢¢êjˆÒAgË–-¬X±¢ÊÑ’%K(..Ž´¯_¿~¹ßÓÌÌÌr¿³;v$>>>ÀI’$I’$I’$©&2$I’$IR‡Ãdff’™™YéúÒÒRÖ¬YSéÅÌÓ§O'??Ÿ‚‚ÊÊʈ‹‹#%%¥ÊÙÒÓÓ ‡Ãu9Di¿QRRBaaaµ³T÷û–••ÅðáÃý}“$I’$I’$I:2$I’$IRˆ%99™äää*ƒÕÝY=;;{—wVÿésאַUe3‡üøuu3‡ôëׯÂïJ»ví…BJ’$I’$I’$IûC’$I’$IÚgÄÅÅEîš^•M›6Uzwöüü|,XÀâÅ‹Ù¸qc¤}8®r6„¤¤$ÚµkGÆ ëbxÒn)**ª4h³3H››Kiii¤}8Ž¼Ÿ3222dH¹÷y›6mˆŽŽpD’$I’$I’$IÚŸ2$I’$IÒ~¥Aƒ» "Tw‘vvvvµiWFð"mÕ”ªB2;_çææV’ùñ,éé餥¥àˆ$I’$I’$I’t 1d I’$I’¤N8&“‘‘Qe›¢¢¢JgCØDÈÉÉaÛ¶mÄÆÆ’˜˜XålÉÉÉ´mÛ–¨¨¨º¢öA[¶laÅŠU\–,YBqqq¤}ýúõ˽233˽§:vìH|||€#’$I’$I’$IÒÁÈ$I’$I’Jáp˜ÌÌL233+]_ZZÊš5k*½X|úôéäççSPP@YYqqq¤¤¤T9Bzz:áp¸.‡¨TRRBaaaµ³T÷~ÈÊÊbøðá¾$I’$I’$I’´Ï3d I’$I’$U"66–ääd’““« "Twçúììì]Þ¹þ§Ï½s}p*›Ùâǯ«›Ù¢_¿~Îe»ví…BJ’$I’$I’$Iúù H’$I’$I{(...rWúªlÚ´©Ê»ßÏš5‹ÜÜ\6nÜi‡«œ !))‰víÚÑ°aúÞ£¨¨¨Ò ÈÎó››Kiii¤}8Ž † Rî<´iÓ†èèèG$I’$I’$I’$ÕC’$I’$IR-jРÁ.ƒÕ]Ÿ]íEð•…¦‹à« qäççï2ÄñãYÒÓÓIKK#666ÀI’$I’$I’$IÁ2d I’$I’$,‡ÉÈȨ²MQQQ… éóóó#A„œœ¶mÛÀiÑÑü³¶M›*ÃmÛ¶%**ª®†¸G¶lÙŠ+ª `,Y²„âââHûúõë—gfff¹1wìØ‘øøøG$I’$I’$I’$íû H’$I’$Iûp8Lff&™™™•®/--¥ðÛo‰½þz'Näã=x­kW–.]ÊôéÓÉÏϧ  €²²2âââHII©r6„ôôtÂáp­§¤¤„ÂÂÂjg!¨®Þ¬¬,†^gõJ’$I’$I’$I C’$I’$IÒ væL’.ºÖ¯‡ÿý_úœ~:}~Ò¦º™²³³w93ÀOŸW73@U3/ì|þã™bccILLŒô߯_¿ ûj×®¡P¨¶Ÿ$I’$I’$I’¤ÿÏ$I’$I’´?+-…?ÿÆŒáé§!9¹Ò¦qqq‘»þW¥¸¸˜¼¼Â$I’$I’´¿š?†‡E‹`Ü8¸újØË‹õ›6mJÓ¦M9âˆ#ªl³zõjòòòÈÍÍe|Úx¾lù%]nèB»ví:t(©©©‘PArr2±±±{U“$I’$I’$I’¤ºcÈ@’$I’$IÚß”•Áÿ7ÜptïsæÀ¡‡ÖÙî[´hA‹-8òÈ#™ÀúÒ—^x¡Îö/I’$I’$I’$©öD]€$I’$I’¤Ÿ!'Ž?®»n¼¦M«Ó€ÁOå’Kií_’$I’$I’$IRÍ2d I’$I’$í/þùOèÖ Ö¬O>Ñ£!::Ð’–³Ü$I’$I’$I’t1d I’$I’$íëÖ¬ÓO‡s΋.‚Y³ {÷ «bÛXÉJ_WY IDATC’$I’$I’$IÒ$&è$I’$I’$Uãwà²Ë ^=xï=0 èŠ" ( ”RC’$I’$I’$IÒÄ™ $I’$I’¤}ц 0r$ G sæìS€\rhMë€+‘$I’$I’$I’TSœÉ@’$I’$IÚ×̘]ë×ÿþ§tE•Ê%—h¢I")èR$I’$I’$I’$Õg2$I’$I’ö¥¥0z4s z(|ñÅ>0XÎrZÑŠXbƒ.E’$I’$I’$IR q&I’$I’$i_0o K–À£ÂW]Ñ.å’KkZ]†$I’$I’$I’¤äL’$I’$IR¶o‡‡‚= A˜={¿ÀŽAiA—!I’$I’$I’$©2$I’$I’‚²l<\wÜx#üç?СCÐUí6C’$I’$I’$IÒÇ$I’$I’„瞃.]`íZøôS=¢£ƒ®êgù†oèÀþŠ$I’$I’$I’´k† $I’$I’¤º´z5œv\rÉŽÇ̙Э[ÐUýlkXCEÊ¡A—"I’$I’$I’$©Å]€$I’$I’tÐxûm1ââà½÷ ÿ +Úc‹Y @G:\‰$I’$I’$I’¤šäL’$I’$IRmÛ°FŽ„Áƒ!+ ¾ür¿|Í×4 )¤]Š$I’$I’$I’¤äL’$I’$IRmúè#¸è"ظ^{ † º¢ñ5_ÓDyI’$I’$I’$é€â7€’$I’$IRmؼn¼Ž9:u‚9s˜€ÀbÓ‘ŽA—!I’$I’$I’$©†2$I’$I’jÚ¼yз/<öØŽÇ[oARRÐUÕ¨¯ùÚ$I’$I’$I’t2d I’$I’$Õ”íÛᡇ 36„Ù³áŠ+‚®ªÆ•QÆ7|át)’$I’$I’$I’jXLÐH’$I’$I„eËàâ‹áÓOáŽ;àºë êÀ¼ÇÇ Vð=ß2$I’$I’$I’@æ·œ’$I’$IR]zî9èÜÖ­ƒ3à†Ø€À×| @G:\‰$I’$I’$I’¤švà~Ó)I’$I’$Õ¶U«`Ø0¸ä¸ôR˜5 ºu ºªZ·˜Å$@ Z]Š$I’$I’$I’¤t’$I’$IÒ~éÕWá׿†Æáý÷áØcƒ®¨Î|Í×Ê¡A—!I’$I’$I’$©8“$I’$I’ôslØ#GÂ/ §œsçTØ1“!I’$I’$I’$éÀdÈ@’$I’$IÚ]Ó§C÷îðÚk0q"<÷ÜŽ™ 2ó˜GA—!I’$I’$I’$©2$I’$I’veóf¸ñÆ3tïóçÃСAWˆldËèLç K‘$I’$I’$I’T b‚.@’$I’$IÚ§Í Ã‡Ã·ßÂcÁW]Q æ12ÊèB— K‘$I’$I’$I’T œÉ@’$I’$IªÌ¶mpï½Ð£4n Ÿ üñ¤U«Vrúé§Ó¸qcRSSùïÿþï ÝL™2…£>š† ’ÀСCY¸pa]¦ÆÌe.ñÄÓ†6A—"I’$I’$I’$©2$I’$I’~êÛoá¸ã`ôh¸ãøðCHO/פ¬¬Œk®¹†k¯½–+VpõÕWó»ßýŽ3fDÚL™2…“N:‰ÌÌL–.]ʬY³Ø´iýúõcùòåu<¨š1—¹Á„]Š$I’$I’$I’¤Z`È@’$I’$Iú±çžƒ.] ¸>þn¸¢*þ7ÚêÕ«¹à‚ 8æ˜cHHHàú믧]»v<û쳑6·Ür <ôÐC´jÕŠ:ðòË/³yófî»ï¾:TÍù’/éLç Ë$I’$I’$I’TK H’$I’$I«VÁСpé¥0jÌœ ]»VÙ<::š–[vøᇳlÙ26oÞ̧Ÿ~Ê©§žZ®MóæÍéׯ|ðAM NÌg>GpDÐeH’$I’$I’$Iª%† $I’$I’¤ÿùÈÈ€yóàý÷áž{ ^½j7iÞ¼9111å–ÅÇdz~ýzŠ‹‹Ù¾};-Z´¨°mË–-Y»vmÍÕ_G–±Œµ¬¥;݃.E’$I’$I’$IR-1d I’$I’¤ƒ×úõ0r$œu _~ dz[›†B¡j×7mÚ”¨¨(Ö¬YSaÝêÕ«iÞ¼ù•¤ÙÌ&Š(ºRõ ’$I’$I’$I’öo† $I’$I’tpš2:w†‰áõ×á¹ç qãë¾~ýúôêÕ‹·Þz«ÜòuëÖ1}útú÷ï_cûª+³™MG:O|Ð¥H’$I’$I’$Iª%† $I’$I’tpÙ¼n¼N<z÷†yó`ÈZÙÕwÜÁܹsùýïϪU«Xºt)çw±±±\ýõµ²ÏÚ4›Ùd’t’$I’$I’$I’j‘!I’$I’$<¾ürG°àñÇá±ÇàŸÿ„ÄÄZÛÝÀyçwøôÓOiÛ¶-ݺu£^½zLŸ>6mÚÔÚ~kËç|Αt’$I’$I’$I’jQLÐH’$I’$IµnëV7n½Ž9Þ|ÒÒö¸»±cÇ2vìØ Ë_~ùå ËÈÀ÷x_ûŠ<ò( À$I’$I’$I’t€3d I’$I’¤ÛÒ¥pñÅ0s&Üq\wD9ÁçÏ5›Ù„ÑnA—"I’$I’$I’$©ùmª$I’$I’LeeðÄÐ¥ lØŸ|7Ü`À`}ÆgÊ¡4¥iÐ¥H’$I’$I’$IªE~£*I’$I’¤OA  £FÁUWÁgŸíhÍ`}ét’$I’$I’$I’jYLÐH’$I’$I5êŸÿ„+¯„„xÿ}8úè +Úïmg;ŸñgqVÐ¥H’$I’$I’$IªeÎd I’$I’¤Ãúõ0r$œsœy&|ñ…ƒ2yl`}èt)’$I’$I’$I’j™3H’$I’$iÿ— —\¥¥ðúëpê©AWt@™Á ш 2‚.E’$I’$I’$IR-s&I’$I’$í¿6m‚o„“N‚¾}aþ|µàc>¦7½‰ñž%’$I’$I’$IÒÏo%I’$I’´úì3¸è"X¹{ ®¸"èŠX3˜Á/ùeÐeH’$I’$I’$IªÎd I’$I’¤ýËÖ­pï½Ð¯¤¦Â¼y jÑZÖ²˜Åô¦wÐ¥H’$I’$I’$IªÎd I’$I’¤ýÇÒ¥;f/˜=î¼®»¢¼Fmúÿ!Dˆ~ô ºI’$I’$I’$IuÀo`%I’$I’´ï++ƒ'ž€.] ¤dGÈà† Ô©L¥ ]hF³ K‘$I’$I’$I’TüV’$I’$Iû¶‚2F‚«®‚iÓà°Ã‚®ê 1•©ô§ÐeH’$I’$I’$Iª#† $I’$I’´ïúç?!#¾ú >øî¹êÕ ºªƒÆzÖó%_2$I’$I’$I’"† $I’$I’´ï).†áÃáœsà—¿„/¿„~ý‚®ê óþÃv¶Ó½$I’$I’$I’t°ˆ ºI’$I’$©œwß…K/…­[á7`ðà +:hMe*dЂA—"I’$I’$I’$©Ž8“$I’$I’êÆÌ™PRRõúM›àÆáä“¡o_˜?߀AÀ¦2•þôº I’$I’$I’$IuÈ$I’$I’jßêÕ;·ß^ùúO?…îÝᯅ¿ÿ^yš5«ÛUNE|Îç `@Ð¥H’$I’$I’$IªC† $I’$I’T»ÊÊࢋ °î¹fÌø¿u[·Â½÷ÂÑGCëÖ0w.\xapµ*b S(£Œã9>èR$I’$I’$I’$Õ¡˜  $I’$IÒîàÝwaûvˆŽ†sÎùó!/oGø`þ|¸óN¸î:ˆòžûŠIL¢½h†3JH’$I’$I’$IC’$I’$Iª=³gÃ7îlÛ0d|ú)tésæ@ÇŽÁÖ© &3™K¹4è2$I’$I’$I’$Õ1o 'I’$I’¤Úñý÷pÖYPVV~yi)L gž Ó¦0Ø-`ËYÎIœt)’$I’$I’$I’ê˜!I’$I’$ÕŽßü–/‡­[+® …à­·`ݺº¯K»4‰I„ Ó“žA—"I’$I’$I’$©Ž2$I’$IRÍ{åxî¹Ê°cvƒï¾ƒ#ê¶.í–IL"‹,¢‰ºI’$I’$I’$IuÌ$I’$I’jÖ’%pÉ%;f+¨Ni)¼ñ<ÿ|Ýԥݲ‰M|ȇœÄIA—"I’$I’$I’$)† $I’$I’TsJKáì³wü,+«º]tôŽÀ£îh¯}¦°™ÍœÂ)A—"I’$I’$I’$)1A I’$I’¤È-·À_À¶m×ÅÆî4hÇÆÁI'A›6u_§ªô¯Ñ‹^$“t)’$I’$I’$I’`È@’$I’$i°yóf6mÚÄ–-[øá‡"?JJJøþûï+ÝnÆ l«ì‚~ I“&Dïœ-àGbcciܸq¹ç111ÄÇÇM“&Möl“'Ã_þò3ÄÄì„BУœzêŽPAfæÿÍb }Êv¶óoq5W]Š$I’$I’$I’¤€2$I’$IÚC›6m¢°°ÂÂBÖ®]Kqq16l`ãÆ‘Ÿëׯ§¸¸¸Ü²ï¾ûŽM›6E‚›7oz(ì DEE‘@½zõˆ'!!¦M›O“&MhÒ¤ ñññ´ŠŠâìÛo'îÿ ¶&%±í”Sˆ2Ž?ö4¸ :õ1S@Ãt)’$I’$I’$I’bÈ@’$I’$éGV­ZEAAyyy¬\¹’‚‚‚rA‚5kÖ°fÍ #3 üX|||äѤIHHH eË–tèÐ!rQ~\\ 6Œü¬W¯5ŠÌ,ðãÙv^è_™}üTYYÅÅÅ•n³s¦ 2cÂÎÙvþ,--å»ï¾cëÖ­lܸ‘Í›7³qãF6nÜHQQùùù,Z´hGhbãFîÏÏçß%%L&KV®„gž!öùçiÞ¼9‰‰‰‘Ç!‡yžššJ«V­"?ëÕ«·g'N5b"é@~Á/‚.E’$I’$I’$IR@ H’$I’¤ƒÆêÕ«Yºt)ß~û-999äçç“—— ¬ZµŠ’’’Hûøøx’““#É'''Ó¥KZ´hQîÂùÏ›6mJ( p„ÿ' ‡+]WÕò=VZ ¡ÄÄpÂwßñ§âb Y½zu$ ±3¤±zõj¾úê+ Y³f «V­*×UË–-Ë…RSSIMM%==víÚ‘––FLŒÿ¥U[&2‘Ó8-è2$I’$I’$I’$Èod%I’$IÒ£´´”¥K—²xñb¾ýöÛH `çÏï¿ÿ€˜˜RSSIII!99™^½zqú駓””¹¸=--F<¢ýDlläiãÆiܸ1©©©»µiII «V­"77·Ü ùùù,[¶Œ3f““SîܵnÝšvíÚE‚íÚµ£C‡têÔ‰øøøZâÁ`! YÄ"žâ© K‘$I’$I’$I’ C’$I’$i¿SRRÂ×_Í‚ Xºt)óçÏgÁ‚,X°€M›6;îÖŸžžNzz:ƒŽšcŽ9†¸¸¸GP7^äE’IæhŽºI’$I’$I’$I3d I’$I’êÌòåËyçwx÷Ýw™>}:4hЀž={ròÉ'sçwrÔQG‘t©:€ÅÄÄ‘‘AFF]t°#xðùçŸ3mÚ4¦M›Æc=Æí·ßNÆ éÕ«Çw§œr ™™™DEE<‚š7 œÇyDqàM’$I’$I’$IÒÏcÈ@’$I’$Õš’’¦M›Æ;ï¼Ã;ï¼ÃüùóiÔ¨Ç<×^{-ýúõ£GÔ«W/èRu‹‰‰¡gÏžôìÙ“ßÿþ÷,Z´ˆ>úˆ?ü§žzŠÛn»C9„“O>™SN9…O<‘æÍ›\ùÞ›ÉL³˜s97èR$I’$I’$I’$í H’$I’¤µeËÞ~ûm^|ñE&MšÄÆéÔ©ƒæÁä˜cŽ!...è2¥]êÔ©:uâ’K.`îܹ‘ÀÌÅ_ÌöíÛéÝ»7gŸ}6çœs­Zµ ¸â=ó/Ñžöd’t)’$I’$I’$I’öÎ.I’$I’öÚ¶mÛ˜2e —]v­Zµâ—¿ü%k×®åî»ïfÉ’%,\¸qãÆ‘••eÀ@û­Î;sýõ×óþûï³fÍ&L˜@‡¸õÖ[IMMåÄOäÙgŸeÆ A—ºÛ¶³Wx…ó9Ÿ¡ Ë‘$I’$I’$I’´0d I’$I’öتU«¸õÖ[iݺ5YYY|ñÅü×ýË—/ç½÷ÞcÔ¨Q¤§§]¦Tã8óÌ3ùûßÿNAA/½ô5â׿þ5-[¶äüóÏç³Ï> ºÌ]zŸ÷É#ó8/èR$I’$I’$I’$í# H’$I’¤ŸmÞ¼y\vÙe´iӆǜK.¹„… 2sæL®½öZRRR‚.Qª3 4ଳÎâ_ÿú+W®äá‡fáÂ…ôêÕ‹c=–×^{íÛ·]f¥žæiúÒ—Ã9<èR$I’$I’$I’$í# H’$I’¤Ý6gÎN9åºtéÂŒ3xøá‡ÉÉÉa̘1têÔ)èò¤À…ÃaFŒÁìÙ³yÿý÷iÚ´)gžy&:uâù矧¬¬,è#Š)æ5^ã2. ºI’$I’$I’$IûC’$I’$i—Ö®]Ë•W^I=(..æ­·Þbþüù\~ùå4hÐ èòjÕôéÓiÞ¼9 ,º”}‚Çc÷ 0€×_¯¾úŠpÉ%—Я_?fÎœti<ÏóDÅÙœt)’$I’$I’$I’ö!† $I’$IR•ÊÊÊxì±Çèر#'Näoû}ô§œr ¡P¨Vö™——G(ªð¨_¿>Gq÷ß?[·n­•}Wfûö픕•ý¬;Ðggg …˜7o^-V¶{žzê©È1|üñÇ÷º¿=9U©ì\GGGÓ¬Y3Ž>úhî»ï>6lØPåöóçÏçâ‹/&--¸¸8RSSéß¿?wß}7‹/.×vÖ¬Yœ~ú餦¦Gzz:W\qS§N­õÙ:vìÈ“O>ɬY³ˆ¥wïÞŒ1‚âââZÝï®<Ã3œÃ9Äh’$I’$I’$I’ö-† $I’$IR¥Ö¯_ÏgœÁÕW_͈#X´hǯµpÁN©©©”••qñÅÓ¨Q£Èí+V¬à‚ .àø¿ùÍojµ†;æ˜cX·nu¶Ïš4bÄ6mÚTcýÕäñ¨ì\—””0wî\.½ôRyäºtéÂܹs+lûòË/“™™I«V­øðÃÙ¸q#³fÍâ7¿ù =ö]ºt‰´ýüóÏ9ꨣHJJâ£>bãƼ÷Þ{4nܘ0kÖ¬½ËîèÚµ+S§NåÅ_äí·ß&33“Ù³g×ɾj³˜Ã.ã²@ö/I’$I’$I’$ißeÈ@’$I’$U°jÕ* À§Ÿ~Ê”)S¸÷Þ{‰önçÍ›7禛n¢oß¾<ýôÓ¬[·.ÐzT;¢££IIIáÒK/å³Ï>`РAlܸ1ÒfÞ¼yüêW¿âÚk¯åÞ{ï¥]»vÔ«W–-[rÎ9çðöÛoõÿíõ·¿ý¸¸8ÆOëÖ­©W¯mÛ¶åþûï§sçÎu>ÆsÎ9‡9sæžžNÿþýÉÎήóžæi:щ¾ô­ó}K’$I’$I’$IÚ·2$I’$Iå|÷ÝwœtÒI|÷ÝwLŸ>c=6è’Ê9ôÐCÙ¾};yyy¼ùæ›ôèуúõëÓ²eK~ýë_³aÃÆO(" 1~üx®¼òJš5kF(âÜsÏ`É’% 6ŒÄÄDâãã9í´Óøøã+lÿÔSOEj¨n›o¼‘йsgB¡mÛ¶­t,?îÿñÇçꫯ&!!””î¸ãŽ í'MšDŸ>}hРÍ›7gøðá¬\¹²B»Gy„6mÚаaCŽ;î8¾þúëJ÷_ݱÛU½;ÇÏÃÏÑ¢E îºë.òòòxüñÇ#Ëïºë.¶oßÎu×]WévGq?üðCäõ¦M›(---·l§/¿ü’=zìU{¢E‹¼ýöÛ 6Œ!C†ðÉ'ŸÔÙ¾à^â%.ã2B”Ÿ™ä_ÿú¡PˆÏ?ÿ¼Âv'žxb¹c5wî\† F8¦AƒuÔQ|øá‡å¶ÉÍÍåüóÏ'))‰øøxzõêÅ„ jg`’$I’$I’$I’j„!I’$I’TÎo~óV®\É»ï¾[åÅñAúú믉ŠŠ"55•‰'2tèP ÄÊ•+™ûlV¯^Í´iÓX½z5?þ8'NdÕªUµ:FI’$I’$I’$I{Î$I’$IŠøä“Oxá…xâ‰'ö¹€ÁÚµk¹ûî»™1c#FŒ Y³f\ýõdddpÇw‡éÚµ+ùË_xï½÷xÿý÷ËmŸ••ÅYgE£F¸êª«xùå—Ù¼y3_|ñgœq-Z´ iÓ¦Œ;–FUYÇžl³;zôèÁСC‰gÔ¨Q4lØÿüç?‘õüãùÅ/~ÁŸÿüg9ì°Ãxâ‰'øæ›oxôÑG#íFM·nݸå–[‡ÃtïÞ+®¸¢Âþ~α«©1ì‰&MšÐ¬Y3rrrØ°aEEE´jÕj·ûøÕ¯~ÅgœÁÔ©Séß¿?)))Œ5ªNg¨Jtt4Ï=÷%%%Üu×]µ¾¿2Êxˆ‡¸ 9„C*¬‰‰áÒK/åÿø›7oŽ,æ™ghØ°!çŸ>°ãýÓºukž{î9ÒÓÓiÖ¬·Þz+}úôáÎ;ï ´´”O>ù„ .¸€öíÛÓ AŽ<òH^|ñEZ¶lYëc•$I’$I’$I’´g H’$I’¤ˆ§žzŠîÝ»3lØ° Kàûï¿'  …HNNæ…^`ìر<òÈ#äåå±xñb Pn›Þ½{0eÊ”rË»téR¡ÿúõëÓ³gOnºé&þùϲiÓ&bbbª½Ëúžl³;:wîyMbb"«W¯ //E‹Ñ¿ÿrÛtëÖ&Mš ÀºuëX¼x1G}t¹v;ÉN?÷ØÕÄöFe3„B¡ÝÞ>66–W_}•÷ߟ /¼ï¿ÿžG}”>}úpÒI'Q\\¼×5îfÍšñûßÿž¿ýíolݺµV÷5‰I|ÅWü–ßVÙæòË/gÆ ¼úê«lß¾gŸ}–sÏ=—øøxJJJxÿý÷9õÔS‰‰‰)·mÿþý™6m°ã¸wêÔ‰»ï¾› &~œ%I’$I’$I’$íC’$I’$)âã?fРAA—ѨQ#ÊÊÊG¤™ IDAT(++cË–-ÌŸ?Ÿ?üáÄÄÄPXXÀøñã#A„P(D‹-X¾|y¹¾4hPé>&MšÄi§Æ5×\CÓ¦Mû¬Âº®]»2a„]öQPP@çÎ#¯ûôéÓO>III 3gÎÜãm¢¢jö¿\RSSéÔ©|ðA¹åsæÌaÆ dee;f,èر#Ó¦M+×nöìÙúÛÛcWV­ZÅÍ7ßLZZ#GŽŒ,¿ù曉ŽŽfìر•n÷À [ÜrË-ü×ýW…v111ÄÆÆF.ðÒâÅ‹hݺu­íãA$t3x—mO=õT’’’¸ï¾ûxýõ×¹âŠ+"ëêׯπ˜8q"Û¶mÛ­}7lØO<‘W^y…¸¸¸½žùC’$I’$I’$IRí1d I’$I’"Ì«¯¾Ê?üt)»eܸq|øá‡ÜsÏ=RXXȵ×^ËÖ­[6lØnõ1oÞ<xà6lØ@qq1ýë_©_¿>={öÜãmvÎ(°páB IJJª2´°»ÆŽËW_}ÅÍ7ßÌÚµkY´h#GŽ¤C‡\yå•‘v£GfÎœ9Œ3†¢¢"¾üòKî»ï¾ ýÕı« Û¶mcÅŠ<óÌ3ôìÙ“¨¨(Þzë-âãã#m?üp^xáî¿ÿ~nºé&–-[Fii)999üùÏæOú<òÍš5‹lóÐCñüƒ5kÖPRR·ß~ËÕW_ͲeËÊ¿ <÷ÜstïÞäääZ鿘bžçy~Ëo‰ÚÿŒ‰‰áÒK/å‰'ž &&† .¸ ÜúqãƱxñb.¸à¾úê+6mÚÄ¢E‹xðÁùÃþÀòåË:t(ÙÙÙ¬]»–7òÄOPRRÂqÇW+ã”$I’$IúìÝy|Tõ½ÿñ÷d„$aÉ$dÂP!TA6•Ä¢eQEÁ KÅj ¨]ÐR…‹PAÀ«Õ"¨íÏ ¥À(" ¶ÚªÄLBV•%d?¿?¸™›!“@’“åõôq0çœ9çóùæ$Œ™ï{€+GȸÜÿý***Ò“O>iZ ™™™²X,zå•WtöìYY, 2Äã¾·Þz«þñèwÞQdd¤úôé£ï¿ÿ^Û·oW@@€Þxã ×äôûï¿_‹E§Nr=¿S§NÚ²e‹¶mÛ&»Ý®¨¨(}ôÑGÚºu«ìv»V­Zåöün¸á’Ï‘.L‚Ÿ3gŽfÎœ©ØØX7N ¨TÿÅõM:ÕÕJJŠV¯^­®]»JºðÉò[·n•ÃáPDD„ ¤nݺéÃ?Tpp°ë˜“'OÖêÕ«õÒK/)<<\³fÍÒSO=%Iš={¶X£±óÄÓxÔ¦‡š|­}}}Õ·o_­_¿^óæÍÓÿû_·;G”?~¼öï߯¬¬,]ýõ ÒàÁƒuðàA9M:Õµï¢E‹ô§?ýI¯½öš~ô£)((HñññJNNÖ?ÿùOÝ|óÍëk(ü±^{í5-X° ÞαZ«å-oM×ô?gÆŒ’¤‰'*$$ÄmÛÕW_íºÆðáÃÕ¶m[ÝvÛmÊÌÌt… ºté¢Y³fiùòåêÑ£‡"##õÊ+¯è­·Þ"d4bÃ0 ³‹Z²µk×*))Éì2ÀeÆ š9s¦6nܨI“&™]Ь¥§§kРAêß¿¿Þ}÷Ýz9ÇYUŒbô€Гªy€h×®]ºé¦›ôñÇ»Â)Ô7þÎĉ%I›7ov[ïcF1 ñºï¾ûôÕW_iêÔ©*((Ð=÷ÜcvI@³”œœ¬ÿøÇ Ó믿^oçY­Õ:§szPÖø9'NœÐc=¦aÆ0Z/³ Ï3Ï<£ húôéš7ož Ì. hV6oެ뮻N‘‘‘úàƒ\/ç)PžÕ³ú©~ª0…Õè9 —ÅbÑË/¿\/uh¼~÷»ßé¯ý«^ýuõéÓGo¾ù¦Ù%M^FF†î¾ûnMš4IcÇŽ•ÃáÕj­·ó½¨•§<ýL?«ñs‡ŠŠŠôÑGÉn·×[m'B J·ß~»¾üòK]ýõš4i’tèÐ!³ËšœsçÎiñâÅêÞ½»þýïkÛ¶mzõÕWXoç,T¡–k¹Ðê õvÍ !P­ˆˆ½úê«Úµk—Ž;¦~ýúiúôéúüóÏÍ. hôŽ?®ßýîwŠÕsÏ=§¥K—êË/¿ÔÈ‘#ëýÜë´N'tBè‘z?€æƒ¨‘áÇëàÁƒZ³f<¨~ýúiĈÚ²e‹ÊÊÊÌ.hT’““õÀ¨K—.úãÿ¨éÓ§ëðáÚ?¾|||êýüçu^Ë´L35S6Ùêý|šB Æ¼½½5}útýç?ÿÑŽ;äçç§1cƨGZ¼x±¾ùæ³KLsêÔ)mØ°A#FŒPïÞ½õþûïkùòåJOO×ÓO?­:4X-Ô•«\=¦Ç윚B Ö,‹µmÛ6}ùå—5j”Ö®]«=zèÚk¯Õ³Ï>«œœ³Ëê]AAÞ~ûm?^:uÒܹsªwÞyGÉÉÉš3gŽZ·nÝ 5Ó1ý^¿×-P¸ÂôÜš>BàŠôîÝ[+V¬PFF†vìØ¡¸¸8-^¼X;wÖСCõôÓOëóÏ?—af— Ô‰œœmØ°A&LPÇŽ5iÒ$åååé…^Ðwß}§·ß~[£G–——9¿zû~£ éa=lÊù4m„ @ðööVbb¢^~ùe}÷ÝwÚ´i“ºvíªçž{NýúõSdd¤f̘¡·ÞzKyyyf— ÔXII‰öîÝ«Ç\ýû÷WDD„æÍ›§üü|-Y²Dr8º÷Þ{bj­‡uXë´NK´D­Õ°wPÐ<ø˜]h~4~üx?^’ôÕW_iË–-r8ºë®»TVV¦=zhÈ!}úTÚöÃ?xœ˜¾k×.¥§§«¤¤DÒ…CÇŽ®ˆˆÙl6Ùl6EFF*<<\6›Maaaj×®ºÅf§´´TÇ׉'ôÃ?(33S999ÊÊÊRVV–²³³•™™©ï¾ûNEEE®çuìØÑ;v¬+4ÒÒïV±I›´E[ôžÞ“¿šÏݘ§e¾û š½:¨C‡8p`¥m%%%®Éì'¸çääèÈ‘#úðÕ™™©üü|·çµnÝZíÚµSûöíÕ¾}{µk×Î@ SHHˆ‚ƒƒÕ¦MµiÓF¡¡¡ Q›6mšÝÝrssuúôi9sÆíÏS§NéÔ©Súá‡tâÄ W àøñã:~ü¸Nž<év???uêÔÉê0`€ÆŒã ztêÔIQQQjÕª•I6^yÊÓÃzX35S7êF³ËÐL2-Žºté¢.]ºT»ß™3g”““Si²ü±cÇ\“æÓÒÒ\ÛòòòT\\ìñX®ðÕj•———BBBd±XêúS’¬V«$¹Ö—×ܦM›JÇ-?ÎÅΟ?¯‚‚‚Jë uîÜ9×ãüü|ëìÙ³***rýyîÜ9×¾………:{ö¬[  *åÁŠ:¸±±±® F‡\ SÇŽ«ù  : µP¥*Õ2-3»Í!€*”‡jãüùó®Éøyyy:uê”ëqùý¼¼<•””èÌ™3*--ÕéÓ§URR"§Ó©²²2åååÉ0 :uÊí¸5 ”óõõUPPP¥õÞÞÞ v=nÕª•üýý¨€€×ŸmÛ¶U`` üýýÕªU+ºîÒ¢ÐÐP×ãàà`ׂ†±]ÛµVkõýEVYÍ.@3BÈ *00P:thðs[,mÚ´I'Nlðs£áÓ1Ý«{u‡îкÓìr43^f f š¡ò‘ÖhÙåh†¸“ÐD¬Ò*mÕV½¯÷e•Õìr4CÜÉh¾ÒWZ¨…úµ~­ánv9š)B@#W¨BMÑõW=®ÇÍ.@3æcvª÷ ýBiJÓçú\>üJ@=âI {Coh•Vé/ú‹¢mv9š9/³ àÙôÍÔL=¤‡t§î4»-! :©“§qºN×é÷ú½Ùåh!L©J5ESTªR½¡7ä#³KÐBðî$ÐÈüB¿Ðú@{µWíÕÞìr´ „ €Fä/ú‹Vh…^ÖËŠW¼Ùåha¼Ì.Àt@35SèM×t³ËÐ2T¥j”Fi˜†i™–™]€Š`²:¡[t‹Ú«½6i“¼åmvIZ(³ Z²ó:¯±«"é} …˜]€Œ`’R•jª¦*YÉÚ«½ê¤Nf— …#d˜ägú™¶i›vj§zª§Ùå!À Oé)­Öj½©75XƒÍ.$2ÜZ­Õ/õK­ÔJÓ8³Ë/³ Z’Wõªfk¶žÐš«¹f—n ä-½¥š¡Ÿégú•~ev9P ! ü?ý?Ý¥»4Wsµ\ËÍ.<"dÔ³íÚ®;u§îÑ=Z¡f—U"dÔ#‡ºM·i²&kÖÈ"‹Ù%@•õä=½§±« š õZ//~ ‘ã]M ¼«w5J£t›nÓËz™€€&w6€:¶I›4^ã5MÓôgýYÞò6»$¨B@zM¯iª¦*IIZ£5ÜÁ@“Â;œ@yA/èÝ£GôˆVi•,²˜]Ô ! ,Ó2ÍÕ\-ýßÿ )ò1» )3dh‘i¹–kµVk¶f›]\6BÀe*R‘îÓ}Ú¬ÍzE¯hª¦š]\BÀeÈW¾&h‚öj¯ÞÑ;ºE·˜]\1BMÐÖ­[uîܹJë?ùäY,·u#FŒPÛ¶mª´!G9ú‰~¢ïôvk·ú«¿Ù%@ dÐmܸQ¯¿þz¥õ+V¬ÐŠ+\ƒ‚‚tìر†,­ÙûJ_éVݪ éc}¬(E™]Ô/³ @íMž<ù’ûøúújܸq h€ŠZ†ô‘†k¸"¡ÝÚMÀ@³CÈ  ºùæ›Rí>ÅÅÅš2eJUÔü½ªWuÓÿþ÷¾ÞW˜ÂÌ. ê!€&È××W“'O–ŸŸ_•û„††ê¦›njÀªš§2•ihº¦k¾æë ½¡qwÍ!€&jòäÉ***ò¸ÍÏÏOÓ¦M“OWÕ¼ä+_ã4NÒŸ´A´LËäůÔ4c¼Ë ÐD :TáááÊÉÉ©´­¨¨H“'O6¡ªæÃ)§ÆhŒ¾×÷Ú¡¦af—õŽ]h¢,‹¦L™"??¿JÛl6›hBUÍÃ^íÕ ’|´_û h14a“'OVQQ‘Û:???MŸ>]‹Å¤ªš¶õ¢nÒM®áúH)JQf— †@Ö¿uíÚÕm]QQ‘&OžlREMW 4C34Gsô¸×&mR+µ2»,hP„ š¸iÓ¦É×××õ¸[·nêÓ§‰5N÷ë~ÑÛ2”¡á®·þ÷¿ßè7²ˆ;Ahy4qS¦LQII‰$É××W÷Þ{¯É5>µQë´Nwé.•¨ÄmÛ6mÓ5ºF…*ÔAÔ83©J0!€&.66VW_}µ$©¤¤D“&M2¹¢Æ%S™š¥Y²È¢Ïô™k±$É¡eZ¦Q¥[t‹>ÒGŠU¬¹Å€É4Ó¦M“$ÅÇÇËn·›\MãaÈÐ}ºO…*”!C¥*ÕSzJ[´Eã4N¿Ô/õ”žÒkzM­ÔÊìrÀt>f€šËÉÉQjjªÒÒÒÜ–o¿ýV’täÈ Ô‡¤A&V !€Fä‡~ð"(_ $I¾¾¾êܹ³¢£££aÆiÓ¦Mš0a‚Nœ8¡ÔÔTýýïWZZšÎœ9#Iòòò’Ífs=çâ BçÎåãÓ|~]ôµ¾Ö"-r HR™ÊtZ§5@4PMª§æó®1@››«ììlåääÈétº-GŽÑéÓ§]ûZ­VÙívÙív;Öõw»Ý®.]ºT Œ?^½zõªñ9·nÝZí9/^<³±*V±îÒ]*SY•Ûwh‡^Ô‹š­Ù \4^Mã]a€&âüùóN§Sß~û­òòò\ûVœÐŸ ¤¤¤+šÐï)`P~«Õª¸¸8Ûsss]5V #8>|Xùùù’.Ü=!,,L6›Íc!**JÞÞÞµª¹¾üZ¿ÖúB¥*­rC†æk¾k°®ÒU X4^„ j¡  @ÙÙÙCN§S¹¹¹®}=…ÂÃÃe³ÙÔ«W/µjÕÊÄNþÕjU||¼âãã=n¯B¨Dp8JNNÖÙ³g%I~~~ŠŒŒ”ÝnwõY1„-//¯zïgŸöé÷ú}•w1¨¨XÅš¢)úTŸ*@õ^4v„ *(,,TVVV¥ð@ùÄúÔÔT†!É=D0xð`M˜0Áõ¸G 2¹›ºQÛBùràÀ¥¦¦êüùó’ÜCž‚111²X,WTëÑdM–—¼ª øÈG%*Q˜Ât›nÓh–EWv^h.€¥¨¨H™™™N§Ócˆ <<\qqq=z´kB|÷îÝÕ¦M“»i.7„àp8”žž®’’I’¿¿¿"""Üî~P1ŒP“ƒzP9ÊQ‰J\ë¼å-I*U©º©›ÆiœFi”k0á¸!ЬT T ”/iii*+»ð ÷nŸ¤Ÿàú{×®]br7ÍCu!„ââb;v¬Òש<„pôèQ•––JªüõºxùÐú¡þGÿ#éÿîVà/%*Qc5V£4JÔ©![€&‡hRª›”ît:«”^1D«ÐÐP“»¯¯¯l6›l6[•!„ŒŒ ‡Ãñ¡‘Ž’å+‹ÔNò?ᯮ_tÕuÇ®ÓïêÕƒÐÔ!Ðèäææz 8N¥§§«¤¤D’äï﯈ˆ!‚ððpÙl6“;Á•òõõu}M=)¿sÅSeOéDÊ …oWéç¥r:úÀùþ'í\w®°Z­®ëââ» tëÖMÁÁÁ Ù4J„ @ƒ»’AÅIâ111²X,&w3ùùùÉn·kÖ]Xq­ûöÂÂBeee¹]cÙÙÙ®;!¤¦¦Ê0 IB‡TÊ÷èÑCAAA Ü4|ø°òóó%I—Øc IDAT¾¾¾êܹ³k2vBB‚’’’\“´£¢¢äíímr7hÉ.+„àp8”’’¢S§N¹ö½8„P1Œ`·ÛØPmÀe#d*)\ p:úæ›otæÌ×¾'V"@ss©Bnn®Çï‡Ã¡#GŽèôéÓ®}« !ØívuéÒE>>üª€ùxç€èüùó•&E—/ß~û­òòò\ûV"`b4Z:«Õ*«Õª¸¸8Û« ì8·ÀŽ¯¯¯ÂÂÂd³Ù<†ìh(Ì *((Pvv¶ÇAJJŠN:åÚ·ªAxx¸bbbÔªU+;š6«ÕªøøxÅÇÇ{Ü^B(_ÊÇCÉÉÉ:{ö¬¤ !„Î;+<<Üc!::Z^^^ Ù€fŠMPaa¡²²²<†Ê')—»8D0mÚ4×$åž={ªuëÖ&v´lµ !”;}ýõ×:wîœ$ÉÏÏO‘‘‘®ïõ‹Ã„Ô!!O!‚òð€ÓéTjjª Ãô!‚ððpÅÇÇk„ ®‰ÅÝ»wW›6mLîÀ府‚Óé”ÃáPzzºJJJ$IþþþŠˆˆp»ûAÅ BLLŒ,KC¶ ‘"d€ ŠŠŠ”™™Y)ðFÐ22 +##Ãc€ÀétV;¹·bˆ 66V¡¡¡&w ©ª.„PÝÏ©Ú„ø94/„ ¸LU}BxmB|B8³øúúº~yRÝWGµw\©¸pÇ i!d@ª ¤§§«¤¤D’äï﯈ˆˆJ!‚ò ·111²X,&wµãççWm¡°°PYYYn?³³³]!„ÔÔT†!éB¡âÏÅŠ!„îÝ»«M›6 Ù€j2´XU…²³³•––¦sçÎIº0Ñ622Ò5!vðàÁn“d£££åååer7аüýýkBp::pà€Þ}÷]åää¸ö-!\|—»Ý®ž={ªuëÖ ÕÐâ24[‡²³³•““#§Ó©ääd={V’äëë«Î;»&µÆÇÇ»Mv%Dµw©BAAë·á””:uʵïÅ!„Ša„˜˜µjÕª¡Úš=B€&«}ÚµoÅ ¦‡˜` MO```µ!„ª‚f‡Ã-h&UB h¸c†À4ÊÎή p:JIIÑ©S§\ûV"—ÝnW`` ‰šÕjU||¼âãã=n/!\Dp8:|ø°òóó%]¸ÛMçÎîºã !´d„ õ¦ºAù„ÏržBå>{öì©Ö­[›Ø  ©©M¡â¿M‡C_ýµÎ;'IòóóSdd¤+Øvq!::Z^^^ ÙP¯.[aa¡²²²ª ¤¦¦Ê0 Iî!‚Áƒ»MÒìÑ£‡‚‚‚LîÌ·oß>3F{öìQïÞ½Í.§Y»œ‚ÓéÔ¾}û”žž®’’I’¿¿¿"""Ü‚Ã111²X, ÙpEªTTT¤ÌÌÌJáòÇiii*++“ta²fù¤Ê¸¸8=Ú5Ù²[·n 6¹›æ!33S;wv[·bÅ =ôÐC—u<‡Ã¡ÄÄD}ñÅêÓ§$iÕªUúéO*Iz饗4sæÌ++ú"žz.LÔíÚµ«î»ï>=øàƒòñ©»_[xêÓ Ð’%Kô駟êرcŠˆˆPBB‚¦L™¢aÆUšˆœœœ¬eË–Éápèûï¿—ÕjÕ AƒôÈ#hèС®ýêúºhN<}í-Z¤eË–I’¶mÛ¦‘#G^öñëòX’TVV&Ã0\-˜çrC‡ã’!„‹Ã@cBÈZ°ââbeddT x ¸Ý} !!Áõ÷®]»*$$ÄänZ†ÈÈH†¡éӧ뭷ÞR~~~ŸcÞ¼yš>}ºÚ´iSçÇ–ªîáĉZ»v­yä%''kíÚµõr~³|öÙgºþúë5cÆ }ôÑGêÔ©“²³³õ§?ýI7Üpƒ>ýôS 0Àµÿ;Ｃɓ'köìÙÚµk—ºté¢ï¾ûNþóŸ5bÄ=ùä“Z´h‘¤†¹.š“¥K—júôéêÕ«W£:–$ :T'Ož¬“c¡~UB(..Ö±cÇ<þÛêp8tôèQ•––JªüïkÅ%66V¡¡¡ ÝZ8BÐŒU7ÉÑétV;ɱbˆÀn·ËjµšÜ š»víÚéÑGջﾫõë×kéÒ¥jÛ¶­ÙeÕ™—_~YþþþZµj•¼¼¼$IÑÑÑúãÿ(‡Ãá¶orr²&Ož¬ùóçë駟v­ïÒ¥‹üq…††jÞ¼yêÛ·¯~ò“Ÿ4h.Í××W6›M6›Íc¡â‚.ú9ŽjC~B~¨^f¸2¹¹¹:pà€Þ|óM-[¶L³fÍRbb¢bccÕªU+EDDhÀ€š6mšÖ®]+§Ó)»Ý®¤¤$mܸQ{öìQJJŠÎ;§””íܹSkÖ¬ÑÂ… 5aÂÅÇÇ0h"RRR4vìX………©M›6ºí¶ÛôÉ'ŸT¹ÿ¢E‹”˜˜(IêÛ·¯,‹¢££Ýö)))у>¨EDDèÉ'Ÿ¬tœ-[¶hÀ€ PÇŽõÀèôéÓ—ÝG·nÝTVV¦ÌÌLIÒöíÛ5pà@ª]»vš6mšrrrjÜû¥ú\½zµ¢¢¢ Aƒéã?–Åb‘ÅbÑÀµjÕ*×ãU«ViöìÙjÛ¶­,‹î¼óÎÃùóçU\\¬sçÎUêù¿ÿý¯Û] –,Y¢’’-X°ÀãÍš5KíÛ·×O5íkݺuµêµÜ¥®g4~~~®@ßÝwß­… jÍš5Ú¹s§RRRtþüy׿ÉÏ=÷œ¦M›&»Ý.§Ó©µk×jÒ¤I0`€BCCÕ¶m[ 0@£GÖ¬Y³´lÙ2½ùæ›:pà€Îœ9cv«h‚@#WUˆ ..N­ZµrM.œ:uj¥Áúõë=NX¬"2dˆìv{&Тq?~¼uèÐ!edd(::Z Uî¿téRíܹS’ôÅ_È0 ¥¥¥¹íó /(!!A™™™zì±Çô›ßüF»wïvmçw4fÌÝzë­ÊÉÉÑŽ;´{÷nÝ~ûí2 ã²ú8r䈼¼¼©-[¶èÖ[oÕˆ#”‘‘¡}ûöéðáÃ6l˜ÛäÙêz¯®Ï7jÞ¼yºÿþû•““£_|Q .tõþÉ'ŸhÞ¼y®s=óÌ3ºé¦›”‘‘¡•+WÖj®½öZhÔ¨QúðëŸü㊋‹«2àããã£ë¯¿^û÷ï×ñãÇ/kœ/V>Ö‰‰‰ÊÌÌÔ¿þõ/ùûûkĈ’¤gŸ}Ö5Žåî¸ã;vÌmݥƫ.Ƴâ1*^£ .t»FkrKRPPJKK¥)S¦¸*ÑÑÑúþûïçúTùê<ùä“=z´222´~ýz­_¿^<òÈeõåi<«ëUªÙõŒ¦¡b!))I‹/®2„°téR%$$(00P_}õ•[!88Øõ:aâĉš?¾[!??ßìVÐù˜]´t¹¹¹r:nKvv¶rrr”œœ¬³gÏJº0á022Rv»]ááኗÝnw-ÑÑÑòò"KÞRè?ÿù{ì1uèÐA’´|ùrmܸñŠŽ;`À3F’4gÎ-Z´H{öìÑðáÃ%I ,P\\œëÕ­V«þð‡?hôèÑÚµk—nºé¦ŸëĉZ»v­>þøc%%%©mÛ¶úùϮ޽{ëw¿û$),,Lk×®U¿~ýôüóÏkáÂ…WÔûâÅ‹uÍ5×è—¿ü¥«þò=IHHЄ $]˜ü=oÞ¼ÃôéÓõÏþSýë_5|øp…‡‡ëöÛo×Ýwß­ë®»ÎuŽÓ§O+77÷’Ÿ:.Ã0”žž®°°°Köz)?ÿùϧßþö·®ukÖ¬Ñ?ÿùÏË>fUãu©íµ¹®*^£sçÎÕ£>êvÖ”———fÍš¥ßþö·ÊËËSHHˆ$éÕW_ÕÌ™3kĺ馛4nÜ8IÒí·ß®3fèÅ_Ô£>ªˆˆˆ+þ~¹T¯µ½žÑtùûû»þý÷¤  @ÙÙÙ•^_ìÛ·ON§S¹¹¹®}­V«Ûë‰ò×6›M½zõR«V­ª-4„  žU ”‡Ê>|Øõ)¾¾¾ “Ífsûôâò QQQòöö6¹4VúÑ~¤G}T‹E£FR`` ¾ÿþû+:nß¾}]·X,jß¾½~øáIRff¦¾ùæ›JÇË'Ì¿÷Þ{—œ4}öìY×äm???uíÚUË—/×üùó•™™©Ã‡kîܹnϹæšk,‡Ã¡… ^vï'OžôXÿ€ª|ÎUW]Ui]MÇÁ××Wo¿ý¶>øà­_¿^ÿûßõüóÏëùçŸ×Í7߬M›6)44Ôu‡ƒ†¼»HUc¤'N\öq=×¥¶×öºªxz{{+,,ÌuÖÖÌ™3õÄOè7ÞЬY³$I›6mªt‡ªT ‹HÒСCµråJ}Úµoʼnz‡˜¨‡+µ}ûv-Y²D=ô¦Nª„„ýú׿®4ñ¹6‚‚‚Ü{yy©¬¬L’tüøqIÒªU«´jÕªJÏMOO¿äñ[·ní Ú\¬üømÛ¶­´­]»v®íÒåõž““ãñøÁÁÁU>'00°Ê:k:7Üpƒn¸ájûöízöÙgµcÇ-Y²DË—/WHHˆBCC•]e뎎®v¿š¨n¬¯„§ñºÔöÚŽçÅר¯¯¯ë­­öíÛëŽ;îІ 4kÖ,}òÉ'ºúê«Z£ç[­V·ÇíÚµ“$egg«sçÎ’®ìû¥º^/çzFËXm¡ª×6‡£Ú×6„šÞá€K¨êÓ~NgµŸöKˆ Íjµê™gžÑ3Ï<£O>ùD?þ¸† ¦C‡)66¶ÎÏ&IZ°`–-[VoÇ?yòd¥m'Nœpëérz÷x|Oç«Iµ3F·Þz«ìv»þýﻶýøÇ?Ö_ÿúWåææVš¸.I%%%úè£4pàÀ: T7ÖyyyI’Š‹‹]ë*N6® õ}]]Êܹsuýõ×ëСCÚ°aƒæÌ™Sãçž9sÆíqù] l6[½÷UW×3 ]ø™jµZçq{Uwir8ÕÞ¥éâ…»44NÌnÐâ(;;ÛcˆÀét*77×µ¯§Axx¸l6›zõê¥V­Z™Ø Z²ï¾ûN‰‰‰úâ‹/$IÔK/½¤ØØXíß¿¿Ê‰öå“Æ/Gdd¤zöì©O?ý´Ò¶«¯¾Z=ö˜&MštEÇïÑ£‡>øà·õŸþ¹NŸ>­„„I5ëÝSŸmÛ¶U÷îݵwï^·õžú¹T5‡_þò—²X,úíoë¶|}}]Ÿz/I>ú¨þö·¿é÷¿ÿ½ž~úéJÇ]³fŽ;¦×^{­ÆuŽ9R‹-Ò 7Üే=zh÷îÝnë³³³e·Û•••¥víÚ©}ûö’.Œy¹ÿþ÷¿5®¡&êúºªí5>hÐ õë×O«V­ÒÑ£GÕ¿ÿ?wÿþýš0a‚ëñž={äëë«øøxÙl¶zý~©«ë¨ «ÕªøøxÅÇÇ{Ü^1„P1ˆàp8”œœ¬³gÏJ’üüü)»Ýîz=U1„}EÿNàò2Ðì*++«Rx |Â[jjª ÃT9DPqÂ[=dr7@Õ¾üòK­X±B3fÌPYY™Ö¬Y£€€ýèG?ªò9åŸ~žœœ¬N:©oß¾z÷ÝwÕ³gÏó™gžÑ˜1c´téRÍœ9S’ôÔSO©¤¤DcÇŽ½âž–/_®±cÇêñÇ×Ã?¬ãÇkÖ¬YêÚµ«fÏžíÚïR½WÕçâÅ‹u×]wiÉ’%š;w®Ž=ª5kÖԺΚŽÃsÏ=§ž={êæ›oVHHˆ²²²´bÅ ¥¥¥é…^píwõÕWkÆ š1c†Š‹‹5{öluéÒEßÿ½^}õU=ùä“zöÙguóÍ7_Ö¸zR>Ö¿úÕ¯ôÐC)??_³fÍÒ=÷Üã @tïÞ]:tÐóÏ?¯ë®»NÇŽÓ+¯¼Rg5”«Ë몪¯}u?ÏçÌ™£¤¤¤Z÷ööÛokÈ!ºá†äp8´~ýz%%%Éf³Õy_žÔÕõ \©Ú†Ê—(55UçÏŸ—äBðDˆ‰‰‘ÅbiÈÖZ€©Ö¬Ycv M^aa¡‘’’bìܹÓX³f±páBcÚ´iFBB‚a·Û ‹ÅbH2$V«Õˆ7Fe$%%K—.56oÞlìß¿ß8}ú´Ù­—”‘‘ẞ˗+V†a[¶l1víÚÁÁÁÆ!CŒ÷Þ{ï’Çœ3gŽbsæÌ16nÜèvü)S¦T:oll¬ëùÛ·o7høûû;v4îºë.###£V= <¸Êý·mÛf\{íµ†¿¿¿aµZ)S¦ÙÙÙnûÔ¤÷‹û,·zõj£K—.F@@€1tèPãÀ†$cݺu†a•ÆC’‘››[©ÎKÙ3gŒ—_~Ù9r¤eøúúV«ÕHLL4vìØá±÷/¿üÒ¸÷Þ{]ûûùùS§N5öïßÉ1õ´ìÚµ«Êq¾x¬m6›ñðÃçÏŸwÛgçÎFïÞ½ÀÀ@cøðáÆþýû]Çÿñ|Éñª‹ñ¬í5zñ×~áÂ…•ž_QVV–fT;^†a¸ëµ×^3&L˜`V«Õ˜7o^¥cT××Ê•+Ýê>|x­{½Ôõ 4'Ož4öïßolÞ¼ÙXºt©‘””äz]çãããºöýýý »Ýn$$$¸½®Û³g‘’’b”••™Ý €ËÀÿ#@Ù0a‚1a„Jë-†ñ¿× Àk×®URR’Ùe4jEEEÊÌÌt»û@ÅO½MKKSYY™$) Àín+.]»vUHHˆÉÝhì>ÿüsõë×O»wïÖ°aÃÌ.Ç%//OÝ»w×| ^½z™]N³¶råJeffjÙ²ef—rÅëõ \®ââb;v¬ÒëÁòåèÑ£*--•TýëB»Ý.«Õjr7<áÿ‘ áLœ8Q’´yóf·õ>f]Éd±„„×ßcccjr7š’uëÖéwÞÑóÏ?¯Ž;*%%EóçÏ×5×\£Aƒ™]ž›­\¹RwÞy§^yå]uÕUòòò2»¬fcÉ’%ò÷÷×wÞ©•+WjçÎf—TkMéz.—¯¯¯l6›l6›âãã+m/..VFF†Ç`ªÃá œ P„ 4ˆÜÜ\§Ó©ôôt•””H’üýýá1D.›Ífr'š“»îºK'NœÐO~ò¥¤¤(,,L Z²d‰|}}Í.¯’‰'*88XsçÎÕ7Þ¨%K–˜]R³²`Á=õÔSzâ‰'ev9µÖÔ®g >øúúº^;zRñYß%ëâ‚Õju½þ¼8„Э[77dk Æb†av@K¶víZ%%%™]Æ»ÜAÅð€ÝnWLLŒ,‹ÉÝ %*,,TVV–ÛkÙŠA„ÔÔT•ÿJÝjµz|=k·ÛÕ£G™Ü Ð45—ÿG€¦`âĉ’¤Í›7»­çN.É0 åää(55Uiii•–ôôtIº"ˆŽŽVtt´bcc5bÄ×㘘uìØÑänÏüýý«½B~~¾ÒÒÒ”ššêöÚxÏž=JKKSnn®kßððpÅÄĸ^ W\¢¢¢äçç×PmÔŠ—Ù€úµoß>µk×N‡2»”Fñ¨{MyL¯¤öÚ<·®ÇèrŽ×”¿Nw2pI‹E6›M6›Mƒö¸Onn®œN§Û’’’¢;w*==]%%%’.|:lDD„ëbív»ÂÃÃe³Ùd·Û#‹ÅÒíuÎáp(11Q_|ñ…úôécv9*++“a2 £ÞÏ•ŸŸ¯°°0½ÿþûºþúë«Ý·¾Ç©ªã7Öñ¨ Œií]Iížž[×c´hÑ"-[¶L’´mÛ69ò²×_§Ï>ûLøô{÷n={V½{÷Öc=¦Q£FÕÛ9ºTXX¨¬¬,·×´ÙÙÙÊÉÉ‘ÓéTjjªë{Èjµº^Ï:T“&Mr=îÑ£‡‚‚‚Lîàò2P'¬V«âããïq»§‚Óé”Ãá¸dáâ0€Ú:t¨Nž<Ù çÚºu«¬V«Ø 绌GÝkÊcz%µ×æ¹—{ž¥K—júôéêիׯ!¾N·Ür‹n¼ñF8p@>>>úÕ¯~¥1cÆhëÖ­ºå–[êõÜ@M)33Óc€Àét*--Meee’.¼¾­†MHHp½.íÖ­›‚ƒƒMî ~2Ð ª !ëرcn¼*†Ž=ªÒÒRIR@@€k¢×ÅKll¬BCCº5üíoÓm·Ý&///³KiºÇ˜6n­ZµÒºuëÔºukIÒÊ•+õúë¯ë¥—^"d€Q\\¬ŒŒŒJáO!‚‹_WV tíÚU!!!&w`Þ`:___Ùl6ÅÇÿöî=:îºÎÿø3Is+½0mI3¹´É´´%áj¸—ŠbºÈÊEZ›úSOA`9« º.GÑ…ýéÙÊ®­îþÄË.—= Reý5 B ÊRE¥Wši›¶Iš^¦÷$Mšïï6c¦™¤i›æÛ¤ÏÇ÷ÌÉw¾óùNÞŸÏ$“IòyͧŠ9sæpß}÷1oÞ<.\H]]{÷®Ž… 2oÞñÄLž<™¼¼<.¹äÞxãäùÝßM¾­­_þò—|ä#9¦qÚ²e _üâ™:u*yyyœ{î¹<ÿüó)5uïÃܹs¹ýöÛ7n|üãïõþÃÃÕÛeÁ‚œþùäåå1qâD>÷¹Ï±sçÎcþÚsLÓé±ÔžîÜ#£þ>.餻¿¦¦¦ä±C/ûØÇíqŠÇãÉ€Àˆ#ÈÏÏgëÖ­‡í—Ôííí444°dÉž}öYyän»í6fϞ͔)S’¯gÍšÅ'>ñ æÏŸO<'‹QSSÃSO=Å[o½Å¶mÛhiiIyyß}÷1gΪªª H’$I’$I’¤“[ I’$)TóæÍ »„!¯­­-¨«« .\Ì›7/øû¿ÿû ¦¦&¨®®b±X‘‘D"‘ ªª*¸ú꫃šššàᇞyæ™à­·Þ vîÜvW4DœsÎ9Á 7ÜlÚ´)H$ÁÝwßœrÊ)ÉÛ.\ÁŸÿüç”ó^xá… 3y­Úg IDAT33xà‚Í›7Ë—/.¸à‚`êÔ©)_Ë—/€ ¼¼Ø¹sgðè£@ðÊ+¯$ÛýÇüGßøÆ7‚mÛ¶o¿ýv0kÖ¬þõ_ÿ5¥ ,N=õÔ`ß¾}Ç4Nwß}wp÷Ýw[¶l víÚüøÇ?rssƒwÞy'¥]WÊÊÊ‚gžy&ؽ{wðøã7ÜpCŸ÷Öx®Þçž{.ÈÈÈþîïþ.yß3fÌ®¸âŠ ³³Ó1=Ncz,µ§;÷Hƨ¿K×÷ú‹/¾Øëý566øÀRÎûÊW¾äååúÓŸõqêî׿þu÷ÜsO¯m¤CmÛ¶-xë­·‚gžy&xøá‡S^×1"ùº.777ˆÅbAuuuÊëº×^{-¨««K~ŸKZüY’$I’$I’Ïœ9s‚9sæô8nÈ@’$I ™(Ž¿ÖÖÖ”Â}÷ÝÌ™3'˜9sf¯!„9sæ÷Ýw_ðÏÿüÏÉ®]»ÂîŠN---<ýôÓÉcíííAAAAòzo“Œ§OŸœyæ™)Çþð‡?@ððÃ'uM(þ⿘Òö _øBlØ°áˆÚõ5±øÓŸþtòXGGG0räÈàßøFòØ´iÓ‚sÏ=7åþñ‹_¤XüéO:¸å–[ŽyœÒ¹æšk‚Ûn»-åXW>ûÙϦ=çh&įñèO½Ó¦Mëñõñ /@ðÒK/9¦ÁÀé±Ö~¬!ƒtÒ=.ý êÕW_ ²²²‚Ç{¬ÏsŽÇãAH$‚'žx";vlPTT455õÙo\z TTTùùùÉ×e999=BO>ùd°páBCÒ0æïÈ’$I’$I’4xz Œ8²u$I’$ièÉÍÍ%‹‹ÅÒÞÞÚÚJCCñx<åR[[K<'‘H$ÛF"‘ä}u]¢Ñ(EEEœqÆŒ9r°º¥äååqÁðå/™ŒŒ ®¾újòóóÙ´iSŸçmØ°•+WrÇw¤?÷Üs3f µµµÜwß})·þù)×gÍšÅã?Îïÿ{Š‹‹¸]:guVr?++‹ &ÐÜÜ À¶mÛXµjwÞygŸuìß¿ŸŸÿüçÌŸ?8úqê͸qãXµjUÚÛÎ>û죺ÏtŽ×x®Þ 6¤½ï‹.º€—^z‰+®¸Â1=cz´µ/}=.ýµ}ûvn¹åþê¯þŠ/|á ý:g §.÷Üs ,ছnâÁ¤  à(z¢¡*‘H¤¼¦jhh ±±‘x<Ί+سg999”””$_OUUU¥¼Æ*++#333äÞH’$I’$I’$| H’$I:éåååõBhiiINŒ;4„°zõjvìØ‘l›.„Ðu™4i#FøkØpð«_ýŠ‡zˆ{[n¹…êêj¾öµ¯%'/§³eËàÀâC?>y{wcÆŒéÑ ¡¡á¨Ú¥3jÔ¨”ëÙÙÙtvvÐØؘ¶æC?Àk¯½ÆÞ½{¹òÊ+“ÇŽfœ–-[ÆW¿úU^ýuš›› ‚8ÈH'??ÿ0½ì¿ã9]ÒÕÛõøÏ;—¹sçö¸½¾¾pLǘmíáH—þúÜç>Gkk+?øÁú}Î@=NÝÍ;—üãý®ACG÷A÷A<gåÊ•ìÞ½8ðu4aŠŠŠˆÅbTWWSSS“|m4yòd²²²Bî$I’$I’$I’åìI’$I:Œüüü>C‰D¢Ç»®»ï¾ËÎ;“m ! ‘H„ï|ç;|ç;ßá·¿ý-_ùÊWxï{ß˲e˘2eJÚs&L˜xGðCmݺ5íyÝWÑèjPTTtTíŽT4zÖœ®?ûÙϸòÊ+SVó8šqjoo§ººšââb^yåN?ýt²²²øÔ§>ÅÛo¿}Lý9VÇ:}éúúøÒ—¾Ä#<Òk;ÇtàÇ4,ÇëqyòÉ'yúé§ùå/9`«Éã¤á¡·×6ñx¼Ï×6‡†|m#I’$I’$I’44ùI’$I:F‘H„H$BeeeÚÛ{{·ßÚÚÚ>ßí÷Ћïö{bhjjböìÙüùÏàâ‹/æ{ßûS¦Lá­·ÞbÊ”)dffö8¯¤¤„éÓ§óÊ+¯¤ûí·Ù¹s'ÕÕÕ=ÎyóÍ7¹ñÆ“×_{í5²³³©ªª:ªvGjܸqL›6E‹¥ÿŸÿùŸmŸ{î9zè¡äõ£§xCgg'óæÍ#// .¸øË»~¯X±‚ÂÂBÎ:ë,^xá¾ýíosÝu×ñ•¯|…{ï½—-[¶pÛm·1uêTn¿ýöŸçå—_æ¹çžãøµµµüÛ¿ý555=V(èo»£ñàƒrÓM7ñÐCqÇw°nÝ:æÍ›—Òæ­·Þ¢±±‘k®¹æ˜Çé¿þë¿8í´ÓøÑ~Ä_ÿõ_‹ÅxõÕWyñÅ)++;¢Ú{{ºOè>RÇ2‡óï|‡k¯½–‡~˜Ï~ö³|ë[ߢ££ƒë®»ŽíÛ·;¦<¦ÇCǨ¬¬lÀ€ŽŽn¾ùf&Ož|\VnèÏãÔ%‘HpÞyçqÑEñôÓOx-:¼ÖÖVÒ†âñxÊ 8éB]¯3Î8ãŒ~¯"I’$I’$I’¤a$$I’ªyóæ…]‚NpÛ¶m Þzë­à™gž ~øá ¦¦&¨®®***‚üüü€ '''ˆÅbAuuuPSS<üðÃÁ“O>,\¸0¨«« :;;ÃîÊ°±`Á‚`öìÙÁøñãƒ1cÆ—]vYðÒK/¥´ùüç?Œ;63fLðùÏ>yüÅ_ .¼ð 777ˆD"ÁÍ7ß444¤œ»|ùòž}öÙàÖ[o FD"‘àÎ;ï Z[[¨Ýã?žü‚Ë/¿<øÏÿüÏ”c7ß|s°~ýú”cS¦LI~ž'žx"˜4iR——Ìš5+X²dIßÿþ÷ƒ ‚/ùËÁìÙ³lœÞ|óÍ`Ö¬YÁ¨Q£‚ÒÒÒ ¦¦&øØÇ>–¬­±±±G€ ‘Hô¨áÐûk<ú[ï¯~õ«àâ‹/rssƒ‰'7ÝtS°~ýzÇô8é±ÔžîÜ#£þ>.÷Ýw_zÒÝß³Ï>Ûc<º.Ó§O”ǩ˶mÛ‚ÒÒÒ`Îœ9= ŒÖÖÖ ®®.X¸pa0oÞ¼à¾ûî æ̙̜93ˆÅbAFFFò1ŒD"AUUU0gΜ஻î ~øáà™gž Þzë­`×®]awE’zðwdI’$I’$Iô¡QWWÇ„ ¨®®æ¡‡";;{À?߉ÎñxŽéÐàãtdú ùÅãñ>C~ÝCS¦LáÔSO ¹7’$I’$I’$I:Ùd]ëjK’$I Åüùó©©© » dÚig [ØÊV¶°…MlbË!Û&6±9ØLsg3Û2·Ñ‘Ñ‘rœÀ¶olK†"‘Ñh4íj§Ÿ~:cÆŒ £«’$‰D"m€ S__OGÇŸ›¹¹¹§]%¨ëç¦$é/üY’$I’$I’Ïõ×_À3Ï<“rÜ• $I’$ih¡…D·­‘FhèõX3ÍìgÊ}ä‘GED‰!B %œ•qEY9Öµ•PÂØÇÒöå66nÜØcråÒ¥K©­­eÍš5teÛ#‘HI•]קOŸÎ¨Q£Â:I’Ò:ÚAuuuÊϹòòr222Bî$I’$I’$I’Ô† $I’$éÔ84,.<Ð@ÛÙžr~y)¡€bĘÉÌ”c]¡‚ q¿"æææ&'U¦ÓÚÚJCCCÉ™‹/¦¡¡ÆÆÆdÛî!„CÃ3fÌà”SN9âú$IêMºA×ϦåË—³wï^rrr())Iþ\ªªªJùYUVVFfffȽ‘$I’$I’$I’Ž!I’$I[ÙJ3Ílf3›n›n›ØÄ–C¶N:SÎ?•S) € · ¨¤2yl<ã™À&2‘ L`'ƪyyyGB¨­­¥®®ŽíÛÿžHBèšð‹ÅÈÏϬnI’†€î!‚®ð@×õ•+W²{÷n²³³)--MÛª««©©©Iþœ™ö¥œŸG^JH FŒ™Ìì&ˆ!J” 2BêéÉ+‰PUUEUUUÚÛ»B]—®0Bmm-+V¬`Ïž=ÀBii)Ñh4m¡¬¬ŒÌÌÌÁìš$ {­­­444¤ ÔÕÕ±}ûödÛÞBÑh”òòrFŽbO$I’$I’$I’¤áÅ$I’¤!e/{i M·FÙ|pë¾ßD;Ø‘rn>ùœÆiD‰rÚÁí,΢€Nã4 ( ÂämYøÎöCÝ‘†º./¼ðk×®eïÞ½äääPRR’œàzhÁ‚$õÔÖÖÆÆÓ>Ïv…¾º"¸õÖ[“ϳ3fÌà”SN ±'’$I’$I’$IÒÉÅ$I’¤ÐuÐA3Í4ÑD#4ÓÌF6¦=¶‡=)çžÖm+¤ó8Ó8‰·îáQŒ ©‡:Qm¡¶¶–úúz:::ÈÍÍ¥¸¸¸Ç ]a„òòr22\éBÒð’.DЈÇã¬Y³† €¿„¢Ñ(UUUÌ™3'ù\9mÚ4Fro$I’$I’$I’$u1d I’$é¸i¡…Fi ‰äþ¡›if?û“çå‘G„E%J|€¤+¢ˆRJÉ&;Äj¸;–ºuëØ¿ÿÀ×u^^^ÊÊé‚’t¢Ù·o6lè躬]»–ÎÎNàÀóe÷U^ª««“Ïs§Ÿ~:cÆŒ ¹7’$I’$I’$I’úË$I’¤#ÒF[ÙÚgx A‚õ¬g»RÎI¢D‰K D‰&ƒÒPÐW¡½½õëק˜{$!„)S¦pꩧv×$úzžŠÇã}>Ouø<%I’$I’$I’$ /† $I’$ qØ$h¢‰€ y^y)A*ªz¬8%ÊD&’EVˆ=”Wvvvrn:}½CxmmmŸïÞýâ;„KêKo+®IˆÀW$I’$I’$I’¤“‹!I’$ië ƒfšÙÀšhJ~\Ïz6±‰l¤ùàÖIgò¼|ò)¤(Q (`:әŬ”cEQ@¹ä†ØCièÊÉÉé3„ÐÖÖÆÆS&744$CkÖ¬!~"‘HÊDàî!„iÓ¦1zôèÁ욤AÔWˆ ¾¾žŽŽrss)..î"èzÞ(//'###äÞH’$I’$I’$I:2$I’†¨fšS‚Ø@#ld# ·MlJ ŒgŸ9sfJ訬¬ŒÌÌÌ{#I’$I’$I’$i(0d I’$`Zh¡‘Fhèõc=õìfwòœ<ò(¢ˆ(QŠ(b&3S®G‰RJ©ái9\¡µµ5¹òA÷Kmm-uuulß¾=ÙöÐB÷0Byy9#GŽ¬nI'CC 466ÇY±b{öì ;;›ÒÒÒdH¨ªª*åûÕ$I’$I’$I’¤bÈ@’$I$m´±•­=/Ïz™gy6¹úÀvþ2ñ7‡Æ3>ˆK (¢(ÄžI:åååõBhiiINd>4„°zõjvìØ‘lÛ[!‹1iÒ$FŒðÏ RoºBÝÃ]—U«V±k×.à@ˆ`„ ÉÕª««©©©I~¯Mž<™¬¬¬{#I’$I’$I’$édà,I’$i$H$ƒqâiWh¢‰€ yN„Q¢cÎç|ª¨JbĈ¥B2ñ]‰% ¼üüü>C½MŒ®­­M™ }‡œ­á.‘H¤ ÄãqÞ}÷]vîÜ™lÛý{åÐI’$I’$I’$I' ÿs)I’$õa/{YÇ:6²‘ l žz6²1¹ê@ lbSJx`(¤RJ)¤ó8B )¡„(QŠ)f"ÉâÀ¤Ûù¿˜OMMMX]”¤´"‘UUUTUU¥½½+„ph¡¶¶–•+W²{÷nàÀ»³—––F“ïÐnACIkk+ =ñxœºº:¶oÿË D½…¢Ñ(±XŒüüü{"I’$I’$I’$IýcÈ@’$I'­Z¨§ž ·®A×þ6 ‘lŸO>¥”R|p;ƒ3(¦˜¢ƒ[1ÅD‰’Knˆ½’¤Áq$!„îa„ÚÚZ–/_ÎÞ½{ÈÉÉ¡¤¤$9ûÐ BYY™™®è¢ã§¯AW€¦KºA××íŒ38å”SBì‰$I’$I’$I’$ C’$I–ÚhK®8ÐH#qâ=ö›hJ®@K.ãGEĈñ~ÞŸÜ#J”B Éĉ®’ÔGBˆÇã,^¼˜úúz:::ÈÍÍ¥¸¸8%xÐ=ŒP^^NFFÆ`vMCL[[7nì5@°fÍ‚àÀëî!‚™3g¦„^¦OŸÎ¨Q£Bî$I’$I’$I’$† $I’4äìc[ØÒkx ‘FÖ²–N:È!‡ñŒO†f23¹%JE”Qf€@’ÑцjkkB84Œ ámß¾}lØ°¡Gx ëúÚµkéì<ðš ‰$¿.*++¹æšk’_/§Ÿ~:cÆŒ ¹7’$I’$I’$I’>C’$I:¡k€ Š*’4 ôBhoogóæÍ)É»‡Ö­[ÇþýûÈËËKy7úî—)S¦pꩧv×t„ÚÛÛY¿~}ð@ºÁ¡wuuurêÔ©Œ;6äÞH’$I’$I’$IÒ‰Ï$I’M;ílfsŸ‚u¬c?&†f“Í&ô ˜Ìd²È ¹g’¤Á”MQQEEEiCÝßÙþЉ鵵µ}NJï~qRúàè+4Çû tÄb1"‘HȽ‘$I’$I’$I’¤¡Ï$I’L+­4Ð@üàÖ=D'ž ˆI*©ä®I†bÄ H’ŽJNNNrÒy:ÝC‡jkkY³f AVTˆÅbD£Ña„iÓ¦1zôèÁìÚ•H$Òâñ8õõõttt››KqqqA×ø———“‘‘ro$I’$I’$I’$ix3d I’¤~k¤‘úƒÛ:ÖQO=kY›¼¾íɶãÇd&3‰ITPÁU\Å$&QL1“™ÌD& $…âp!„¶¶66nÜØc2üÒ¥K{ !¤ "LŸ>Q£F f×BÓ[ˆ ¡¡5kÖÐÒÒû’’’äÍœ93eÌ H’$I’$I’$IRø H’$ €vÚÙÌ攕º¯F°’•ìfw²}×*1b\ÎåÉÕbĘÊTÆ26ÄÞH’tôrssû !´¶¶ÒÐÐÐcBýâÅ‹‰Çã$‰dÛî!„CÃgœq#GŽ¬n“t!‚® V¬XÁž={€ÔA4¥ªª*¥ïeeedff†ÜI’$I’$I’$IR_ H’$D$RÝ·u¬c?ûÈ!‡Jˆ¥ˆ"ª©¦†bĈ¥œrF24&EJ’4Ðòòòú !´´´ÐØØØcR~mm-«W¯fÇŽɶéB]—I“&1bÄàüé¦{ˆ +<Ðu}åÊ•ìÞ} h˜Í„ ’«TWWSSS“¬yòäÉde¹R‘$I’$I’$I’$ e† $I’†‰N:i µ¬eÍÁmm·m=ëé €|ò)§œ2ʘÆ4®äJÊ(cÒÁ­Â{#IÒЕŸŸßg!‘Hô˜ÈßBx÷ÝwÙ¹sg²í@…zûœñx¼ÏÏyhˆ`0ƒ’$I’$I’$I’¤pø_aI’¤!$ÝJ 4ÐH#+XÁöY‰ FŒJ¸ ‰uÛÊ(#“Ì{#IÒÉ)‰‰D¨¬¬L{{ss3kÖ¬aíÚµ)—矞µk×ÒÚÚ XU ´´”²²2ÊËË)++cÒ¤Ilݺ•µkצÜÇ®]»ÈÌ̤¨¨(y·>ô!ÊÊÊ’—ÒÒRC’$I’$I’$I’t’ó¿Æ’$I'=ìé"èÚÖ²–VN*$›IL¢ìàv%W&(§œ(Q2ȹ7’$éhPPPÀE]”ööÆÆÆ´!„W_}• Ö®eÕ¸qäL™Byy9×^{mJˆ`Ò¤Iäää r$I’$I’$I’$IC‰!I’¤A°‘½ 6±)Ù6J4¹òÀ\\ Œ2Š)&‹¬{"I’ÂF‰F£\zé¥=oÌȀǃë¯üÂ$I’$I’$I’$IÂ!I’¤ÖF[¯A‚•¬d7»È!‡Jˆã,Îâ:®K† ¦1ÑŒ¹'’$I’$I’$I’$I’¤“!I’¤£°•­¬>d«£®ÏÕ®æjîâ®äõ"ŠBì$I’$I’$I’$I’$I=2$IêE3Í=‚][‚p`5‚rÊ™Â.ànà†dˆ FŒ|òCî…$I’$I’$I’$I’$IýgÈ@’$Ô$ˆwÛ–²”e,c5«ÙÁà@ „bÄ8óø(%FŒ *˜ÎtFø’J’$I’$I’$I’$I’4L8#N’$ {4²êàVG]ÊŠ{Ø@>ùL=¸]ÁÔP“¼^B ™d†Ü I’$I’$I’$I’$I’Ž?C’$iXØÅ®d`«XÉJV±Šwy—ì`£’Á«¸*¹?…)”Pr$I’$I’$I’$I’$I Ÿ!I’4dtÐA=õÄnKYÊ2–'ÎÖ0‚Lb1bTQÅ­ÜJ%•ÄˆQF™+H’$I’$I’$I’$I’ÔC’$é„“ Ñ#DÐu½•V"Dˆ£‚ ª©&vp« ‚|òCî$I’$I’$I’$I’$IC“!I’ŠvÚYÍj–³œ¬`ËXÁ V²’Ýì` c˜vp»–kù[þ–iLãtNg4£Cî$I’$I’$I’$I’$IÃ!I’t\íf7+nËXÆJV²ŒeÔQG;íd’Éd&3ƒ¼÷q·1iLg:…†]¾$I’$I’$I’$I’$I'C’$i@$H'ÎR–²ŒeÉkYK'd“M)¥TPÁu\GŒTp.ç2ŠQa—/I’$I’$I’$I’$I’0d I’ŽP=õÉAת+XÁV¶0–±Lg:TPC gÜÊ)g„/=$I’$I’$I’$I’$I:¡9ÓO’$¥• ÑcU‚?òG6³€*¨ ’J®ášä~9ådrõ’$I’$I’$I’$I’$éh2$é$·íÔQÇR–²„%,cïðM4©a‚«¹šJ*9‹³˜ÈÄ+—$I’$I’$I’0<õ© IDAT$I’$IÍ$I'‰ì`5«{¬N'ÀXÆ2•©TPA5ÕTPÁùœO”hÈ•K’$I’$I’$I’$I’¤ÁbÈ@’¤a¦v–³œ?uÛ–²” l`4£© ‚39“+¸‚39“ *(¦8äÊ%I’$I’$I’$I’$IRØ H’4„5ÓÌŸøäÉ@Á2–±}äCœÍÙ)a‚2ÊÂ.[’$I’$I’$I’$I’$  H’4tÐA=õ,e)KnËXFœ8"TPÁ{y/·s;Tp>ç“G^È•K’$I’$I’$I’$I’¤¡Ä$I'˜ílçÞI –²”ßó{Zha#˜Æ4*©äVn¥Š*Îç|¢DÃ.[’$I’$I’$I’$I’$ † $I Q=õü¾Ûöþ@ PÀ9œÃ%\ÂmÜÆÙœMd“rÕ’$I’$I’$I’$I’$i¸2d IÒ ‰g KRB[ØB&™Le*ïá=ÜÍݜ˹œÍÙRvÉ’$I’$I’$I’$I’$é$cÈ@’¤ã –tÛ~ÇïØÌf²Èb2“© ‚;¸ƒ*ª¸”KÏø°K–$I’$I’$I’$I’$I2d IÒ±ØÏ~V°‚e,c)KYÂÞà ¶²•Œ`Ó¨¢Š¯ðª¨â<ÎãN »lI’$I’$I’$I’$I’¤´ H’tÖ²–ßñ;Þ<¸ýžß³—½ä’ËYœÅ{xß䛼‡÷p6g“KnØ%K’$I’$I’$I’$I’$õ›!I’z±íÉ0AW° ™fF0‚39“‹¸ˆÿÍÿæ=¼‡J*É&;ì’%I’$I’$I’$I’$I’Ž‰!I’€:XÉJ–°„Å,f‹XÁ :é$J”*ª¸Û¹ŒË¸”KÉÈ°K–$I’$I’$I’$I’$Ip™a IRhà^à~îç2.c c8“3¹“;YÊRª©æ)ž¢‰¦dÛyjª H’$é„÷·û·²eË>ò‘0jÔ(JJJxì±Çz´}饗¸ì²Ë9r$cÇŽåÚk¯eÅŠ!T-I’$I’$I’$I:2$ {-´ð*¯ò-¾ÅÕ\ÍxÆSL1壼̘˹|—ﲜåì`‹XÄ£<Êæ0‘‰a—/I’$• ¸çž{¸÷Þ{Ù¸q#wÝuwß}7o¼ñF²ÍK/½Ä•W^IUUñxœ%K–ÐÒÒÂÌ™3©¯¯±zI’$I’$I’$IRXF„]€$I­™f^çu±ˆ×y%,aû(¢ˆË¸Œ¯ñ5.äBÎã<òÈ »\I’$é¸hnnææ›ofÖ¬Y|éK_â»ßý.?øÁ¸ä’KøêW¿Jee%>úhò¼§žzŠI“&ñÿøÌ;7”Ú%I’$I’$I’$Iá1d IòâÄYÄ"³˜E,b9Ë ˆc&3ùŸb&3©¤2ìR%I’¤A“••ÅìÙ³SŽqƬ]»€ÖÖVÞ|óMî¿ÿþ”6ãÇgæÌ™¼òÊ+ƒT©$I’$I’$I’$éDbÈ@’4¤tÐÁùc2Tð ¯°™Íd“ÍÙœM5Õ<ȃ\ÁŒg|ØåJ’$I¡?~<#F¤þégôèÑÉÁöíÛéì줠  Ç¹'NäÏþó`”)I’$I’$I’$I:Á2$ÐZhá Þà×üšßðþ‡ÿ¡•V&2‘K¹”û¹ŸK¹”*ªÈ&;ìr%I’¤FFFFŸ·Ÿzê©dff²yóæ·5773~¼¡]I’$I’$I’$I:2$Pö±ßñ;^æe~ͯù-¿¥6¦0…÷ñ>>ͧ™ÉLNçô°K•$I’†´¼¼<.¼ðB~ñ‹_ðÐC%o۶ŋóÉO~2Äê$I’$I’$I’$Ia1d I Õ~öó6oSK-‹XÄ«¼ÊNv%Êe\Æc<Æ_ñW”Qv©’$IÒ°óõ¯«®ºŠ¿ù›¿áþûïgÏž=Ü~ûídggó¥/})ìò$I’$I’$I’$I!0d ITt²œå,f1µÔ²…lg;™È{y/ÿ‡ÿÃLfRIeØ¥J’$IÃÞìÙ³yñÅyðÁ)++#;;›Ë/¿œÅ‹3yòä°Ë“$I’$I’$I’$…À$é¸[Îrþÿ_ók^åU$8Óxïã[|‹÷ó~f0#ì2%I’¤aãÛßþ6ßþö·{ꩧz›={6³gÏŒ²$I’$I’$I’$IC€!IÒ€ÛÅ.^â%þûඎuDˆp9—ó ò~ÞÏ™œIa—*I’$I’$I’$I’$I’¤n H’Dœ8/ð XÀk¼F;íœÇy|œSM5—s9Ùd‡]¦$I’$I’$I’$I’$I’ú`È@’tTv³›_ók°€y‘õ¬gx?ïç1ãj®¦ˆ¢°Ë”$I’$I’$I’$I’$IÒ0d Iê—N:ù öàö~C'œË¹ÜÄM\ÍÕ\Ê¥d’v©’$I’$I’$I’$I’$I:J† $I½j£—x‰Ÿñ3~ÎÏi¦™"Šø ä'ü„jª‰ »LI’$I’$I’$I’$I’$ C’¤»ØÅ‹¼ÈOù)/ò"»ØEUÜÃ=|ˆq6g‡]¢$I’$I’$I’$I’$I’ŽC’$¶²•_ð °€_òKZiåb.æà£|”©L »DI’$I’$I’$I’$I’$ C’t’ª§žÿæ¿yø¿"‹,.ã2¾É7¹(¤0ì%I’$I’$I’$I’$I’4È HÒId5«yš§ù?c KËXþš¿æ'ü„«¸ŠQŒ »DI’$I’$I’$I’$I’$…È$ s[ØÂÓ<Íù1¿å·Ld"×qñWp9ä„]¢$I’$I’$I’$I’$I’N† $ij¥•…,äGüˆçyž,²¸š«y€ø $›ì°K”$I’$I’$I’$I’$IÒ È$ tò:¯ó#~ÄS<Ånvs —ð8s#72šÑa—(I’$I’$I’$I’$I’¤œ!Iâ–²”gy–òCÖ°† *x€ø$Ÿ¤Â°Ë“$I’$I’$I’$I’$IÒbÈ@’†  ~Èùwþ?ñ'Ê)çæƒÛ f„]ž$I’¤Áð÷MM©Ç&L€'Ÿ„—^êÙ¶¨hðj“$I’$I’$I’$ Y† $iYÄ"æ3Ÿgy–l²¹‘y‚'˜ÉL2È»ùÉpê‘$I’$I’$I’$ i† $i´ÓÎ|æs:§ó5¾ÆMÜÄ*VñM¾Ia—'I’$i(+-…‹/†ÌnæÙ¿®¿>¼š$I’$I’$I’$IC–!I:Î~ÊO9ƒ3¸‹»ø(¥Ž:åQÃ’$I’έ·BFÆýÌL¸ì2(.·&I’$I’$I’$IÒdÈ@’Ž“?òG®à >ÆǸ˜‹YÅ*åQ ) »4I’$IÃÍœ9ÙÏÈ€O|"¼Z$I’$I’$I’$ICš!I`[ÙÊmÜFUìe/¯ó:?æÇLbRØ¥I’$I®&L€êꃌ ø_ÿ+ìŠ$I’$I’$I’$IC”!I@Oó4•T²€ü_þ/oðsqØeI’$I:Ür \y%Œv5’$I’$I’$I’¤!jDØHÒpÐH#wr'?ãgÜÂ-ü3ÿÌ8œÔ#I’$i`lݺ•M›6±yófhnnfÓ¦M466²yófÙÓÔÄ€¿yí5y&D£Q (,,¤°°ÓN;¢¢" (((`Äÿ4$I’$I’$I’$IJå’%éý„ŸpwP@/ó2ïã}a—$I’$iH$455¥š››SBMMM477³oß¾äy#FŒà´ÓN£   ¨¬¬¤°°†çžã=þ0…;vÐÔÔDSS¿ûÝï’û---)5t… &NœHaaa2˜0qâÄ„¬¬¬Á"I’$I’$I’$IR HÒQÚÍnîäN~ȹ‹»øþ|òÃ.K’$IRˆZZZH$466ÒÐÐüØýX"‘`ýúõìÚµ+åÜH$B4%‰PTTÄ¥—^šÜF£É}®@ðÉO2e\﫪Zß¡µ¾ùæ›$ 6nÜÈŽ;RÎÍËËëQK÷úºö'Mšä ’$I’$I’$I’4„ù_I: ¿ç÷ÜÈ$Hð/ð!>vI’$I’Ž“þ6lØÀÎ;SÎ=48PUUu|'æ÷0ÈÏÏ'???YK_×ï%K–ô»ß‡î÷+0!I’$I’$I’$I …ÿÅ•¤#ô,ÏòI>É¥\Ê+¼B”hØ%I’$I:B‡{Gÿ®cýyGÿãÉ‘z _t$ôg‡Þ 'N$++ëxvY’$I’$I’$I’„!I:"ò(÷r/Ÿå³<ÁŒðiT’$I:aôè¾ßÐÐÀöíÛSÎíˆD"TVV¦Lpïšô^ZZJvvvH=ŧxžçù9?ç| ì’$I’¤R[[[·níWp ©©‰ ’çö'8‰D˜4i£G±—:‘m !Ý×iW ¡¾¾žŽŽŽäy¹¹¹Œ7®×B÷cÑh”ŒŒŒãÙeI’$I’$I’$I:. HR>Ççø)?e x?ï»I’$iP¥ ô"Hè>ùº·à@ii)cÆŒ ±—:I !‘HôºÊ†I’$I’$I’$IÑ!IêÅã<οóï<Çs $I’4lìÛ·-[¶ô:aºûþ¦M›èììLž{hp RUUÕcÂtII cÇŽ ±—ÒÀ‰D"D"‘~µ=\ aÙ²e444°yóæ”@B×÷Vo! ’$I’$I’$I’“!IJãe^æ^îå|ƒk¸&ìr$I’¤ÃêšÜ|¬ÁH$BeeeÉÍÅÅÅœzê©!öP:ñk ¡û÷lW ¡¹¹™ýû÷'ÏK÷=Û×¾$I’$I’$I’$)C’tˆìà|‚ð¾Ì—Ã.G’$I'±ÃMBîÚïÏ$ätÁ¢¢¢~Oˆ–4°N¤@‚!"I’$I’$I’$IÝ2¤CÜÏý´ÑÆo D ¡´´”1cÆï.K’$I’$I’$I`† $©›‡xˆË¸Œ›¸)ìR$I’²–––!t“qëëëˆÅbi'à–œœŠŠŠ(**:l ¡­­­[·ö¤Z²d ‰D‚¦¦&‚ Hž{h !]0!‰0iÒ$F}¼»-I’$I’$I’$© HÒAïðÿͳ€a—"I’¤ã¤¿Áõë×ÓÞÞžêS+¶úª¶RiD_÷ èÛ‚("xÅ*jEP"KH ,d !÷šû!@ È÷sÇaf~3×or‡ûÄÌu¤jË’$ýÇ“}*ôøsçÎ%%%…%K–Ð¥K— =¶þ×(,,,vCøé`òäÉÜ~ûí@Ù¾OÒÓÓ1b‹/f÷îÝ´hÑ‚ÌÌÌ2ëðkX‘ߥÍcÑ¢E<úè£ÌŸ?Ÿ]»vѹsgî¿ÿ~®¹æšã>×±”§——ÇÖ­[K-}ZHÈÎÎ.6¶,…„øøxZ¶lIDDD¥ÍW’$I’$I’$I:Y2¤ÿ˜Å,†1Œ0‚Ž"I’Tí•V8¼D™™I~~~±±EÅ¢MK+ÄÆÆR³fÍ€fX>›7oæ¿øiiilݺµØ'x¿ûî»Ìž=»XÉ`çÎôîÝ›—^z‰–-[žôÌEÆŽËM7ÝD§NË wñųuëÖ cT¸Ûn»n¸¨¨¨2í?dÈÎ:ë,fϞͺuëèß¿™Ïuø5¬ÈïÒæqÕUWÑ»woÒÓÓ ç _¿~¼õÖ[\uUðû¢¢@||<Ý»w?ê¾e-$dee±mÛ¶ÏSR áÐu ÔªU«2§,I’$I’$I’$,H°Œe¬cWpEÐQ$I’N[YèÞ½û)_(¯>}úðâ‹/òþûïÓ·oßÐúwß}—„„æÎKaa!aaK³;wîdÛ¶m ¤ÓI^^Ë—/çÞ{ï¥nݺtêÔ‰+V먢££™}úçXµjýû÷§I“&Ô«WŸþô§|òÉ'ÇÌ6{ölzõêETT \wÝu,\¸°Ôýï¹ç ã믿àõ×_­{ùå—Ë”iôèѤ¤¤ÐµkWÂÂÂhÓ¦MhìÌ™3éÑ£‘‘‘ÄÆÆrÇw°cÇ&L˜:ß„ ¸óÎ;iÔ¨aaa 4¨ÄÜes´ó–皺OãƹñÆÉÎΖHŠ²„……Ñ«W/V¯^]l}E\‹C·Ož<ùˆ1O?ý4#FŒ Aƒ´hÑ‚?üáG\»¿ÿýï´nÝšÈÈHÎ?ÿ|>þøã#²—¤,¯ÝãÍMïÞ½ùî»ïJ=ÿ¡ç(zJÀðáà £OŸ>Ç•±èMY^Ce™GFFF¨`NTT[¶l9f†SYTTT¨ŒÐ·o_†ʨQ£?~j®™3grõÕW“’’Bff& .¤víÚ\~ù奎yâ‰'˜3gN±u×]w›6m:bߣe;vlè8K–,¡°°Õ«W0}útúõëÇÕW_Mvv6ï¾û.óçÏçÚk¯¥°°»ï¾›~ø€ÇœË.»ŒuëÖñä“O–š»,cŽuÞ²^³¢}.¿ürÖ­[ÇG}Äòå˹ä’Køᇨ[·.´hÑ‚!C†„ŠmÚ´aãÆtèÐTȵ8t{I×⩧ž"99™ÌÌLFŃ>ÈüùóCû¦¥¥q÷Ýwsûí·“ÍÓO?ͨQ£BcVd)Ëk÷x³¬_¿ž¿üå/Üwß}¥ž¿¤s<óÌ3òÎ;ïWÆc)Ëkèxç1oÞwî\RRRX²d ]ºt~"|Ó¦My饗¸á†Ê”©¤ãtèЈˆ–,YZ7sæLúöíË{ï½Çe—]ÆÎ;©W¯·ÝvÏ<óL™¾ÇS–ó–åšuìØ‘Zµj;ÎâÅ‹9çœs;vlè&ýûï¿Ÿ'žx‚ììl4hÀc=Æž={xà*ìZmæ™g¸í¶ÛŠ­»å–[xöÙgØ¿?õë×ç·¿ý-¿ûÝïB玎fÑ¢E¡ã½ýöÛüä'?ᩧžâŽ;î(Óµ/røk÷D³üóŸÿäg?ûÙ1³”t Ž'ã¡ãKúÞ(Ë׫¼óضm¯¾ú*÷ß?uêÔá‹/¾ 66ö¨sЉٳgO‰ï¹‡/¯[·îˆJLLL©ïµ‡¾ÇÆÆžÖOð‘T}ù3²$I’$I’$O‘Ã_»'š¥èZT¤£e<š²|½ºuëVîyÜsÏ=Ìœ9“믿ž1cÆЬY³rgSùDEE‘˜˜Hbbâ1÷=V!!==ììl233ÉÏÏ/6¶¤BBIÅ ’$I’$I’$I*K’RH jC’$©Ì¶mÛFLLL©Û£££éÒ¥ ^x!ݺu£M›6¡› ›6mjqàÕ«W/4hÀìÙ³0`û÷ï'>>8X2xæ™g˜7okÖ¬áÊ+¯ [¶l¿ûÝïø÷¿ÿMNNE5ìÖ­[±ãÏž=›‡~˜{n¸ääd~ÿûß—zóòæÍ›ƒ7TW–òf:4ׄ ˜0aÂÛ×®][ìïG+c”¦¤1e9oY®ÙÑöiܸqh;À™gžÉE]ÄsÏ=ÇðáÃùä“Oˆ‹‹£U«VeÎt¬yKݺu‹ý½V­Z8p€ìììçR¿~ý2»¬¯ÝÉrV/Ü IDAT¢¯ßòf<š²|½Žw&L`РAåΤÊWžBÂæÍ›ÉÉÉ!''‡5kÖ°téR¾úê+-ZÄG}Tê¸I“&qûí·WdlI’$I’$I’$Æ,H’$IÒ)¨N:<÷Üsdgg“““ÃÆ‹-oݺ•O?ý”O?ý4´||<Íš5£Y³f¡åØØXš7oN³fÍB%„ã¹ÉX'Gxx8—]vsçÎeÖ¬Y¤¤¤„¶]~ùåÔ¬Y“wß}—Õ«WóÈ#°oß>’““iÑ¢óæÍãÌ3ϤfÍšÜtÓM,^¼¸ØñcbbxüñÇyüñÇùä“Oøïÿþo.¹ä–-[FÛ¶mÈÓ¤Iàà'Ä—W5BùŠìرãˆýÊ›éÐ\÷ÝwãÆ+w¶ãU–ófffG¿fG»®[¶l9bÞ7ß|3·Þz+Ë–-ã¹çžã–[n)W¦Ê9—²¼fÊóÚ=‘,Û·o/÷±*+cY¾^Eù+rª: Ø´i999dee…Þ×7lØ@NNÙÙÙlܸ1´Ï¡¢¢¢hÞ¼yè}½h¹iÓ¦\~ùåÍH’$I’$I’$I§"K’$I’t ªU«7ß|s©ÛóóóÙ¼y3¹¹¹dgg“••UlyÍš5|òÉ'dee±qãÆÐ'}DFFC||++‹ÄÄDÖ¯_OãÆKœGÓ¦MCç-òÕW_•;SQYáð\;vä³Ï>;bÛÙgŸÍý÷ßÏÀKÌu"ÊzÞc]³¢ë:oÞ¼bû,^¼˜;vœœ\l}jj*#FŒ`„ Ìž=›¿ýíoåÎTY5jDûöíY°`A±õ%å9\Y_»'šå‹/¾8®ãUFƲ~½Ê;çŸþ¸ò¨âäææñ^|èrÑ7mÚDAAAh\íÚµiÔ¨Q轸E‹œ{î¹G¼?ÇÄÄGXXX€³”$I’$I’$IÒéÂ’$I’$†"""ˆ'>>>t³yiöîÝË–-[Žzãczz:¹¹¹lØ°ÂÂÂÐØà %,$T¬+¯¼€µk×rñűíÁ,öIömÚ´¡iÓ¦¼ôÒK\}õÕ$&&òÁ0kÖ,Ú´iSlü×_Í_ÿúWn½õV8Àĉ‰ŒŒäÜsÏ-5Ïc=Fÿþýyà¸çž{عs'Çç¿øE©€öíÛÓ¬Y3þñгgO6mÚÄ /¼pÄ~ÇÊTôéôß~û-Í›7§k×®¼ùæ›<þøãôë×±cÇrÛm·ð?ÿó?пÿ£\áS–ó–åšíóßÿýßÜ{ï½lÞ¼™áÇӮ];î¼óÎbç¬[·. à駟æÎ;ï$22²Ü™*Ó˜1c¸þúëyøᇹ뮻X³f 'N<æ¸ò¼v7˺uëøóŸÿ|\Ǫ¬Œeùz•g¹¹¹œsÎ9ôìÙ“×^{í¸2©dÇ*-¯]»ö¨Å¸¸8K|/µ8 I’$I’$I’¤ „zwˆ$US©¤0•©'‘$UG“&MbØ°aAÇʤ¤BBi7V–TH(­„pèr«V­¨W¯^€³¬ú:vìÈgœÁ¬Y³Š­_¸p!½zõbÊ”)Å>þ³Ï>ã׿þ5‹-"&&†«®ºŠ­[·òúë¯MóæÍyë­·?~<_|ñûöí㬳Î⡇â²Ë.;jžwÞy‡|/¿ü’Æ3hÐ yä"##=z4ãÆ í;dÈ^~ùeæÎËÈ‘#ùþûï9ï¼óxüñÇéÑ£p°0ñÎ;ï”)Ó]wÝÅ+¯¼Baa!7Üpÿûßx÷ÝwyðÁY´h 6äòË/gܸq´lÙ’)S¦0xðàbóÈÍÍ¥aƥγ¬cŽvÞ²\³’ö‰ŽŽæꫯæÑG +õá‡rÉ%—ðÙgŸ…®aY3k^&Là—¿üehÛ¥—^ÊwÜQlÌ!C;v, ¡umÛ¶eåÊ•üãÿ`ܸqäääpî¹çòÄOн{w&OžÌ­·ÞZê5/ËkwÞ¼yÇ•eãÆœsÎ9üå/á‚ . gÏž|òÉ'Gä8ü]ó‹.º¨L_ýõ#®a¯^½JýÞ(Ëk¨¬óÈÍÍåì³Ï¦W¯^LêϼDzgÏžc¾¿egg³nÝ:öíÛAãÆúþV´Ü¼yóŸÂ"I:ÈŸ‘%I’$I’$éäIMýÏý³‡ý.Ñ’$aÉ@’,o Ðé*//­[·–©]llY ñññ´lÙ’ˆˆˆ€f(éx-^¼˜sÎ9‡ùóçsÉ%—G§¹C‹G{O:¼8„ž&`q@’NF–$I’$I’¤“§´’Axa$I’$I§¿¢¢@||6m :™$I’$I’$I’$U[– $I’$I’ª¸4Ò¸ŒËˆ'>è(•#9/†É“aÆ èÐƃ¼¼ “I’$I’$I’$IRµcÉ@’$I’$© ÛÁf1‹Á :JåªQ†…•+aÄx衃eƒ_„ ÓI’$I’$I’$IRµaÉ@’$I’$© {ƒ78À~ÆÏ‚ŽrrÔ©cÆÀwßAŸ>pË-Ы,Xt2I’$I’$I’$Iª,H’$I’$Uai¤q5WÓ†AG9¹Z´€‰aáBˆŽ†‹/†¾}aÕª “I’$I’$I’$IÒiÍ’$I’$IR•Cïó>ƒt”àtïï¿sæÀêÕЩ ›7L’$I’$I’$I’NK– $I’$I’ª¨)L!Š(®áš £/9-‚ `útèÐƃ½{ƒN&I’$I’$I’$I§K’$I’$IUTi\˵Dt”ª!<† ƒ•+á—¿„1c kW˜6 ƒN'I’$I’$I’$I§K’$I’$IUP,d!ƒt”ª§n݃ƒ+ wo4.¸>ú(èd’$I’$I’$I’tʳd I’$I’T¥‘FšLrÐQª®„˜8.„Úµáâ‹!522‚N&I’$I’$I’$I§,K’$I’$IUЦ0„t”ª¯G˜7¦O‡E‹ S'9¶o:™$I’$I’$I’$r,H’$I’$U1_ñ_ó5ƒt”SKß¾°l<ù$L™mÛ¸qŸt2I’$I’$I’$I:eX2$I’$IªbÒH£­8ŸóƒŽrê©U † ƒo¿…Ûnƒ1c kW˜6-èd’$I’$I’$I’tJ°d I’$I’T…RȦ0„!„tœSWL Œ Ë—CÏž0p \p|üqÐÉ$I’$I’$I’$©J³d I’$I’T…|ÄG¬f5ƒt”ÓC«Vðâ‹°p!„‡Ã…Bj*¬^t2I’$I’$I’$Iª’,H’$I’$U!i¤Ñ‰Nt¥kÐQN/çž |Ó§Ã_@çÎ0z4lßt2I’$I’$I’$IªR,H’$I’$Uðop7åôÕ·/|ó <ñ<÷´m ãÇCAAÐÉ$I’$I’$I’$©J°d I’$I’TEÌaÙH*©AG9½ÕªÆÁòåpÛm0jtéÓ¦L’$I’$I’$I’gÉ@’$I’$©ŠH#ó9Ÿv´ :JõcÇ,œw ÉÉ°hQÐÉ$I’$I’$I’$)0– $I’$I’ª€<ò˜Á 38è(ÕOëÖðâ‹ðñÇ—=z@j*¬Yt2I’$I’$I’$I:é,H’$I’$U3˜ÁNv’JjÐQª¯ž=áÃaÊHO‡Îaôhر#èd’$I’$I’$I’tÒX2$I’$IªÒHãr.'–Ø £Toaa0`|ó üÏÿÀĉж-Œû÷N’$I’$I’$I’*%I’$I’¤€í`ïðƒt‰ˆ€‘#aÕ*¸õV5 ºv…™3ƒN&I’$I’$I’$I•Ê’$I’$IRÀ¦1 €k¹6à$:B£F0v,|õté}ûBJ |ùeÐÉ$I’$I’$I’$©RX2$I’$I Øßþù7òÂòÈX”qĶ+®¸‚=z„þ¾dÉú÷ïOLL QQQ\pÁ|ðÁÅƬ[·Žë¯¿ž¸¸8êÕ«ÇyçÇk¯½Véó8­µoS§ÂÇÃîÝð£ÁС]a§ø¯ÿú/š7oÎæÍ›¹öÚk©[·.-[¶äoûÛû¾÷Þ{\tÑEDGGÓ AúõëÇ·ß~[aY$I’$I’$I’$U_– $I’$I’´ ,í»”˜¸ž}öÙbÛÖ¬YÃ{ï½Çí·ßÀ—_~ÉùçŸO:uHOOgýúõôéÓ‡””ÒÓÓCãRSSÉÉÉaÁ‚äääðôÓO3}út6nÜxRçvZêÕ ,€)Sþ÷Ì3aôhøᇠ9|aa!÷Üs÷Þ{/ëׯgĈŒ9’?þ8´Ï{ï½Ç•W^I÷îÝÉÈÈ ==={öpá…²víÚ É!I’$I’$I’$©ú²d I’$I’ )L¡Nx†Ý2ŒW^y…¼¼¼Ð¶çž{Žèèh®¿þzî»ï>ZµjÅ‹/¾Hbb"5â÷¿ÿ=½zõâü#ûöícáÂ… 2„¶mÛÅ~ô#^}õUbcc™ãi',  €o¿…G§Ÿ†ŽaÒ$Ø¿ÿ„““Ã!C¸øâ‹iР÷ÝwgœqÏ?ÿ|hŸßýîw$%%1~üxš7oN»ví˜2e yyyüùÏ>ÁÉI’$I’$I’$Iªî,H’$I’$(4~Æϸóö;Ù±co¼ñàùçŸgРAÔ«Wüü|Þÿ}®¹æÂÃËãÒK/eÁ‚ÔªU‹:ð§?ý‰×^{mÛ¶ô9U0r$¬Z×]wÝgo¿}܇¬Y³&)))ÅÖuêÔ‰Õ«W——ǧŸ~Ê5×\SlŸÆsá…2oÞ¼ã>·$I’$I’$I’$%I’$I’¤À¬bŸñƒLëÖ­¹âŠ+xöÙg˜3gk×®åöÛo 77—}ûöñè£VìÏÿøG¶nÝ:î¿þõ/Ú·oÏСCiܸ1çŸ>S¦L dŽÕBãÆ0~<|ý5$%ÁO~))ðÕWÇq¨ÆG”HêÕ«ÇöíÛضm Y³fGŒeË–-Ç7I’$I’$I’$IúK’$I’$Iy•WiJS.ã2†μyóXµj“'Oæì³Ïæ¼óΠAƒÔ¬Y“‡zˆÂÂÂ#þ8p tÜ:0sæLrss™5k-Z´`ðàÁ¼õÖ[̳ÚèЦN…÷Þƒ-[àœs`èPØ°¡Ì‡ ;êö† R£F 6mÚtĶœœ7n\îØ’$I’$I’$I’t(K’$I’$Iy×È@Â9øÉõ×\s qqqüùÏfÆŒ 6,´odd$?þñ™>}:û÷ï/Óñ£££¹âŠ+˜:u*µk×fáÂ…•2æ²Ë =¦L?„ví`ôhعó„ÉyçwDadëÖ­|ôÑG\zé¥'|I’$I’$I’$IÕ›%I’$I’¤,f1KYÊ`‡Ö…‡‡sË-·0iÒ$ÂÃÃ2dH±1?þ8+V¬`È!|óÍ7ìÙ³‡åË—óÄOðë_ÿ€µk×Ò¯_?æÎË–-[øᇘ4iùùùôîÝû¤Î±Z ƒ`éRxàxê)èØ&M‚2–DJó‡?ü%K–ð«_ýŠ7’‘‘ÁàÁƒ©U«÷Ýw_M@’$I’$I’$IRueÉ@’$I’$)i¤ÑšÖô¢W±õ·Þz+©©©4hРض³Ï>›Ï>û €K/½”FñÓŸþ”ÌÌÌPÉ U«V >œÇ{Œ:вeK^xá^ýuKAˆŽ†Q£`Õ*øùÏá®»à¼óàý÷û)))Ìš5‹O?ý”6mÚЭ[7"""øè£hݺu†—$I’$I’$I’T…B’‚–J*S™pIRu4iÒ$† t IÒITH!‰$r=×óÛöþûïsÙe—ññÇÓ«W¯RŽ SÖ·ßÂïÓ¦Ar2üõ¯Ð¥KЩ$Iª2üY’$I’$I’NžÔÔÿÜ?;µøý³>É@’$I’$é$[ÀV³šÁ .¶~Ë–-Üÿý\rÉ% NW;ÂÔ©0glÞ çœÇÃÆA'“$I’$I’$I’$À’$I’$IÒI—Fgq]øßO°ONN&..Ž°°0þïÿý¿¦ÓI‘œ ééðì³ðæ›Ð®ŒyyA'“$I’$I’$I’TÍY2$I’$I:‰ (à Þ8â)sçÎ%??Ÿÿûß$&&”N'U0t(¬\ ¿ûüå/pæ™0i8t:I’$I’$I’$IÕ”%I’$I’¤“è]Þe›ÈÀ £¨ªˆŽ†Q£àÛoáê«áÿüèÙæÏ:™$I’$I’$I’¤jÈ’$I’$IÒI”FpgpFÐQTÕÄÇÃĉ°d ÄÆ ))°tiÐÉ$I’$I’$I’$U#– $I’$I’N’Ýìæ_ü‹Á :Šª²N`æL˜3rràœs`øðƒË’$I’$I’$I’TÉ,H’$I’$$oò&yäq×E§‚ädX´&O†3 C7òò‚N&I’$I’$I’$é4fÉ@’$I’$é$I#d’‰%6è(:UÔ¨C‡ÂÊ•0z4<ü0´o/¾……A§“$I’$I’$I’t²d I’$I’tä’Ë;¼Ã`E§¢:u`Ô(øö[¸ê*¸åèÙ>ü0èd’$I’$I’$I’N3– $I’$I’N‚7xƒ0ÂèOÿ £èTÖ¢Lœ_~ MšÀ%—@ß¾°jUÐÉ$I’$I’$I’$&,H’$I’$i¤Ñ—¾4 AÐQt:HJ‚·ß†9s`ÍèÔ †‡M›‚N&I’$I’$I’$égÉ@’$I’$©’e“Í|æ3˜ÁAGÑé&9¾ø&L€éÓ¡C7öî :™$I’$I’$I’¤S”%I’$I’¤J6…)Ô¡WqUÐQt: ‡aÃ`åJ1zÚ·‡_„ ÓI’$I’$I’$I:ÅX2$I’$Iªdi¤q×IdÐQt:«[ÆŒ+ O¸ùf8ÿ|X° èd’$I’$I’$I’N!– $I’$I’*Ñ*Vñ9Ÿ3˜ÁAGQuѲ%LœŸ~ ‘‘pÉ%š A'“$I’$I’$I’t °d I’$I’T‰^æeb‰¥7½ƒŽ¢ê¦{w˜7Þ}–-ƒN`äHض-èd’$I’$I’$I’ª0K’$I’$I•h*SÈ@jR3è(ª®’“aÑ"xòI˜2Ú¶…qã`ïÞ “I’$I’$I’$Iª‚,H’$I’$U’/ø‚e,c0ƒƒŽ¢ê®V-6 V­‚_þÆŒ³Î‚iÓ‚N&I’$I’$I’$©Š±d I’$I’TIÒH#‘DÎã¼ £HÕ­{°`°bôì Âùçÿÿt2I’$I’$I’$IU„%I’$I’¤JPH!Ó˜Æõ\OaAÇ‘ŠKH€_„… !".ºRSáûïƒN&I’$I’$I’$)`– $I’$I’*Á|ÀÖ0ˆAAG‘Jwî¹0>LŸ‹AÇŽ0r$lß~ì±……ðÕŸQ’$I’$I’$IÒIeÉ@’$I’$©¤‘ÆÙœMIAG‘Ž­o_X¶ ž|ÒÒ m[7òóKóúëð“Ÿ@^ÞÉË)I’$I’$I’$©ÒY2$I’$íF IDATIª`ûØÇë¼Î`E*»Zµ`Ø0X¾n» ÆŒ³Î‚iÓŽÜwï^øõ¯áÃaÈ8pà¤Ç•$I’$I’$I’T9,H’$I’$U°ÙÌf+[ÈÀ £HåcÇ,œw —_‹ýï>O> YY—§O‡»î &«$I’$I’$I’¤ gÉ@’$I’$©‚¥‘Æ…\HÚE:~­ZÁ‹/Â'Ÿ@~>tï©©ðå—ðÇ?Âþý÷Û¿&N„Ç6¯$I’$I’$I’¤ aÉ@’$I’$©íf73˜Á`Eªç|iiðùç0dìÞ]|ŸÂBøÍoॗ‚É(I’$I’$I’$©ÂX2$I’$Iª@Ó™Ny\ÇuAG‘*NX o¿ Ë—CAÁ‘ûÂ-·Àܹ'?Ÿ$I’$I’$I’¤ tI’$I’¤ÓIi\Á4£YÐQ¤Š7jÔÁÂAi€þýáßÿ†³Ï>y¹±ÿ~Ö­[Çwß}ÇÊ•+Y¹r%›6mâŒ3ÎàÌ3Ϥ]»v´k׎&Mš’O’$I’$I’$Iªê,H’$I’$U\r™ÍlžåÙ £HïãáÍ7>± 4@~>¤¤ÀçŸC«V•eÿþý¬]»–•+W+|÷ÝwdddŸŸ@LL íÚµ£iÓ¦|öÙg|ÿý÷ìÝ»€† Ò®]»bŃ¢å¦M›VJnI’$I’$I’$éT`É@’$I’$©‚Lc5¨A?úEªX……p÷ÝeÛ· rsáŠ+à“O aÃã>mVVË–-###£ØŸo¾ù†Ý»w‹‰‰‰$&&Ò¿ÿÐrÑŸÃåææ²téÒbÇ}ûí·ùöÛoÙµk‘‘‘$&&’””tÄñÎ8ã ÂŽö4I’$I’$I’$égÉ@’$I’$©‚¤‘FúSŸúAG‘*ÖÚµ˜;vÀ÷ßÃþýP£DDÀÞ½G>Ý  22 o_˜;j×.ñ°¬]»öˆAFFË–-cÏž=@ñ"Arr2Æ ;j‘àhbbb¸è¢‹¸è¢‹ŽØVRaîܹ,_¾œ;w$I’$I’$I’tú³d I’$I’T²ÈâC>ä Þ:ŠTñZ·†iÓ.ççòe°t),Y_}uðÏúõ·‡‡CÍšË P8dßKÆêÕÇU$HJJ"..î¤LóX„ŒŒŒb%„à µkצE‹tîÜùˆ‚I’$I’$I’$*,H’$I’$U€4Ò¨G=úÐ'è(R劈€nÝþù‚‚2—-cÓ¼yä}ö5¿ù†˜5kHÈÍ¥îoð¯7Þà×”^$èÒ¥ Í›7nNeC÷îÝéÞ½ûÛN´€Ð¦MjÔ¨q²§$I’$I’$I’$•È’$I’$IRH#븎ÚÔ:ŠT)öíÛǺuëŽxAÑÍõyyyÀ!E‚Þ½>‰ I®gÛµ×Ò uë€gQ9ÊR@8´„0wî\V¬XÁ?üX@$I’$I’$IRÕbÉ@’$I’$é­d%é¤ógþté„”V$Xºt)+V¬   (ù‰;w¦k×®4hÐ àYT-‡ Pl[y ‰‰‰G”, H’$I’$I’$©2X2$I’$I:A/ó2qÄq)—E:¦ã)ôíÛ7t“»E‚ŠSÞÂG}ÄóÏ?ÏŽ;ˆˆˆ eË–$I’$I’$I’T¡,H’$I’$ )Laƒ¨IÍ £H@ÉE‚¢Õ׬YÃþýû#‹E7ªŸyæ™Ô¯_?àYToe- }mV@8¼„`A’$I’$I’$IGcÉ@’$I’$é|Îç,g9/ñRÐQTÍäçç“™™Y®"Á€,œ- ®¤Bzz:¯½öÛ·oŽ^@hݺ55kZ˜’$I’$I’$IªÎ,H’$I’$€4ÒhK[zÐ#è(: O‘àÆo Ý0n‘ ú)O¡èõt´¡% ’$I’$I’$IÕƒ%I’$I’¤ãt€Le*·p a„G§¨Ã‹E%‚ŒŒŒ2 Ú·oO½zõž…N$I’$I’$I’T– $I’$I’ŽÓ|æ“I&tUqG+¬^½š œò222˜1c6l V­Z$$$X@$I’$I’$I: X2$I’$I:Ni¤qçЙÎAGQph‘àÐAiE‚Î;“œœº!»C‡Ô­[7àYHG:ž›o¾Ivv6Pr¡è{ C‡„‡û¿©%I’$I’$I’ª{#I’$I’tö±ÿÇÿc£‚Ž¢“hïÞ½¬_¿Þ"ô$I’$I’$I’N?þ†F’$I’$é8Ìb[ÙJ*©AGQ+*^"(©Hйsg’’’Š :vìH:už…¼ã) Ì;—ŒŒ  ôBbb"IIIDFFžì)I’$I’$I’$U – $I’$I’ŽCi\ÌÅ´¦uÐQtªj‘ 33“„„„bëþú׿rÏ=÷TÈñ'OžÌí·ßÀSO=ÅwÜQ®ñsçÎ%%%…%K–Ð¥K— ÉžžÎˆ#X¼x1»wï¦E‹dffsÜèÑ£7n³fÍ¢OŸ>%îw¢ó>UÈ×kÑ¢E<úè£ÌŸ?Ÿ]»vѹsgî¿ÿ~®¹æš Év¢„ððpZµjUb¡sçÎDEEUHNI’$I’$I’¤êÈ’$I’$IR9íb3™É£<tÅÑŠßÿ=………@Õz"AË–-),,䦛nâõ×_gçÎzüÛn»n¸¡ÊÝ€=dÈÎ:ë,fϞͺuëèß¿™Æ;–›nº‰N:u¿ª:ïªìª«®¢wïÞ¤§§Î<@¿~ýxë­·¸êª«*õÜG+ lÛ¶U«V•X@(ú¾¶€ I’$I’$I’tb,H’$I’$•Ó¿ø{ÙËu\t”j///U«VQ"8¼HGRR‰‰‰ÅŠ:u":::àYToyyy,_¾œ{ï½—ºuëÒ©S'V¬Xt¬j/::šÉ“'‡Š6O>ù$¯¼ò Ï<óL¥— Ž¦aÆ¥Jû÷À‚$I’$I’$IRùÔ:€$I’$IÒ©&4®à šÐ$è(ÕB^^K—.eÚ´iŒ7ŽáÇ“’’BÛ¶m‰ŽŽ¦K—.¤¦¦2~üx222HLLdØ°a¼öÚk|þùçìÚµ‹¬¬,æÌ™Ãĉ5j  {÷î§TÁ`ÕªUôïߟ&MšP¯^=~úÓŸòÉ'ŸsÜßÿþwZ·nMtt4½{÷æ»ï¾;bŸÍ›7ó›ßü†víÚI·nݘ>}z±}FMJJ ]»v%,,Œ6mÚ”y|I&L˜º©{øðá„……ѧOŸÐöÙ³gÓ«W/¢¢¢hܸ17Þx#ÙÙÙÇû,ûöíãî»ï¦Aƒ´nÝšgŸ}6tÜÑ£G‡Æ¾òÊ+¤¦¦R¯^=5jĈ#Ø»woh¿Ã¿^Mš4 £W¯^¬^½ºØz€ŒŒŒbOò'**Š-[¶”éú!22’¤¤$ À¨Q£˜8q"sæÌaÕªUìÞ½›¯¿þšW_}•aÆ‘˜˜HFFÿŸ½;¯ªº÷>þIBÈD&fB€‘Qq@E«5\E´¨´j•‚µ>ö–ZõÖÚëcAÅ^+ÞRE_¥ŠS[Zh/Zèu¨¢EfCÂœIHÈyþ0ÙMÈ`@àù¼×ë¼ÎÙëì½öoåœö÷¬§Ÿ~š1cÆpꩧOZZ#FŒ`„ L™2…W_}•O>ù„={ö„{z’$I’$I’$IGŒ!I’$I’¤°ƒ¼Ë»\Çuá.å¸R\\Ì'Ÿ|Òlàúë¯ç駟n4H°gÏžã"HÐœ+¯¼’¸¸8–.]J^^äää4{̬Y³˜8q"ßûÞ÷(((àñÇgÒ¤I ö{衇¨¬¬äÃ? cÆŒaÉ’%Á>“'OæÝwß`ñâÅ„B!Ö­[×âã3qâDJKKxæ™g…B¼õÖ[À—é9’ .¸€¼¼<.\ÈŠ+8÷Üsƒc¾î¼ßxã .»ì2FŽIaa!ï¼óï½÷W\q¡P¨^}S§Nå›ßü&yyy<ùä“Ížÿ@Æ>}:_|1\{íµ|ï{ßcüøñ\xá…äççsíµ×rë­·²~ýúàuX¶lÿñÿÁرc)((àÙgŸåÙgŸå‡?üa“¯×¶mÛØ·o½zõbìرAH%##ƒÍ›7“Muuu£sš?>[¶lá”SNivîG«–^yå~ðƒ4@HHH0€ I’$I’$I’ZˆPíšñ’ÔŠ]Ã5¼Â+a®D’Ô=ýôÓŒ?>ÜeH’Zh:Óù!?d3›iG»p—sL)**"77·É|ùmé={ö$33³ÁmÀ€Á·Þïnºé&^{í5víÚ|¹šC\\/¿ü2×\óåŸa«ªªèÞ½;›7onrœ¬¬,âããY´hQÐ÷Ç?þ‘o}ë[üú׿æÖ[omòØË.»Œ´´4¦OŸôÍ›7#F°xñbN:é¤fçÐØñÙµk‰‰‰<óÌ3Œ7.èïׯÑÑÑ,^¼8èûì³Ï2d“'OæÇ?þ1Ë—/§ÿþÌ;7X¡¥óÎÊÊ¢mÛ¶õÎ1gÎFÅ_þò¾ùÍoõ7Žgžy¦Ù¹Ô:qo¹åf̘@AAéééŒ;–ßýîw’––Æ‹/¾Èu×]WoÎ?úÑxä‘G‚sÜyçLŸ>µk×Ò½{÷&_¯_üâüçþ'………$''ðØcѦMîºë®zs)..æÅ_äÞ{ï%!!O?ý”.]º´èçp<¨¨¨   €%K–°téÒzŸYk×®¥ö¯×SSS0`ÙÙÙõ>·úõëWoEI_Í?#K’$I’$IÒ‘Sûo¯¼RÿúÙ6á(F’$I’$éX5‹Y\Æe šÐ’ Att4=zô.ÂÍÉÉ gggæY}bcc9í´ÓøÉO~BDD—^z)qqqÍ vìØÁÊ•+™8qb½þ3Î8£Eçlß¾=+W®<èš¿Îñùùù¬X±‚;^ÿàÁƒIJJbÞ¼yAÈ`-w~~~³ûÕ†j 8°Åµȸu/þïÔ©Sƒ¾Ú ú·nÝÚà\§žzj½ísÎ9‡'Ÿ|’O?ý”îÝ»7Yã¸qãxðÁy饗˜0a/¿ür°òA]wÝusæÌáúë¯çg?û;wnrÜãQLLLðù4jÔ¨zÏ5@˜7oëÖ­ V…0€ I’$I’$I’Ž5† $I’$I’Zh#YÀþÈÃ]JXmß¾Õ«W³zõjV­ZUï~ÇŽ´mÛ–ŒŒ úôéCvv6—_~9}úô¡OŸ>ôêÕ‹6mük©õöÛoóÐCq×]wñío›œœ~úÓŸ6(,,¾¼Ø¿®ý·–.]Êý÷ßÏßÿþw¶lÙ|;ûàÁƒ[TÛ×=~Û¶mk²Ö:Ï7¦¥ó®cÚ´iL›6­Á86l¨·ÝÒU4tÜvíþXŠŒŒl²¯ö‚õº’’’êmwèЀ76[c§N¸êª«xî¹ç˜0a|ðƒ "%%¥Ñý§M›Æµ×^Û옭Qs„Ý»wŸ‹µ·+V0gÎœàõ‰ŒŒ¤GÁgcííÄOä„N0p%I’$I’$I’Â&2ÜH’$I’$+^à’IæB. w)j…RSS™:u*¼÷Þ{”——sî¹ç²fÍšF÷ïÖ­@ü¨µsçÎzÛ•••äää——Çüù󩬬$ qã7aæ|ÝãÓ±cÇFk‡/C.µÏ7¦¥ó®cÒ¤I„B¡·™3g~­Úõ¸)**ª·½}ûvÒÒÒ¾òØ;>úˆ¥K—òÜsÏqûí·²º$I’$I’$I’tló+ã$I’$I’Zh³¸Š«ˆ!&Ü¥„U‡èСC£ß _TTDnnn½Û’%K˜={6¹¹¹DGGÓ£GàÀëÞ²³³ýöîFlÚ´‰#F°xñb† Æ3Ï<à 'œÀ?þñN8á„Ç´oßž¾}û²`Á‚zýŸ~úi½íÜÜ\ ¹ûî»éׯ_Ð_QQÑ`ÌÚoÕ?Øã[*==¬¬,æÏŸ_¯ÿ³Ï>£¤¤„œœœ&mé¼ÓÓÓéׯüqƒ1 Ľ÷Þ˘1cªöÃ1nc>úè#®»îº`ûoûÑÑÑ :hüõªuæ™g2dȦM›Æúõë9å”SÝïÿý¿ÿwHj=UTT°xÓb­^ĺë(XUÀ†UX¿l=ëÖ­ VŸHMMeÀ€dggsñÅŸwýúõ#!!!̳$I’$I’$IjÈ$I’$IR ,g9‹XÄT¦†»”£Zjj*C‡ .r®«¸¸˜5kÖ4!Ì›7/ ´iÓ†ž={6@0`qqqGzJG/¾ø‚_þò—ÜrË-TWWó›ßü†ØØXN;í´&ùÙÏ~Æõ×_ÏC=ÄwÜA^^<òH½}222èÔ©3gÎdäÈ‘dffòþûï3wî\222êí[»JÀòåËéÚµ+'Ÿ|2¯½öZ‹?=ö£Gæ¾ûîãî»ïfÛ¶mL˜0>}úpÛm·5{lKæ 0uêT.»ì2&OžÌ¸qãxøᇩªªbôèÑ]ûáwýë_yýõ×¹à‚ ˜7oÏ>û,ãÇV2hìõš={6§žz*·ß~;ãÇçùçŸotü¢¢"† ÂgœÁË/¿|Èê>–TTTPPPÀ’%KXºti½Ï®µk×z,w4<¶mu[â#≈¥BŠ(â>!–Xâˆ;ä÷‰$ÒÆ¿ò—$I’$I’$I‡@Dè`×,—¤ãÈ5\À+¼æJ$I­ÑÓO?ÍøñãÃ]†$é+<ÀÌ`Ø@Qá.ç¸ÓT!¸7j„üü|zôèQ¯ï—¿ü%wÝuúÓŸxâ‰'øôÓO©¬¬dàÀ<øàƒ|ó›ßlvÌÿþïÿfÊ”)lÞ¼™!C†ðøãsÖYgpÆgðÁðñÇóÃþE‹‘ššÊÅ_ÌŽ;xíµ×(,,¤k×®ÜqǼð „B!¾ýíoóÔSOÐñuM›6ïÿûõúþö·¿1|øpÞzë-xà>ÿüsâãã9r$>úhpñü=÷ÜÔ)S‚cÇŽËï~÷»ÏàwÞá`Ñ¢E¤¤¤pÁ0eÊÒÓÓy饗ê­_^xŸ’’ÒìÏü@Ç;v,7Ýt#FŒú®¼òJ&NœÈùçŸôÕ† –/_NÿþýyõÕWyóÍ7yýõ×iÓ¦ cÇŽå±Ç#&æ_«­4özÕÚ¸q#ƒ "??¿Þ1uç:hÐ † Æ+¯¿gR^^Κ5k„ê~þÀ—¡ìììzŸ=íúµ#éÄ$B±!Ê)§Œ²ºßÃ*¨øÊû–Ú?|ÐŽvA!‘D∣íH"‰8âH d’‰#ŽxâI!…¸š–Jjð8…¯~ÏK‡ŠF–$I’$I’¤#çškj®ŸÝïß‚ H† $Iáå’tlÈ"‹K¹Ô• Âàë\œ™™Iÿþý‰ó,¤C§6d0wî\.ºè¢ƒçÉ'Ÿ$??¿^Pãxu¬Žì:ØÍnö²÷+ïK(¡Œ2v³›ì¤Œ2ö°‡bŠ)«iE¡‡æÄ×´$’ /è°òÏÈ’$I’$I’tä42pídI’$I’¤¯ð1³’•\Çu_½³¹ØØX²³³ÉÎÎnð\SÏ›7pX:Òzè!bbb¸öÚkyòÉ'y÷ÝwÃ]Ò!Ó’ Áþ+¢äää+¢Ô^àŸJêa=Omà Œ2Š)f{(£Œìd7»)£ŒJØÅ.Ê(£´¦ía[ØÒd¨á«æVVH&™R‚íDI"‰RH")Ø®Û_û8–ØÃú³‘$I’$I’$©51d I’$I’ôf1‹>ôáTN w)ÚOs„ŠŠ X²dI½‹Ž÷ ¤¦¦2`À€!„~ýú‘p¤§$5ëž{î V¸øâ‹;v,¿ûÝïhŒI“&ñðÃóàƒÒ«W¯ÃQæaS\\Ìš5k„Ž‡ ÁÑ µ¦jÅSNy“«(”PB)¥ì¬i¥”RL1yäQJ)EûTPÑè9¢‰&‘DRH!™d’H B µ­îvíãTRƒ¾H"ùÜ%I’$I’$I:2$I’$IjF5Õ¼Â+Œc\¸KÑŠ‰‰ .,5jT½çš ¬[·ŽêêjÀ‚Ž>“'OfòäÉ}üý÷ßÏý÷ß+:ôŠŠŠ ÔÞ¢££éÑ£G£A‚ììlbcýVû£I )‡l¬½ì¥¤¦SL)¥A¡6œP÷ùì —\v²“âšVJi£c׆ %Ôn·¯i©¤ÛÓž8 ¯H’$I’$I’Ž† $I’$I’š1ŸùPÀ5\îRtµ$€››[/„`A:t è`µ¥-kÚÁÚǾ tPDQ>¨D¨»½ Áöv4Rˆ%¶^è`ÿBS}É$‡$I’$I’$I‡…!I’$I’¤fÌbCÊ„»!u999õž;BfffƒBVVíÚµ Ç´¤#®© Á’%K(,,š 0€¬¬,Ú´ñ¯°uèE\ä0*©d;(¢ˆuÚþÛ…²”¥õúB„­¥CMëHG:ÐN5­n_G:Ò‰N‡teI’$I’$I’ã¿ÐH’$I’$5a/{ùà^î w):J4@Ø»w/ùùù  .4€ ãVSA‚/¾ø‚M›6 tü‰&š.5í@5LØÎv¶±íl§€>ã3¶Õ´=ìipþýÃéÜ ¯¶¿ˆ#îPM_’$I’$I’Ô ø¯7’$I’$IM˜Ë\Š)f cÂ]ŠŽmÛ¶mQ¡naáÂ…¬_¿ž}ûöÿ ìBèÛ·/‰‰‰á˜–Ôh`É’%,Y²„ââb ñ Aí{¸W¯^DEE…yÒÑ!µ¦À ->¦œò PÛ )d#ƒí<òø'ÿ¤B ( ‚ŠzcÄK*©¤‘F7ºuÔÝ®}Ü….Dáï¬$I’$I’$µf† $I’$I’š0‹YœË¹¤“îRtŒ«@Ø_S„™3g@ÐÓTà‹/¾`çÎÀ—ïãôôt233ÉÎÎfÔ¨Q ¤# –XÒjZK•PÂÖýZ!…Áãld‹‚íjªƒcÛІNt¢#éJ×`5„ÚÕ:Õ´ît§3iKÛÃ1mI’$I’$IR2$I’$IjÄnv3‡9<Îãá.Eǹ Ô^øÝÒ‰'žHRRÒ‘ž’ŽR$¸ú꫃÷“AéØ‘TÓZºbBeõVGØÿñÇ|\o…ºjCÝèÖì}*©‡cª’$I’$I’¤ÃÀ$I’$IR#þÀØË^®äÊp—¢VìPê† ŸöÔ®Œ±zõjƒ’šG™5í«ìa›ÙL!…lf3°…-ä“϶ð>ï³¹¦Õ]!!ÒI§3éNwºÒ•n5­+]I#.t9œÓ”$I’$I’$µ!I’$I’¤FÌbqèîR¤F5@¨¬¬$//¯ÁÅ毾új‹}úô!99ùHOI-ÔT`ÕªU”””õƒC‡å;ßùN$ÈÈÈ 222̳t¬Š'žÞ5í«QÄF6+#Ô]!á ¾àÞ!têÔ‰ÜÜ\Ö®]KEE)))ôéÓ‡O<‘‘#Grçwrâ‰'ûK’t°¾*„PDQƒB.¹ÌcëYÏ>öJj0ÎM6™d’Eíhw$§$I’$I’$©•1d I’$I’Tãe^æz®'’Èp—"©QQQôîݛ޽{3bĈzÏíÛ·¼¼¼ |ûÖ[\óÖ[tìØ1LÕK’Z»TRZÓöWAkYK.¹¬b+YÉ*Vñ<ϳ TSMô 'r"}éÜ÷¥/dMtf%I’$I’$éxbÈ@’$I’$ øYÅ*®ãºp—"ékˆŠŠ"##ƒŒŒŒ/:À›oòàƒ†»4I’¾R 1ô«iûÛË^òÉ'—\–°„¥,e«x›·YËZB„hCzÒ3X¡î*½èEQa˜•$I’$I’¤c!I’$I’$`³èG?Ná”p—"I’$5Ж¶Ap ‡œzÏ•PRo僬`‹x…W(¦€XbÉ"‹~ôc@MëOúÒ×Õ$I’$I’$ÕcÈ@’$I’$µzÕTó*¯2 á.E’$I:`I$1´¦ío+[YYÓV°‚e,c&3YËZö±h¢9È&›~ô#›lúÓŸ~ô#–Ø0ÌF’$I’$IR¸2$I’$I­Þ_ù+Ùȵ\îR$I’¤CªSM;›³ëõWRÉJV²”¥ä’Ë–ðo1•©”S@7º‘M6Üb‰$†c*’$I’$I’ŽC’$I’$©Õ›Å,Nã4úÒ7Ü¥H’$IGD4Ñd×´º*©d«XÊR–±Œ¥,å=Þã7ü† *èE¯z« bÙdO|8¦"I’$I’$é3d I’$I’Zµ *øà§ü4Ü¥H’$IaM4jZ]ûØÇZÖ²´¦-cïó>¿á7ìf7QDч> d ƒÄÀšÖ‹^aš‰$I’$I’¤ƒeÈ@’$I’$µjæÏ”PÂ5\îR$I’¤£Vmˆ }¸ŒËê=·‘|Â',e)KX‹¼ÈOù)ÕT“D's2Ùd3€ e(CB aš‰$I’$I’¤¯bÈ@’$I’$µ ¡šId½þYÌâ<Σ;ÝÃT™$I’tlK«i£ô•RÊJV²„%|RÓ^äEv± €ntchM« ô§ƒÿ_—$I’$I’tä2$I’$I­ÂnvÓŸþ\Çu\Ïõ f0¥”2‡9üŠ_…»ážæiÊ)'‘D2Ðà$I’$I’t2$I’$I­BíJM=WE‘D²ƒô§ÿ¬L’$IRK4<¨ ‚òÏ tð>ïók~M%•$“Ì)œÂéœÎ™5­3Ã< I’$I’$éègÈ@’$I’$ ˆ$’Dyƒ7ˆ#.ÜåH’$Ijb8­¦Õ*§œÏù<Ìaò(ÕTÓ‡>Aàà,Îâ$N"Š¨0Î@’$I’$I:ú2$I’$IªñGþHoz‡» I’$I_C,±œQÓj•RÊç|ÎB²€ÜÏýì` $0˜Á g8gs6gr&éÆê%I’$I’¤ð3d I’$I’Z½"˜ÊTÎçüp—"I’$é0H$‘á5íÇü˜}ìc9Ëù„OXÈBf3›Gx„!2ÉälÎf(CÎp†0„H"Ã=I’$I’$éˆ1d I’$I’Zµ6´áj®æ.î w)’$I’Ž(¢È®i7p[ÙÊÿÖ´¿ów~ÏïÙÃRIåÌšvg1ŒaÄæH’$I’$I‡!I’$I’Ô*„5è‹&š9‘ÌCE’$I’Ž&èÄe5 V;XÈB°€YÌâ?øÚІA "§¦ÍÙÄæê%I’$I’¤CÇ$I’$Ij•"‰$Ž8f3Ûo!•$I’Ô@ÝÕÆ3€|ò™_Ó^ã5¦0…b8ƒ38Ÿó9óÆ0b‰ sõ’$I’$IÒÁ‹ w’$I’$Iáò ¯If¸Ë$I’tŒH'oómf0ƒÕ¬f#™ÉLúÑWy•ó9ŸD9•S¹‡{˜Ç<Ê)wÙ’$I’$IÒq%I’$I’ÔêDÁ£<Ê…\îR$I’$úѫk@!…,`ó˜Ç«¼Ê¦Ð†6 b95m8Ã]é@’$I’$IG5C’$I’$©ÕÃîæîp—!I’$é8³è`ë˜Ï|þ‡ÿa³˜ÂâˆãLÎä.à".bCˆ "Ì•K’$I’$IÿbÈ@’$I’$ÕS^^NYYUUU”––PTT<¿{÷nöîÝlïܹ“êêjB¡ÅÅÅÁs•••ìÚµ«ÁØûÛµk•••-®qÿ¾JDDñ]ãá HݘJü/â™°wB³Ç¤¤¤QÿBŸ¨¨(’’’‚혘âããƒíøøxbbb‚í¤¤$¢¢¢‚íÔÔÔàq»v툎Ž&66–¸¸¸cK’$I:öeÁM5 —\æ×´iLã>î£ ]¸ ¹ˆ‹Á:Ò1¼EK’$I’$©Õ3d I’$IÒQ®¢¢‚={ö°sçN***صk¥¥¥TTTPRRž={(//§¸¸8¸ˆ¿¸¸˜P(„JKK©ªªj6@p ú7%99™ÈÈH"##INNžkÛ¶- ŽiÓ¦ ‰‰‰->Çþã~•={ö°eÝ¢K£pÏ6nhvÿÚŸùþöîÝËîÝ»ƒí²²2ÊË˃íÚŸó×Up¨ "Ô†êj÷IHH mÛ¶$&&CRR ÄÄÄ’’„RRRˆ‰‰!!!¤¤$bbbèg.I’$éëˬi7s3ðeè`6³™Ãnâ&ª¨bCÈ©ißàDæª%I’$I’ÔÚ2$I’$é0عs'¥¥¥”””÷ÅÅÅ”””Ôë+--¥¨¨(x\VVÆÎ;ƒ‹×ë® Ð”¸¸8bccIMM .F¯ýýÚ þ»téBLLLp‘Ý‹ôk÷©ýþæö‚‹ÖkÕ^´ÛÅ.>àr~›sÄÎY÷õÛ·o%%%ÁvII ûöícÏž=TTT«>Ô] ¢%ûc———³k×.víÚEEE;wîüÊÛµkGLL ÉÉÉÄÇÇKJJ ÉÉÉ$&&’””ܧ¤¤””T¯/11‘ÔÔÔ«6H’$Iúj™dòƒš¶›Ýü/ÿËlfó/1…)´§=p9ä0’‘¤“î’%I’$I’Ô 2$I’$©ÕÕÕìرƒ¢¢"vìØÑä­öùº¡‚æ.ìnêÂíôôt’’’ˆ‹‹#)))¸à¿np 99™˜˜ÚµkWïÂpµL;ڑÑ ¤¦¦ÖÛîرã=?ükµ…Ú•.öìÙCII ”––²{÷nÊË˃pKíJ%%%lÙ²…5kÖ4ÄìÛ·¯ÑsÕ\jßÛíÛ·oô–ššÚ /&&æÿd$I’¤£K Á Oð¹ä2yÌf6?àL`À(F‘Cçr.miî²%I’$I’t2d I’$IjvìØÁ–-[غu+[¶laÓ¦MlÛ¶­[·6lß¾=ø–øºÚ¶mÛà"éŽ;Ò·oßzáääd’““|Û{JJJf®Ö...Ž¸¸¸‡¯£6¨P¬)**j°rGIIIð;µ|ùòz¿c»wïn0fBBB£A„Î;Ó©S':uêDçÎéÒ¥K°Ý¦µ%I’¤ãW&™Œ¯i{ØÃÿð?¼Å[üžß3…)$“¬p0ŠQt¢S¸K–$I’$IÒq‰•$I’$“öíÛǦM›Ø¸qcؼy3›7ofëÖ­lݺ•M›6+++ë_{‘rÇŽéСééé 8°Éo^oß¾=íÚµ Ól¥£K||<ñññtíÚõ Ž¯¨¨h°Hc·õë×óñÇ7ù{ܱcG:wîLÇŽéÒ¥K½B×®]ƒûîÝ»w(¦.I’$…E<ñ\RÓV³š·y›¹Ìe"Ïx†1ŒÑ5­/}Ã\±$I’$I’Že† $I’$IG²²2 Ù¸qc½ûÜÜÜàñ† ¨ªª Ž‰%55•ÔÔTÒÒÒèÖ­YYYÁãºÏõèуèèè0ÎPjÝbbbèÖ­ݺu; ãÊÊÊ(** >ö¼|ùrþú׿RTTÄ–-[Ø·o_pllllðy––Ffffð¸ö¾W¯^DEEêéJ’$I‡\ŸšvwPFó˜Çæð83‰Id’É¥\Ê(FqçÑÆ–$I’$IÒðo“$I’$IGTUUùùù¬]»–uëÖ±nÝ:Ö®]KAA………äåå±k×®`ÿÚ ƒÓÒÒHOOçôÓO§gÏžtëÖîÝ»“––F—.]HHHã¬$ qqqÄÅÅ‘––ÆСC›Ý·²²’­[·!„¼¼¼à3¦°°9sæPPP@qqqpLÛ¶méÚµ+éééAè ##ƒŒŒ z÷îMïÞ½‰?ÜÓ”$I’HqŒªiÿͳ€¼É›¼ÁüŠ_Ñ….\Á\ÅUœÇyDa°V’$I’$IÍ3d I’$I:¤ª««)((Âuï×­[G~~~°A\\\pï 'œÀ¹çžK÷îÝéÖ­[$èرc˜g$éX”š $ìÙ³'äçç³qãF ÈÏÏgþüù¬[·ŽíÛ·ûwîܹ^ð öqí-66öHLO’$IjTQ|£¦Me*_ð¯ó:¯ñÓ™NG:ƒó9Ÿh\åO’$I’$I 2$I’$”;v°bÅ –-[ÆÊ•+ƒÇk×®eïÞ½ÄÄÄßÞ·o_.¼ðÂzãvíÚ5̳ÔÚÅÇÇ“••EVVV“û”––Ö[y¥68õöÛo³nݺz«!¤§§“••Eß¾}éß¿0vÏž=‰ˆˆ8S’$I’'Õ´û¹ŸU¬âµšö ÏОö\Îå\ÅUäcà@’$I’$IC’$I’¤&íÛ·µkײ|ùr–/_Ί+X±bË—/gëÖ­À—«Ô^D;fÌúöí|Ãw·nݼ¨VÒ1/11‘“O>™“O>¹Ñç‹‹‹ƒÂêÕ«Y¹r%‹/æÕW_eÛ¶mÀ¿Â }ûö¥_¿~ôë×/øìŒ?’Ó‘$IR+u"'ò“š¶žõ¼Îë¼Ê«\Â%¤Â¥\ÊÕ\ÍHFET¸Ë•$I’$IR2$I’$PYYÉÊ•+ùä“O‚ÛgŸ}ÆîÝ»HMMeÀ€dggsÉ%—™™É€èß¿?‘‘‘a®^’Â'%%…Áƒ3xðàÏ‘››Knn.K–,aéҥ̙3‡)S¦P^^NTT½zõbÀ€ :”¡C‡rúé§Ó¥K—0ÌD’$I­E/zñƒš¶Žu¼ÄKÌb3™Iz0†1\Ïõ aH¸K•$I’$IR2$I’¤V¨¨¨ˆO?ý”Ï>ûŒÏ>ûŒE‹±bÅ ªªªHLLdàÀ 2„ï~÷» 4ˆ¬¬,Ã]¶$sRSSƒðÀÕW_ôWVV’››ËâÅ‹ƒÏâgžy†|€ôôtÌ!C‚ûÞ½{‡k’$I:ŽeÁ=5m K˜UÓã1úÑ븎oóm2É w©’$I’$I:B H’$IR+°iÓ&>þøc.\ȼyóX´hÕÕÕÁê\p“&MbèС®L IG@tt4YYYdeeqÕUWýÅÅÅ|ñÅÁŠ2¿ÿýïyøá‡Ù·o]ºtá´ÓNcøðáäää0dÈ?¯%I’tHe“ÍC5íC>d³ø5¿ægüŒs8‡›¸‰«¸ŠDü"I’$I’¤ã™!I’$I:­ZµŠ¿ýío¼ÿþû,X°€5kÖЦM†ÊyçÇOúSN?ýtºvíîR%Iu¤¤¤0|øp†ôíÞ½›E‹±`Áþö·¿ñ‹_ü‚{ÔÔTÎ>ûlÎ9çÎ9çN=õT¢££ÃX½$I’Ž'gÔ´Çyœ¿òW~Ëo™ÈDnçvF1ŠïðF2’(¢Â]ª$I’$I’1C’$I’t(++cáÂ…Ìž=›×_ 6ÍÀ¹êª«‚‹PSRRÂ]ª$é%$$Áƒ{êêj–-[ÆÂ… Y°`O=õ?þñ‰ç¬³ÎâÒK/å[ßú=zôwé’$I:DINM{‚'˜Å,žçy.ã2zÑ‹¸›¹™ 2Â]ª$I’$I’×S—$I’¤cÔ®]»˜5k—_~9íÛ·ç /äïÿ;7Ýt ,`×®]üãÿ`òäÉŒ5Ê€,\¸:°téÒp—"´ÈÈH²³³?~<¿ýíoY¿~=Ë—/gêÔ©ÄÆÆrï½÷Ò³gON;í4}ôQÖ¯_î’%I’tœH%•Û¹ù%,a c˜Á Nà.æbþÈ©¢*ÜeJ’$I’$ék2d I’$Iǘ÷ߟ±cÇÒ¹sgn¼ñF***xꩧظq#ü1>ø gŸ}6mÛ¶ w©äççÑè-==o¼‘M›6…»Ì#êž{î ~o½õÖ9çŒ3‚sNŸ>ý°ž+ókNqq1ßþö·Ù²e ÕÕÕ„B!B¡P½ý>ùäÎ>ûl‚÷çþ{?ÿ×ýיǡ2oÞ<"""øâ‹/‚¾iÓ¦ó™1cÆ!?çñöž¿é¦›X²dÉ!óPÈÊÊâÖ[oeöìÙlÛ¶?ÿùÏ 4ˆÉ“'Ó»wo†ÎóÏ?OYYY¸K•$IÒqb˜ÂòÈãmÞ&‘D®æjzЃ{¸‡\rÃ]¢$I’$I’’!I’$I:TVVòì³Ï2`À¾ño°zõjþë¿þ‹ÂÂBæÎËÍ7ßL—.]Â]fééé„B!n¼ñF‚‹»wïÞÍSO=Åk¯½ÆèÑ£©®®w©GÌäÉ“Y¶lÙ=ç¸qãŽØ…Åá˜_SJJJ8÷Üs9ÿüóéܹ3çœs;vì ;;»Þ¾cÇŽ¥{÷îlÞ¼™¥K—ß`¼ÆÞÏwÝu×™Ëá4qâDJKKÛøÇÛ{þûßÿ>_|1ŸþùaÿPˆ‹‹ãâ‹/fÆŒlÚ´‰9sæžžÎøñãéÞ½;“&M ‚7’$IÒ×E9äð ¯°šÕÜÂ-ü–ßr"'rñ:¯³}á.S’$I’$IÀ$I’$å^xáúõëÇm·ÝÆYgŧŸ~ʇ~ÈøñãéСC¸Ë;(ñññŒ=šï|ç;|ôÑG,Z´(Ü%é84iÒ$ºwïÎ-·ÜÒì~ååå¬X±‚œœÚµkGÿþýY¹råªRÇš¡C‡2aÂÆŽ˾}Gÿ…RÑÑÑŒ9’—^z‰ 6ð£ýˆ™3g’™™É¤I“kÀD’$I­O<ÄCl`¯ñDp%WÒ‡><ÆcQî%I’$I’Ô† $I’$é(µ~ýz.¼ðBn¸áÎ?ÿ|V®\ÉŒ32dH¸K;d222€/çZל9s8õÔS‰¥K—.Üzë­”””0mÚ4"""ˆˆˆ`úôéÜyç$%%Ñ£Gž}öY*++™8q"ÉÉÉôêÕ‹gŸ}¶Áyß~ûm† F\\:tà;ßù………ìÚµ+?""‚aÆ°nݺzý-©õ@}ÕXÛ¶mãG?ú}úô!66–ÁƒóÆo4:ÖSO=E¯^½ˆçüóÏgÕªU|κ?ëiÓ¦qÛm·Ñ¾}{"""¸öÚk[<¯éÓ§×ûÙ½ôÒK_yþ}öWZZÊóÏ?Ï÷¿ÿý ¯î|f̘ôÅÅÅ0aÂ"""¸è¢‹Z<·¦äääÔ«³ªª €ÿûÿ/ ö»ÿþûƒ}zè! å¯óš5k=z4;v$11‘Ë/¿œ>ø Éšî¹çFŒÀÉ'ŸLDDDð;X«ªªŠ;３äädºwïÎÏþóãøž‡Ûo¿åË—3þüƒšw¸téÒ…Ÿüä'¬Y³†Ÿÿüç<÷ÜsdggóÖ[o…»´#âßÿýßéÚµ+Û¶mãŠ+® ]»v¤§§ó«_ýªÁ¾ùË_>|8ñññ$''sÙe—±|ùò0T-I’tljC®à æ2—•¬d cx˜‡éF7nàþÉ?Ã]¢$I’$I’šaÈ@’$I’ŽBŸþ9Æ #??Ÿ 0cƌ֭[Pono¼ñ—]v#GŽ¤°°wÞy‡÷Þ{+®¸‚P(ÄĉƒoÞž>}:_|1\{íµ|ï{ßcüøñ\xá…äççsíµ×rë­·Ö 1Ì™3‡‘#GrÁ——ÇÂ… Y±bçž{.¥¥¥´k׎ªª*ºwïÎرcƒ‹¶322ؼy3YYYTWW·¨ÖÑ’±zè!*++ùð˯njÒ%Kê5kÖ,&NœÈ÷¾÷= xüñÇ™4iÒŸ³îÏzêÔ©|ó›ß$//'Ÿ|ò€æöÝï~—Q£FñÔSO …‚‹µ›;BBûöí£W¯^¾ÙÙÙÁëИ7ß|“òòò œÔ›Oc}Ï<ó ¡Pè\p=oÞ<®¹æN>ùdB¡mÚ´ êZ¶lkÖ¬¾|Mñ‹_0}útî¿ÿþ ¯%¯ó•W^I\\K—.%//ŒŒ rrrš¬iòäɼûî»,^¼˜P(üÖúõ¯MNNùùùÜ{ï½<ðÀ¼÷Þ{Áó¾ç¿”ššJ¿~ýxùå—hÎG‹øøxî¾ûn–-[Æðáùä’K˜6mZ¸Ë:"B¡wÝuwß}7Üyçüà?àÿ÷ƒ}þò—¿pá…2tèPrssùä“O(++ãì³ÏfÆ a¬^’$éØt'0™É¬g=ññƒÄ\À›¼I5MÿÙN’$I’$Ia’$…®®i’$…Ão~ó›p— £ÌæÍ›C;v åää„JJJÂ]Î!qã7†‚íÝ»w‡ÞxãP|||èòË/¯·oß¾}C'tR½¾Ù³g‡€Ð_þò—P( •––†€Ð-·Üì“ŸŸBcÇŽ ú6nÜB/¾øbЗ••Õ`üE‹…€ÐäÉ“ƒ¾Ÿüä'¡¸¸¸PqqqÐ÷裆~þóŸP­Y¶lYÍ;÷k5jԨЄ êõõíÛ74xðàz}øÃB@è׿þõ³ög=nܸ&khn~{öì ]tÑE¡gžy¦Á~-9ÿÃ?ÜèëðË_þ²Ùî¾ûîPbbbƒþÚùÔ­§±¾æìÿ~nÊ /¼B¹¹¹¡P(*,, eff†"""êÕæ™g† škÿ×¹¬¬,„^~ùå ¯²²2Ô¹sçfÇy÷ÝwC@hñâÅõúk7ß|sÐW]]j×®]è?ÿó?ƒ>ßóÿr饗††Úì>ÇŠ)S¦„"""B¯¼òJ¸K9<^~9‚ÐøÃúóŸÿ\ïéÞ½{‡Æl6,4pàÀzûlÛ¶-ºãŽ;ŽHÉ’ÔšùgdéøWª½z+4242Š õ õ ÍÍ•‡ÊÃ]š$I’$IR«sõÕW‡®¾ºáõ³®d I’$IG™ûî»ääd^ýuÃ]Î!³{÷n"""ˆˆˆ !!ë®»Ž'žx‚ßÿþ÷Á>ùùù¬\¹’óÎ;¯Þ±gœqðå·K×uÒI';uêÔ ¯K—.lݺ5ÅŠ|ãߨ7ÎàÁƒIJJbÞ¼yAßw¿û]ÊÊÊx饗‚¾çŸžo¼ñ jmÎ׫}ûö¬\¹2ØÞ±c+W®døðáj×· IDATŽu°ç8p`‹æR×îÝ»¹ä’KHNNfܸquþqãÆQ]]]ïuxù嗹馛š=÷¦M›HII9àš¥‘#GÍo¼ÀìÙ³¹á†8í´Ó˜={6ðe¡Pˆ´´´fÇÚÿuŽå´ÓNã'?ù ¯¾ú*eee´iӆ͛7­šO>ùäàqDD:ubË–-€ïùý¥¤¤°iÓ¦f÷9VLš4‰Ûn»‰'RYYîr«¨¨(FŒQ¯¯ÿþÁªååå|ôÑG\zé¥õöéСgŸ}6óçÏ?B•J’$¿"ˆàB.äOü‰¬àßø7&2‘žôägüŒìw‰’$I’$I­ž!I’$I:ʼóÎ;Üzë­$$$„»”C*!!P(Duu5«V­"++‹x€íÛ·ûlÛ¶ €iÓ¦„ˆˆ:wî À† êÙ®]»àqddd“}ÕÕÕõÆoß¾}ƒú:tè<pâ‰'2|øpž{î9>øàºuëFÏž=ªÖæ´t¬¥K—ò­o}‹®]»IDDÏ?ÿüðÃ:§N¸êª«ê½ƒ úÊAQQÑÑÑ\ó¡”’’¹çž[/d0zôhFÍûï¿ÏÎ;yóÍ75jT½ãZò:¼ýöÛ\~ùåÜu×]¤¤¤pÉ%—4ø9¨º¿?ðåïÐþ¿?¾ç¿ÍŽÇÏÅOÿþïÿΖ-[øì³ÏÂ]ÊaÕ¡CÚ´iS¯/11‘;wP\\Luuu𾨫K—.õþ›%I’¤¯¯}x‚'XÃnäF~É/éMo~Ä(  ÜåI’$I’$µZ† $I’$é(\Ô{<Šˆˆ OŸ>¼ð lÞ¼™ûî»/x®cÇŽÀ—ߪ …ÜfÎœùµÎ];~coß¾=x¾Öw¿û]>úè#–.]ÊsÏ=ÇÍ7ß|XjmÉX•••äää——Çüù󩬬$ qã7 …‚±ºuëÖèk/ =õ7åþûïçÍ7ßdРAÜpà ìÙ³ç ÎÇwÔ{n¿ýö¯¶M-+­ÖC§U73;šiºšõm«ÝR·ý~S;¨Ôj©¿J)ÍCZŠÊaQ†ƒ ˆ0¿?\îuM…׳Ç<¸çžë¾ïÏu3‡®÷}ÕØ×´iÓ´eË͘1C‡C?ýô“^zé¥s>æ…¯^xAóçÏך5kÎëø=ôþú׿jĈõ:æï~÷;•””Ô:kECŠUçεlÙ2—>Ýzë­Z¾|¹n¾ùf—öçò{Þ¾}»fÏž­ââbjáÂ…2™LêÚµkõT_õ×®]:tè,ËC §ã9ÿ_ûöí«Wàår0sæL-\¸Po¾ù¦|||Ü]ŽÛýå/ѶmÛôØc)//Oééé6l˜|||\ÂR¸xü䧱«=Ú£·ô–R•ª6j£‘©=Úãîò?'À™òŸÿp‡… º»\‚~úé'§Õju¶oßÞ¹~ýzw—sÞ233’\n÷ß¿K›‡~ØxlöìÙN§Óé\µj•³GN???gdd¤óž{îqfff:N§óÃ?tÙßðáÃkÖ¬qYwçw:ÿýﻬ»ñÆc~þùçÎnݺ9ýüüœf³Ù9|øp§Ýn¯µ_ýµS’sÓ¦Mµ>~¦Zk3yòäõ×w_ßÿ½óÚk¯u8cbbœ£Gv2ÄØWNNŽÑöÍ7ßtÆÆÆ:ýüüœ=zôpnܸÑh×½{÷zóôs-Éép8êì›Óét¾üòË.íÇçüüóÏ]ÖuêÔéœÏ_vv¶3<<ÜYVVvÆãW+..všL&çgŸ}f¬›;w®K}úô©±N’ó›o¾©uŸµ=Ÿ«Ÿ³g2uêTgHHˆ³¢¢ÂX·víZ§$çÚµkk´¯ïïyåÊ•Î~ýú9ÃÂÂœAAAÎÞ½{;¿üò˳ÖóÐC9ƒƒƒAAA·z¨Ö×Ôé}mݺµ±=Ïy§Óáp8½¼¼œ©©©g=ß—²ÜÜ\ç]wÝåôòòr.X°ÀÝå\Ðk¯½¦nݺéw¿ûî¿ÿ~ 2DëTn7iÒ$Íœ9SÓ§OW\\Ü9m¬uëÖéá‡ÖÀy­à‚xã7ôÙgŸ©C‡î.¥^ÊË˵jÕ*½÷Þ{Z¾|¹‚ƒƒõÀè±ÇS‹-Ü]p^|å«ñ¯{u¯fi–¦išj¡ž×óº[wËCî.à²æá¬k„ 4!wé?Ó½héYZpá-Z´H£Gvw—ÒÒÒá€úNçåå¥ÐÐP™Íf—Ÿ¡¡¡ 1‚Õ˧†ª×{x4ܯ7nܨ… ê_ÿú—JKKuà 7(%%E”Õjm°:—¦cÇŽéË/¿ÔǬO>ùDÅÅźöÚkõÀhÈ!òóósw‰ kéRièP‰¯Aà²À¿‘œLejª¦êoú›zª§ÞÐê¢.î. à’w×]ÿ?»Ôuü,3p+§Ó©C‡)??_‡ÒÁƒ•ŸŸ_gX úgiii}5oÞ¼FX ::Z;vtYær?88Ø =?={öTÏž=µ`Á}úé§Z²d‰{ì1;V]ºtÑ-·Ü¢ßÿþ÷JJJjzI ‰Úµk—¾úê+­\¹Rk×®UYY™ºuë¦?ÿùÏJIIQtt´»K.šÅè½£GôˆÕ£JR’†k¸^Ñ+Š³Úœ+B.¸‚‚íÝ»W»wïÖ®]»´{÷n#¨`¼GÞzë­š4i’ñ¾i2™ÜÜ iˆT¤>Ô‡¡zPêj]­wôŽú«¿»K¸¤2.¢ÊÊJåääèÀÊÈÈPff¦233ûÙÙÙ:tèÑÞÛÛ[‘‘‘Š•ÅbQRR’n»í6#P`±X-7ö M‰§§§l6›l6›ú÷wýƒ{aa¡~ùå#|°{÷nýßÿýŸöìÙ£òòrI’¯¯¯bccÕªU+µjÕÊT/[,wt ÎIqq±öíÛ§ýû÷a‚êûûöíSII‰¤“ï™111jÛ¶­5bÄ#„ãæ^¨v‹nÑvm×DMÔ Ð½ºW ´@ÍÕÜÝ¥\¿ÃáPff¦222ŒAõrFF†ìv»Nœ8!éd€Àjµ*66V±±±ºù曥]»vé¾ûîStt´Z¶lÉUßqÙ Q·nÝÔ­[7—õN§S999.ƒp÷ï߯ôôt}õÕWÊÌÌ4fæ0™LFè **JÑÑѲZ­²Z­ŠŽŽ–ÅbQdd¤;º ‰8räˆ1“Pvv¶1“PVV–8 ýû÷³HRË–- Ô-·Üâ Š‹‹“¯¯¯{ ¾‚¤…Z¨~꧱«Žê¨÷ôžz©—»Kp;BÀ8—+ŸúóÀ:räˆÑ6,,L111Š‰‰Q—.]4xð`EGG+66Vqqq²X,òòòªqŒE‹Õ¤ \Î<<<Œ @¯^5ÿ0_YY©¬¬¬Z¯¾qãFeeeW—NΆP=‹GTT”@hÙ²¥bcc)‹Å¢ÀÀÀ†ì&€K\yy¹òóó•­œœœ:ƒ§¾ßøùùÉb±ï5}ûöu \qÅÌ&42C4D=ÔC£4J7èýEÑdM–‡<Ü]€Û2@“vôèQc€sma‚¢¢"I’§§§¬V«1À011Q±±±F¨ ..NÍš5sso€Ëƒ———âââ§>}úÔÚ¦¬¬Lv»]v»]999.?·oß®åË—»Ìˆ *³Ù,³Ù,«Õ*‹ÅRërtt4W.C¥¥¥Æ{ÃáÃáp¹êrnn®œN§±­Éd’Õj•Íf“ÕjURR’,‹ñþPý9ÏŒB@Ó­h­Ñ½¦×ô¤žÔFmÔ{zO! qwinAÈšÓéTvv¶öìÙ£½{÷jÏž=JOO7BùùùFÛˆˆãJÅ7ß|³(¸âŠ++???7öhZL&“l6›l6[mªªª”——§ÜÜ\åææ*??_ùùùÊÍÍÕÁƒuèÐ!mØ°AyyyÊÏÏWyy¹Ëö¡¡¡ŠˆˆP‹-jª—k»_ì®M±cÇTPP ‡Ã¡‚‚—[õºÃ‡«  @‡2^ß'Nœ0öáéé©-Z(<<\-Z´ÅbQ||¼úôé£ÈÈHãõ] àsÀ™xÈCOè ]§ë”¢uR'-Ó2u3€¦‡.{•••ÊÈÈ0B§ öìÙ£²²2IRóæÍÕ¦MÙl6õîÝ[#FŒ0BW\q3—OOOY,Y,–zµ/..®3ŒPPP ÜÜ\íܹÓesUU•Ë>¼¼¼j "˜Íf)((H!!! V`` ±.((HÁÁÁ æ*éhJJJT\\lÜJJJäp8\î×(((0>›OXãµuå•WªgÏžF˜Àb±¨E‹Æ×€ ­«ºj³6ëÝ£>ê£9š£Ñíî²!\Nœ8¡ýû÷ë×_5ÂÕ·ýû÷ëøñã’¤µiÓF­[·Ö­·ÞªÖ­[«M›6jÓ¦M½"hœªû·mÛ¶ÞÛÖ¸Êzm¦8` ®.,,Tqq±*++kÝg@@€‚‚‚ŒBpp°BBBŒuþþþ ‘Éd’¿¿¿‚ƒƒåç秀€ÈÏÏOÁÁÁò÷÷—ÉdRHHˆ<<<.ÔiB#TRR¢òòrëرc*//—ÃáPyy¹Ž;¦ââb•——«¤¤DGUYY™ŠŠŠŒçòé‡ÃQç±N Öœˆ‰‰©Ì9=PàíÍ×T. á ×çú\ÏêYÕXmѽ¡7äÍŸ×@Á· ¸¤jïÞ½JOO׎;´sçN¥§§ëçŸÖ±cÇ$If³Y6›M6›M·ß~»±\}€ %$$D!!!çõÞrôèQ—«ºª¨¨¨Æ í¢¢")''G»wïVYYYàuªùúúªyóæ ”ŸŸŸ‚‚‚Ô¬Y3ùùùaoooJ:ù>*¼r¼···XðññQ@@€<<<"éä q///c’\—ä²oüWõï°ÚÑ£GP\UU•ŠŠŠ$ÉøWŽ?®£Gº´)**RUU•±ê}WVVª¸¸Øh[ZZª²²2ª¼¼\G=kAAAòóóS`` š7o.cV«Õê 2fî8u¶ŽÀÀ@ãy—¼4S3ÕU]5B#´Oû´TK(þý ?Bhp'NœÐ¾}û´k×.íÞ½Û¸íÚµKùùù’$“ɤ¶mÛª]»vºùæ›5aµk×NmÛ¶Upp°›{g×¼ys5oÞü‚Ì¢RQQ¡#GŽèÈ‘#*//¯1˜¼¬¬¬Ö+Ò?~ܸ~ìØ1åçç×kàúoUl¨Ö¼ysùúú÷ƒƒƒåééYc;OOÏsz¯IÔGu_ë£úÜÖ¦¤¤D'Nœ0îŸzUÿ'N¨¤¤¤^Ç8“s „x{{!³Ù,???5kÖì¬3`œÚP·Ûu»6j£j zª§Vj¥âçî².*B¸hÊÊÊôóÏ?ëçŸÖöíÛµk×.íÚµK{÷î5®älµZÕ¾}{%$$èŽ;îPûöíÕ®];ÅÅÅÕ:š"™Íæ»R|õÕñ¥“3Ì8N9rD’j¦?Óû%¹ÌÆàt:UXXXëq«õq®ú###]‚gr¦°Ãé‰ê™¤³Ïðpzø"$$D OÇ—†Nê¤oõ­i®Ñ5Z®åJR’»Ë¸hà7;~ü¸&ضm›*HOOWee¥|||Ô¶m[ÅÇÇëŽ;îÐUW]eÌJäîò§ñòò2 làR¥(­ÕZ¥(E×ëz-×rõU_w—pQ2@½UTT(33S;vìÐÎ;ŸÛ·oWyy¹¼½½k„ âãã• „„™L&w—ç-HAúTŸj¤Fj êc}¬þêïî².8B¨UNNŽ¶nݪ­[·jË–-ú駟ô믿ª¢¢BÞÞÞjݺµ:tè hâĉJHHPÛ¶måëëëîÒà¢ð–·Þ×û­ÑºM·éú‡îÔî. à‚"dÐÄUTTh×®]Úºu«~úé'mÙ²E[·nÕÁƒ%I111êÔ©“n»í6]}õÕŠWûöí h’¼ä¥ÅZ¬@j¨†ê½£‘éî².BMHQQ‘¶mÛ¦;wjÇŽJKKÓ?ü ÒÒRy{{«mÛ¶JHHЃ>¨ÄÄDuëÖM‘‘‘î..)òÐlÍ–|ôýAþòWŠRÜ]ÀAÈ ‘:|ø°6oެ͛7+--MiiiÊÈÈ$…‡‡«S§NêÑ£‡F­N:)>>^>>>n®.òÐËzYªÐYf%+ÙÝeüf„ ÂÂB¥¥¥¡‚Í›7kÿþý’¤ØØX%%%i̘1êܹ³:uꤨ¨(÷ ÄlÍV‘Št«nÕ­QOõtwI¿ !€ËÌ‘#G´eËcv‚´´4íÚµKUUU²X,JLLÔ}÷ݧÄÄDuëÖM‘‘‘î.-yh‘é êVݪ¯õµ®ÒUî. à¼2¸„8qBÛ¶mÓÆõí·ßjóæÍúå—_TUU¥ÈÈH%%%)%%E‰‰‰JJJ’ÅbqwÉ€ûäåIï¾ëºî§ŸNþ|ñE×õf³4ztƒ”€ÆÏG>Z¦eJV²ú«¿6i“Z¨…»Ë8/„ .!‡Ò·ß~«ÿ÷ÿþŸ6nܨM›6éèÑ£ R=tçw*))IIIIŠŽŽvw¹À¥%,Lzé%©¨Hò>å«O__iêÔÿÞ//'`€vækÕ IDAT ®™ši…V¨«ºj¨†jµVË›?É€Ëßh¸Iee¥víÚ¥´´4mØ°Aëׯ×Ï?ÿ,§Ó)›Í¦^½zé¶ÛnSïÞ½Õ¥Kyzzº»dàÒæí-Ý}·ôÖ['ƒgrÏ= Sš”0…éÿôê©žš¤IzM¯¹»$€sFÈ kÆ úöÛoµqãF}÷Ýw*))Q`` ºvíª;î¸C=zôÐ5×\£ÐÐPw— \ž† “æÏ?s›-¤Þ½¦49ÕQoé-Ý£{ÔQ5J£Ü]À9!dp‘”””è»ï¾SjjªÖ¯_¯ï¿ÿ^²X,êÝ»·ž{î9%&&ª[·nòõõuw¹@ãЫ—dµJv{íûúJ#GJ^^ [š”a¦4¥iœÆ)Q‰ºZW»»$€z#dpäççë›o¾Ñºuë´nÝ:mÛ¶M’Ô¡CõéÓG=ö˜®½öZEDD¸¹R óðFŒfÏ–**j>~üøÉÙ€‹l–fé;}§‘©ïô|ÅÅfÀåÀyÊËËÓ÷߯ 6(55U?þø£<<<Ô®];õîÝ[O?ý´úöí«°°0w— 4-ÆI/½TûcqqRbbÃÖ€&É[ÞzOï©“:é¹ÿüp9 dPOúꫯ´fÍ­[·N»wï–···Õ·o_MŸ>]½{÷Vpp°»Kš¶Î¥+¯”~ýÕu½¯¯4j”[J@Ót…®Ð,ÍÒx×@ TwuwwIgEÈ úöÛoµzõj­Y³F›7o–$uëÖMwÜq‡úôé£^½z) ÀÍ•¨aäHé/‘**þ»îøñ“³ èA=¨ÿÕÿj”Féý ù»»$€3"dpŠôôt¥¦¦*55UkÖ¬Qaa¡®¸â õë×OO<ñ„’““e6›Ý]&€³6LúóŸÿ{ßÃCºúj©];÷Õ€&ÉCz[o«ƒ:è½¢gõ¬»K8#B I;r䈾ýö[­X±B+V¬Ð¾}ûÔ¼ys]sÍ5š2eŠ’““•˜˜èî2œ«Ö­¥Î¥­[¥ª*ÉÛûäì€Ä*VS4E34C£4J1ŠqwIu"dšœ-[¶hÅŠúôÓOµyófIR×®]uï½÷ꦛnR÷îÝåíÍÿ&—½‘#¥‰O† Nœ†uwEhÂ×ãzKoéY=«wõ®»Ë¨£ç@£W^^®µk×jùòåZ¹r¥222dµZuË-·hâĉºñÆâî2\hC‡JO}ºn¿ývÅÆƺ»DQii©222d·Û•••¥¬¬,Åyyi¨§§¦oݪ°9s-«ÕªØØXEFFÊÛ›¯HÐp<ä¡çôœúª¯¾Õ·ê¡î.  þ‚.[N§S›6mÒÒ¥KõÉ'ŸhÏž= ×-·Ü¢?üP7Ýt“Ý]&€  °°PÙÙÙÊÌÌTvv¶"°ÛíF°   Àhïçç'«Õªv‘‘Š‹ŒÔç߯¬ýKyyyr:’$///EFF*&&FV«U111.!«Õª¨¨(™L&wuÐ ºAÝÕ]¯é5-ÕRw—P!pÙùᇴdÉ-]ºTû÷ïW›6mtÇwhРAºæškäåååÃ!»Ý®œœ¥§§ËÕ?÷ìÙ£¢¢"£½Éd’Õj•Åb‘ÕjÕ-·Üb,WÿŒ‹‹ûï¿ ²²´):Z’TQQ¡üü|—ýWsçÎZ¹r¥233UQQaÏl6×Ø¿ÍfsY6›Í zÎpy{\ëÝ£=Ú£6jãîr\2—…;vhÙ²eú裴{÷nÅÆÆê¶ÛnSJJŠzõê%w— ‡Ã%8pzˆ ##CGŽ1ÚWªñ'$$èÞ{ïuYg±XÎíßÿ H’¬V«¬V«ÏXwm5§§§+--MYYY*..®Qw]!‹Å¢–-[ÊÓÓóÜN ¥;u§®Ðš­ÙzSoº»„ À%kçÎZºt©–.]ªŸþYÑÑѺãŽ;´xñb‚€›•——ëðáõ¯^ÎÈÈЉ'ŒmNÀf³©W¯^.ó¯¼òJ¹±Wÿe6›e6›•Pg›ÒÒÒ:ûŸššj¬«æçç§ÐÐÐ:CV«U±±±òöæk[€ÆÎK^zDèi=­—õ²š©™»K0ð×*pIùõ×_õÑGiéÒ¥Ú¾}»¢¢¢4dÈ-^¼X×\s Á ”••Én·×9û@NNŽöï߯ªª*c³Ùì2ûÀ AƒŒó‹E­ZµRóæÍÝØ« Ïßß_6›M6›­Î6g:—iiiZ¹råÏåé!„Æz.š¢a¦?éOZ®åº[w»»!àv¥¥¥Z¹r¥-Z¤/¿üR¡¡¡0`€fÎœ©þýûsOà:ÓÕ÷O T;ýêû‰‰‰5¾ÇÅÅÉËË˽ºt™L¦³Ž?®C‡Õú»HOOWjjêYg…85„`µZÕ¦M7DpžZ¨…’•¬õ!!pIaÄp›´´4-Z´HÿøÇ?TQQ¡~ýúiÉ’%ºýöÛ çÁápÔ¨^ÎÌÌTII‰ÑÞd2¹ LONN®1pÝb±0£ØEæëë+«Õ*«ÕªÄÄÄ:Û9ŽZ!éééZ¿~½222täÈ£}õï·¶¿_€KÃ0 ÓõGÖa…)ÌÝåH"dXvv¶>øà-^¼X{öìQ||¼žyæÝÿý wwyÀ%éÔ+Ý×"ÈÌÌTEE…±Í©Wº·X,JLLtlÞºuk…„„¸±W8Wf³Y‰‰‰g "ÔB°ÛíÚ¹s§öîÝ«ÂÂB£ýéA“ÚfG`¦ €‹ç6ݦ±«ÿÕÿêú£»ËDÈ4€²²2­X±Bï½÷ž>ÿüs)%%EK—.U—.]Ü]àVeee²Ûíu ÏÉÉÑTYYilc6›Aà6›M½zõr§€€7ö îb6›e6›•Pg›ÒÒÒ:Ÿk6lÝnWnn®œN§¤“3-„……ÕB°ÙlŠ‰‰‘OCu ÑT úª¯Vi!pÉ d.š;vhÞ¼yúðÃuìØ1 0@ü± Àà4 Õƒ¹OÄ}úÀîºsÛl6%&&ÖØ+oo¾ÖÃùó÷÷—Íf“Íf«³Myy¹>\ë¬;vìPjjj­á—ºB‹E±±± lˆ.\Vú¨^ÒKrÊ)y¸»Bબ¬ÔŠ+4wî\ýûßÿVÛ¶mõç?ÿY#FŒPDD„»Ë.‡ÃQë•à«îÙ³GEEEF{“ÉäHNN6–«¶jÕJžžžnìp’ŸŸŸ¬V«¬V«ëlçp8j„ìv»ÒÓÓµ~ýzíß¿_ÇŽ3ÚW¿j !œúZhJ®×õš¨‰úY?+^ñî.€¸0ŠŠŠôî»ïjÎœ9:pà€úöí«O>ùD”‡W^À奮ÓÕ!‚èèÑ£FûÓN'$$hôèÑ œF£g6›•˜˜xÖ Bm¯¥ôôt¥¥¥)++KÅÅÅFûÓ9µ…ä€Æ¤‹º(D!Z§u„ À%øMvïÞ­ùóçëí·ß–§§§† ¦ñãÇ+>ž?„àÒS^^®Ã‡×:عz9##C'Nœ0¶1›Í.ƒ{õêå2ð9&&FAAAnìpi3›Í2›ÍJHH¨³Miii¯ËÔÔTÙívåææÊétJ:9ÓBhhh!›Í¦ØØXy{ó8¸ôyÉK½ÔK_ëk=¨Ý]!pªôÕW_iÎœ9úôÓOÕ¦M=ûì³3fŒBBBÜ]š¨êAÊu]1Ýn·+//OUUU’$…‡‡“4hÐ —AÊ111òññqsÏ€ÆÏßß_6›M6›­Î6eee²Ûíµ¾ÆÓÒÒ´råJíß¿ßxK'§†N$ÄÅÅ)  !ºpFÕQŸêSw— ‰8ÇŽÓâÅ‹5{öledd¨ÿþúâ‹/Ô¯_?yxx¸»<4b‡ÃL\WˆÀápíýüüe &>}ö«Õª¸¸8yyy¹±WÎ…Éd:káøñã:tèP÷»Ý®ôôt¥¦¦žu¶’ÓgEhݺ5zpÑ]©+õ«~U•ªä)Ow—š8Bଠôæ›oê7ÞбcÇtÿý÷ëÑGU›6mÜ]‡ÃQc0ð©„333URRb´7™L.ƒj ¶X,aÝhÆ ÿüsÝ|óÍtÿ©©©êׯŸ¶mÛ¦:\Ð}צªªJN§SN§ó‚l[Wýç{œºÎ÷o©û\,^¼X<ð€$iÁ‚;vìE=Þ™Ôö\÷ôôTpp°âãã5xð`;VAAAµn¿cǽôÒKúꫯtðàAµhÑB­[·ÖÍ7߬;ï¼SmÛ¶5Ú¦¥¥iƌڴi“òóó¥ääd >\×]w1@ÿÇÔË/¿¬uëÖéèÑ£Š×SO=¥^°~›Íf%&&ž5ˆPWøiçÎÚ»w¯ ö§Ï RÛìÌ êr¥®”$ý¢_·#djÈÏÏ׫¯¾ª7ÞxCAAAzâ‰'4a„Ì€ËCYY™ìv{·§Æ=pà€*++mÌf³1ØöÔAõº¸¸8¸±Wÿ-§Ó©Q£FéŸÿü§Ž9rQ7kÖ,5JW]uÕE=NC¹öÚkUPPpÑ·=ßãÔu¾KÝçâü£FŒ!ÿ‹~¬³©í¹^YY©ÜÜ\­ZµJÓ§O×üùóµbÅ ]}õÕ.Û~ôÑG5j”Ư¯¿þZQQQr8Z»v­&Nœ¨éÓ§«¬¬LÒÉà@Ïž=uÿý÷kãÆjÙ²¥ìv»Þxã ]ýõÚ´i“’’’$Iýû÷× 7Ü ´´4y{{ëÙgŸÕàÁƒõ駟ªÿþ vnÌf³Ìf³êlSZZZk!''G6lÝnW^^žªªª$I>>> ¯3„`³Ù#Ÿ†ê&¸DD(BA ÒíQ_õuw9 ‰#d ÕႹsç* @S§NÕøñãe2™Ü]Põ Ùºf°ÛíÊÍÍ5®öîëë«°°0c€lbbb´±±±òöæ«(àràå奨¨(ýáÐÀÕ­[7 0@;wîT`` $iûöí5j”üqÍœ9ÓØ622RC‡UBB‚ºuëf¬ÿŸÿùùùùiÞ¼yòôô”$µjÕJ¯½öšRSS]Žß¬Y3-^¼XÍ›7—$Í;WÿûßõÖ[o5hÈ >üýýe³Ùd³ÙêlS^^®Ã‡×x_MOO׎;”ššzÖY]N_Ž‰‰©sv pù U¨ Uxö†™§» îWPP I“&©U«Vzÿý÷5kÖ,8p@“'O&`ÐÈ8cPë¢E‹4mÚ43Fƒ RRR’‚ƒƒÕ¬Y3µnÝZýúõÓ˜1c´hÑ"¥¥¥I’’““5yòd-Y²Dß|óöîÝ«ÒÒRÙívmÞ¼YK—.Õœ9s4yòd9RÉÉɲÙl*`°wï^Ýzë­ W`` n»í6}ûí·gÝnÕªUêÑ£‡üýý£!C†è»ï¾«³ý„ äáá!mß¾]’ôÏþÓX÷ÁÔ«¦)S¦¨_¿~’¤«¯¾ZjÕª•±íÊ•+•””$“ɤÈÈH;VÅÅÅ’¤yóæÇ›7ož|ðA…††ÊÃÃCwß}w­uŸºÍâÅ‹k¬ûë_ÿªG}TÁÁÁŠŠŠÒ_þò—3n[Wýµµ•¤C‡iâĉjÓ¦L&“:wî¬O>ù䬿ŸÚö—››k¬;ý6dÈzÃjo¾ù¦âââÔ¬Y3Ýpà úõ×_ÏZSµSŸ;aaaº÷Þ{•““Skíg:¿ç#""B3gÎTVV–þú׿ëgÎœ©ªª*Mœ8±Öí:tè cÇŽ÷KKKUQQá²®ÚO?ýdÌb IéééFÀ@’¼½½åïï¯Ã‡ÿ¦¾¸‹ŸŸŸ¬V«•’’¢ñãÇkÖ¬YZºt©Ö¯_¯½{÷ª¢¢BÚ¼y³–/_®Y³fiРA²X,JOOײeËôøã«_¿~êСƒ‚ƒƒåïïo¼W9RS¦LÑ¢E‹´bÅ ¥¥¥Én·»»ëà*PGtqgQ¨B4a¥¥¥z饗ԦM½ûî»zþùçµgÏ=òÈ#„ .C‡CiiiZ±b…-Z¤)S¦häÈ‘êׯŸ ÐÐPuèÐAýúõÓøñãõþûï+==]f³YÉÉÉzùå—µ|ùrmÞ¼YÙÙÙ*--ÕÞ½{µ~ýz-]ºT³fÍÒøñã•’’¢Þ½{Ëf³W%o*î¼óNùûûkçÎÊÌÌT«V­”œœ|ÆmV®\©¨_¿~ÊÊÊÒwß}'???Ýxãunóúë¯kÍš5.ë† ¢üüüsªiÖ¬YÆ~¶mÛ&§Ó©ýû÷K’>ùä ¯z“’’4xð`IÒ¸qãôä“Oê›o¾QŸ>}Îk§{ýõ×]î>\K–,Ñܹs]®Ä_-[¶Tjjªqÿ›o¾Ñ¬Y³4{öl]}õÕ’êw§M›¦Î;ë™gž1ÚŒ=º^¿ÿ?ýéOŠ×óÏ?/I ×¢E‹Ô¥KÍŸ?_“'Ovi1ÎoPPBCCuàÀIRqq±‡zôèQï}Œ5J_|ñ…þõ¯©OŸ>²X,ºýöÛ5räHuïÞ½Öm õüCO=õ”¬V«¦L™rÞ}h,Ìf³Ìf³êlSZZZã3¢úgjjªìv»rss ‡¯¯¯ÂÂÂ\>#¬V«ËgHlll£š‘€K3€K ‰Y»v­Æ¯;wêþûï×Ô©Se±XÜ]@“U=´¶à@õr]ƒA-‹4hÐ —Á 111òññqsÏ'“ɤ®]»êÉ'Ÿ”‡‡‡(åååÕ¹MVV–vïÞ­qãƹ¬ÐáÇÝRSu]¿üòK°@õ€ï/¿üÒ%dбcÇß\«$cp¾$yyy)<<\¼ û®Khh¨~ùå—ß´ÂÂB1B7Ýt“yäIõ;‡;w>c›3©ë¹Ó¹sg)55µFÈàbßÚfððð¨÷ö>>>úøãµvíZ½ýöÛZ¾|¹æÏŸ¯ùóç릛nÒ’%Kâ²Í„ ´råJÝsÏ=š6mš¢Á™ùûûËf³Éf³ÕÙ¦¬¬Lv»½F!==]iiiZ¹re­áµºB‹Eqqq hˆ.Ðh1“¸T2 ‰8pà€&Nœ¨eË–iÀ€Zºt©Úµkçî²µƒ*;;[YYYÊÊÊ’ÝnWff¦²³³•­ŒŒ =zÔh߬Y3ÅÄÄÈjµ*::Z¿ÿýïeµZ£¨¨(EEE)22òœöâÂ[µj•f̘¡ &hĈJNNÖŸÿüç::tHÒÉî—JM§Ö5oÞ<Í›7¯Æã.÷ýýý/H­§BöññQUUÕÙ·$íܹSÏ<óŒ6nܨƒƒã;wîü›ö;vìX•••éÝwß5ÖÕçæääHªùû¯ÏóáLÏ°°0ãñS]Œó[TT$‡Ã¡N:I:9³Ùl6úv.®¿þz]ýõ*//תU«ôúë¯kõêÕš1c†^yå•íçÍ›§»ï¾û7ÕšL&ÓYƒÇ—Ýn7>ò³³Ï°íÛ·ë³Ï>Snn®Nœ8ali|†EGGËjµ*66VQQQÆróæÍ¢‹\–¼ä¥*]¸ÿ78_žî.\\¥¥¥š>}ºâããµeË­\¹RŸ~ú)à<™Íf½úê«ÊÎÎÖºuëTVV¦ë®»N{÷î­µ}xx¸$©  àœåéyò뻊Š c]qqño®éÔº&Mš$§ÓYãöþûïŸs½îVQQ¡äädeffjíÚµª¨¨ÓéÔ}÷ÝWë•øëëoû›–,Y¢wß}×åjúõ9‡Õ3Çþû/**:ëqÏôÜ9|ø°ñøŶ|ùr9N 0ÀX7`Àmß¾]………çµO??? ù¤üüüÜ]@“¡ˆˆuéÒ¥Î6¥¥¥ÊÉÉ‘ÝnWNNŽÒÓÓåU«VÉn·+77× íëë«°°0Y­VY,Y­VÙl6—嘘ùøø4T7›”ÜÜ\õë×OÛ¶m“$õèÑCo½õ–Z·n­Í›7«uëÖ5¶‰ŽŽV»ví´nÝ:—õv»]6›MÙÙÙ «õx-Z´0Ž[í§Ÿ~:皪à §×Õ¾}{mÚ´©Æc:uÒSO=¥¡C‡žét4˜Úê¯Mzzºrrrôøã«}ûöÆúòòòó>öÞ½{õÈ#èÑGUÿþýõ:tÐöíÛëuÛ¶m«õë×»<þÃ?œõØÕϵk׺¬ß²e‹Š‹‹•œœ|~:yyyzú駣1cÆëŸ~úi}üñÇzå•W4cÆŒÛÍž=[Ó¦MÓ¾}ûªgžyFzî¹ç\Úy{{ËÇǧÖ×À©³FàÂ*++“Ýn7>ojû :pà€*++mÌf³ñyÓ¡Cõë×Ïå3(..®ÆLàòDÈ€F(//O'NÔ| [n¹EkÖ¬QLLŒ»Ë@-üýýe³Ùd³ÙêlS^^®Ã‡×šžž®;v(55µÖÁ u…,‹bccØ]lt¶o߮ٳgëþûïWUU•.\(“ɤ®]»Ö¹Í+¯¼¢[o½UÏ>û¬&L˜ #GŽh̘1ºï¾ûê HRÛ¶m¡ùóç«{÷îÊÏÏ×ßþö·s®©újú»víRË–-uõÕWkÅŠzõÕW5xð`Íš5Küã%I3gÎÔ‰HÎlç IDAT'të­·þ–ÓtAÕUÿ©AIjÕª•Z´h¡÷ß_ ÍfÓ×_­Ï?ÿ\­Zµ:çãž8qBÇW\\œ^|ñÅZÛÔçN›6M÷Üsf̘¡qãÆ)33S/½ôR½j¨~î<ýôÓzüñÇuèÐ!3Fmڴу>xÎ}ªÊÊJåææjÕªUš6mš<==µbÅ —÷Œ«®ºJ|ðî½÷^UVVj̘1ŠŠŠ’Ýn×| 3fè­·ÞRhh¨±Íœ9sÔ¾}{ÝtÓM Vvv¶fÏž­ýû÷kÁ‚.58uéÒEÝ»w×’%K.J?«êðÚ©Ÿ§‡ê ¯Ùl6%&&Ö²ÅÆÆÊÛ›?'ÐTx8Ë<áÐHÜ¥»$IKµÔÍ•¿ÓéÔ;ï¼£‰'*00PóæÍÓ AƒÜ]€³X´h‘Fíî2€Ãá¨B8u€étôèQ£½Édª3„pêÓ¦(++«FPwöìÙš0a‚>ýôSÍ™3G?üðƒ***Ô±cGMŸ>]}ûö=ã>¿øâ M:U[·nUXX˜î¾ûn=ÿüó2™Lš2eŠË öáÇëƒ>$¥¦¦jüøñÚ·oŸºuë¦W_}UIII’¤ßÿþ÷úâ‹/êUÓ¸qãô÷¿ÿ]N§S#FŒÐ›o¾)IZ½zµ¦NªüQ!!!ºñÆõâ‹/*::Z}ô‘† æ҇á:û9oÞ<=òÈ#Æý>}úhìر.û>|¸fÍšårŽ[·n­ &Ôضú*þ§×ÕUWÕÚvÓ¦Mzâ‰'ôã?Êl6«ÿþ*((Ð?ÿùOIRNNŽ^ýõç»G5ö÷ðÃëÿ³wçñQÕ÷þÇ_ IØ€2!‹Š­Ú ¢Á¥ ®lZÁ¥Ö*jk‹í¯¯í½U»\Ѷ·VÛklK75XÀĺ[[®¹ÞÖ†Z•(!LØ&,I~ÐÌ%+Q’œ$¼ž<ò8ÃÌ÷œ|¾'3ç$3ß÷ùNŸ>½Å~Ž=š·Þzë€û°ÁOúSæÍ›Çúõë9ñÄùáÈi§À)§œÂË/¿Üê>Ýÿ¹Ó¯_?.¼ðBî½÷ÞXø¢éÏ©µýûî»ï6ÛvKÏõ¸¸8Ì1ÇÃÔ©S¹ñÆ4hP‹µ­\¹’»ï¾›¢¢"6oÞÌ'>ñ N9å¾úÕ¯’——k·cÇ-ZÄc=Æ?þñ¢Ñ( `ìر|ýë_gâĉ¶[UUÅñÇϸqãxüq߯iPUUÕê1¾²²’wß}—mÛ¶ÅÚ7çÛ:Æggg·{¶©+ø7²¤C™ŸWI’$I’¤®6cÆ¿ÞhòyŒ!IÂ7mÕ;D£Q®»î:ž{î9æÌ™Ã]wÝEÿþýƒ.KR;8€B’Ô•4@õwÞaûöí±öP•¤®aPLÚÇ¿‘%Êü¼J’$I’$uµÖBÎo,IR/ŸŸÏM7ÝDjj*üã9ãŒ3‚.I’$IÝT(" ‘““Ój›êêêVCEEED£QÖ­[Gõ+úöíËСC[à‰DÈÌÌ$!Á·¢$zvíÚÅæÍ››W÷¿]^^ÎÞ½{cë„B¡FÇм¼¼FÇØŒŒŒVg—$I’$I’$I:X~²+IRVUUÅìÙ³yâ‰'øâ¿È¼yóèׯ_ÐeI’$©‡KII!‰‰DZmSSSC4mB(++£¤¤„ÂÂBV¯^MmmmlP(Ôj!--¬¬, Ð]”¤ÑÊj)8Ðp{ýúõÔÕÕ˜˜Èa‡;æää0yòäFÇÃŒŒ î™$I’$I’$I:”2$©‡Z±bW\quuu,[¶ŒsÎ9'è’$I’tINN>`a÷îÝlÚ´©ÅYÊÊÊ(**jóêÝMCáp˜‘#G2xðà®è¢¤C\UUU‹Aª†Û«V­bëÖ­±öIII¤§§ÇŽWMg‡ÃdeeѧOŸ{%I’$I’$I’t`† $IêaêëëùñÌ׿þuÎ:ë,~õ«_1lØ° Ë’$I’šéÛ·/áp˜p8Lnnn«íªªªš…öŸaÍš5|ðÁ±öÉÉÉÍî6 $¤¥¥×ݔԵu܉F£<îäääpõÕW{Ü‘$I’$I’$I½’!I’zÍ›7så•Wò /p÷Ýwó•¯|Å ’$IêñB¡¹¹¹ "´6xåÊ•”••QUUkßôŠâ-ÍŠàÅ¥Þ§­Tîkk•H$› á¾Q£F‘šš`¯$I’$I’$I’º–!I’zˆ7ß|“©S§R[[Ë‹/¾ÈÉ'ŸtI’$IR— …B„B!rrrZmS]]Ýb¡²²’åË—FY¿~=uuu$&&rØa‡µBˆD"ddd˜˜ØUݔԆšš¢Ñh‹¯ñ†åûï¿{þcGÃë:‰0a„Fᣬ¬, `¯$I’$I’$I’ºC’$õ………Ìœ9“O~ò“,Z´ˆaÆ]’$I’Ôí¤¤¤‰DˆD"­¶Ùµk›7on6@¹¬¬ŒÒÒRŠŠŠx•ó¦·3224hPWtQêµBBM_—û‡Ö­[G}}=°o¶’!C†Ä^‹¹¹¹Í^£™™™$$ø¸$I’$I’$IÒGå',’$ucõõõÜu×]ÜyçÜtÓMüèG?ò*ª’$IÒAHJJ"‡ÉÍÍmµ]UUU‹ƒËÊÊ(..fõêÕìܹ3Ö>99¹ÕÂþ3%H‡¢ªªªƒ ·+**ؾ}{¬}Ãë©áuÓtöp8Lvv6ñññöJ’$I’$I’$©÷2d IR7µwï^fϞͯ~õ+|ðAfÏžtI’$IÒ!# ‘››{À BkW\_¹r%ï¾û.Û¶m‹µo:pº¥‚§ÕÓ´Èi¸½fÍöìÙkß4“››k G’$I’$I’$©›1d IR7´sçN.»ì2^xá–,YÂE]tI’$I’š…B„B!rrrZmS]]Ýb¡²²’¢¢"¢Ñ(ëÖ­£¾¾€¾}û2tèÐVC‘H„ÌÌL|[Ok×®]lÞ¼¹ÕÙ*++Y½z5µµµ±uB¡P£çj^^^£çrff& °W’$I’$I’$Ij?”$©›Ù¸q#^x!åååüéO⤓N º$I’$ISJJ ‘H„H$Òj›šš¢Ñh³BYY%%%¶8˜»µBZZYYY 0 +º¨¨!üÒÚìm…_ÒÒÒÈÉÉaòäÉžw$&&Ü3I’$I’$I’$uC’$u#6làœsÎáÃ?dùòåŒ5*è’$I’$u²ääävïÞͦM›Z^VVFQQkÖ¬aÏž=±uö¿ª|ÓB8fäÈ‘ <¸+º¨.TUUÕb`¥áöªU«Øºuk¬}rrr£çHÓÙÂá0YYYôéÓ'À^I’$I’$I’$©+2$©›hìرƒ?ýéOdgg]’$I’¤n¢oß¾„ÃaÂá0¹¹¹­¶Û€yÓ«Ó—””°fÍ>øàƒXûý˜ËÆ€qøTí§ 2OKK#..®+º©¨ªªjôsmú³.//gÇŽ±ö ?߆ŸeNNW_}µ?_I’$I’$I’$µÉ$IÝÀºuë8ûì³Ù»w//¾ø"#FŒº$I’$I=P(" ‘““Ój›êêêØÀôw6½Ã ^àµô×xyÌË°–»ŒÊòÊXû¤¤$† Òìêö^é¾ãì?SEk!‚òòröîÝÿ™*"‘Hl‚†ûFEjjj€½’$I’$I’$IROeÈ@’¤€mÛ¶óÏ?Ÿ½{÷ò§?ý‰ôôô K’$I’Ô‹íHÙAq¤˜üH>ÏñµÔ2ŽqÌaÓ“¦^¦¦¦†h4Úâ¬%%%òþûïSWWÛn(j5„––Fvv6ýû÷°çÁhk_6,ÛÚ—‘H„ &¸/%I’$I’$I’Ôe H’ Ý»w3}út6nÜÈòåË H’$IêïñOò$ùäó/‘Dçpó0S˜Â`7jŸœœL$!‰´ºÍ]»v±yóæfçËÊÊ(--¥¨¨è€Wßßà|8æÈ#dРA¶:Úþ³B´"hÐtVˆÜÜÜfû"33“„߶•$I’$I’$IRpü´J’¤€ÔÕÕqÕUWñÊ+¯ðç?ÿ™ììì K’$I’Ô‹”RJ>ùRH % aqs˜Ã\ÀÔö“’’‡Ã„Ãarss[mWUUÕâ ü²²2Š‹‹)//gÇŽ±öÉÉÉ­†îKKK#..î ê?ªªªƒ ·+**ؾ}{¬ýøëá‡S~Ä„ÃáF³4Ô=|øpâãã;µnI’$I’$I’$é`2$) ·ß~;<óÌ3œp A—#I’$©‡«£Ž¬ B~Ïïy‡wÈ ƒ ¸€oñ-Îç|IìòºB¡¹¹¹ "´6ÀÊ•+y÷ÝwÙ¶m[¬}C¡­Y²²²èÓ§O³ïµgÏ6nÜØh¦¦!‚5kÖ°gÏžF}Øû¹¹¹¾g$#ƒÐÍ7Ã’%ðoÀ¾Ð±;Q’$I’$I’$IêB† $I ÀâÅ‹¹çž{X°`gžyfÐåH’$Iê¡v±‹y‘ È'ŸJ*‰a“x„GÈ#8:÷Šÿ!  …ÈÉÉiµÍÖ­[Y»v-kÖ¬aíÚµTTTPQQA4婧ž"²eË–Xû†™ÒÓÓ …BTVVRQQÁúõ멯¯ OŸ> 6ŒŒŒ Âá0cÆŒáÜsÏ%“™™[?99ùÀxì1¸çøÒ—àÿxúö=è}#I’$I’$I’$u5C’$u±·Þz‹k®¹†›o¾™Ï}îsA—#I’$©‡ùyžçÉ'Ÿ¥,e;ÛÃf3›Ì` c‚.±S <˜Áƒ·D¨®®¦¼¼œh4Ú(„°eËŽ:ê(2221bD,D0lØ0:è-Ò¸8˜;Ž;®¼JKá‰'`øðŽÙ¾$I’$I’$I’ÔE H’Ô…>øà¦L™Â'?ùI~ðƒ]Ž$I’¤b3›yŠ§È'Ÿe,c/{Ç8îâ.>ÃgÁˆ KìRRR=z4£G®ˆ /„W_…©SaìXX¼N:)¸z$I’$I’$I’¤È$I]è–[naëÖ­üùÏ&111èr$I’$uc«YÍ–PH!/ð $0 ü˜3•© cXÐ%ª5GË—Ãå—ÃgÀÏ~³f]•$I’$I’$I’Ô.† $Iê"K—.å‘G!??Ÿ´´´ Ë‘$I’Ô •RJ!…PÀ VJ*™Èp13A—¨ö2ž~¾ñ øìg¡¤~øCèÓ'èÊ$I’$I’$I’¤62$© ¬_¿žë¯¿žk¯½–K/½4èr$I’$uuÔñoP@ñoñ‡qps™ËyœG_ú]¦>®>}àî»áøãá󟇷߆ßýº2I’$I’$I’$©U† $Iê_ýêWéß¿??úÑ‚.E’$IRÀj©å%^"Ÿ|žà Ö²–l²™Âîç~Îâ,|Û®w¹â 8úh˜6 N: –.…1c‚®J’$I’$I’$Ij‘ŸVJ’ÔÉŠ‹‹ùÝï~ÇâÅ‹8p`ÐåH’$I @5ÕQD>ùPÀV¶2†1\ÅULbãt‰êl'ž/¿ —\ãÆÁ¯~S§]•$I’$I’$I’ÔŒ!I’:Qmm-7ß|3&L`ªƒG$I’¤CʶPø¯Oó4ò!'r"s˜Ã•\ÉQt‰êjiið ð…/ÀÅÃü|ë[te’$I’$I’$IRŒ!I’:ÑÃ?Ì?þñüñ K‘$I’Ô6°gx†|òy–g©§žS8…ïðf0ƒ4Ò‚.QAKJ‚ à”Sàæ›á­·à‘G _¿ +“$I’$I’$I’C’$uš]»vñÝï~—n¸£Žò ¥’$IRoUFO>+XA )œÍÙÌg>S™J*©A—¨îhöl8úh¸ôRÈ˃ŋ!;;èª$I’$I’$I’$C’$u–ùóç³qãFn½õÖ K‘$I’ÔÁJ)%Ÿ| )¤„†2” ¹9ÌáB.¤?ýƒ.Q=ÁgÀK/Á´ipÒIŸgtU’$I’$I’$I:ÄÅ]€$I½ÑîÝ»¹÷Þ{¹þúë1bDÐåH’$I:HµÔRL1·qGr$Çr,?ççä’Ë“Þ,Ÿ$I’Ô”RJ!…PÀr–"Ä&°€\Â% `@Ð%JÿgèPxæøæ7aÖ,øûßá{ßÿÖ”$I’$I’$IR'3d IR*))¡¢¢‚iÓ¦]Š$I’tÈ«£Ž7xƒ ø¿ãmÞæ|‚ó9Ÿ¹Ìå<Σ/}ƒ.Sj]BÜ}÷¾™ ®»Þ|~û[HM º2I’$I’$I’$õb† $Iê@K—.%++‹ã?>èR$I’¤CR-µ¼ÄKäÿë_%•ÁLf2 X@yÄt™ÒG3s&D"ð™ÏÀÉ'ÃÒ¥pôÑAW%I’$I’$I’¤^Ê$I詧žbÊ”)ÄÅ9hI’$Iê*ò!Ïó<ùäó$O²mŒa ³™Íd&“KnÐ%JïÔSáõ×áâ‹á”Sà׿†É“ƒ®J’$I’$I’$I½P|ÐH’Ô[|ðÁüío㬳ΠºI’$©×Û²Ìàpçb.¦Œ2þãmÞ¦”Rîàê]ÂaøË_à’Kö… æÍ º"I’$I’$I’$õBÎd IRyíµ×¨­­å”SN ºI’$©W*§œgx† x–géCÆ3žïò].ã2†3<è¥Î—”?ÿù¾™ ¾øEøë_aÁHI º2I’$I’$I’$õ† $Iê /¿ü2¤§§]Š$I’Ôk”QFä“Ï VB gs6ó™Ï4¦1ˆAA—(cölÈÊ‚+®€¼üðCúõëp%’$IRר£ŽbŠ¹Û8Š£8–cy„GÈ%—'y’u¬c! ™ÎtúÓz7!!k¯½–ßüæ7ÔÔÔÄîä‘GèׯW^y%·Þz+™™™,\¸H$Â!CøÿøÆÇ·¿ýmöìÙÃ+¯¼ÂÌ™39r$)))|êSŸâ·¿ý-Æ ëÜ"uG p÷Ýðàƒð_ÿS§Âöíí^}Æ Ìœ9“ÓO?ÔÔTn½õVŽ8â~ñ‹_ÄÚ|ó›ß$''‡ûî»áÇ3jÔ(}ôQjjj¸çž{:¡S’$I’$I’$Iêl† $Iê ƒ¡’’’®D’$Iê<5ÔPDs˜C:éœÎéä“Ï\À‹¼ÈjVó3~Æd&“Hb»·{ýõ׳}ûvžxâ êêêøÅ/~Áå—_ÎÀÙ½{7úÓŸ˜4i Ö=óÌ3).. 11‘Ñ£GóŸÿùŸ<öØclݺµã:/õd³gÃÿ¯½ãÇCYY»VëÓ§'Nltß1ÇÃûï¿ìû[øÕW_eÒ¤IÚ :”¼¼<^xá…Ž¨^’$I’$I’$I]Ì$I ¾¾€¸¸¸€+‘$I’:Ö‡|HÌbÃÆD&RD7p¥”²ŠUÜÇ}Œgzð;Bêéòòàõ×!) N: ŠŠ¸ÊСC›{ȶmÛغu+uuu~øáÍÖ6l›7oî˜Ú%I’$I’$I’Ô¥ H’$I’$©‘Ílf! ™Ìd†0„‹¹˜2ʸ‹»XÃJ)åî` c:ì{Þpà ¼ð ¬ZµŠùóçsüñÇsòÉ'ššJŸ>}¸óÎ;©¯¯oöUWWÛÎèÑ£),,¤ªªŠ§Ÿ~šôôt®¸â žzê©«Uê±FŒ€_„‹.‚óχyóÚl~  ýàÁƒ‰gãÆÍÛ°aC‡=¨r%I’$I’$I’ C’$I’$Ib5«¹û˜ÈD†3œ¸€ócÖ²–bŠ™ÃF0¢S¾ÿ¤I“HKKãž{îáÉ'Ÿdöìٱǒ““9묳Xºt)µµµíÚ^¿~ý8÷ÜsyüñÇIJJâ•W^锺¥'9~ùKøîwáöÛ᪫ ºúcn*™“O>¹YˆgË–-,_¾œ3Ï<³#*–$I’$I’$IR3d I’$I’tˆ*¥”yÌc<ã9‚#¸“; b ØÀ (`6³Æ°N¯%!!k¯½–‡zˆ„„fÎœÙèñüà¼ýöÛÌœ9“üãTWWóÏþ“ýèGü¿ÿ÷ÿ(//gÊ”)±yóf>øàzè!vïÞͧ?ýéNïƒÔcÄÅÁܹPXO=眕•kSwÝuo¾ù&_ùÊWX¿~=eee\qÅ$&&rë­·vpá’$I’$I’$Iê † $I’$I’uÔQB wpGs4Çr,?àDˆ°”¥¬có8³˜Å@vy}Ÿÿüç˜1c©©©;þøãyíµ×8óÌ32dÓ¦M£¢¢"2ÈÌÌä†nàûßÿ>£GfĈüò—¿dÑ¢E† ¤–\p¼ú*lÝ cÇî»ýMœ8‘§Ÿ~šW_}•ììlN8áúöíËòåËÉÊÊê„¢%I’$I’$I’ÔÙ‚.@’$I’$I§–Z^â%òÉg‹ˆåŽ`2“™Ï|Nã4â»Éu(Þ{ï=n¸á†?úè£yôÑGÛÜÆE]ÄE]ÔáµI½Ö‘GÂË/ÃUWÁgÀƒÂ5×ðýïŸïÿûÍš·ôœ8q"'NìŠj%I’$I’$I’Ô H’$I’$õ2ÕTSDùäS@[ÙÊÆp=×3™ÚSþ& IDATÉä’t‰ÍlÞ¼™Ûo¿3Î8ƒqãÆ]Žth4/†o|>÷9xé%øÉO Á·%I’$I’$I’E~J$I’$I’Ô la …ÿú÷4Oó!r*§r·q —p$G]b«&L˜À_þòÆŽËÏþó Ë‘M}úÀÝwà 'Àç?ï¿> ¡PЕI’$I’$I’$©‹2$I’$Iê¡Ö°†§yš x–g‰'žÓ9ïðf0ƒ4Ò‚.±]ŠŠŠ‚.ARƒË/‡£†iÓàä“aÉÈÉ º*I’$I’$I’$u!C’$I’$I=HeP@>ù¬`)¤p6g3ŸùLcƒt‰’zºN€×^ƒéÓáÔSaáÂ}¡I’$I’$I’$ H’$I’$us¥”’ÿ¯+YÉP†r!2—¹œË¹$‘t‰’z›O|ž{¾øE¸ä¸õVøÏÿ„¸¸ +“$I’$I’$IR'3d I’$I’ÔÍÔRËK¼D>ùüžßSAYd1•©ÜÇ}œÅY$ø¶Ž¤ÎÖ·/<ü0œtÜ|3”•ÁÏýû]™$I’$I’$I’:‘ŸFK’$I’$u5Ô°ŒeRÈR–²žõŒa 3™É$&‘GqxqI˜=Ž9.½N; –.…ìì «’$I’$I’$IR'1d I’$I’­leË( €%,a';9‘¹‘¹œË9š£ƒ.Q’ö9ýtxýu¸øb;Î>;èª$I’$I’$I’Ô H’$I’$u¡Mlâü|òyŽç¨¥–qŒãÛ|›K¹”tÒƒ.Q’Z–‘/¼×\çßùÌtU’$I’$I’$Iê`† $I’$I’:Ù{¼Ç“ù¼ÄK$‘Ä9œÃÃ<̦0˜ÁA—(Ií3`äçÃ=÷Àí·CYÜ?ôíte’$I’$I’$Iê † $I’$I’:A)¥ä“O!…”P†p1‡9\À `@Ð%JÒÇ·oƒc…™3¡´-‚áîL’$I’$I’$IÀ$I’$IR¨£Ž¬ B³˜·y›Ã9œó8oñ-Îç|I ºLIê8]¯¼S§Âر°x1œtRÐUI’$I’$I’$é 2$I’$Iú˜v±‹y‘ È'ŸJ*‰a“XÀòÈ#Ž¸ Ë”¤Î3z4¬X—]gœ?ûÌštU’$I’$I’$I:† $I’$I’>‚ùçyž|òy’'ÙÆ6Æ0†ÙÌf2“É%7è%©k Ï<ßø|ö³PR?ü!ôéte’$I’$I’$Iú H’$I’$Àf6óO‘O>ËXÆ^ö2ŽqüÿÆgø £t‰’¬>}àî»á“Ÿ„뮃·ß†ßýº2I’$I’$I’$}D† $I’$I’Z°šÕ,a …ò/@ãÏ<æq9—3ŒaA—(IÝÏ•WÂÑGôipòÉ°t)sLÐUI’$I’$I’$é#ˆºI’$I’¤î¢Œ2îã>Æ3ž#8‚oð ’If XÏz–±Œ9Ì1` ImùÔ§àå—aÈ8åxòÉÖÛÖ×Æ ]W›$I’$I’$I’È$I’$I:¤•RÊÜÁ1ÃHFò¾C„KYʶP@³˜Å ]ª$õá0üùÏpé¥pÉ%0oÞ¾@AS÷ÜÓ§·ü˜$I’$I’$I’‘t’$I’$I]©–Z^â%òÉç ž`-kÉ&›)Lá~îç,Î"Á·L$éà%%Á#À¸qpóÍð¿ÿ @¿~ûÿÃàßþm_Àà׿†«¯¶^I’$I’$I’$† $I’$IÒ! šjŠ(¢B–°„ l` c¸Š«˜Ä$Æ3>è%©÷š=FÞ7cA^,Y55pÙe·¯Í-·À¤I [«$I’$I’$I’ H’$I’¤Þ©Š*Š(¢€³˜ù9‘›¸‰+¹’£8*è%éÐqæ™°bL §œ}û®]PW·ïñ>Ø7«Áƒ[§$I’$I’$I’ H’$I’¤Þc#yš§É'Ÿgy–:êÇ8¾Ãw˜Á ÒH ºDI:tµ/h0f TVÂÞ½ÿ÷Øž=ðÐCðÙÏ©§W£$I’$I’$I’ H’$I’¤ž­Œ2 ( Ÿ|V°‚R8›³™Ï|¦2•TRƒ.Q’Ôà»ß…õ롶¶ùc}úÀç?û$øÖµ$I’$I’$IRPü¤F’$I’$b KèC&3ù#¯[J)ùäSH!%”0„!\ÄEÌar!ýéß K’Ê¢Epï½­?¾w/¼ý6üä'0gN×Õ%I’$I’$I’¤Fâƒ.@’$I’$Zv³›9Ìá.á—ü²]ëÔRK1ÅÜÆmÉ‘˱üœŸ“K.Oò$ëXÇB2é $©;*)™3!.®ívµµpûívM]’$I’$I’$IjÆ™ $I’$IR—y÷¸”Kù+¥žzþÀ¨¡†d’›µ­¡†bŠ) €Çyœu¬#B„ILb:ÓÉ#80XU’Ô=¼ù&dg 1öìi½íž=ð¥/ÁOtYy’$I’$I’$Iú?† $I’$IêuuulØ°uëÖ±víZÒÓÓ>|8‡~8ññ½²Á'y’«¹šùZj¨¦še,c2“ØÉNþÈÉ'Ÿ¥,e;ÛÃnà.ã2Žá˜ » Iú¸®¹fßWi)üêWðÈ#°q#$$ÀÞ½ÛîÙ¿ÿ=<õ\tQ§”SSSCEEÑh”-[¶––FFFÇ?$ÎÉ’$I’$I’$Im1d I’$IÒAØ»w/ëׯ'²nÝ:¢Ñ(•••¾¢Ñ(6l`oÓA”ÿ’Àá‡N8&--­ÑW8føðá„Ãa† FBBÏûS~/{ù&ßäî!Ž8ꨋ=–H"¿å·TQE>ù<ÇsÔRË8Æqwq)—’Nz€ÕK’:TNÜ}7|ï{°b,\¿ù TWC|<Ôî ¡_øüóŸÜ|¶›¶TWWS^^ÎÚµk©¨¨`Íš5Ínoܸ±Åu>|8™™™¤§§“žžÞìöðáÃ{äùX’$I’$I’$©½ü$D’$I’¤ìÞ½›M›6ÅB ˪ªªF÷­^½šÚ†‘@rrr,,‡9í´Óý?--P(D(j¶­ý—%%%±àB}}}lû¡P¨Ù¶šn?33“±Ûš)§œK¹”ÿá¨ÿ׿ýía¿ç÷,e)çp÷s?Ó˜ÆáPÅ’¤.ãÇïûú¯ÿ‚Å‹÷žþÿÂååûÂwÝ[­¦¦†h4;g–••5»½ÿ¹³oß¾ :”p8L$áŒ3ΈÝÞÿÜÙpNnº½W_}•¥K—6;߇B¡FÛhº½ììlú÷ïßå»U’$I’$I’$©#2$I’$RjjjزeK«ƒû–ëׯ§®îÿ®¸ß4<››Ûlpÿˆ#HMMmw-)))±mµf×®]lÞ¼¹Õ:W®\Ù®z[Z~Ôz?ª'y’«¸Šjª©¥¶Õv»ÙÍ3<Ãyœ×iµH’º±þý᪫¨þÌgØ𷿱뗿äð‚WTPûÝïòÅâb^ß¾=vþk””DzzzìÜ–——×lÀvv6ñññ,¡=ç䪪ªC eeeQ^^ÞhÖ¢ýƒ-²²²0`ÀÁí;I’$I’$I’¤N`È@’$I’Ô+TWW·:ÛÀþËÍ ““Ól†€ &%%‡8ð±éÌ M÷AII ………Í@¶Fh:KB{íe/ßä›ÜÃ=ÄGum¶O$‘?ðC’Ô‹5œ§[¤_YYɪU«Øºuk¬}rr2gŒÁUÀeï¿Ïà3š ÒOKK#..®Ëú …ÈÍÍ=¨ š5kسgO£m¶6B8æÈ#dРA]Ñ=I’$I’$I’¤C’$I’¤nmÿð@kËh4Úh`"´Øà^FF‰‰‰õ¬cõíÛ·]aØ7²µýÙFh:299¹Yè ¥åîá»™?ƒÿá¨ÿ׿ÙÃåQ~Ĉ£ë‹J’:Æþç•ýØ7,ßyç¶oßkßpkTŸ““ÃÕW_Ýz€ ®ŽOÇÅA >®öZÚ_eeeS^^ÎŽ;bí›î¯¦·GŽÉàÁƒ»¢{’$I’$I’$éaÈ@’$I’ˆö„*** JìÛ·/C‡ hOKK#77·Ù`÷ÌÌLü“·5¡PˆP(DNNN›íZ #4\¥¹Ù@È‹_ƒ ®6Žøºø}Dã Ž8êãë‰#ŽÚ¸Úfრl „Æ2¶Sú,IúxÚF8 >''‡Ù³gÜ€øøøîU°Úsnk¿—””4û©¥ BKI’$I’$I’¤öpÄ…$I’$©Cµu•ü†eEE»w“””Ä!Cbâ"‘yyyÍ®’?|øpâ{Ù@Ãa„êêjÞ]ÿ.Kk–RùÏJ6mßÄ–[ØR³…m;·±­n|ð»úíÚ·Â@ 'îûêŸHüÀxþýÕçüŠó ŒHRiµ6 ÁêÕ«Ù¹sg¬}ÓY‚rss `5j©©©ö¨÷hÏ9¸!°ÙÒÏpåÊ•¼ûî»lÛ¶-Ö¾!ˆÐRø Å$$I’$I’$IÒ!ËOé%I’$IíÒžð@yy9{÷î­“œœL(Š hËÉÉa„ ÍÂhëÙRRR8.û8Žã¸6Û5›½be“çO´œyU󨬬l´^ÓA­--322HLLìÌnJRÒV€ ¬¬¬Ù9{ÿcíþa¿†Áç™™™ 80À©©””"‘‘H¤Õ6mŠŠŠb÷5HJJ"==½Å BÃ2;;ÛЧ$I’$I’$I½œ!I’$I:„íÚµ‹Í›7·¨ªª¢²²’Õ«WS[[[¯é•psss[ø …캛ö †„}"ž{-=/W®\I4eݺuÔ××ÇÖk »°¥éÒA²’zºÝ»w³iÓ¦VŽ·ú …B±Áâ-²³³éß¿€½RgiϹ·¦¦†h4Úâs©!ˆ°ÿ9·oß¾ :´ÕÙÂá0YYYôéÓ§«º)I’$I’$I’:˜!I’$Iê…jjjزeK›³D£QÖ¯_O]]]l½¦áH$ÒlvFFƒ °wêíRRRHII‰XZs ÌòåËÛõ÷Lj¿ö„ÿ‚-½NJKK)**júkDhHÈÎΦÿþ]ÕMI’$I’$I’ † $I’$©µ'<°fÍ>øàƒØ:ûmúš+++k5ˆÐÚliiidff2pàÀ®è¢$I’$I’$I½‚!I’$IúÚºÚyò¢¢‚Ý»wÇÖizµóH$xèÕÎ¥ž¥½a„¶‚F a„µkײmÛ¶fÛo+ˆ‡ÉÌÌ$!Á·v¤ƒq Á»ï¾Ûèõ™œœÜèu8a„fƒ™ÓÒÒˆ‹‹ °WRÏ …ÈÍÍýØA„ââbV¯^ÍÎ;m³µÙÂá0£F"55µ+º'I’$I’$IR·ç'Ñ’$I’´Ÿö„š^9µix ''‡ &4ì CéГ’’B$!‰´Ùî@³ž”””PUUÕèjéо0BFF‰‰‰ÙM©[ÚÿœÞÒ@䊊 ¶oßkß htœ““ÃÕW_Ýl ²¤î¡½A„ÖŽ%%%Ífjzhz{äÈ‘ <¸+º'I’$I’$IR  H’$IêõvíÚÅæÍ›[ÀÛ0pwõêÕÔÖÖÆÖkzµâÜÜÜfxÓÓÓhÔC-_¾œ)S¦ðâ‹/2f̘ ËÑ!®½a„šš¶lÙÒêñlåÊ•D£QÖ­[G}}}l½†0Âþ¨¦ËÌÌLØÙ]•:Ä8œ››ëÀaéО™‡Z;ž4œWßyç6I­ÍŒ I’$I’$IROfÈ@’$IRu Á¶ Ëõë×SWW[¯ix ‰4l›‘‘Á Aƒ:­öŠŠ 222ÝOjj*cÆŒaÊ”)ÜxãZCG˜?>×_=ÿýßÿÍ7Þø‘Ö/**bâĉ¼ùæ›{ì±Qb«5ÖÕÕQ__ßh öGuÛm·1oÞ<ž~úiÎ?ÿün±­ÎÒ?¯¦:{¿tfŸÞxã î½÷^þüç?³sçNÆŒÃí·ßΤI“>ö6Ž_ Á§Ö(\µ|ùr¢Ñ(6lh3\ÕÒÒp•:[UUU‹Á†ÛkÖ¬aÏž=±öûÏè‰DÈËËkô¼=òÈ#;í|ÚSÏç-ûxà¾ô¥/ððÃsÝu×uy]=ù˜ßÔ¡°?[r°¿¶¥3Ϋо BÃŒC-›V®\ɪU«Øºuk¬}Ko4½í,g’$I’$I’¤îÌ$I’¤n§aOK³ ì_kWênГ““Óèÿ¡Pˆììlú÷ï`ïö1bõõõ\sÍ5,Z´ˆ;vP[[˺uëxöÙg¹óÎ;ùéOJAAÇw\Ðå¶êºë®ãª«®"%%%èRZÕZ§Ÿ~:[¶l9¨mß}÷Ý\sÍ5sÌ1µŽÞVoÒ“÷Ë\À§?ýiJJJHHHàßÿýß™2e O=õ\pA§~襤¤v…vïÞͦM›Z<ÆVVVRRRBaa!åååìÝ»7¶^Ka„–fIp¥š:P€ és­µAà ݠgàè-çs€›o¾™k®¹&ÐýÙ“ùMªû³37 ò¼Úž‡Ú "Åîk””Dzzz›³!dggß©}“$I’$I’$©%† $I’$u™–ÂM—Ñh´ÑU@áÀáp8LFF‰‰‰õ¬côéÓ‡ôôt®½öZ&MšÄÉ'ŸÌ…^ÈÊ•+ &éãéׯóçÏ›î¿ÿ~~ó›ßððÃwú`ÈöêÛ·o,ŒÐÖœaßàðÖŽß%%%TVV6»Â|rrr‹áƒ¦ËáÇ;ˆ²‡k¬´6À¶²²’Õ«W7š9# ÅÓF"&L˜ÐèyÑ]‚•çs©st÷ój{‚555±¿yš#fÚ?HÝ·o_†Úêláp˜¬¬,úôéÓUÝ”$I’$I’$"üôV’$IÒA«ªª¢´´”ââbòóó¹ï¾û¸í¶Û˜5k'N$''‡Aƒѯ_?FŽÉ駟ÎUW]żyó(**¢ªªŠH$ÂôéÓ¹ãŽ;xüñÇyñÅYµj{÷îeË–-”––²lÙ2.\ÈÝwßÍœ9s˜>}:ãÇ'‰ôø€AS‡~8ßûÞ÷¨¨¨àÁlôسÏ>˸qãHIIaèС\}õÕ®ŠÚ °°±cÇ’œœÌ°aøñÆÙ¾}{ìñU«V1uêT;ì0È´iÓxùå—XÛO~ò²²²èׯŸþô§yçwšµÙ´i_ÿú×5jÉÉÉœp ,]º´Q›Ûn»‰'pÜqÇGvvv»×?˜xàââ∋‹cþüùÍî{à¸é¦›2dqqq\~ùåüž;vìàsŸû©©© 2„/ùËìÚµ«Q›ýLZs Ÿùþµ?øàƒ|ùË_&55•ôôtîºë®V÷Orr2§žz*/½ôRlýqãƵXC[?¯öÔØÞþeddp饗òÊ+¯´Úþ–[n‰Õû÷¿ÿ€E‹Åîûõ¯kÛÖóü@}jëçÕžçKYYY£Ò ¤¤¤°yóæví—î& ‘““Ä ˜5ksçÎå¾ûîãñǧ¸¸˜U«V±{÷n¶lÙÂßÿþw–-[ÆÏ~ö3æÎË„ …B”••‘ŸŸÏ×¾ö5¦L™ÂرcIOO§_¿~„ÃaÆŽËäÉ“¹á†¸ãŽ;x衇((( ¸¸˜²²²FÔÕuvíÚE4¥¤¤¤Ñy~ÆŒŒ?ž‘#GÒ¯_?ÒÓÓ;v,3fÌàÎ;廊¨ˆêêjrrr˜={6<òË–-kô\yýõ×)((ˆ=Wf͚ń ÈÉÉ鑃¦ºãùü@Ç>€½{÷ð\Òç´ÞtÌoËög{ïù¸¿»µä@?¿ò»X{~7<Ð÷™Ç{¬Óû!I’$IÿŸ½;ªºÿ?þξ ¬d!"¥A-Å¥VD6k«Â¯j…¯í×Xq¡–¶`µuÿªÅ½­•ŠUÁZ%¸ ›Res!Bö„ {Îï˜i&™$–Ü,¯'<îpsïäsîÜ3IÎûÝœ˜'þ'ëÕW_5t«NÖÓO?mu ­*++3;wî4k×®5/¾ø¢Y¶l™¹í¶ÛÌŒ3LZZšILL4>>>F’ãÃßßßDEE™””3}út3þ|sß}÷™§Ÿ~Ú¼õÖ[fÛ¶m&//Ï444Xݼ.áÇ?þ±éÓ§ËÏUTTsÑE9Ö­^½Úxzzš»îºË”””˜={ö˜±cÇšÁƒ›Ã‡;¶{ã7Œ‡‡‡¹çž{LYY™Ù±c‡9ûì³Í”)SLcc£1Ƙ‘#Gš™3gš¢¢"S^^n.\Øj-vûÛߌ$ó›ßüÆ”••™Ï>ûÌ\zé¥F’ù¿ÿû?Çv .4 .4¥¥¥¦²²Òüå/1~~~fçÎNÏ·víZ#É|ùå—NëÝÝÿTj¬¬¬4’ÌŸÿüçëÌÊ•+Í‘#GÌc=ffÎœÙê×Û³g‘dFŒaV­Ze***Ì믿nÍ-·ÜâØÎsb®wÞyDZŸ»çÜ^û¹çžkÞ|óMsøðaóÈ#IæÃ?lõøìرÃLœ8±Åñq¥µóån®Ø÷½ûî»Mii©ÉËË3³gÏvz-º:.®j)))1’ÌË/¿ìX×Þë¼µ6¹s¾:úzùàƒŒ$³hÑ¢6IoqìØ1³oß>³~ýz³råJóðÛŒŒ 3oÞ<“žžn†nBBBœúIÆf³™áÇ›ôôt3oÞ<“‘‘a~øa³råJ³~ýz³oß>SWWgu󺪪*§óд¯OII1QQQÆÓÓÓqü}}}ýüŒ3Ìm·Ýf–-[f^|ñE³víÚ^{ü»[ÞÚµÏU_òøã·èKÜ©Ë•ÞtÍïÈñt÷}ÏÉœkWÇÓöº[“»ï»èWO¯êêj“——g¶mÛæÔ‡6ý9ÍËË«EÿÙôg´æ×îÚÚZ«›åЕF€3m†™a®®»ÚDEE9ýNÁcöïßo<==ÍSO=eŒ1fÇŽ¦OŸ>fÖ¬Yfß¾}æàÁƒæ׿þµñõõ5Û¶msì7~üxó½ï}Ï|ûí·æرcfûöífÖ¬Y¦°°°SÛº¦3f˜3ZŽŸe4$B8u„ œ +P¸ðöönHLL4iiim†ÚX†–Ú”hŒ1aaa&11Ññÿ¤¤$sÎ9ç8móùçŸIfÙ²eŽuC‡m±ÝêÕ«$³nÝ:SUUe$™W_}Õñùºº:Þf½C‡5£FrZ÷ú믻5@ý²Ë.3 ,pZ×Ú`?w÷?•Û Üxãí~;ûཅ :­ÿùÏn|||Lnn®£®¶ÎIÓçj:ÐÝsn¯ý§?ý©c]}}½ 4¿ùÍoë\Ÿ·ß~û”BîÖèJRR’1b„ÓºÊÊJÓ¯_?ÇÿOvÀ©;¯óÖÚäÎùr÷õR^^nžxâ b¢££ÈÑAm…¦OŸî2a„šš«›wF5?v®Žcæççç2@Ðô˜Õ××[ݬ.©»õçí… šö%&((¨E_ÒÞ5Ò•ÞpÍoÞ6wŽ§+Íß÷œì¹vu>>JJJÒïÿ{ê’K.áw|À-„ €ndË–-Ú´i“òòòTXXè4€²é@BIŠˆˆPxx¸¨ˆˆ9RŠ‰‰Qdd¤¢££)‹ZƒöTTT¨¼¼\#GŽ”$•––J:>À«¹°°0ÇçíËÇ\?þx‹m³³³%Iï¾û®–.]ªE‹iîܹJOO×½÷ÞÛê€ü‚‚—_ßU=»wïÖÝwß­7ª¸¸Ø1¸rÔ¨Qí7üöïHm èÐöÒñWM………I’òóó+©ýsÒœ»ç¼©   §ÿûøø8T·v|NeðôÉÔèξ§KG_çMër÷|¹ózyüñÇuíµ×v°zt„=,ÐÞ`Êêêj*??_………ÊËËSQQ‘cùÁ¨¸¸¸ÅÀݦƒ*£¢¢©˜˜M˜0AãÆ;­m)--U^^žrrr”››«¼¼¿ÍfÓC=¤‡zHŸ~ú©/^¬I“&i÷îÝŽçj***Êåׯ¨¨pú]]ÒÓÓ£?üPC† ‘———®¿þzíر£ÝvŸÊþîÖx&TVV:ýß~Wæèèh·ÏIsîžswµv|\=¿»N¥Æ¶ömýnëMï¬{øðáÛuôuÞ´®Žž/tþþþJHHPBBB‹ÏUVV:Âß}÷¾üòKmÞ¼Y_|ñ…ÊËËU^^®Ý»w;ísóÍ7Ÿ–ÁîÝ»aÂo¾ùFùùùÊÎÎVUU•Ëí£££uþùçk„ 6l˜cð&w%ízºZ~ªÎDŸÖžž|ÍïÈûžÓq®Ýi¯»5¹û¾ËêcŒSãåååñ5ÕØبÂÂBåææ*77W[·nÕæÍ›µuëV•””¨¤¤¤Åk¸ÿþŽÁˆ#4vìXMœ8Ññ8[°`®ºê*íÛ·O+V¬ÐÈ‘#?{„„„ÈËËK÷Þ{¯î½÷Þ6Ÿ'))Ik֬ѱcÇ´aÃ=óÌ3š5k–‚ƒƒõƒü 3šº!B@7òÄOhéÒ¥ŽÙ \-ßxã =öØcNƒ°üýýe³Ùw_nméÄkiñâÅŠÕ‚ $ITRR’>üðC§mwìءÇ+==ݱÝÙgŸ­­[·¶xÞ‘#Gê®»îÒäÉ“5uêT}ùå—’¤ñãÇëÏþ³Î:ë,mÛ¶Íå@µ~ýúièСڰaƒÓúÏ>ûÌéÿYYY*((Ðí·ß®³Ï>Û±¾¦¦¦Åsºz½udÿ“­ñLضm›f̘áøÿúõëåã㣔”EGG·{NfÎœÙâsîžswµv|\Õ劫óu*5Ú÷ý補Öççç+11QyyyŽ!š0`€$©°°Ð±î‹/¾pÚ¦°°°Ý×ykm:™óÕš^xÁímqfTUUµÚoÚ—ùùù:tèÓ~öÙ RSS[í?í3•œªáÇkøðáºþúë]Öž••å¨×þø»ï¾Sff¦SÝöìõ%&&¶xLßyºb.¹¾ž»ëd¯‘½åšßQî¾ïq§}îp§½£Fr«&wßwѯvOååå.ûû㜜§Ÿ9í}æøñã]ö=±±±§4{ôFÓ§OWTT”xà½õÖ[úÓŸþäøœ¿¿¿.¼ðB½ùæ›Z¼x±¼¼¼Ú}¾ÀÀ@]|ñÅJOOW`` 6oÞLÈ´ŠÐÍØl6Ùl6%''·¹]yy¹ÓàÉòòrÇ㬬,mØ°A999Nw_÷óóS¿~ýZ !؃ ñññnýñÓÐРÂÂB½ûî»Z²d‰<==µzõj§™(|ðA]qÅZ¼x±n¿ýv•––jÁ‚ÿÐCé‚ .Ðœ9stß}÷)!!AÙÙÙzçw”““£‡zHÙÙÙºõÖ[uÛm·iôèÑòõõÕ_ÿúWÕÖÖꢋ.²¨e [03ãÄ?àd½ú꫆nÀÉzúé§-ýúÇŽ3ûöí3ëׯ7+W®4?ü°ÉÈÈ0óæÍ3éééføðá&$$ÄHrú°Ùlføðá&==ÝÌ›7Ïddd˜‡~ج\¹Ò¬_¿ÞìÛ·ÏÔÕÕYÚ¶®,''§Å1õðð06›Íœþùfùò妢¢Âå¾ï¼óŽ7nœñóó36›ÍÌ™3Çäçç·ØîÝwß5ãÇ7~~~&""ÂÌž=Ûäää8>¿fÍ3uêTfúöík.¸à³nݺvkâ‰'L\\œñóó3ãÇ77nt´!55ÕcÌ–-[ÌĉMPP‰5óçÏ7W_}µc»‚‚ÇóÝ|óÍ&$$ÄôíÛ×Ü|óÍÞÿdj|ì±ÇœŽýäÉ“Í+¯¼Ò✔——·ùu222Ûþå/13fÌ0AAAÆf³™[o½ÕTWW»}Nš>—$3gÎÇ~íóæµÏ™3§Åk쬳Îjq|üýýÍĉÍöíÛ$³bÅŠ6ÛÛÚùr§Æ¶4Ý7::ÚÜ~ûí¦ªªªÝã²víZ3|øp`&Ožl¶mÛæØî’K.1Ƹ÷:o­Mm/w_/eee&66Ö̘ÁÏQVVfvîÜiÖ®]k^|ñE³lÙ2sÛm·™3f˜´´4“˜˜h|||œŽ¿¿¿¿‰ŠŠ2)))fúôéfþüùæ¾ûî3O?ý´y뭷̶mÛL^^žihh°ºy–¨ªªrêï›Ó””e<<<ÇÓ×××q©ˆˆíÛ·O .Ô¨Q£4a«ËC7TSS£ƒ¶ÈÏÏWqq±û5¿î§¤¤´¸îÇÄÄ(44ÔÂÖõ.~~~íöõµµµ*--m1°5++K»víRff¦8àt®Ûä+ŸÎh"tIÕÕÕÊÏÏw°?.,,tÜÜÂ××WaaaŽë¨½mz}————Å-œŠƒê®»îÒ¤I“€NGÈÀie4j0Úšö¥~òÉ'n JuµdP*кٳgëàÁƒúÁ~ }ûö©ÿþJOO×Ò¥Kä 'í…ÆìËöBciii„Æz__ßvûùºº:•””¸¼ÓvVV–233[ÌXÑtæ#W„øøxuV3à´i>#\{???õë×Ïq ´÷£ wIOO×Ǭ1cÆèù矷ºÐ 2` ???·Â’T^^Þê×íÛ·kÍš5-+6äj³ÙZ $DEEÉÃÃãL7è2•‘‘¡ŒŒ «KEšvlº,//wZ×TÓAàQQQJNNnq]8p |}}-jº ·úøòòr—ƒmíA„œœÕÕÕ9¶o/ˆ@x@g³÷©®®eÚ·oŸ:äؾy/==½Åµ,22Ržžž¶ ÐdffZ]èåèòl6›l6›’““ÛÜ®­0®]»\Xô÷÷o3„`_2Ø@W×VxÀ¾ÌËËSEE…Ó~î„âââäíͯ‘pzÙl6¥¤¤t(ˆÐtï† tàÀ=zÔ±½}¯«Btt´¬Îh€nÎÕÏM¯Gß~û­SŸê*@0oÞ<§ëg@wÁ_‡ô #4¿c·ýîÉ6lPNNŽ*++ûøùù©_¿~-ÂÍ ñññòòò:ÓMЋ4äèêÚÕÞ5+11Qiii-®]\³ÐÕ¹Dp58++KÛ·ooñýÑ^Á¾ÐsµuÝpÕ¯6¿n$''kþüùN×›Ífa‹8½èuìa„ö´uWp{!??_‡jñümÍŠÀ]ÁHmϾb_¶7ûJó𳯠7r'dØÖÉwïÞÝîÉ]¸#9Ð5µ5J~~~»3 ¤¤¤0  ×cD ´" @‰‰‰JLLls;Wa„¦wß½{·òóóUXX(cŒc¿öÂ6›M êӧϙn*€Ó¤¶¶V¥¥¥.ghºÌÎÎV}}½c¿æšSRRZ̔ fàä¹D°÷ç®îjž™™©¬¬,•——;¶w'ˆ@à8½šš?nÎkú~»i0Ïþ=§àà` [@×DÈN‘»a„êêj•••µ:ðØF(**Rcc£c¿æƒ]-cccÕ·oß3ÝT ×ª©©ÑÁƒÛ äç竸¸X Žý\…šÿÆÄÄ(44ÔÂÖÜëÏ› ›pÎÌÌl*ôóóS¿~ýZ !DGG+>>^^^^ÕL Kª««SIII‹ï«¦ßkÍzíâããda«è¾@'±6¶4nM{ƒ™·oß®5kÖèÀmfvµd03ଽð}ÙÚL$öÙì›~¿qwd çq'ˆP]]­üü|—A„O>ù¤Å5Å××WaaaÐcÙgùim¦‚‚‚ïkm6›ã{ 11QéééNß±±±òññ±°Uôl„  ‹ñóós+Œ Iååå­Œ¶‡šßõÕßßß10º­@BTT”<<<Îts3¢ùÝÆ]-ËËËUPPà´_Ó»"GEE)99¹Å÷´Åßß¿Ý BMMòòò\Þ­ÝÞïß¿ßif£¦ƒ®]âââäíͯ{ѹÚz-Û7}-ûøø¨ÿþNýìe—]Æk€.†ßÔ@7f³Ùd³Ù”œœÜævm…víÚ¥ÌÌLåä䨮®Î±»a„ÈÈHyzzžé¦’Ü äåå©¢¢Âi¿æá”””¯g5è,~~~íÚºû»½ïnëîï̓QQQ4h;£‰èÚš•Ãþ¸­Y9RRR˜•€nŠ¿œ@/àn¡­ÜYYYÚ°aƒrrrTYYéا進¦›T†¶4ÂØghº.;;[GŽqìãçç§~ýú9^g‰‰‰JKKkñZŒˆˆàµ Ûñõõmwf£ºº:•””¸¼ƒ|VV–233[ÌhÔ4tå*¯   Îj&,bÏ×48à*L`×´ÏmÚß6} %$$< ‡ dph÷ÎÊ’{a„üü|:tÈi¿æw“wµŒ•Ï™l&:Q[³hØ—íÍ¢ÑZx€Y4ôv>>>í¤ã×bWw¡·š_‡íýuÔÀ( ;@IÞINÊcccÕ·oßÎh"N‚«AÓÇûöíszæïïïÔǦ§§·ŸÐçл2tØ©„šÞ¡~÷îÝÊÏÏWaa¡Œ1ŽýÚ #Øl6î´l¡ÚÚZ•––ºœm é²ùݳ›bLIIi1ã…} 8}l6›RRRÚ "دß[‹·êß1ÿÖ§ç~*Se”05¡Ål2ökº«Ù¢££uÖYg)44´3š×«4=O®f!øæ›otøðaÇö®óæÍs:_QQQòðð°°U «!d8cÜ #TWW«¬¬¬Õëö0BQQ‘û58çjÉÝ–ÝWSS£ƒ¶pç<¤¤¤´8111 6€.¬¯­¯¶Û¶ë™ägôOýSýÕ_·êVÝd»Igí:KRëܳ²²´}ûvåä䨲²Òñœí–9këøæçç·{|“““5þ|§cm³Ù,lè®,g$g ÞšöÁoß¾]kÖ¬ÑÔÐÐÐâùÛ #DGG÷ØxUUUíÎ:ÐÖŒöÙÒÒÒZ·¸¸8[Ø:À©ÈQŽžÒSzVϪD%ºD—h¥Vjº¦ËG>NÛÚl6Ùl6%''·ú|öYŒ\ ’ß½{·¾ýö[UTT8¶oÞG» $ô„;í———»œyÀ¾îÀ:zô¨cûæ³:¥¤¤8›Áƒ+$$Ä€žŒ Ûðóós+Œ 9ß ¸ùÒFÈÎÎV}}½cÇ€ú¶ ]e°£} gkíü&î=pT¥ëJök>p199¹E;cccåããÓÊWtgFFïë}=¡'ô–ÞR¸Âµ@ tƒnPœâNé¹Ý™Å¨­ Bff¦²²²T^^îØÞÏÏO111mΆ OOÏSªýd54œ““£ºº:ÇöMûáÄÄDGˆÏÞ.|Àj„ =’;w[–Ú#ìÚµK™™™ÊÍÍUmm­c???õëׯÝ0BddäI xl/<ŸŸ¯¼¼<§;AÛÛÜôë¸ý€ ’ ´pûB]]wµ¢££'oo~½Q¥*õŠ^Ñãz\_êK¥(EÏé9ÍÒ¬³œI "¸¸Ÿ™™Ùbž¦}³«Ù¢££////·ë¬­­Uiii‹¯ß´¦æE›Íæøš® êÓ§ÏÉ<€NÀ¨@¯æn¡¸¸XEEEÊËËs, •ŸŸ¯]»viíÚµ*((Puuµc___EDDhàÀ ×À¡˜˜Ir3ÂðáÃ[ßèškŽ/W®ì”šfi–†k¸®ÒU£1zM¯)U©òµ«Qz[oëQ=ªLej´Fë9=§Yš%õ¾»ìhÈ!2dˆë V®”fÎÔû÷wj]]™§Õ€3o¤Fj«¶j¸†k²&k…VX]à4ªQžÖÓ¢!ºRWªúè#}¤Ïô™®Óu½2`€“CÈ€^¢Ÿúé_ú—îÔZpâ_­j­. p Žê¨þ¨?*Q‰Z¨…šª©Ú«½zCoh’&Y]º!o« ÇK^Z¢%JQŠæižvi—þ¡(JQV—è€JUê9=§eZ¦JUêÝ ÿÕÿj Z]º9f2 ºL—i³6«Le£1Ú¨V—pÃAÔ¯ô+ Ô@-Ñݤ›´_ûõˆ!`€Ó‚½T’’´Y›•ªTMÖd-×r«K´âéÝ£A¤gõ¬~¥_é€è~ݯþêouyèA¼­.X'XÁZ¥Uz@è.Ý¥Ú©gôŒ`uiIGtDOè -×ryÈC·Ÿø×W}­. =!z9y(C¡š£9JSš^×ëJP‚Õ¥@¯uTGµB+ô{ý^GuT·èÝ©;ªP«K@çiu k˜¦iÚ¢-ªSÆj¬ÖiÕ%@¯S¯z=¥§t–ÎÒ=ºG7êFe+[Ë´Œ€:!à0DC´I›t¡.Ô¥ºT˵Üê’ ×xSoj„Fh¡êZ]«,ei©–Ê&›Õ¥ !dœ)H+µRKµT‹µX³5[ÇtÌê² ÇÚª­ºPêJ]©Á¬]Ú¥‡õ°ú«¿Õ¥ "dZð‡2”¡LejÖé|¯,eY]ô(YÊÒLÍTªRed´Y›µZ«5Xƒ­. º#A IDAT½!Ъ u¡¶i›|䣱«÷ôžÕ%@·wLÇ´DK”¬dýGÿÑ«zUé#Ó8«K€¶Å*Vëµ^—ërMÓ4-Ñ«Ë€niµV+YÉzDh‰–è }¡šauY€!Ð.ùëy=¯'õ¤~§ßéJ]© UX]t_é+]ªKu…®ÐDMÔ^íU†2ä+_«Kœ2n›¯ùZ§uÚ¬ÍJUªöhÕ%@—V¡ -Ò"£sT¦2}ªOõ’^R„"¬. p‰艚¨mÚ¦P…*U©ú§þiuIÐ%­Öj£sô²^ÖƒzP›´Iã4Îê²€626Põ‘>ÒLÍÔô#Ý©;Õ¨F«Ë€.¡P…š¡º\—k‚&è+}¥…Z(/yY]Ð.Bà¤øÉOÖŸõ”žÒŸô']¦ËtH‡¬. ,cdô’^R²’õ™>ÓZ­ÕJ­Tõ·º4Àm„ À)™¯ùz_ïës}®q§ÚiuIÐé¾Õ·JWºnÐ š«¹úB_(]éV—t!pÊÒ”¦ÿè?ŠQŒ&h‚^ÓkV—¢A z@è£Ã:¬­ÚªGôˆú¨Õ¥'…8-h€Öj­nÑ-ºF×èNÝ©5X]œ1YÊÒ…ºP÷ê^ݯûõ©>Õ(²º,à”2§·¼µLËô’^Ò£zTéJW‰J¬. N»—ô’Fi”ÊU®MÚ¤;t‡¼äeuYÀ)#dN»¹š«Oô‰ök¿ÆhŒ¶i›Õ%ÀiQ¬b]©+õ“ÿ¶k»Fk´Õe§ !pFŒÖhmÕV ÑMÔD½ ¬. NÉëz]ÉJÖNíÔÇúXèùÉÏê²€ÓŠ8cú«¿þ­k¡ê'ú‰hêTguYÐ!5ªÑÏõsýH?ÒUºJ;´CiJ³º,àŒð¶ºÐ³yË[Ë´L#5R7êFíÑýCÿP„"¬. Ú•­lÍÔLíÒ.½¢Wt­®µº$àŒb&,òË_þR‘‘‘*--ÕøCiàÀzôÑG[l»nÝ:]pÁ THHˆ.¿üríݻׂªOÞ,ÍÒFmTžò4Fc´Y›­. Úô–ÞÒ(Rµªµ]Û ô`½­Oh !,dŒÑ¢E‹tûí·+//O·Ýv›.\¨M›69¶Y·n.¹ä¥¤¤(++KÛ·oWUU•ÒÒÒ”maõ7R#µU[5\Ã5Y“µB+¬. Z¨W½îÔºRWjº¦ë}¢!buY8Ãz[Ÿ ÐBX¨¸¸XsæÌÑĉ¢;î¸Cƒ Ò /¼àØæî»ïVrr²yäEFFjðàÁúûßÿ®êêj=ðÀÖ’ú©Ÿþ¥éNÝ©'þÕªÖê²@’”«\¥)MOêIýUÕKzI ´º,t‚ÞØ'¸BÈ yyyiêÔ©Në† ¦ýû÷K’ª««µeËMŸ>Ýi›°°0¥¥¥éÃ?ì¤JO//yi‰–è ½¡Wõª¦hŠ T`uYz¹MÚ¤±«£:ª­ÚªYšeuIèD½µOhŽ “···Óºàà`UTTH’:¤ÆÆF…‡‡·Ø7""Bì”:Ï”Ët™6k³ÊT¦1£ÚhuIz©WôŠ¾§ïi´Fë}¢$%Y]:Yoï“ì`!6?*OOO•””´ø\qq±ÂÂÂÎTi&IIÚ¬ÍJUª&k²–k¹Õ%èEÔ ;u§fk¶nÒMZ£5 QˆÕeÁôÉÇ2  ó÷÷׸qãôöÛo;­/++Ó'Ÿ|¢É“'[TÙé¬`­Ò*-ÕRÝ¥»4OóT¥*«ËÐÃUªR?Ôõ°Ö‹zQèyò+S´¢·ôÉüÅ €.îþûï×—_~©ÿùŸÿQQQ‘²²²4kÖ,ùøøèŽ;î°º¼ÓÆCÊP†VkµÖhÒ”¦ýÚouYz¨}Ú§q§íÚ®õZ¯ëtÕ%¡è-}2èÝÐÅM:Uï¼óŽ¶lÙ¢„„5J¾¾¾úä“Oouy§Ý4MÓmQê4VcµNë¬. @ó™>SšÒÔG}´U[5Vc­. ÝDoë“@ïämuôV>ø |ðÁëÿþ÷¿·X7uêTM:µ3Êê†hˆ6i“~¢ŸèR]ª¥Zª eX]€àC}¨+u¥R”¢êŸê«¾V—„.€>࿘ÉtIA ÒJ­ÔR-Õb-ÖlÍÖ1³º,ÝØzCß×÷5ESô¶Þ&`¸@ÈtYòP†2”©L­Ó:¯ó•¥,«ËÐ =©'õ#ýH7车×ä/«Kº$B Ë»Pj›¶ÉG>«±zOïY]€nä>ݧ[u«î×ýz\Ë“_‹­â¯i [ˆU¬Ök½.×嚦iZ¢%22V— ‹û¥~©ßê·zFÏh±[]Ðåy[]€»üå¯çõ¼&h‚nÕ­ú\Ÿë%½¤…X]€.è.Ý¥‡õ°^Ð š«¹V—t Ìdºùš¯uZ§ÍÚ¬T¥jöX]€.f±ë= çõ< €ni¢&j›¶)T¡JUªþ©Z]€.ânÝ­åZ®çõ¼æižÕåÝ !Ðm Ô@}¤4S3õ#ýHwêN5ªÑê²XènÝ­eZ¦çôà$2ÝšŸüôgýYOé)ýIÒeºL‡tÈê²XàÝ£eZ¦çõ¼®ÓuV—tK„ @0_óõ¾Þ×çú\ã4N;µÓê’t¢'ô„~«ßj…V0ƒp €#Miúþ£Åh‚&è5½fuI:Áj­ÖB-Ôïõ{]¯ë­.èÖ€e€h­ÖêÝ¢ktîÔjPƒÕe8C¶h‹®ÕµºQ7*CV—t{„ @ã-o-Ó2½¤—ô¨UºÒU¢«Ëpš}«o5]Ó5ESô„ž°º Gð¶ºz´·Þ’Þ~ÛyÝöíÇ— 8¯ÿÁ¤Ë/z‰¹š«d%ë*]¥1£UZ¥1cuYNƒ•èûú¾iþ®¿ËK^V—„®./Oºÿ~çuYYÇ—ÍûäÈHé׿ºBœIaaÒ3ÏH^^’g³ Ÿþø²±Qjh®»®óëëFk´¶j«®Õµš¨‰ú?ýŸ®×õV—àÓ1]ªKå)O½­·ÕG}¬. ÝATÔñð_qññ~ÙÎÇç¿}²$ÕÕI‹u~}]„gû›€“vþùRtôñA]놆ãÏ?ßêj{¬þê¯ëßZ¨…ú‰~¢Z :ÕY]€“´@ t@ôŽÞQõ·ºtžžÒܹ’·wë}r݉¾aölkk°!Î$ã3øø´¾týõÇ·Åã-o-Ó2ýMÓ_ô}OßS‘Š¬. @=¦Çô7ýM/ëe%*ÑêrÐÝÌš%ÕÖ¶½Ml¬4fLçÔÐ2àL›5ë¿wFv¥®îø6è³4KµQyÊÓÑfm¶º$nÚ¤Mú¥~©ûu¿¾¯ï[]º£óΓnýóÿpÆ{®””Ôúç–FŒè¼z ‘©­Úªá®Éš¬ZauIÚQ¤"ÍÐ ]¢Kô+ýÊêrÐÍÛú CuuÒÌ™[@CÈ€Î0ožë>>ÒO~Òùõ@ýÔOÿÒ¿t§îÔ‚ÿjUkuY\¨W½®Ñ5 P€^ÒKòäך8sç¶>ÃPròñ€^Œ¿ÆÐfÍ’êë[®¯«“®¹¦óë$ÉK^Z¢%zCoèU½ª)š¢X]€fþWÿ«íÚ®×õºBju9èîÎ:K9Ròðp^ïã#ýøÇÖÔÐ…2 3$&Jçç< ÑÃC3F<غº IºL—i³6«Le£1Ú¨V—à„÷ôžÑ#zJOi„FX]zŠë®“¼¼œ×Õ×K3fXS@BÈ€ÎÒ|@£—×ñuè’”¤ÍÚ¬T¥j²&k¹–[]ÐëU¨B7êF]©+5Ws­.=ɵ×Jÿý¿§§”š*%$XV@WAÈ€ÎÒ|@cc#wLîb‚¬UZ¥¥Zª»t—æižªTeuY@¯õsý\5ªÑSzÊêRÐÓDGKçŸ<\ _üDÈ€Î.Mšt|//iòd)2ÒêªÐŒ‡<”¡ ­Öj­Ñ¥)Mûµßê²€^çM½©—õ²žÒS W¸Õå 'š7ï¿‘~ô#ëjèBЙšhœ;׺:Юiš¦-Ú¢:Õi¬ÆjÖY]Ðk”ªT ´@7èýP?´ºôTW_}|é{ß;!:ÕÕWŸÅÀÓSºê*««A;†hˆ6i“.Ô…ºT—j¹–[]Ð+üL?“|ô ´ºôdýúI_||ƒ¦!@€^ÎÛêèêJKK>ŠŠŠTRR¢ÊÊJ:tHGŽÑÑÊJ=|XåêèÑ£ª­­UUuµªkjÏÓÐبÃÇŽéÍÿ¿ÂfSßÀ@yyþ÷þ~~ ð÷—¯¯¯úôé#[X˜úôí«>ÁÁ Rhh¨‚ƒƒ®ððpõïß_ýû÷×€ÖÉG¦wRVj¥ÐZ¬Åúþ£Z¡@Z]Ð#­ÔJ­Ò*e*S¡ µºœauuuŽþôðáÃ:räˆ*++ëì«««ÿÛ¿VU©ººZ555:vì˜êêêtäÈ544èðáÃŽç>|ø°Z|McŒ:$Iš%é9Iáóæ©rÞ<ùøø(((Èe­þþþ $yyy©oß¾òððPhèñ×ihh¨<<<Ô·o_yyy)((H>>>êÛ·¯‚‚‚¬àà`G_ÞôÃþ]!@¯VQQ¡ï¾ûNÙÙÙÚ¿¿rrr”}à€²÷íÓTrèê› P õöV„——‚%…66*¨¡AAŠ*©$?I¾'7e“4àÄã•’ÊsúüÑ#GT+©FÒQI‡$‘tÌÓSÅ^^:äé©Ã’Št¨¾Þi_o//…ÛlŠ‹‹SÜYg).>^qqqJHHPll¬ ¤S>f½‘‡<”¡ ¥*U35Sçë|½®×•¨D«Kz”c:¦;t‡~ªŸjŠ¦X]:¨¬¬LÅÅÅ:xð Ó‡=¤gl__QQ¡š&a¼æl6›‚‚‚¤ÀÀ@Èßß_~~~ TPP"##]ø—äØÞ•àà`y{{Ë«¦FÏ?¯gÿßÿ“$G€Á•ÊÊJÕŸè{›š²³³eŒq„ìÄ#GŽèèÑ£­¶788X!!! s ºú0`€"""¡€Ó‰ Ç3Æ(++K_}õ•öîÝ«¯¾úJ_ïÚ¥½{ö¨°¬Ì±]¸â<=[W§ÔÆFÍ”.)BǃýO,}êë¥füϸÆÆãMÔI*‘T*©XǃE¥¥:PZªì;ô··rŒQq]cŸÈ~ýtö°aJ:ç :TÆ SRR’ $ÎlQ·t¡.Ô6mÓUºJc5V¯è]¬‹­. è1~«ßª\åZª¥V—‚&JKKUPP ÜÜ\)//ϱ,,,T~~¾ [ ÎïÓ§ú÷ïï¦Ñ£G;ʇ††:îðßôîþöufÚ4 êׯS¾Tcc£***tøðaUVV:fn°ÏÚpèÐ!§`ÆÎ;µµµNϪèèhEFF:–111Šˆˆp,ãââ#€!dèQ´wï^}öÙgÇ?6oÖŽÿüG‡OÌåë«$c”TW§Ë%-)QRœ¤€&ƒñ»IÑ'>Zhl”N D¬’t@Òw’ö”•éëO>ÑW[¶èMžØ¦o` F¥óRSuÞyçé¼óÎSRR’¼¼¼:¥-ÝI¬bµ^ëõ3ýLÓ4Mwënݧûä!BÀ©Ø§}ú£þ¨eZ¦HEZ]N¯RVV¦ýû÷;}|÷ÝwŽÇGŽql èèhEEE)**JcÆŒQTT”c]DD„#DÐÚ,]N' $ÉÓÓS6›M6›í¤ö¯¬¬Tii©Š‹‹U\\¬üü|8>öîÝ«üü|©¡ÉLL‘‘‘JHHpùß}Îè„ ÝÚ‘#G´qãFmØ°A¿ÿ¾¶nÛ¦c55òõôÔoo¥ÔÖj¶¤Ñ’’$…4»po ãaŠ³%}ß¾òD ¢BÒ^IŸ;¦Ï6nÔÇÛ¶é‰úzÕ66ª¿¿ÆŽ£IS¦è‚ .Є dEºùëy=¯ š [u«>×çzI/)D!V—t[‹´H‰JÔͺÙêRz¤ªªªã3ù|ýµ¾úê+íÙ³G_ýµ¾ùæ>|X’äáᡨ¨( 4H ºâŠ+¯„„ 8PÑÑÑ á:g%ûlƒ js»ÆÆF©°°PÙÙÙNÁ‘wÞyGû÷ïWEE…cûèèh%%%ièСJJJÒ°aÃ4tèP%$$ÈÓÓóL7 t1„ ÝJMM>þøcýûßÿÖú÷ß×ç_~©ú† öóÓµµšgŒÎ“tNc£|za  £B$¥žø$ÕÖªNÒNIÛ««µaÃýuófÝ_W'o//wÎ9š4uª.¹äMš4I¾¾¾V•Þ%Ì×| Ó0ÍÐ ¥*UÿÔ?5Lì. èvÞÓ{Z£5zWïÊG>V—Ó­UWWkçÎúüóϵsçNíÝ»W_ýµ8 cŒ¼½½• ³Ï>[]t‘æÏŸïtG{???«›€ÓÀÓÓÓ1ÛÄèÑ£]nS^^®ýÿŸ½;‹ªÞÿ8þD”Å MÑÐ E—PC3!×r-o™¥¦Õ-­ì¶ùSË,++Óºnmnå‚ûRîâ’š¸–¦¢â† Š"ËÌïëšhÀàýìqÃœ9çû}Ÿ3sç\áû9߃9tèû÷ïgß¾}ìÙ³‡Ù³gsâÄ sŠjÖ¬IPP>ø U«VÅdÒ >"""""""""""E•Š DDDDDDÄæ%&&2þ|Ì›ÇÒ¥K¹xù2%JÐ,-—‡Ÿ+WŒŽYd85óÃÀÓééVef²fûvìÞ͇~ˆ›³3-Z¶¤uT­[·ÆÇÇÇÈ؆iB6³™Žt$Œ0&3™ÇxÌèX"…F `éHKZ§PIJJbëÖ­lÛ¶íÛ·³mÛ6öìÙCFF®®®rÿý÷Ó¬Y3¨Y³&ÕªU+öb’ÅÃÃ[!œ;wŽ½{÷²wïÞœB•Y³f1bÄ233qss#((ˆàààœ¥víÚ*R)"Td """"""6éäÉ“LŸ>ï'OfÖ-8ÛÙÑ ø 3“6@%ÍRP |dzÒÒ8,¸|™ù±± ˆå‹…ðºöêEçÎñòò22n«@V²’þô§x•WÎpì°3:šˆÍ›ÊTþâ/²Ðè(6-##ƒßÿuëÖ±~ýzÖ­[ÇÁƒðññ!88˜¶mÛòæ›oÌ}÷݇¾ƒäÞ¸»»FXXØuë/]ºD||<Û¶mcëÖ­lÞ¼™‰'’’’‚Ùl&88˜ððpÂÃÃiÔ¨•*U2èDDDDDDDDDDDäŸP‘ˆˆˆˆˆˆØŒK—.1sæL¾Ÿ:•å¿ü‚³ÉDŒÅÂV+Í33q6: ä¨<<Ÿ™Ée`90}óf†üöƒ^x?Ì=zбcGœ‹Ç;çˆ#˜@}ê3€ì`ßñî¸MÄf¥“Î0†Ñ‹^T¥ªÑqlÊÅ‹Y¹r%ëׯgíÚµlÚ´‰””ÜÝÝ §wïÞ„††LùòåŽ+ÅDÉ’% %444gÅbá?þ`ëÖ­¬_¿žõë×3nÜ8ÒÓÓñóó#<<œ† Ò¤IêÖ­«â‘B@E""""""b¸cÇŽ1~üxÆ~ü1ç.\ ™ÉÄ7 íW£ÃÉ9m¶V+©™™,¦®XÁÓË—3à¹çèõ¯ñÒK/Q¹reƒ“Œ>ô!@:щPBù‰Ÿ¨Mm£c‰Ø¤)L!^çu££Ø„;w˲eËX½z5W®\ÁÇLJÆóÿ÷4nܘ|Pƒ´Å¦ØÙÙ@@@?þ8éééüþûï¬Y³†µk×2bÄ^zé%<==iÞ¼9‘‘‘´jÕJ3ˆˆˆˆˆˆˆˆˆˆˆØ(ˆˆˆˆˆˆˆaÖ¯_χ|Àœ¹sñ6›y)-g/«Õèhrœ€( *3““Àø”¾üòK¾;–˜èhþ=x0aaa§ÌhÄv¶Ó™Î„ÎD&Ò‘ŽFDZ)é¤3œá<ÅSøãotC$''3oÞ<,XÀ²eË8yò$ÞÞÞ´hÑ‚o¾ù†-ZP®\9£cŠÜ5êÕ«G½zõ8p ;vì`ñâÅ,Y²„’ššJ:uhÙ²%QQQ4iÒD4"""""""""""6B¿±‘·cÇ¢Û¶¥aÆ‹å;‹…¿ÒÒxð2:œä™rÀPà`z:S-bciРµkÇÎ;Ž—ï¼ðb)KéG?:Ó™! !“L£c‰ØŒ‰L$†0Äè(êâÅ‹üðÃ<öØc”/_žÞ½{sâÄ ^~ùe¶mÛƱcǘ2e ݺuS)uêÔáßÿþ7K–,áÌ™3,Z´ˆÈÈH.\HÓ¦M©X±"díÚµXUp*""""""""""b(ˆˆˆˆˆˆH9zô(=ºu#8(ˆ„%KX¬ÏÈ  à`t8É7ÀãÀ†ôtæ.ä:uèÕ½;ÇŽ38]þ2cf$#™Â>åS"‰ä§ŒŽ%b¸4ÒÎpþÅ¿ŠÅ,™™™Ì;—N:Q®\9zöìÉåË—ùâ‹/8~ü8Ë–-ãÕW_%((“Édt\‘|çììÌ#<ÂG}ÄÎ;Ùµk}úôaéÒ¥4nܘʕ+óòË/³k×.££ŠˆˆˆˆˆˆˆˆˆˆK*2‘1iÒ$j׬Ɇ3øÞjeKz:J \k`KFßY­¬™>À€&Ožlt¬|×î¬e-9H!lf³Ñ‘D 5ƒå(¯ñšÑQòÕÙ³gùðé^½:=ögÎœa̘1$&&²hÑ"žzê)<<<ŒŽ)b¸ZµjñÖ[o±k×.~ÿýwzöìÉœ9s¨]»6-Z´`îܹX,£cŠˆˆˆˆˆˆˆˆˆˆ*2‘|•˜˜HÛGå_½{óTJ ¿§§ÓÐ}š‹/;²f6Ø‘žN¯‹éýÔSD=ú(Ç7:Z¾zÙÄ&ªS&4a“ŒŽ$b˜Où”ö´§•ŒŽ’/þøãúöíKÅŠyï½÷ˆŠŠbÏž=,_¾œ>}úP¶lY£#ŠØ¬:uê0lØ0öíÛÇüùó1›ÍÄÄÄpß}÷1zôhRRRŒŽ(""""""""""Rä©È@DDDDDDòM||¡zõêFÇ)TìììxôÑGY¸p!{öì¡M›6¼óÎ;Ô¨Qƒ‰'jf‘|¤"É+V¬ Ix8OŸf}z:Àâ, IDATŒô7ÖžÀ.£ƒØˆ‚>M€MééøŸ:Exh(‹/. žaÆÌHFò=ßó-ßò0s‚FÇ)0Ÿò)u©K#›¾2ÜôôtÆO­Zµøþûï5j»ví¢ÿþ¸¹¹ï–Ö®]‹§§'»vÝùÛþn¶Íë¾EjÔ¨ÁgŸ}Æ_ýEÇŽéÓ§uêÔaáÂ…FG)’Td """"""ynÆ ´nÕŠÖ—.±"#ƒ²yØöÀ”‹eÌ]´i¬Ù ÀkÚY”'©ïÝ××dW@}Þx>–egˆÏ§>=€Å´IM%&*Š7æSO¶ã ž`ë8ÊQB!Ž8£#‰ä»cc³È@££ä™={öÆ AƒèÝ»7û÷ïgàÀØÛÛç[ŸGŽÁd2]·Œs7W>°X,X­V¬Öÿ}Û/[¶ “ÉD||ü·Í!C†ää[´èWÔ{m¯8¹Ý¹Ë+·{¯m§§'Ÿ|ò ñññÒºukzöìIJJŠÑÑDDDDDDDDDDDŠˆˆˆˆˆˆHž:qâbbhi±0ÕbÁ1Û¯@Öà÷^€ ÿ í2ø.Ûlœ³ŸvçEØ<ð4p¹€û¼ñ|Gà;‹…H‹…˜6m8vìX'(xA±‰MÜÏýDÁ×|mt$‘|õ_àélt”<1þ|BBBptt$>>ž‘#GRªT©|ï·B… X­Vzõê…‹‹ V«•AƒÝUMš4áÌ™3ÞùÛþn¶½ÖÈ‘#Ù½ûæ+꽶WœÜîÜI–€€¦OŸÎÌ™3‰¥qãÆ$&&KDDDDDDDDDD¤ÈP‘ˆˆˆˆˆˆä«ÕJ—p9s†©™™úG§Ü5;`jf&nçÏÓýñÇ‹Å]®ËP†,`Cx6û¿4ÒŒŽ%’çÒIgèK_œp2:Î?6cÆ bbbèÚµ+«W¯¦jÕªFG)v:tèÀ–-[HMM¥Q£F=zÔèH"""""""""""E‚Æ{ˆˆˆˆˆˆHž™1c«×­ãûôtJ˜c$08 ¼Ü8ÁÀœ¶ ˜²—{¹‡üÝö1x( øïÞ¢ÍÏÊ@I ðÇ]ä‰B²³”úÉÙ¯…\“Ã;{]«kÖ ãÖçcÐ"ûç:Ù¯U¹‹LwËø.=•kÖðóÏ?çcO¶Ã{Þæmf3›iL£9ÍIĸ;2ÿüóϘL&¶nÝzÓk-[¶$$$$çùŽ;ˆŽŽÆÃÃggg6lȪU«®Û'!!®]»âãホ›¡¡¡L›6-ßClËpŠS<É“wÜÖÖ?ƒ6l gÏžôë×ñãÇc6›ï¹­¼4vìXL&&“‰qãÆñ /Pºtiüüüx÷Ýwo¹Ý×_g}Û2„-²¾íëÔ©ƒÉd¢J•*·ÜàôéÓ¼òÊ+Üwß}899Ìœ97^ÿ>ãÕöŽ?ž³îÆ¥cÇŽ9ûÆÆÆ‚““åË—§oß¾$''ß®«\ŸÜ´y]®ŒŒ þïÿþûï¿?§¡C‡æl3lØ°¿=‹/¦Aƒ8;;S±bE:vìH\\Üm·4hPNÛñññÌœ93gÝ·ß~›³íþýû‰ŽŽ¦lÙ²¸¹¹Æ €Û¿×¹9מӱcÇòÜsÏQ¦LL&?þøßo~ñ÷÷gÍš58;;Ó¶m[RSS É!""""""""""R”¨È@DDDDDDòÌ'£GÓÁd¢®AýŸãƒç!kÐ|:ÇÿŠº;¯Ù¦?páôy·}| DG€ÁÀ[ÀÊk¶ý!{ûg€£ÀGÀ«¹Ì2h´%Ùm?XÍÀ§d \†8€¡Üú|Œ–fÿ¼#»­ƒ¹Ìt¯B€“‰1£FåsO¶%Š(âˆã g!„u¬3>>>>|óÍ7×­?tèË—/ç™gž`ûö턇‡ãââ–-[8zô(­Zµ¢E‹lÙ²%g¿Î;sòäIÖ¬YÃÉ“'7nsæÌáĉz\b¬©L%‚üñ¿ã¶¶üLOOç駟&""‚>úè®÷ÏOýû÷çÂ…¬oñ/¿ü’ÈÈHŽ9ÂàÁƒyë­·X¹råMÛ]5räH–.Íú¶ß±cV«•ƒÞr[€aÆ‘žžN\\\NÁA—.]عsçMÛÞ.ãµ~øa¬VkÎòÆoàääÄ[o½Àœ9sh×®­[·&11‘%K–°råJ{ì±ÛÎz“Ûó‘›ö—-[FçΩS§V«5§°dîܹìÞ½›ýû÷眗#F0nÜ8†zÛóKëÖ­iÑ¢GŽ!..GGG~øáÛî3f̘œ÷誎;rêÔ©›¶íСÎÎÎìÚµ‹„„ªT©Bdd$pû÷:7çáÚs:zôhš7oNBBŸ}öÙmsOOObccÙ¿?#FŒ04‹ˆˆˆˆˆˆˆˆˆˆHQ "É'Oždý¦Mô²X ¬Ïþw×}àqÃëc²OÀè´òrÜÝöBV!€Ь٠V_óúÛd͆0”¬ãyè“Ë,¯dÍŽàAVñÀ(`ðKö6ýæÀSÀy`@ö~rÙGAzÒbaM\Ü-OeGa„Aïó~g0›ÍôîÝ›ï¾ûîº;Bÿ÷¿ÿ¥dÉ’tíÚ€W_}•J•*1eʪV­J™2eøÏþCƒ xï½÷r!wëÖjÕªáììLݺuùþûï)_¾|›ã,g‰%–ôÈÕö¶üŒeÏž=|þùçØÙÙî¯WCBBh×®nnnôë×’%K²zõê;ï˜KcÆŒa̘1xzzâêêJ·nÝhÙ²å= 6÷ööfÙ²e9ÏW¯^ÍÈ‘#ùàƒ¨S§õ^òî»ïâááAPP£FbÅŠüòË/·k:ÇÎGnÚŽŽfÇŽüõ×_@Ö §OŸÆd21oÞ¼œ¶æÎKTTÔßæù÷¿ÿM`` ï½÷žžžøúúòÕW_áèè˜ûw©©©lß¾öíÛS®\9ÜÝÝùðÃqqq¹ã¾wsž###éÔ©...ôïߟüñgÿ'üýýyã73f —/_64‹ˆˆˆˆˆˆˆˆˆˆHag»‘BeË–-X­V`Ÿ.dÝUÿêr6û”öåg¨;ôQ皟í²ÀÉìçg²÷k|Ã>a¹èóHö¾Mo³ïòìG0‘¬Ù š©@×\´o„Æ€Õjå·ß~3:JsÃYÌbÃx×éA.S°&Ÿyæ’““™5k‹…I“&ñøããææFZZ¿üò m۶͹£÷U¬Y³1bÓ¦Mãܹszb¦1 ;ìè@‡\ïc«ŸÁyóæѤIªU«öÚÉoWçØÛÛS¶lYNž<ù7{üseÊ”aß¾v•=wîÝ»w§eË– 0€#GŽ°oß>š6mzݶaaYW¹åË—ßØÌMþî|ä¶ýÖ­[ãààÀœ9s€¬ÏBÏž=©_¿~N‘ÁñãDZZ­øúúÞ6Ë‘#GØ»w/=ôÐuë]]]IJJºã±Ü‰““õë×çµ×^cÆŒ\¾|³Ù|Ç™;îöL5îiß‚äêzýÇÁÁKÎ6´k×.Ú·o··7vvv˜L&&OžÌÙ³ÿì*Û·o_RSS™4iRκӧ³®rcÇŽ½î}.W.ë*wøðá[5u¿;¹mßÝ݇zèº"ƒèèh¢££YµjçÏŸÏÕ,Wû+S&7Wø{³xñbbbb4hîîî´iÓ†¸¸¸\åÊíyvvvΟðÿ@ùòåñððàСCFG)ÔTd """"""yÂÅÅ…4‹…4£ƒdK"à×ìçV Wö£-öá“ýxæ†õçs±oÙìÇW¹~v‡«ËÔk¶Íf/ó¿â[sH·XnZÜ´¦5ÙH:éÔ§>˹ó»óʳÏ>˯¿þÊþýûùúë¯ "44€Ò¥KcooÏ;＃Õj½i¹v0s@@±±±œ={–… âççÇO<Áüùó ìXÄ8ò'ëYOzÜõ¾¶øôððÈ“»ÍféééDFF’À¯¿þJzz:V«•^½zaµÞûUvòäÉL›6I“&å l([6ë*÷ꫯÞò½ž:uêíšÌ•»i?::š5kÖpôèQþüóO‚ƒƒi×®,\¸9sæ«þ®-„É-;»¬_駧§ç¬KNN¾i;FÍÑ£GY¹r%©©©<ôÐCìß¿ÿŽ¹òë<„+W®páÂ…|-à)Td """"""y" +°Ýè Ù5x¾ P°Ï^ņû(ÔÖÜ°þ·\ì[!;æ[¼L»æù{@;à' ð¯\´oÄ/¶‘U Q³fMz·-Õ©ÎzÖÓ”¦´¢ïó~ôÛ¶m[|||øàƒ˜;w.}úôÉyÍÉɉ¦M›2gÎ233sÕ^É’%iÙ²%Ó§OÇÑÑñŽwÕ–¢aÓ(G9"‰¼ë}mñ3بQ#–-[Æ¥K—îz_[wuû8p€ÄÄDºtéBÍš5±·Ïº^¹rïWÙýû÷3`À^xá}ôóìÔ®]› *P³fM6mºù*Ä´iÓnZ7î¦ýèèh222xñÅiÑ¢ENƪU«òÃ?pðàAïØ_@@+W®¼ný±cÇprrúÛ"///ŽÿßÜM¿ÿþûuÛ?~œ:uêäL»víX¶lIII\¸pñãÇ“––F³fÍò5¿Ø†ÙÌ&šh̘ïz_[ü vëÖ «ÕÊÈ‘#ïz_[çã“õm¿gÏNŸ>OΠôkU©R///¦NÊ®]»HMMeÉ’%,\xoWÀŒŒ ºuëFåÊ•yÿý[Q=šU«V1räHNŸ>ÍéÓ§y饗ÈÈȸã̹‘Ûö+UªDpp03f̸n}tt4sçÎ¥U«V¹êïÃ?d×®]¼ùæ›$%%qèÐ!z÷îM¯^½ðôô¼í~5jÔ \¹r|ñÅœ9s†½{÷2yò䛶‹çã?&99™sçÎñÕW_áääDýúõÛ¿×ù}žóSZZo¾ù&íÚµ£B… Fǹ'N8‘JªÑ1DDDDDDDDTd """"""yÃd2ñÌóÏ3ÁlæL>ös0“”ìŸkßb;G`>P#k†€Y@ ²f[0DZ€[ö>ÏM!@­ìuÝo“%·}üxCݯ9ŽýÀçÀ}Ù¯?‘ý|YùŸ†g¿öÐà6YZ €9dÍlP8,œ²û­OV!Â[ÙûDe?öÍÞþVçƒìóñ<ð4P h„üM–* øÚl¦O¿~˜L¦|ì©p1ab0ƒYÆ2–³œ†4äòµÏý+k®‹Î;Sºtéë^ ʹãuDDeÊ”!&&†#GŽä ð®T©Ï>û,~ø!T¨PÉ“'3sæLG9ʶͽN¶µÏ ——ÇgĈ,X°àžëŸ8rä&“‰É“'“’’‚Édb̘1üø㸹e}‹?óÌ3tïÞ=gÛýû÷óùçŸsß}÷1vìØë¶kÚ´)U0øüóÏóôÓOS­Z5Ú·oφ nÚÖÑÑ‘ùóçSºti¨Q£³fÍ¢E‹lß¾“ÉÄñãÇ2dµje]Q}ôQºwï~˾gÏžM\\ñññ8;;c2™r–ŒŒ¬;ù¶nÝš 0gÎ*T¨@íÚµ9qâ‹/ÆÉÉé–ç)·çãnÛŽŽÆÝ݇zèºuíÚµËÕ{ضm[æÏŸÏ’%Kðóó£aÆòÉ'ŸÜòÜ8::òÝwßqàÀ*T¨À³Ï>ËСCèÑ£­ZµÂÛÛ›ØØX.\HÕªU©\¹2ëÖ­cþüùT­Zõ¶ïuHHÈÏÃçÔd2qîܹ\s~{î¹çHHHàã?6:ŠˆˆÈ=S‘ˆˆˆˆˆˆˆØ “ÕjµBDÄhé Àt¦œD «éÓ§Ó¥KtY‘{1~üxúôéct ‘<‘œœL ZŸ;Ç-£ãH!eºÚÙñ«‡<ˆ«««Ñ‘lR ´§=8Àü@KZæK?¿üò Í›7gýúõ4hðwe."7ûœÏÂNq 'n=üNlõ3øÜsÏ1iÒ$¦NJÇŽŽ#Rleff2`ÀÆÏO?ý”ëB±]ú7²ˆg/ó2ëYÏ:ÖEDDDDDDDŠ‰Î³ÇÏN¿~ü¬f2‘þøc>=šWM&ž3™¸dt(±Y—€gM&†˜LŒùäF}ø¡Ñ‘ ¥&4a3›qÇ0Âø™ŸŽ$ÅÜ"шFxàat”|ĪU«øñÇYµj¼öÚk9rÄèh"EƪU«ˆŠŠâá‡ÆÏÏßÿÏ>ûŒR¥JMDD$ϸâÊ.CDDDDDDDDE""""""’^|ñEfÌœÉ..—(Á£‰ÍY980ÃÕ•Y?ýÄ /¼`t¤B­XÉJºÐ…t`C°`1:–S¿ò+ÍintŒc2™èÔ©»wïæ­·ÞbòäÉøûûÓ¹sgV­Zet<‘BéòåË|ýõ×Á©S§X¼x1óçϧV­ZFÇÉsÞxs†3šÍ@DDDDDDD §"ÉWíÛ·gçÞ½ÔhÞœ“‰L&’Œ%†; ôš™LÔŠŒdçÞ½ÄÄÄ«HpÄ‘ L`ãø˜‰"Šsœ3:–3ð‡8T¬Š ®rttä•W^áàÁƒLž<™Ã‡App0Ÿþ9'Nœ0:¢ˆÍÛºu+¯¼ò +V¤ÿþ<ðÀlܸ‘ 6вeK£ã‰ˆˆä_|±bå8ÇŽ""""""""ÅœŠ DDDDDD$ßùúú»p!ÿ8‘iîîT3›\4:˜¸‹À»@5³™™eÊ0iòdæ.X€ÑÑŠœ>ôa+ØÊVB %žx£#I1²‚¸âJ(¡FG1L‰%èÚµ+6l`ãÆ<ðÀ <???"##™0aII*»¹*>>ž7ß|“€€êÖ­ËÌ™34h‡bÊ”)Ô¯_ßèˆ"""ùÎ_ŽqÌà$"""""""RÜ©È@DDDDDD L¯^½Øø0/ÿç?Œrqá>Fç&ùîð!PÍÁ\\xå­·øóÐ!zôèat´"­ØÎvüð#œpf2ÓèHRL¬`ñ8Å&Ô¯_Ÿ)S¦pòäI~øáÜÝÝ8p >>>´nÝšqãÆqàÀ£cŠ¨ŒŒ Ö­[Çþój×®M:u˜4imÚ´aÆ 8p€¡C‡R¾|y££Šˆˆ|°ÃŽDŽ""""""""ÅœŠ DDDDDD¤@¹ººòæ›o²ÿàAz È;ÎÎT°·§ŸÉÄ£ÃIžÛ#“Lì±7:ŽˆˆˆˆˆˆˆSú«’ˆˆˆˆˆˆØ´Š+2`À @JJ Ë—/gÁ‚L™7w£„!öö4IO§ Ðp7:tqX ¬V;8°93“t‹??‰Šâ­6mhÞ¼9%K–48©Ü 3fF2’ ‚xš§ÙÍnf0ƒò”7:šbç9Ï>öJ¨ÑQŠ4³ÙLýúõ©_¿~κÌÌLþøã¶oßÎÖ­[Ù¾};Ÿ~ú)GŽ D‰T«Vš5k@@@µjÕ¢zõê”)SƨC`±XHHH`ß¾}ìÝ»—Ý»wçü|µ˜ÀÁÁêÕ«óÀЯ_?‚ƒƒ ÂÛÛÛàô"""EOmj“J*ò'GDDDDDDDŠ)ˆˆˆˆˆˆH¡áââB»víh×®ŒGbb"kÖ¬aÍš5,[¾œvíÂjµâãà@½Œ êC»<` IDATY­Ôê÷÷ÎñÀ`‹[J”`Ï•+X¬VªV¬H£¦My²qcZ´h¡;1Oð÷s?íiO!Ìd&a„K ©Ml‚…úÔ¿óÆ’§ìíí©Y³&5kÖ¤K—.9ë¯\¹ÂŸþÉ®]»8pà;wîdéÒ¥|öÙg¤¤¤àä䄯¯/U«V½nñññÁ××L&“Q‡&yàìÙ³8pà¦åرcø 4 lÙ²&–‚D›ØÄ:;;qˆÅZRR‰‰‰;vì¶G%==ÈúÍž={سgäðáÃìøã:ÄÑ“'IÏŒPÊlÆÛÞž²V+e32ð²XðÜwÀ%{qJeÿì Ø¥¯éÓ”½ýµÎÞðü`.)@2p!ûç”ìדÉ*&8egÇi³™S&'23IÎÈÈiÇÁÞ¿òå©T¹2•ï» ì»׬Y“š5kjv¹%{ìy›·©G=zЃìd3ðÁÇèhRHle+}ékt ÉŽŽŽÔ¨Qƒ5jÜv›ÌÌLNœ8qÝóOœ8‘38þäÉ“ìÞ½;gÐüùóçojÃÞÞžR¥JQºtiÜÜÜpuuÅÙÙ™2eÊàî«+nnn¸¸¸P¢D \\\0›Í¸¹¹aggGéÒYWZ «ÐÁÎÎo¥T©RØÛÛß´þòåˤ¦¦ÞòX“³ RSS¹|ù2iii¤¤¤‘‘Á… °X,9ÇyöìÙœÇ .pñâE.\¸À… 8{ölÎó‹/^×öUf³ù¦Â ???‚‚‚ðòò¢\¹r9¾¾¾9ç@DDD ¿ ‚ø–oŽ!"""""""ŘŠ DDDDDD¤X*[¶,7¦qãÆ7½f±XHLLäСCœ##ãº;ò'''sáÂ’““9wî/^$!!Ù³gS¡B|||rá_ºt)§àêÀ~[uu°ÿÕB†«W///ÜÝÝsŠ*ÜÜÜ(Uªîîî”*U OOO¼¼¼r (DDD¤øiBÞáH "Ž#"""""""ÅŠ DDDDDDDn`gg‡ŸŸ~~~¹Þ'==‹/rîÜ9RRR¸råJκ«n¼ 2Ü|'eWWWprrÂÅÅ%g`¢Ù¬ÂKÁ €8âèE/"ˆ`ÃÌ`£c‰ ‹'+VÈm™ÍfÊ—/OùòåoùúêÕ«éСUªTaÎœ9TªTéoÛû»Ùà³ ÜèÚío•ÑÍÍí–¯];KÂßͪ """òO…N J°šÕt¥«ÑqDDDDDDD¤Ò‘<ààà€‡‡GÎÝ‹EŠ7ܘÅ,>à^çuâ‰g<ãqÆÙèhbƒv²\¨Le££H!4~üxú÷ïOLL “&M¢dÉ’wÜÇl6ç\w===ó;¢ˆˆˆH)IIBa%+Ud """""""†°3:€ˆˆˆˆˆˆˆˆØ.&3˜yÌ#–Xшƒ4:–Ø ì$@ìô+G¹ ¼ð ôíÛ——^z‰ü1W""""E]¬d¥Ñ1DDDDDDD¤˜Ò_üDDDDDDDDäŽZÓšl$têSŸå,7:’ؘxâ©Mm£cH!ræÌyäþûßÿ2sæLFŽ‰~e-"""YE{ÙË1ŽEDDDDDDDŠ!ýÅFDDDDDDDDr¥:ÕYÏzšÒ”V´â}Þ7:’Øxâ $ÐèRHìÛ·† ²oß>V®\IûöíŽ$"""bSÑ3fV±Êè("""""""R ©È@DDDDDDDDrÍW¦3a ã Þ +]¹Ä%£c‰ÁÎqŽœ µŒŽ"…ÀÂ… ÅÓÓ“Í›7S¯^=£#‰ˆˆˆØW\iD#æ1Ïè("""""""R ©È@DDDDDDDDîŠ ƒÌ2–±œå4¤!8`t,1Ð~öPj'[÷É'ŸÐ¶m[Ú¶mËòåË)_¾¼Ñ‘DDDDlV{Ú3y¤’jt)fTd """"""""÷¤)MÙÌfp >õY£#‰Ap;ì¨Le££ˆºrå O>ù$/¿ü2ÇçÛo¿ÅÉÉÉèX""""6­¸ÈEV°Âè("""""""R̨È@DDDDDDDDîYE*²šÕ´£­iÍÛ¼«Ñ±¤€íg?©ˆ#ŽFG”˜˜HDD?ýô?ÿü3ƒ6:’ˆˆˆH¡à‡¡„2‹YFG‘bFE""""""""ò8áÄD&ò_0œáÄÃyÎK ÐP•ªFÇ´mÛ64h@RRqqqDEEIDDD¤Pé@æ0‡ 2ŒŽ""""""""ňŠ DDDDDDDD$Oô¡ËYNq„ÆnvI ÈPjFÇ3cÆ 5jD@@7n¤V­ZFG)t:Ò‘$’XÅ*££ˆˆˆˆˆˆˆH1¢"É3MhÂf6ãŽ;a„ñ3?I À~öã¿Ñ1ÄFX­VÞÿ}ºtéB÷îÝ™?>FÇ)”üñ§.u™ÊT££ˆˆˆˆˆˆˆH1¢"ÉS¨ÀJVÒ….t C‚‹Ñ±$ŸX°p„#T¡ŠÑQĤ¤¤Ð±cG†ʧŸ~ÊW_}…ƒƒƒÑ±DDDD µ§yšiLã,gŽ""""""""Å„Š DDDDDDDD$Ï9âÈ&0Žq|ÌÇDÅ9ÎKòÁIN’A~øE väÈ"""X¹r%K–,¡ÿþFG)zÐ3f¾å[££ˆˆˆˆˆˆˆH1¡"É7}èà V°•­„J<ñFG’""""""""R ºÓµ¬å  !„Íl6:’ü‰$âƒÑ1¤ýù矄……ñÛo¿±jÕ*zöìit$‘b¡=ðÆ›QŒ2:Šˆˆˆˆˆˆˆq*2‘÷ ²‰MT§:MhÂ$&IîÑINRžòFDztéRBCCqrrbýúõ„††IDDD¤ØpÀ—y™ñŒ'‘D£ãˆˆˆˆˆˆˆH¦"1DYʲˆE d OñÏò,é¤KîRIxâit )ãǧM›6´lÙ’5kÖP©R%£#‰ˆˆˆ;Ïð e(ÃÇ|lt)ÂTd """"""""†1cf$#ùžïù–oy˜‡9Á £cÉ]8ËYÊPÆè’222èß¿?}ûöåõ×_ç‡~ dÉ’FÇ)–œpâE^ä ¾à§ŒŽ#"""""""E”Š DDDDDDDDÄpOðëXÇQŽBqÄIré,gñÀÃè’O’’’hÑ¢“'O槟~âí·ßÆd2KDDD¤X{Žç(IIÆ0Æè("""""""RD©È@DDDDDDDDlBAlb÷s?Dð5_Irá gTdPDíرƒúõë³ÿ~V®\ILLŒÑ‘DDDDpÁ…—y™Oø„£5:ŽˆˆˆˆˆˆˆA*2›Q†2,`C³Ùÿ¥‘ft,¹4ÒH!…2”1:Šä± иqc|}}Ù¼y3uëÖ5:’ˆˆˆˆ\cƒðÁ‡×xÍè("""""""R©È@DDDDDDDDlŠ=ö¼ÍÛÌf6Ó˜Fsš“H¢Ñ±äÎr@3!V«•÷ߟ¨¨(:wîÌŠ+(W®œÑ±DDDDäŽ82’‘|Ë·¬aÑqDDDDDDD¤ˆQ‘ˆˆˆˆˆˆˆˆØ¤(¢ˆ#Ž3œ!„Ö±ÎèHrƒóœ 4¥ N"y!55•^½zñÆo0|øp&L˜@‰%ŒŽ%""""·ÑDÉ aÁbt)BTd "’íèÏG1™Llݺõ¦×Z¶lIHHHÎó;v‡‡ÎÎÎ4lØU«V]·OBB]»vÅÇÇ777BCC™6mZ¾‡ˆˆˆˆˆHQ@qÄFDð>ïI®‘B .¸œDþ©cÇŽAll,‹-bðÿ³wïq:Ö‰ÿÇ_÷Ì`fCr µ”mÑÈi0äåÐ)›ŒúFçš©´é›ÝÈ~;©vC[ÙCÛ¶1”39צ¶ÃÔJƒrsøý±Ü¿ä…kƼž×c³Ý÷纮÷çž™¶™ûó¾®´´ #I’$é(<Á¬d%ctI’$I’t ±d IûTëRªU«ò /ðøþóf̘A¿~ýX¹r%Í›7§téÒ,]º”o¾ù†Ž;Ò¾}{–.]Ù¯{÷î|ûí·Ì;—o¿ý–¿üå/¼ù曬_¿þ¤ÎK’$I’Šº²”åu^gC¸ŸûéMov±+èXv²€xâN¢_bùòå4k֌͛73þ|ÂápБ$I’t”êSŸ¾ôå>îã{¾:Ž$I’$I:EX2¤}B1!n¸áÆŽËîÝ»#ÿõ¯%>>žž={pï½÷R³fMÆŒCbb"*Tà÷¿ÿ=Íš5ã‘G`ïÞ½,Z´ˆ^½zQ§Nâââ¸à‚ xõÕW©\¹r ó“$I’¤¢,Dˆ4Ò˜Ä$&3™d’ùŠ¯‚ŽUìY2(úÆOrr2çwï¿ÿ>õêÕ :’$I’ŽÑ†°—½ÜÅ]AG‘$I’$I§K’ôýúõãûï¿çõ×_ ??Ÿ—^z‰ßþö·”-[–œœÞ}÷]:wîLLLÌû¦¤¤0wî\J”(AݺuyôÑG?~<[¶l9és‘$I’¤SÑ¥\Êû¼Ï^ör!2ƒAG*Öö— ∠8‰ŽUAAƒæšk®¡wïÞLž<™òåËK’$I?ÃéœÎHFò/0…)AÇ‘$I’$I§K’ôµjÕââ‹/æ…^`úôé¬^½š~ýú°yóföîÝËðáà …B|<òÈ#|÷Ýw‘cýãÿàW¿ú×]w+V¤yóæŒ7.yI’$IÒ©äÎa hC:Ò‘a :R±µ“DM)JEÇ`ûöí\yå•<ú裌=šçŸþ ‹)H’$©h¹ŒËèNwúÑÍl:Ž$I’$I*â,HÒôïߟ÷Þ{/¾ø‚Ñ£GÓ Aš4i@¹r刎Žæᇦ  à üüüÈqêÖ­ËäÉ“Ù¼y3S¦L¡zõê\sÍ5¼õÖ[AMM’$I’Ne(C að=鹪¾Nžì$žø cèdeeѼysæÎË´iӸᆂŽ$I’¤ãäYž%—\îáž £H’$I’¤"Î’$ýHçΩZµ*=öÿüç?IMM<K›6mxóÍ7ÉËË;ªãÅÇÇsñÅ“‘‘A©R¥X´hщŠ.I’$IÅJˆi¤‘I&3˜A ZEVбŠ•Ýì&–Ø cè(Í›7æÍ›ÅâÅ‹III :’$I’Ž£ŠTä9žã¯ü•©L :Ž$I’$I*Â,HÒÄÄÄpà 70räHbbbèÕ«×ÏÿßÿýŸ~ú)½zõâã?f×®]üûßÿæÉ'Ÿä®»î`õêÕtíÚ•ÌÌL6mÚĶmÛ9r$999´mÛ6ˆiI’$IÒ)« mXÂJP‚ ¹iL :R±‘K.1ÄCGaÔ¨Q\tÑE4iÒ„9sæpÖYgI’$I'À•\É5\CúMvÐq$I’$IReÉ@’¡oß¾tïÞråÊð\ƒ X¼x1)))T¨PË/¿œ5kÖDJ5kÖ¤ÿþüéO¢nݺœy晼üòËLœ8Ñ’$I’$5¨ÁæЕ®\Ê¥ f0ë”—GÑDCG——Gzz:ýû÷çŽ;îàÍ7ßä´ÓN :–$I’N çyž Tàj®f/{ƒŽ#I’$I’Š /3&I‡ðå—_пÿC>_¯^=ÆwÄctêÔ‰N:÷l’$I’¤C‹%–y‘æ4gXÎrÆ0†r”ûéõ³X2(ܾûî;zôèÁܹs3f ×^{mБ$I’t”¡ dД¦ bÃt$I’$I’TÄx'Iú‘M›6qÿý÷Óºukš5ktI’$IÒ1J%•Ì`‹hJS>æã #²,^Ÿ}öÉÉÉ|üñÇÌž=Û‚$IR1ók~Í(F1œá¼ÁAÇ‘$I’$IEŒ%Iúp8LÕªU …B¼øâ‹AÇ‘$I’$ýL­hÅ–Pžò4¥©‹jNK…Ó;ï¼C“&M(_¾èX…^iJ³ƒAÇ8å½öÚk\tÑE4lØ÷ߟºuëI’$I§€h¢y‰—è@ºÐ……, :’$I’$I*$,H’$I’$Iû\Ã5Ìg>ßð iÌ"©P+CòÈc»‚ŽrJ*((`ðàÁôìÙ“~ýú1yòdÊ•+t,I’$BJP‚qŒ£ mhO{¦3=èH’$I’$©°d I’$I’$ý@°˜ÅœÇy¤ÂhF©Ð*C¶³=à$§žmÛ¶qùå—3tèP^|ñEžzê)¢££ƒŽ%I’¤SPIJòwþNwºÓ™Îdt$I’$I’°˜ H’$I’$I…M*ð6oóПþ,f1#AIJ­PùaÉ •Nsêøâ‹/èÚµ+6l`Ú´i´nÝ:èH’$I:ÅEÍhF“@=éÉ‚OpîìsƒŽ%I’$IR±U§NÚµkØù-H’$I’$I‡M4ƒLIô¦7ñ˜@Uª­ÐØ_2ØÁŽ€“œ:æÌ™ÃUW]EµjÕX¼x1µjÕ :’$I’Š‰!þÄŸ¨D%Ò륓V/¡ :–$I’$IÅÒÈ‘#-DvfI’$I’$©èB±ˆïøŽÆ4f>óƒŽThüðNúåöÿ±¸M›6ÌŸ?ß‚$I’‘F=çöd8ù…[È%7èH’$I’$é$³d I’$I’$ý„ºÔe‹hJSRHaÂŽT(X28>rss¹í¶Û¸é¦›¸óÎ;7nñññAÇ’$IR1–²*…‰Läe^æ.a3›ƒŽ$I’$I’N"K’$I’$IÒQ(KY^çu†0„û¹ŸÞôf»‚Ž(K¿Üwß}G‡xá…˜8q"C‡%*Ê?ÛJ’$)xWpó™Ïg|Fšð1I’$I’$$¾[%I’$I’$¥!ÒHc“˜Ìd’Iæ+¾ :V`JP‚R”²dð3}ú駴hÑ‚O?ý”Y³fqå•WI’$I:Àoø KXB5ªÑ”¦LfrБ$I’$IÒI`É@’$I’$I:F—r)ïó>{ÙË…\È f)0e(cÉàg˜2e Mš4¡bÅŠ,Y²„¤¤¤ #I’$I‡t:§óïp—qWð8S@Aб$I’$IÒ dÉ@’$I’$IúÎá°€6´¡#Æ° #Â’Á±{ꩧèܹ3:ubÆŒT®\9èH’$IÒÅËßøऑƕ\ɶK’$>ÿüsB¡¡Pˆž={rÌСC …BŒ7ÿ¥—^" 1qâÄãzÜ©S§ …xòÉ'ëq%I’Ž–%I’$I’$ég*C2È`Cx€èIOv²3èX'Ô^ö’M6+YÉT¦’K.Ó™ÎÝÜÍõ\O:ð~Ã,fµÐÙ³g}úôá®»îâü#cÇŽ%666èX’$IÒQ»—{™Á ³˜4`! ƒŽ$IRĸqãøàƒ‚ŽqJ™;w.¡Pˆ!C†z I’tòÅ@’$I’$I*ÊB„H#¦4¥=hA þÎßI$1èhÇ]i<Æc<EØÀ{¼GÞ¾-Š(Ò0 ”…ÓÚµk¹âŠ+Xµjo¼ñ]ºt :’$I’ô³´¦5+XAozÓŠV a÷r/!BAG“$cµk×æË/¿ä`Ò¤IAÇ‘$I*ò¼“$I’$I’t´¡ KXB Jp!2iAG:înäƃå“Oξ-<Îç|ÊQ.ˆˆ…ÒŠ+hÖ¬›6mbÑ¢E $I’TäÎé¼Å[<ÄC<Àt£ßñ]б$IÅØo~ó®¼òJ&OžÌ‚ ‚Ž#Ij> ? IDAT’TäY2$I’$I’Ž“Ô`sèJW.åR3˜ ‚ŽuÜœÃ9t¥+%(qØ1%)I:œÄT…Û„ HNN¦nݺ¼ÿþûœ{î¹AG’$I’Ž‹(¢Ä 2Éd! 9Ÿó™Â” cI’Š±!C†Åý÷ßTãwíÚÅÃ?̹çžKll,åÊ•£]»v¼óÎ;ݲe   jÕªÄÅÅѸqc&Ož|Øcð׿þ•-ZP¶lYâââhРÏ>û,Çö÷Â3fТE âãã9ãŒ3èׯ6løEó9š±C† ¡U«V<øàƒ„B¡È@^^#FŒ ))‰„„Ê—/OãÆyüñÇÙ¹sçQcêÔ©„B!ž|òIfÍšEJJ eË–¥qãÆ‘³gϦW¯^œ}öÙ”*UŠJ•*Ñ¥KæÍ›wÀœ~x¬£}Í$IÒáÅ@’$I’$I:•ÄË‹¼Hsš3€,g9csÊ\ÙÿîáMÞ<ìó9äЖ¶'1QáTPPÀc=Æ}÷ÝG¿~ýxæ™g(Qâðå I’$©¨jC>â#îå^.åRzÓ›gy–²” :š$©˜9÷Üs¹öÚk3f Ó§O§}ûö‡›““CûöíX¨¾gÏfΜɻï¾ËsÏ=ÇM7ÝÀîÝ»iÛ¶-+V¬ˆŒ]ºt)]»v¥{÷î»  €Þ½{3vìØÿàƒ0`+W®däÈ‘G5§ùóçs÷Ýw“—÷ß;ˆîÚµ‹Ñ£G3wî\/^L™2eŽy>Ç2öHî»ï>†~ÀcK—.eéÒ¥”,Y’Õ÷Ïóž{î!77€üü|Ö­[GJJÊc7nÜÈäÉ“™:u*3fÌ uëÖëh^3I’tdÞÉ@’$I’$I:RIe3XÄ"šÒ”ù8èHÇE2É\È…D}È磉&™ä“œªpÙ±cW_}5ƒ â駟æù矷` I’¤SZ9Êñ<Ï3 La ¿á7¼Ç{AÇ’$C?ü0%K–ä8â¸gžy†yóæQ³fM&MšÄÖ­[Y½z5ƒ& qÇw°nÝ:FŒÁŠ+¨[·.™™™lÛ¶¬¬,n¾ùfÆб_yåÆŽËùçŸÏÛo¿Í¦M›Ø¾};³fÍ¢AƒŒ5Š Õ|&L˜@ïÞ½ùì³Ïؾ};³gÏæüóÏç“O>aØ°a?k>G;vРAÌ™3€Gy„‚‚‚ÈÀ?þñJ—.Í믿Ζ-[رc+V¬àî»ïŽ,äÿ©cüpž×]wÿþ÷¿ÉÍÍeÙ²e„B!Ú·oϤI“øúë¯ÉÉÉaýúõdddPªT)†ú³_3I’td– $I’$I’¤¤­XÂÊSž¦4å Þ:Òqq/÷’Oþ!ŸkD£b}ÅÒ5kÖ’’¬Y³˜6mÚ1]±M’$I*ê®æj>äC~ͯiG;îænv²3èX’¤b䬳΢_¿~,^¼˜¿ÿýï‡7aÂÆOçÎ9í´Ó¨Q£=ô7Þx#»wïæŸÿü'¯¿þ:¡Pˆ×_víÚQ¦Lj׮ͳÏ>K»ví:ö‹/¾Htt4ï¼ó—\r *T téÒ´nÝšW_}€7ß<üB¨I“&üõ¯åì³Ï¦téÒ´jÕŠüã”(Q‚‰'þ¬ùËØ#9óÌ3©V­]»v¥\¹rÄÇÇÓ A†NŸ>}Žj~û5k֌ѣGó«_ýŠèèÿq“Ê•+óè£2fÌš6mJ||<•+W¦{÷îìرƒ?üðg¿f’$éÈ,H’$I’$I'ЙœÉ,fу\ÅU¤“~ØúEÅ\A j"tÀã%)I:”*xóçϧqãÆäää°xñbÚ¶mt$I’$餫B&1‰‘Œä^ >õy›·ƒŽ%I*F D||<>ø ùù‡þ;ÜçŸNÅŠiÖ¬ÙAÏuîÜ92fÿçêÕ«S¿~ýƒÆvìØñ Ç>úè#òòò¨Q£111DGGETTTä«W¯>ª¹\|ñÅ„Bþ .11‘_ýêW|ñÅ?{>G;öHžxâ òóó9ûì³éß¿?Ï=÷Ë—/?ªyýX8>hžðß¿·µhÑ‚ &MnnîÏïÚµë }Žö5“$IGfÉ@’$I’$I:ÁJQŠQŒâ/ü…'x‚.ta [‚Žõ³EÍÝÜMÔþ¼˜Cm)ž ëÇŽK»víHJJbÎœ9Ô®];èH’$IR úÒ—Oø„V´¢èB¾æë cI’Š*Uª0pà@V­ZÅ+¯¼rØq‡ZÔ~<ì/6äåå‘——G~~>DÆäää÷óË|ŽÇÜ4hÀ'Ÿ|˜1c¨]»6sæÌ¡cÇŽÔ¯_ÿw8’Š+òñ¡C‡’““ÃC=ÄçŸή]»"¯gݺuñ$IÒáY2$I’$I’N’TR™ÉL–³œ&4á_ü+èH?[_úRšÒ<C ÍiP¢çHoúæåå‘žžÎµ×^Kjj*“&M¢\¹r'1$I’TxU¦2cÃLfò)Ÿr>çóO‘G^ÐÑ$I§¸´´4Ê•+ÇàÁƒÙ»wïAÏŸ}öÙlܸ‘÷ßÿ çÞ~ûíȘýŸ¿ùæ>ú裃ÆN:õ ÇêÕ«G||<[¶l‰” ~ü1qâÄ£šÇ´iÓ('deeñ駟R§NŸ=Ÿ£õßå…?¾ƒÀ~111´nÝšôôt^{í5¾üòK¾ÿþ{úöíóSÇ8’¬¬,*W®ÌàÁƒ©S§±±±„B!¾øâ >ûì³Cîs´¯™$I:2K’$I’$IÒI”L2+YIuªÓœæLäðo(~ÅWü›ŸÄtG/žx2”ˆù$/¿ü2O=õTäMSI’$Iÿ_[Ú²œåÜÄMÜÃ=´  Yt,IÒ),!!»ï¾›/¿ü’—^zé ç»uë@=xûí·ùþûïY³f <ò#GŽ¤T©RtíÚ€«®ºŠ‚‚®ºê*fÎœÉöíÛùòË/¹å–[˜1cÆAÇîÛ·/;wî$3yòd6lØ@NNÿùÏxë­·¸êª«¹ß¡¼ÿþûôíÛ—Ï?ÿœ;v0wî\®¸â öîÝËÕW_ý³æs,c+T¨Àœ9sØ´iÓÙZ´hÁ_þòV­ZÅ®]»Øºu+S§NeÓ¦MdeeEÆé?¥fÍš|ûí·<óÌ3lݺ•­[·òöÛos饗Fîñs_3I’td1A$I’$I’Š›JTb:ÓÄ ºÓ{¹—?𢉎ŒÙÅ.ºÐ…Ó8¹Ì%ĉ¹}û/q+·òP’’\ÌÅ':þn¾ùf-ZD¯^½xã7"·’ÿüóÏéÚµ+[·neöìÙ4iÒ$ऒ$IRáOI•*U8p ¯¾ú*+V¬ ]»v‘1¡Pˆ=z0~üøö½þúë™5k/½ô]ºt9d¶~ýúÕ®¾újÆŒË/¾xÀãõêÕ#--ígÍçXÆžsÎ9T¯^™3grúé§GÆ°lÙ2,Xð“ó;Ò1~Jÿþý™2e dàÀ‘Ç5jįýkÖ®]{Ð>GûšI’¤#ó²Z’$I’$IRbˆa(CÞæi„ÙÀ†ÈóýèÇ'|ÂBò/˜ôðÎà ®åZbˆ!‡ÚЀ]»v1}útÒÒÒHJJâ´ÓNãÒK/åñÇçƒ>8ª7 ƒ‰'2sæL&MšÄC=ÀôéÓiÒ¤ ±±±,X°À‚$I’t Îç|Þå]ÞäM²ºÔ%t¶±-èh’¤SL™2e¸ÿþûù\É’%ÉÌÌ䡇¢nݺ”,Y’²eËÒ¶m[¦L™ÂM7ÝË»ï¾ËÿüÏÿP¹rebcciÔ¨o¼ñ;v<èØ¡Pˆ_|‘ñãLJIHH dÉ’$&&rùå—óÆo‡jÉÉÉL™2…&MšÇ駟Nß¾}™={6eÊ”ùYó9–±ÑÑÑLœ8‘–-[Rºté²-Z´ˆ[n¹…óÎ;/’-99™Ñ£GóÄOÕ1~Êe—]ÆرcùÍo~C\\U«V¥ÿþ̘1ƒR¥Jý¢×L’$Y¨ ¨¼£'I'PwºAFÀITTeddУG"³PFRá2räHRSSƒŽ!I’´œå\É•ä“Ïë¼Î<æqwPÀÇ(Mi>åSªQ-à¤û„O8óˆ.ˆæ÷ú=ïM}ùóç³{÷nêÕ«G»ví8çœs˜7o3gÎdÓ¦MœqÆ´k׎p8L8¦fÍšAOã ;wîäœsÎaݺu‘[¯‡B!úöíËK/½D÷îÝ=z4qqq'•¤S‹¿#KRáp²þ}¼‡=<É“ü?Pžò<Ê£\Ã5Dy½DI’tŒ¦NÊ%—\ÂO<Áí·ßtI’~±“õ»y÷îûÖÏf¸~6æ„ŸY’$I’$IÒ5¢‹XDwºÓšÖä)äÃÜÁxÆá('WVV™™™dffss {£÷òÌŸž!%%…§žzŠ:P«V­ÈøÛn»üü|>þøcæÍ›Gff&·ß~;Û¶m#111R8h×®*Tpfÿ5xð`¾ýöÛHÁ`¿—_~™þýû3bÄB¡P@é$I’¤SC)J‘F×s=ƒÄu\ÇŸøCB':O’$I’¤bËú¿$I’$I’TœÁü¿Q‚=·—½dÁd&ì¿6lØÀ„ èß¿?µkצN:ÜqÇlÞ¼™ë×_ÏÍgß̺uëÈÈÈ 55õ€‚Á~QQQÔ¯_ŸÔÔT222øî»ïX²d ©©©deeÑ«W/*UªDãÆIOO'33“Ý»wŸô¹®ZµŠÇœÜÜÜ/((   €ŒŒ ²³³Oz.I’$éTU…*Œf4ò!çp]èBsšó.ïM’$I’¤bÉ;H’$I’$I…Àöp9—³‹]ä‘wÐóQDÑŸþü›S†2'<ÏÎ;™?~änË—/' Ñ°aCzôèA8¦U«V”*U €5¬!ı]Ù?&&†¤¤$’’’HKKcûöí,\¸0rÎaÆGrrräN5"*êÄ^;åæ›o&**Š¼¼ƒ¿¹¹¹lÙ²…N:±`ÁâââNhI’$©89óÈ ƒ÷yŸx€‹¸ˆ0a†2”$’‚Ž'I’$IR±aÉ@’$I’$I*ná>àrÉ=äóùäó-ßò03œáÇýüyyy¬X±"²ÀÎœ9ìÙ³‡ÄÄDÂá0iii´oßžòåËrÿ39óg(S¦L¤L°nÝ:æÌ™Cff&#FŒ ==Š+Ò¼ysZ¶lI8&)éø.4;v,sæÌ¡  à°cöîݡ~H¿~ýxå•WŽëù%I’$Aš0éÌ`ðr!—s9ð€eI’tH;v<âßô$IÒ±9±—ü’$I’$I’ô“žçy^à…à öË%—Çyœå,?.çÍÊÊbäÈ‘tïÞJ•*Ѹqcžxâ xúé§Y½z5_|ñÏ?ÿ<ݺu;lÁàD©R¥ ݺuãùçŸgÍš5|ñÅüñ$..ŽG}”ÆS­Z5ºwïÎÈ‘#ÉÎÎþEçÛºu+·ß~ûOŽ+Q¢ùùùüë_ÿâ³Ï>ûEç”$I’txíhÇðoð5_Ó˜Æ\Ê¥Ìc^ÐÑ$I’$I:¥Y2$I’$I’vgq ×GQDs„FÅïøyäóy¾ýö[&L˜@ÿþý©U«uêÔáÎ;ïdóæͤ¥¥±dÉÖ®]KFF©©©Ô¨Qã—Lë¸KLL$55•ŒŒ 6nÜÈ’%K¸í¶Ûؼy3¤zõêÔ©S‡þýû3a¾ÿþûc:þƒ>È–-[yų%JP±bEn¾ùf–-[ÆŠ+8çœsŽËÜ$I’$Zˆ—q‹YÌæC-÷m“˜tá%^âù_ÒI§+]I%•0á cJ’$I’ThY2$I’$I’–••)¼óÎ;|ÿý÷$&&‡IMM墋.¢bÅŠAÇ,6*UªD·nÝèÖ­pà×ç±Ç#==3Î8ƒ””Âá0;v¤fÍš§–$I’t(õ¨ÇP†òð ¯ðgþL{Úsð;~GOzR AÇ”$I’$©P±d I’$I’$dk×®eîܹdff2yòd²³³9ýôÓiÛ¶-ǧ}ûöÔ®];è˜Ú'11‘ÔÔTRSSÉËËcÅŠ‘ÒÁm·ÝÆîÝ»#¥ý AÇ–$I’ôe)ËÍû¶¹Ìe4£I'»¹›Ë¸Œ>ôáb.&šè £J’$I’8K’$I’$IÒ ¶iÓ&æϟϼyóÈÌÌdéÒ¥ÄÅÅ‘œœÌ­·ÞJ8¦Q£FDEEU?!::š¤¤$’’’HKKc×®]‘¯kff&£G& Ñ°aÃHá U«V”*U*èè’$I’öi¹oÁ&0y‘Nt¢Õ¸ŽëèC~ů‚Ž)I’$IR`,H’$I’$IÇÙž/_¾ü€…çC‡¥eË–ÄÆÆU¿P\\\¤L°qãFÞ}÷]233ÉÈÈ`Ø°aÄÇÇÓ¢E‹È8 %’$IRáP–²Ü°o[Íj^ã5žçyåQÎã<ºÑ빞Úx§9I’$IRñbÉ@’$I’$Iú…òóóY¾|y¤T0wî\vïÞMbb"áp˜´´4Âá0 AGÕ vúé§Ó­[7ºuë@VVVäûbøðᤧ§S©R%Ú´iC8æâ‹/欳Π6´$I’$jR“4Ò¸‡{x—wy•WyŠ§x„GhMkzÒ“«¸Š T:ª$I’$I'œ%I’$I’$égøáâñ3fðÝwßqÆg’’ÂSO=E‡¨U«VÐ1°ÄÄDRSSIMMü¾¹ûî»Ù¶m[¤Œ‡i×®*¸hI’$I JQ´Û·=ÇsLa ¯ñ·qÀÅ\L7ºÑ….$`‘\’$I’tj²d I’$I’$… 6ðÞ{ï‘™™É´iÓøꫯˆ§E‹Ü{ｄÃa.¸àB¡PÐQUˆý°t››ËÊ•+#¥ƒ^½z‘——G£F"¥ƒ–-[tlI’$©X*E).ß·mcÿàŒg<©¤’O>miË•\Éå\Ne*W’$I’¤ãÆ’$I’$I’t;wîdþüù‘àË—/' Ñ°aCzôèA8¦U«V”*U*è¨*¢bbbHJJ"))‰´´4¶oßÎÂ… #ßsÆ #..ŽäääHé Q£FDEE]’$I*vÊR–Þû¶­lå-Þâïü;¹“[¸…d’#…ƒZxW;éçÈËËcýúõ|óÍ7¬]»–5kÖ°nÝ:¾þúkjÔ¨A•*U8óÌ3©Zµ*Õ«W§råÊDGGœ\’$I:õX2$I’$I’øïÙ+V¬ˆ,ðž3g{öì!11‘p8LZZíÛ·§|ùòAGÕ)ªL™2‘2Àºuë˜3g™™™Œ1‚ôôt*V¬HóæÍiÙ²%áp˜¤¤¤€SK’$IÅO9ÊÑs߶‹]d’É&ðq;·“H"éLºÐ†6ĸ4CbË–-|óÍ7dgg“}@`íÚµ|óÍ7¬[·Ž¼¼¼È>*T ZµjT¯^€%K–Íwß}M•*U¨^½:U«V= ˆP­ZµÈþþ=G’$I:6þ&+I’$I’¤b++++R*ÈÌÌdóæÍT®\™Ö­[óôÓOsÉ%—P£F cª˜ªR¥ ݺu£[·nÀ߯>ú(éééT­Z5R8èܹ3ÕªU 8µ$I’T¼ÄG—}[9Ìb“÷mOó4•¨Ä¥\Jg:s1s§Y:®rrrظq#k×®%;;û€ÏYYYdggóÍ7ß°uëÖÈ>¥J•ŠªV­JÆ ¹òÊ+©Zµjä±5jpÚi‡þyÙ³g›6m:ä¹>ùäfΜɚ5køþûï#ûÄÆÆ’@µjÕHLL<à\û?׬Y“²eËžð×L’$I* ,H’$I’$©ØøöÛo™5k™™™L:•Õ«WSºtiš7oNZZáp˜ .¸€P(tTé ‰‰‰¤¦¦’ššJnn.+W®Œ”Hÿþý#wÞ‡ÃtèÐá° 2$I’$%)Iû}ÛS iH˜0éL ZETб¥ÃÚ¼yóAÅ/ê_¿~=ùùù‘}X¸Ÿ””tТþ*Uªõó¿÷K•*¹KÁ‘îð·k×®Cf^»v-K—.eòäɬ^½šÜÜÜÈ>±±±G,"T«VZµjý³óK’$IE%I’$I’$²vìØÁ‚ " ±—-[Ftt4 4àšk®!ÓºukJ–,tTé˜ÄÄÄ””DRRiii}¯5*ò½¾¿t’’B‰%‚Ž.I’$‰$rÛ¾m›x‡w˜Æ4Æ0†a £•æâ}[5¼3™NŽ/¾?Tà믿fïÞ½‘}ö/¾ß¿Ð>99ù Åø5jÔ(T¿wÆÅÅ‘˜˜HbbâÇý°LñÃ"Bvv6«V­";;›uëÖQPPÙ'!!áˆE„ýŸ%I’¤¢Ê’$I’$I’Nyyy¬X±"²ÐzöìÙäääD®îž––æÕÝuJ*]ºt¤L°~ýzfÏžMff&¯¾ú*Æ £L™24kÖ,2λvH’$I'OE*ÒsßV@ò!Ó˜Æ;¼ÃÍÜÌnvSŸú´Ý·¥BE*[ELNN7nüÉÁæÍ›#û”,Y’Š+FÆ'&&T ¨^½:åË—pf'VBB Ô¯_ÿ°cvïÞÍwß}wÈ"BVVsçÎeõêÕlß¾=²Ï‹‡ú|ÖYgQºté“1MI’$é˜X2$I’$IR‘–••)L›6­[·RµjUZ¶lɈ#èÔ©Õ«W:¦tRU®\™nݺѭ[7àÀŸ“¡C‡’žžN•*UhÕªáp˜K/½”3Ï<3àÔ’$IRñ"Äoömws7;ÙÉlf3ƒ¼Ë»ü™?S@çs>miËE\D+ZQžSw‘·~Ú¯¶¸ÏÿùÏÈËË‹ì“pÀ‚öúõëGþy J•*DEE8³¢aa Zµj$%%vÜþ»Dü¸ˆ°víZ–.]ÊäÉ“òët¨ÏµjÕ"::údLU’$I,H’$I’$©ˆY·nsæÌ!33“·ß~›5kÖD®Ð~ß}÷‡øf¯T%&&’ššJjjêAwü¿óÉÑ IDAT¸õÖ[éß¿äŽáp˜‹/¾˜råÊ[’$I*≧㾠`;ÛYÈB2÷mOó4!BÔ¥.I$Ñ’–$“ÌyœGïNVÔíÞ½›ììì#~ê ùIIIüsbb"5jÔ D‰άxŠ‹‹#11‘ÄÄÄ#ŽÛ¼yó!‹ÙÙÙ¬Zµê'ï8q¨ÏuêÔ9¥ï8!I’¤“Ë’$I’$I’ µíÛ·³páÂÈ‚èeË–Mƒ èÕ«áp˜”””BùÆùš5k¨Q£Æ=ñÄÜ~ûí%:²Ñ£GÓ¯_?þüç?sÓM7ýìc=óÌ3 8€Q£Fqã7—ŒG’žžÎ°aØ2e ;þwNff&íÛ·çÃ?ä׿þõ É·|ùr†άY³Ø±cçw÷ß?;wþEÇ=¢££IJJ"))‰´´4vîÜÉüùó#?c£F"**Š† FJ­[·¦dÉ’AG—$I’Š…2”!¼oØÈFf3›yÌc ÏxrÈ¡2•iNs’I¦9ÍI"‰Xb{žmlã+¾â|Î?îÇ>Õi!ù?ÿPBBBä.‰‰‰$''rA¹Š¶„„„Èï懳ÿ®‡ú¾Ùǯ¿þš½{÷FöùqåÇŸ- H’$éhY2$I’$IR¡’››ËÊ•+# žgÍšE^^5"óÐCѦMÊ–-tÔŸtæ™gRPP@Ÿ>}˜8qâW,Œn¼ñF®½öZâââ~ñ± @Ÿ>}Nê×ièСôéÓ‡sÏ=÷'Çï|—\r mÛ¶eéÒ¥ÄÄÄðàƒÒµkWÞzë-.¹ä’ãrŽ%>>>R&Ø°aï½÷™™™Œ7ŽaÆQºtiš7owÁ yÅTI’$éd8Ó¹rßK.+YÉ\æ²”¥<Ã3ÜÃ=ÄCL2I$ÑšÖœÅY¿øüð­iMoz3„!œÉ™¿ø˜EÝþÅßG*¬^½šÜÜÜÈ>ûï/Ô¯_ÿ àµjÕ":::À™©09–»"îûpÞ¼ydgg³~ýzòóó#û$$$±ˆPµjUªT©BTTÔ‰ž¦$I’ )K’$I’$I Üþ«¯effòÎ;ïðý÷ß“˜˜H8&55•‹.ºˆŠ+S:¬øøxFMéÒ¥1bcÇŽeÔ¨Q…¾dðc•*U¢[·ntëÖ 8ðçó±Ç#==3Î8ƒ””Âá0;v¤fÍš§–$I’ŠbHÚ·í÷_0Ÿù,`³˜Å³µjÕ⫯¾úÉ|Û·o?àNM›6eáÂ…|õÕW|••uÀycbbˆ‹‹cÓ¦M?ùúv‰‰‰¤¦¦’ššJ^^+V¬ˆ”n»í6vïÞ)íÿHHH:¶$I’T¬ÔÙ·õ¦7ÛØÆ"EŠÿËÿ²‰M„q6gÓèGÛœqØc¯bÑD“CÿÇÿñÏq?÷s·KìI™ã/u¨«½ÿ¸@°nÝ: "ûüðj‰$''T ¨ZµªwzS¡W²dIªU«FµjÕHJJ:ì¸ýwé8ÔÏÊG}Dff&_ý5{÷îì³ÿ.‡+"T«Vš5kã25I’¤¢Äÿz“$I’$IÒ ·iÓ&æϟϼyóÈÌÌdéÒ¥ÄÅÅ‘œœÌ­·ÞJ8¦Q£FÅêìW]uõêÕcÕªU”,Y’Áƒ‡Ù¾}û!Ç0€>}úP¶lYþüç?óÈ#ð‡?ü_|‘Ûn»””RRR€ÿ.Ø¿ì²ËHOOgòäÉlܸ‘ë®»ŽÖ­[³lÙ²ÈÂù×^{ðÈ#pË-·ðÕW_qï½÷tî7ß|“+®¸‚Aƒ1}útV¯^Íoû[®¸â 233r1ÅóŽ3†÷hŽ?dÈ-ZD©R¥xóÍ7éÑ£K—.=â•ú†J8¦}ûö|øá‡üú׿>¦|yyy$&&Ò²eK^yåÎ:ë,Ö¯_ÏE]ć~xÈó¾÷Þ{|ûí·ôìÙóˆ¯MQMRRIII¤¥¥±k×®ÈÏuff&£G& Ñ°aÃHá U«V”*U*èè’$IR±R–²„÷mûe“ÍR–²ŠU|ÄGü¿q?÷S@U©J}êsçEî’pç"Ä|)ìÝ·ÝÏý<ÅS f07r#Qó;ý‘Eïÿß«W¯&777²ÏE‡ÃaEK@\\‰‰‰$&&qÜ‘J;™™™dgg³~ýzòóó#ûü¸´óãŸ9K;’$I…K¨à‡lI*¦ºÓ€ 2N¢¢*##ƒ=zàÿ­Jú9FŽIjjjÐ1$I:®~¼ðxùòå-}˜8qb¤@°{÷nâââ?~<Ý»ÿ÷wÒÜÜ\ªW¯Îúõë{œýWÖ¿á†xá…ÈËËã´ÓNã¾ûîcРAÔ«W%J°~ÅŠ4jÔˆ¡C‡’––@ݺu‰ÿìÝ{\”uÞÿñ×0ÊI< 2bc[–¥à䪀`r™‡Lí½¤™]·ÜÑVM·]­-ÛlÒ-ËjS·Ýj;دûÞ»MSkÜ< Ù¯DØZî¶ßJ ¨ ”Ãïåº9‹Š èû9#Ã÷º®Ï÷º``˜ïûûõ÷gïÞ½f»¿þõ¯üèG?âå—_fÖ¬Yf;ooï:û[¿~=wÝu[¶laÔ¨Q-®·ººšnݺ‘œœlÖ{©ûŸ8q"aaaüá0ûú믹馛ظq#cÇŽÀív72hi}O?ý4O>ù$………tïÞ€åË—Ó©S'æÏŸ_§¦£GòÖ[o±xñbºtéBFF½zõj´þ«Qqq1Ÿ|ò‰ù½Ÿ““ƒ¿¿?111æ÷þµ(‘‹£×È""탞¯Å³·Þíÿñÿ¨¢ 6¢‰æ ¾à8ÇÝÞ /ª©f ù=¿g¿~»ååå”””4 8pàÇŽ3·ñññ!00°ÙYÕ#""Z´*Ÿˆ\žÆ¾‡ë¯’ŸŸÏñãÿûüRû{¸© BŸ>}ê¬<)"""rµj«×æ5ïU¾ûnÝñ³Š\‹ˆˆˆˆˆˆˆÈe«ªªbïÞ½æÀâôôtÊÊÊp8†Arr2†a`³Ù<]j»àëëË!CX´h‹… &àçç×lÀ ¶[o½Õü¿Õj¥gÏž:t€üü|þõ¯1{öì:ÛDEEÑ­[7Ün7ÉÉÉ9r„o¾ù†9sæÔi7tèÐ:ççç7Û®¹@cõZ,‚ƒƒëÔ{©û ä›o¾iöØ-Ñ\}3fÌà‰'žàwÞaæÌ™¬[·Ž?þ¸Á¾æÏŸÏúõë™:u*K–,!$$ä²ëëHzöìIRRIIIäää˜Ï +W®$%%…àà`FŽ‰aÜqÇôíÛ׳E‹ˆˆˆˆ\ÃzÒ“Ñço5Nr’,²ØË^þ/ÿ—¿ó÷&·¯âÜ,åÿäŸ$’ÈHFòÏ1ˆAÍ·±YÐëBnnt»ÝŽÓél0Y³ ‹´>>>„……†Óél²]ýÕHjöìÙÃúõë›\¤© BXX‘‘‘X­Ö¶èªˆˆˆÈUI!¹$5ƒ‡Ýn7[¶láÈ‘#„„„ÀÊ•+3f ‘‘‘ž.³ÝÚ¼y3K—.eþüùL›6 Ã0xì±Ç òoL@@@;wîl¼(..Î À¯/((Èü|aaa£íê\Ó~ÕªU¬ZµªÁ>sss/º^//¯õ^hÿÙÙÙ<úè£ìÚµ‹C‡™+ÉEEE]ðø—S@pp0?þñyõÕW™9s&»wïfРAôèÑ£Ñý­ZµŠ)S¦\v]W‡ÃËå2gÚ©ý¼ñë_ÿš'N˜a$Ã0HLLlôkWDDDDDÚNÄž¿}Æg¼Áܦ‚sƒÓI'šhÆÇÄÏ'R¶¯¬Áàá¼¼<Ξ=kn[3X¸f`plllƒÁÃtîÜùŠõYD<ÇÏχÃÃáh¶]ípRý²³³)(( ¨¨Èü›œ '5D¨¹‘†2‘9|ø0[·nÅívóÑGñÝwßáïïOLL .Ä0 ¬[Èf³±bÅ V¬XÁîÝ»yä‘Gˆ';;›~ýú]ò~{öì À‘#G|®¤¤ÄÜ·Ýno´Ý±cÇÝßÂ… IMM½äº.Tosû?{ö,†aлwo¶nÝÊ~ð¬V+Ó§O'33³ÕkjÌìÙ³‰‰‰!;;›W_}•_üâmrÜ«MíÐAEEYYYfèà¾û²’èèh3t‡¯¯¯§Ë¹fe“+•T^¸ñY¨°T@'ø°×‡l¼a# ¤ïþ¾ØívGƒAxx8Ý»w¿âý‘ŽÏf³a³Ù0`@“mÊÊÊ8räH£A„œœÒÓÓÉËËãĉæ6õƒNÝGFF6˜¤BDDDäj§ˆˆˆˆˆˆˆˆ4êûï¿g×®]æà½{÷b±XˆŠŠbòäɆÁˆ#ðñññt©NQQ£GæË/¿`Ø°a¬Y³†~ýúñÅ_\VÈ <<œo¼‘­[·Öy<33“ãÇcpnÅ‚n¸ôôô:í222ì¯ÿþ|þùç Ž5hÐ /^ÌäÉ“/«Þ í?**ŠÂÂB~øaú÷ïo~¾¼¼¼EÇðòòºäúj >œèèhV­ZÅþýûø€[o½Õ3E¶Ðœ9s˜>}:]»võX Ë–-cúôéÜtÓM«¡µ´‡óy9xð`,‹§Kiw^xáòóóÍ ƒxNEEYYYfè`ûöíœ9sÆ\yÅ0 ÆŒS'T#"ƒ^#‹ˆ´z>öœÓ§OSXXØl€`ÿþýTVVšÛØl¶&ªÖÜGFFbµZ=س+¯öJõ½õÖ[Üwß}¤¥¥±`ÁóñÍ›7óøã“••eôÓÒÒÌñj¬_¿ž%K–ðÕW_ѽ{wî¾ûnÒÒÒÌ×ûöíãá‡fçΔ——“˜˜HJJ Æ k´Ö”””¯-###ùî»ï8yò$]»våå—_&;;›×^{€€fÎœÉc=vQu5¥v¿{öìÉСCY°`C‡έxÓM7±qãFÆŽËüùóY¹r%_~ù%·Ür ï½÷ž ã7˜6mÚÏEsý¾PV­ZÅܹss¯ÏÿùϲnÝ:JKK™ÊÇLnn.S¦Láî»ïÆívc±Xøÿøú÷ïOvv6ÞÞÞ,Y²Ã0 <,[¶ Ã0=z´9h¿¾—_~™'Ÿ|’§žzŠ×_9sæ@BBB‹ëjLM¿/^̆ (//gÁ‚$&&6YïsÏ=Ç„ =z´ùØüc>Lppp¶Í‹æú}¡þÌ™3‡éÓ§ÓµkWV¬XAZZË—/çOúéééM]úŸÏ¥K—ðÙgŸáããÃûï¿ÏäÉ“Ù³g hQÿšÒÒ}·TK®}KùöÛo3gΞ|òIfÏžÍwß}ÇÂ… /ú˜—r}¶nÝÊ¡C‡˜:uêEŸƒŽÈf³a³Ùš½ægÎœ¡¸¸¸É Bvv6ûöíãèÑ£æ6>>>6D§{÷îmÑM¹Šh%´’\>­d "—C³‚‰ˆHkª¬¬$33S³{‹´’¥K—ò›ßü†=zðÄOðÐCyº$iƒ²}ûö:«¶0lØ0­Ú"ÒÎé5²ˆHû çã‹S{vê¦EEEuÞGª=;uÍ ÐúDív»~g½Í­dгgOºwïξ}ûèß¿?;wæË/¿4ÛdffͲeËHNNàÆoÄÛÛ»N»õë×s×]w±eËbbbðóócݺuæ  ôîÝ›ƒ6Y¯Ûínt°}ÍÌû÷ß?ü㨮®¦[·n$''ó裶¨®Q£F5zÜþýûãííÍ?þñ:ÇŒŒŒ¤¤¤h|öüÆê-..&88Ø\É ¬¬ì‚碩~·¤?5çfÆŒ¬Y³¦És[[KÏgc&NœHXXøÃZÔ¿–ª¿ï¦4v-.õÚ7vÌo¼öîÝk>ö׿þ•ýèGuV2hÍësôèQÞzë-/^L—.]ÈÈÈ W¯^Íž©«fÕ›ú?ojÿ ÊËËãìÙ³æ6õƒký êÓ§:iÎb‘öB+ˆˆˆˆˆˆˆˆÈeÉÉÉ1C}ôÇŽÃn·Ç /¼ÀwÞIïÞ½=]¦H‡ôè£6;à@Ú§^½z‘””DRRP÷yrÙ²e¤¤¤ʈ#0 ƒñãÇîáªEDDD¤½©?ˆ³±Ann.æ6õq†Ñ`@§qzFíG~~>ÿú׿˜={v6QQQtëÖ ·ÛMrr2ùùù|óÍ7Ì™3§N»¡C‡˜ƒ«‡ ¢E‹°X,L˜0??¿‹t^ß­·Þjþßb±Ì¡C‡Ìú[RW}Mõ; À \__ßK:ÛŸ^tmÍϦòÍ7ߘ_jÿZ²ï–ºÔkßØ19Òì¾.õ˜º>óçÏgýúõL:•%K–Òl{iÈÏχÃÃáh¶]í\ýŸe;w €ƒRUUenS;×TN!8‘kƒ^µŠˆˆˆˆˆˆˆt0yyyæ`Ù-[¶pðàA5j”¹ìüõ×_ïé2EDÚ ‡ÃËåÂårQQQÁgŸ}Æ–-[p»ÝÌ™3‡ŠŠ n¹ås•ƒøøx<]¶ˆˆˆˆ\!eeePPPÀÌŸ‹ŠŠÈËË£°°púôis???z÷îÝn'""‚¡C‡Ahh(ááá„……†¯¯¯{&M9v쥥¥ 487?œt]_PPùùšûU«V±jÕªmsssؼy3K—.eþüùL›6 Ã0xì±Ç Ö¾õ_“xyy™[ZW}Íõ»µ\ʹ¸Øþøùù]t]ÍO€ììl}ôQvíÚÅ¡C‡ÌPJTTTí.¥-ÝwK´ô\µä˜………@ï‡ú_‰ë³jÕ*¦L™rÁvryl66›4Ù¦öÏÄüü|ógaQQß}÷Ÿ~úé&†††Ýn§wïÞú™(""r•ðòt"""""""""ÒvvîÜIPPÙÙÙž.EDDDDDD¤Müío£ººšñãÇгgOàÜLîõ•””˜Ÿ¯¹_¸p!ÕÕÕ þ½ñÆÀ¹¼+V¬àÀlÛ¶²²2âããÙ·oßéOKëjj»Æú}!^^ç†={Ö|ìøñã Ú]ʹ¸Ôþ´–³gÏbyyylݺ•³gÏR]]ÍOúÓ:+`\Jÿ.fß-Ñ’sÕÒcÚív á×ñcÇ.ú˜""""rõÑJ""""""""LDD?ûÙÏøÙÏ~@NNŽ¹²AJJ ÇŽÃn·‡aÜyçôîÝÛÃU_ýŽ=Êœ9søýïß®—y¯ªª2ßn+ùùùDDD4xÜÇLJ믿žûï¿Ÿ‡zˆNZïÏ•n·›Ñ£Góå—_rË-·´Ú~[ÓôéÓY°`A³³ÉIë¨ý<ùñÇsôèQBCC1b«V­büøñ„‡‡{ºLi#¾¾¾8G³íNŸ>Maa!’““cþÿ³Ï>ã¿ÿû¿ÉÍÍ¥¢¢¢Î¾Ã°Ûí„……áp8Ìÿ×Ü÷éÓ§U_ÿHó<È#ÿüóû4h‹/&!!Á|ý 0lØ0Ö¬YC¿~ýøâ‹/èׯ_£uÕ Ú¿-©kòäÉnwã7²mÛ¶:àp88pàAAA38ÎýÅ’ IDAT8€¢¢"ó±üãuÚ]ð\4ÖïKíOkÉÉÉ¡°°‡~˜þýû›———×i×’þ]ê¾[ª%ç***ªEÇ ä†n ==½Îã}Ì‹¹>þóŸ[ÜV.Oii©ùs«±Ÿe}:ï½÷'OžÎ͹zõj~õ«_ñõ×_³zõê6­ËÓæÎ˸qãøàƒ4h§Ë¹ªøàºvíj¶Y¾|9“&Mâ‘Gáᇦ¸¸˜™3grýõ×óàƒšíV¬XÁĉY¶l3fÌàw¿ûLš4‰£GòÕW_ñì³ÏòÀPUUÅ+¯¼‚¯¯/C† i²ÆšYä¿þúkBCC¹õÖ[ùàƒê oÎ…êjJM¿ó›ß0þ|Nž<ÉÌ™3ùéOÚdÀà†n $$„—^z‰¡C‡røða^{íµí.t.šê÷¥ö§5ôíÛ—àà`Þxã ÆÃá`ûöílܸ‘¾}û^Tÿ.gß-u¡se±XZ|Ì%K–0uêT–.]ÊìÙ³ÉËË#--í¢ÙR¥¥¥DGG3tèPÖ­[wIý—†!¸Æ~åååÕYy¤~.66¶ÁÏ …àDDD¤6Ku[NY&"ÒNÝÃ=¼Ë»®D:ªwß}—É“'·éL "rõX½z5.—ËÓeˆˆÈUêÔ©S|úé§fè ##«ÕÊ AƒÌÐA||<ÞÞÞž.µC›5kû÷ïgãÆž.¥]«2¨ÃgŸ}ÆáÇ l•cu„• žzê)Þ~ûm²²²°Z­ž.§ÃjÉs]BB;wöt©"rz,"Ò>èù¸õ•——SRRÒh¡æ>//'N˜ÛÔÚØ}dd$ì™ç5¶zžÅb¡GÜtÓMLš4‰Y³f5:á¦M›xüñÇÉÊÊÂßߟñãÇóÌ3Ϙák|ôÑG<þøãìÝ»—=z˜˜Hjjª¹"Ú† X¹r%œ={–òÄO0jÔ¨fkŸ={6o¾ù&ÕÕÕL›6#Fpï½÷šŸ¿ï¾ûX¶lYþõë×ÿûß-ª«)µûÄ”)Sxꩧðõõ%%%…ÔÔÔ:5üå/ν֞7oß~û-?üáY±b·ÝvcÆŒaÓ¦M-:õûýâ‹/^°?ï¼óNsç¬÷èÑ£É~Öߦ¹óùùçŸó«_ýŠ½{÷b³Ù7nGŽá½÷Þ °°ÐÐÐKºÖ-Ýw}Í]‹ ]û‹9æK/½Djj*$::šßÿþ÷ÄÄÄ0tèPvïÞÝjק´´”Aƒ1lØ0Þ}Wã3ê;sæ ÅÅÅjî÷íÛÇÑ£GÍm||| lôçDM€ <<œîÝ»{°g"""r)Úêµù=÷œ?[ï÷3… DDPÈ@.ŸB"r9ô†ˆˆ´¥C‡±mÛ¶:³{wéÒ…áÇ›q5»÷Å9qâ!!!üçþ'ãÇ`ÕªUÌ;€^xþóŸ¬[·ŽÒÒR&OžLhh(+W®0Á¿÷Þ{$%%ðÆo0mÚ´:ûyùå—ÉÎÎæµ×^# €™3gòØc58^KÛ­Y³†3f´xÛ/¾ø"iii|ã¹1M… ~úÓŸòú믓••ÅÀÙ¼ysƒAiiiuyìÛ·‡~˜;wR^^Nbb")))æLõµß„ˆŒŒä»ï¾kQ¦M›Öìõ{çwX¿~=K–,᫯¾¢{÷îÜ}÷ݤ¥¥™Wš«±Fii)ÁÁÁlÞ¼™ÄÄÄ&ÏÔUQQAVV–*ؾ};gΜѪ-"W½Fiô|ì95³S7D(,,dÿþýTVVšÛÔ^¡©ûÈÈH›ED<¨öª7MÝ÷ÝwTUU™Û4÷ü^  ÅËË˃=‘+E!‘v@!¹\ ˆÈåÐv""âI999æ ]·ÛMii)½zõ">>Ã07n\ƒÙ¥®7ß|“iÓ¦QRRRgþ“'OÒµkWúöíKZZãÇçOúééé¼óÎ;δ_\\l.g?mÚ´:û8p O>ù$·ß~;úÓŸ˜7o[·n%!!á’ÚÕ„ .fÛ·ß~›©S§òä“O2{ölrss™;w.;vìàå—_fÖ¬YÍž«–¬d°k×.&MšDJJ ¿üå/)..æ'?ù ¥¥¥dddеkW¢¢¢èß¿?Ï?ÿ<ÞÞÞ,Y²„µkךûnj%ƒ–öáB×ïý÷ßçî»ïæÑGå—¿ü%¹¹¹L™2…°°0Ün7‹å‚5Ö¸å–[ˆ‰‰aõêÕ-ý²»&Õ~¾úè£8vìv»¸¸8 Ã`„ „……yºL¹Lz,"Ò>èù¸ý+--m6ˆPs_ÃÛÛ›   fƒ‡›ÍæÁ^‰ˆt œ›M-%%…;v\RŸJJJX½z5Ÿ~ú).—‹ÀÀ@~ýë_sóÍ7óÔSOгgOV¯^Mtt4/½ôÉÉÉ”••‘••ÅâÅ‹ `ùòå¼ýöÛ<æÅö¡©ë·páB Àoû[s?Ï<ó wÝuŸ|ò 111-®ñºë®###ãbNÝ5¡¨¨ˆ;vàv»ùðÃÉÏÏ'((ˆádzhÑ" ÃÀétzºL°Ùl8Îf'nnàkNNéééäææÖ BkૈÈÿºÜ@—Ãá 66¶Îóh¿~ýô÷Ué2¬V«9@!99™ï¿ÿž]»v™¡ƒµkך³³×„FŒ§K÷¨¢¢¢fß8p`«çÖ[o5ÿoµZéÙ³'‡ºäv{Œ#GŽðÍ7ß4IÜvÛmÕS§Na±X€so¼^ýõ,_¾œyóæ‘ŸŸÏ¿þõ/fÏž]g›¨¨(ºuë†Ûí&99___† ¢E‹°X,L˜0???<Øì±/¥]¿üüüF÷3tèP¶lÙ¨Q£Z\c=Ø»wo³µ_ Nž<ÉîÝ»Íçœ={öàççGll,sæÌÁ0 ¢££ñòòòt©"""""‚¯¯/‡‡ÃÑl»ÒÒÒ&ÎîÙ³‡õë׳ÿ~*++Íml6[³A»ÝNhh¨~‘véôéÓ6 ¸ÐóÞ€<ÿEFFbµZ=Ø3‘Ö£ˆˆˆˆˆˆˆˆ4Êßßß >|˜­[·âv»Y·n©©©øûûc¶Ÿ››{Q5vîÜ™#GŽ4YÃÕª¢¢‚¬¬,3T°mÛ6*++‰ŽŽÆ0 –-[F\\¾¾¾ž.UDDDDäªf³Ù°Ùl 0 É6gÎœ¡¸¸¸É™¼³³³Ù·oG5·©?£wcA„ððpºwïÞÝ‘k@YYGŽi6@——lj'Ìmê¯àât:öü£ÕêjmÜpà ¤§§×yüóÏ?oµc„‡‡sã7²uëÖ:gffrüøqsU¢¢"FÍ—_~ À°aÃX³f ýúõã‹/¾ _¿~fˆãJô!<<œþýû7ºÝ AƒX¼x1 ¬±ÆÑ£G ½¨:ŠšADééé¸Ýn fäÈ‘,_¾œ;¾}ûzºLi%~~~8G³íj®¿sçN 8xð`•ùl6[³A»ÝŽÝn¿æVXéèꇓ äææRQQanSNªy0`@ƒç‡ÈÈH¬V«{&"""Ò±)d """"""""—ÍËË §Ó‰Óé$99™Ó§O³sçN3t°víZ, QQQfà ..___O—~Ù̉'8räÈEÍÀ~à 7ÂK/½ÄСC9|ø0¯½öÚ¬ôò-Y²„©S§²téRfÏžÍþýûyå•WZõË—/gÒ¤I<òÈ#<üðÃ3sæL®¿þz|ðA³ÝW_}ųÏ>Ë<@UU¯¼ò ¾¾¾ 2øßU ¾þúkBCC¹õÖ[ùàƒZ­+V¬`âĉ,[¶Œ3fð»ßýŽŠŠ &MšÄÑ£G/Xco¿ý–˜˜˜K=eíJqq1Ÿ|ò‰ù½Ÿ““ƒ¿¿?111Ì›7Ã0ˆŽŽn4""""""כ͆ÍfcÀ€M¶)//§¤¤¤Ñ BNN{öìáÀ;vÌÜÆÇLJÀÀÀfƒtëÖ­-º)rMkì{¸~€ ??¿ÎÊžµ¿‡N§³ÁÊ}úô¡k×®왈ˆˆÈµÁR]{(‘kÔ=ÜÀ»¼ëáJ¤£z÷Ýw™|ûí·üð‡?dÅŠÜvÛmŒ3†éÓ§×ÙÏ}÷ÝDzeˈˆˆ0ëׯK—.mQ»ùóç3wî\󱄄fÍšÕ¢mÿýïðÒK/‘ššÊ¡C‡2dÏ=÷N§“µk×òÀ4zŽòóóëì 66¶ÁŠ56mÚÄã?NVVþþþŒ?žgžyÆ lØ°•+W’‘‘ÁÙ³g8p O<ñ£F2ÛÌž=›7ß|“êêj¦M›Æ‹/¾Ø¢>´ôú}ôÑG<þøãìÝ»—=z˜˜Hjj*ááá-®ñèÑ£ôìÙ“Í›7“˜˜ØèùhÏêŠöîÝÛ P4bÄ|||<]ªˆtz,"Ò>èùX:’ú³ 76˜¹©YЛ "„……ѧO:uÒÜ"il5’ú‚æV#iì{N«‘ˆˆˆˆÔÕV¯Íï¹çüøÙw뎟UÈ@D… äò)d "—Co؉ˆÈµ¨°°ôôtÜn7ëׯ§  €ž={rûí·c£Gæºë®ót™-6kÖ,rssùðÃ=]J›ËÌÌ$::šmÛ¶ïér.‰'ûðôÓOóæ›o’••…ÕjmÓc_ŠÊÊJ233ÍPAzz:eee83T`6›ÍÓ¥ŠH¥×È""탞åjÔ’AÑEEEuÞïÓ h¹Ö´$´“——ÇÙ³gÍmÚ¹2<2Ðoo""""""""Òæìv;III$%%““cZ^°`ǯ3hyÔ¨Qy¸ê¦¥¦¦2bÄ^}õUî¿ÿ~O—sŬ]»–÷ߟ—^z‰^½z±oß>æÍ›GTTÇ÷ty-Òžú°gÏ^~ùeþö·¿µë€AíïÏ-[¶päÈBBBHHH`åÊ•Œ;–>}úxºL‘fÙl6l6 h²Myy9%%%®ÎÉÉ!==¼¼<ÿüó|øá‡ÜrË-mzÜ 9|ø0[·nÅív³yóföïßO—.]>|8 .Ä0 ¬™:EDDDDäªãããCXXaaa8Î&ÛÕÌö^%„ÂÂBöìÙÃúõëÉÍÍ¥¢¢Âܦf¶÷¦‚aaaDFF¶ëº´O­ÒQ?@pðàAªªªÌmj¯Òa·Ûq: V" ÅËË˃=ORÈ@DDDDDDDDÚ•N:át:q:$''sòäIvïÞm†ÒÒÒ°Z­ 4È\é !!ÁãƒÜm6o¾ù¦Gk¸ÒüýýINN&99ÙÓ¥\²öÔ‡×^{ÍÓ%ðý÷ß³k×.ó{,##///¢¢¢˜2e †a···§Kiüüüp88ŽfÛ•––6D((( ;;Ûü¸6›ÍÖl¡æ^®~5a–æyyyœ={Öܦ&ÌRóu[çc‡ÃADD„ÇÿŽ&""""íŸB""""""""Ò®˜a€¢¢"vìØÛíæÍ7ß$55•€€† f¶kn¶A‘k]ee%™™™f¨`ÇŽ”——ãp80 ƒäädî¸ãºwïîéREDDDDD:4›ÍfN¤Ð”²²2 Hž““Czz:¹¹¹œ7C~i©¹··7AAAÍz÷î}ÍÏ_VVÆ‘#Gš \hÅ §ÓÙà<÷íÛ—.]º4yÜŸðNsšì$tv²“Õ¬¦’J¢‰ÆÀ –XF2’®t½âçBDDDD® ˆˆˆˆˆˆˆˆÈUÃjµât:q:$''sêÔ)>ýôS3t°fͬV+ƒ 2Cñññx{{{ºt‘ËRÿk=##ÃüZŸ:u*†a@çÎ=]ªˆˆˆˆˆˆ\!ÞÞÞ„……™Ú›R3Û~cåkVÂËËËãìÙ³æ6õÌ×"„……Ñ!_wÖ^%¢©AQQÕÕÕæ66›Íì»Ãá 66¶ÑóÓZüðÃ88ÉIv³÷ù[iX±2ˆAf»èLÇ»""""Ò>(d """"""""W­.]º˜a€C‡±mÛ6Ün7o¿ý6©©©téÒ…áÇ›í4»»tdee™¡‚íÛ·sæÌsÕŽääd­Ú!""""""òóóÃápàp8šmW{ð}ýû;wRPPÀÁƒ©ªª2·±ÙlÍìv;¡¡¡xyy]én6S4 ÈÍÍ¥¢¢Âܦ&LQSó€ô'22«ÕzÅëoNuBE±ƒ¸qó6o“J*0Œaf»Á Æ‚þæ%""""-£ˆˆˆˆˆˆˆˆ\3BBBHJJ")) ÀœÏív“ššJJJ ½zõ">>Ã07n®ZäœÚ_¯}ôÇŽÃn·Ç /¼À„ Zu–D¹¶Ùl6l6 h²Í™3g(..n4ˆ““Þ={8pàÇŽ3·ñññ!00°Ù BDDD“ÁùòòrJJJš äççsüøqs___l6›y,§ÓÙ`å>}úеk×Ö;m(”P’ÎßrÈ1W9H%•RèE/â‰ÇÀ` cˆ$ÒÃU‹ˆˆˆH{¦ˆˆˆˆˆˆˆˆ\³.— —ËEee%™™™æ î‡zˆòòrsfxÃ0=z4=zôðtÙr(**bÇŽ¸Ýn6lØÀ bøðá,Z´Ã0p:ž.SDDDDDD®aÞÞÞ„……ÖìkÔ£GràÀ Ì@QQyyydff²aÊŠŠ¨¬¬4·© "ôîÝÀÜþÈ‘#f«ÕJhh(½{÷Æn·Ó¿FŽIxx¸YWïÞ½¯¹¿ç8pà:«¤’L2ÍÐÁ<æQFæ*‰$H §Ë‘vD!ν)ít:q:$''óý÷ß³k×.3t°víZ, QQQfè`Ĉøøøxºt¹Jœª¨"Š(3tG¾øzºtñ … DDDDDDDDDáïïo† >ÌÖ­[q»Ý¬[·ŽÔÔTüýý‰‰‰1Û <‹ÅâáÊ¥£¨¨¨ ++Ë lÛ¶ÊÊJ¢££1 ƒeË–‡¯¯ÞÔ‘«ŸÕj5W¨ãë¯ÏÝÿö·m_ÔUÈ?3Lp’“ìf·¹ÒAiX±2ˆAf»èLgW.""""mI!‘&))‰¤¤$rrrÌÁáiii¤¤¤BBB†a0fÌ"##=\µ´7µ¿n6mÚĉ'p8†Ëå"11‘ÀÀ@O—)"""""""׈ê„Š(b;pãæ-Þ"•T`ÃÌvƒŒM´!"""r5SÈ@DDDDDDDDä8\..—‹ªª*öîÝkŸ7oeeeæàñš6›ÍÓeK«*øûßÿNII ÁÁÁŒ9’åË—sÇwзo_O—)"""""""@(¡$¿äc®rJ*)¤Ð‹^ÄÁXÆÒ‡>®ZDDDDZ›B"""""""""—ÉËË §Ó‰Óé$99™Ó§O³sçNspùÚµk±X,DEE™ƒ¸¸8|}}=]º´²ââb>ùäÜn7ü1ß~û-þþþÄÄÄ°`Á à ::///O—*"""""""rA¸Îß*©$“L3t0y”Q†‡¹ÊA"‰¢UEDDD::… DDDDDDDDDZ™ŸŸŸ&())a×®]fð 55???bccÍvxÞ1Õ”ìÝ»× ”ÜsÏ=†Áˆ#ðñññt©""""""""—ÅŠçù[2Éœæ4;Ù‰7é¤ó*¯RM5QD™¡ƒ8âðEmˆˆˆˆt4 ˆˆˆˆˆˆˆˆˆ\aAAAÜu×]Üu×]’žžŽÛíæùçŸ'%%…ž={rûí·c£Gæºë®ópÕÒ˜ÊÊJ233ÍPAzz:eee8 à 99Ã0°Ùlž.UDDDDDDDäŠòÃÏ œàŸñ™¹ÒA*©øáÇ`G $Йή\DDDD.D!‘6f·ÛIJJ")) €œœsÐú‚ (;~œðóƒÖ Ã`Ô¨Qy¸êkWíë³eËŽ9BHH ¬\¹’±cÇÒ§OO—)"""""""âQ]éZ'tPD;Ø7oñ©¤@Ãf¶Ì`,X<\¹ˆˆˆˆÔ§ˆˆˆˆˆˆˆˆˆ‡9\..—‹ÊìlÎÜ}7Ûn¹…99Üwß}TVVaÄÆÆ2räHºvíê鲯Z‡fëÖ­¸Ýn6oÞÌþýûéÒ¥ ÇgáÂ…†ÁàÁƒ±Xô¸ˆˆˆˆˆˆˆHSB %éü ‡œ:«¤B/zO<cØ3¶ IDATK4‘ƒˆˆˆH{ ˆˆˆˆˆˆˆˆH{ñúëXgÏƯƦ¥1¶_?Nž<Éîݻ͙ôÓÒÒ°Z­ 4È\é !!εÌü¥úþûïÙµk—yŽ322ðòò"**Š)S¦`ñññx{{{ºT‘Ë×ù[%•d’i†æ12Êpà0W900°aótÙ""""×$… DDDDDDDDD<íôiHI^€¹sá™gàü€ö€€3LPTTÄŽ;p»Ý¼ù曤¦¦À°aÃÌvN§Ó“½i÷*++ÉÌÌ4C;vì ¼¼‡Ãa$''sÇwн{wO—*"""""""rU²bÅyþ–L2§9ÍNvš¡ƒµ¬Å‚…(¢ÌÀAqøâëéÒEDDD® ˆˆˆˆˆˆˆˆˆxRv6Lž EEð·¿Á„ Í6 %))‰¤¤óËÌç䘃åŸ~úiRRR°ÛíÄÅÅawÞy'½{÷n‹ž´kµÏÓÇÌÑ£G eĈ<ÿüóŒ?žððpO—)"""""""rMòÃÏ œàŸñ™:H%?üˆ=‹#ŽèŒV÷¹2ñ”×_‡„ÛnƒM›à‡—Ë…Ëåj0Cÿܹs™9s¦9C¿aŒ3†nݺ]δ/dûöí¸Ýn>ø×n+4W|HIIÁ0 ŒÅbñt©""""""""ROWºÖ QÄvàÆÍ«¼ÊÜl×Qf÷¯¨¨ ++Ë lß¾3gΘ«6$''›«6<Â#úˆcÇŽa·Û‰‹‹ã…^`„ „……5Øn)K)¤ÿà?؆1ÌÕ‹ˆˆˆˆˆˆˆÈ•`ÅŠóü-™d¾ç{v±Ë ¬e-,DeâˆÃ_O—."""Òn)d """"""""r¥> 3gÂ[oÁo ))àååéªp8¸\.\.•••dffšƒøzè!ÊËËÍ• Ã`ôèÑôèÑ£Íê+**bÇŽ¸Ýn6lØÀ bøðá,Z´Ã0p:Ü ¯ð 0‘‰ìd'?àmÐikþø›a€bŠù”OÙÉNܸI%?üˆ=‹#ŽèLgW."""Ò~(d """"""""r%åçÃ~ûöÁ† 0fŒ§+j”ÕjÅétât:INNæûï¿g×®]fè`íÚµX,¢¢¢ÌÐÁˆ#ðññiµNž<ÉîÝ»ÍcîÙ³???bcc™;w.†a×%4:Ó™ÿÃÿa$#Ç8v±‹Úv% i{=éÉ]ço…’N:nÜü‘?òO@Ãf†3 W."""â9 ˆˆˆˆˆˆˆˆˆ\)Û·Ã=÷@¯^ðùçàpxº¢ó÷÷7ÇfëÖ­¸ÝnÖ­[Gjj*þþþÄÄĘíŒÅÒò7_+**ÈÊÊ2CÛ¶m£²²’èèh Ã`Ù²eÄÅÅáëÛ:K×w¥+›ØD 1L`Ÿð ]èÒ*û‘ŽÁŽ¤ó7€rpŸ¿-c)¤J(#Á8ÆA„‡«i[ ˆˆˆˆˆˆˆˆˆ\ «WÃœ9çV1øã¡KÇÌLRRIIçß|ÍÉ1Ãiii¤¤¤BBB†a0fÌ"##ë죪ªŠÿùŸÿaçθÝn6mÚĉ'p8†Ëå"11‘ÀÀÀ+ׂÙÈFb‰e2“ùoþ›Nú3©ˆˆˆˆˆˆˆÈ5Ë×ù[%•d’i†â!Ê)ÇÃ\åÀÀÀ†ÍÓe‹ˆˆˆ\Qz÷LDDDDDDDD¤5••Á¬Yð—¿ÀSOÁÂ…p³ûw‡—Ë…Ë墲²’ŒŒ 3t0oÞ<ÊÊÊèß¿?‰‰‰üà?`çÎüý理¤„yî¹ç0 ƒ>}ú´ií×s=ð£Å/ø«YݦÇ‘öÉŠçù[2É|Ï÷ìb—:XËZ,Xˆ"Ê Ä‡/­³§ˆˆˆH{¡ˆˆˆˆˆˆˆˆHkÉË;·rÁ·ßÂÆ0z´§+jV«•!C†0dÈ-ZÄéÓ§IOO7C¯¿þ:qqq,^¼Ã0¸õÖ[±x8xñC~È;¼ÃÝÜMúð(z´iüñ7ÃÅó ŸN:nܤ’Š~Ä‹A,± c˜VΑO¿Íˆˆˆˆˆˆˆˆˆ´†­[aòd°ÛáóÏáºë<]‘Çøùù1zôhF·óÅ&ð"/2‹Y„ÆýÜïé’DDDDDDDD¤ëIO’Îß )4«XE )t¥+Cj†œ8=\µˆˆˆÈÅóòt"""""""""Þ3Ï€aœ[¹`×®k:`ÐѸp±ˆEÌb›ØäérDDDDDDþ?{÷U}çü•; „;;ñRI/jRª‚ZX°j)êÚ€ŠZ×®@ÝU¬k…ög]Û²­ÔÖ®·V¢U[ÝJƒ>Ô¢‹.Qi¬¶d­«bk›!päúûfšQ“œ$¼žó˜ÇdfΙùœ\&sÎ|Þ߯$©É$“B YÄ"Ö³žrÊù!?$™dîäN ( “L¦3"ŠXÏú K–$Iêg2$I’$I’>¬†˜=ü@Ðàk_ º"} XÀF6r)—ò2/sg]’$I’$I’ú¡!f<5ÓÌŸø%O7r#ûÙOˆPd–ƒÉL&™ä Ë–$IêÀ$I’$I’ôalÛ—^ «VÁ3ÏÀ¿tEú¢ˆb‹¨¦šiLc%+9‘ƒ.K’$I’$IýØ ‘ð4yìe/¯ñZ$tð0E§rj$p0 $té’$I† $I’$I’¤ìoƒ©SaÏøíoá´Ó‚®HQ,±,a Ÿãs\À¼Æk¤‘tY’$I’$I ’HŠ„ j©åU^¥”RJ(a! I$‘ñŒ,w§MtÀ•K’¤c‘ï@$I’$I’¤¢´Î<†?0‹ƒc(Cy‘‰&š©Le{‚.I’$I’$IÔHFRH!÷p«XE5ÕüœŸ"ÄýÜO g8S˜ÂBRFYÐ%K’¤cˆ!I’$I’$©«~ñ ˜4 &N„W_…ôô +R7K%•e,cë˜Á šh º$I’$I’$2ɤB±ˆõ¬§œr~ÈI&™;¹“ È$“éL§ˆ"Ö³>è’%IÒfÈ@’$I’$IêŠ{îk®¯~/†ÄÄ +RÉ%—¥,e+¸žëƒ.G’$I’$IÇ !f1‹bŠ©¥–U¬â&n¢Ž:näFF3š\r™Íl–°„ílºdI’4€Ä]€$I’$I’Ô§57à 7@QÜwüË¿]‘zÁ8Ʊ˜Å\Â%Œf4·q[Ð%I’$I’$é5ˆAä<Íc{ÙËk¼FÉÁÓÃx:›³‰'>èÒ%IR?eÈ@’$I’$I:œ½{á²Ëàþž| ƒ®H½h*Sy€˜Ã²ÈâZ® º$I’$I’$‰$’"a€Zjy•W)¡„¥,e! I"‰³8+²ÜiœF4ÑW.I’ú C’$I’$IRg¶n…©SáoƒW_…3Ï º"`³XǺHÐà|κ$I’$I’$©‘Œ¤ðà `#)¥”J¸û˜Ï|F2’38ƒ L`2“É'?àª%IR_f4Q’$I’$I:TUœslÚ+W08Æ-`Wr%—r)¯ózÐåH’$I’$IG”I&…²ˆETQE9åüÿA"‰|ŸïS@Yd1éQDUA—,I’úC’$I’$IR[kÖÀç>­­ð»ßÁI']‘E‹XÄÙœÍ4¦ñWþtI’$I’$IR—…1‹YSÌV¶²ŠUÌe.uÔq#7’C¹ä2›Ù,a ÛÙtÉ’$)`† $I’$I’¤°wÞ  9~û[5*èŠÔGÄË–Cp5Ô]’$I’$I’ô bùä3y,g9ÛØÆr–SH!e”q—1’‘PÀ|æSB ûÙtÙ’$©—2$I’$I’ÞxÎ=÷ÀÌ/¿ #G]‘ú˜¡ åE^$šh¦2•=ì º$I’$I’$é#I"‰ÉLæNîd«ØÄ&žäIòÉg K˜ÂF0‚)La! )£ŒZ‚.[’$õ0C’$I’$IÒK/ÁĉB/¾ÇtEê£RIeËXÇ:f0ƒ&š‚.I’$I’$Iê6©¤RH!‹XD9ål`ñ!BÜÇ}P@:éLg:÷pe”]²$Iê† $I’$I’tl{á¸è"øÒ—`Ɉº"õq¹ä²”¥¬`×s}ÐåH’$I’$I=&‹¬Hè Š*Ê)ç?øþ§€²Èb:Ó)¢ˆ*ª®X’$uC’$I’$I:v½ð\z)Ìœ =ƒ]‘ú‰qŒc1‹y”GYÀ‚ Ë‘$I’$I’zEˆ³˜E1Åle+«XÅ\æRG7r#9äK.³™Í–°íA—,I’>„˜  $I’$I’ñÔSpÅpÍ5ðàƒíxú`¦2•x€9Ì!‹,®åÚ K’$I’$I’zÍ ‘ð4yìe/¯ñ%Oó0QDq*§2ùàélÎ&g“•$©¯3d I’$I’¤cOqñÙ ¾òøÉO èC›Å,Ö±.48Ÿóƒ.I’$I’$I DI‘0À¶°‚”PB1Å,d!I$qgE–;ÓˆÆã³’$õ5† $I’$I’tlY¼®¼n¼~ô#ˆŠ º"õs XÀF6r)—ò2/sg]’$I’$I’¸TR)æ3Ÿ‘Œd"Ïx&0|ò®Z’$!I’$I’$Kžy®ºê@Àà®FDQ,bÙÈ4¦±’•œÈ‰A—%I’$I’$õ)¡ƒ§«¹8:(9xúwþì “L&0ÉLæB.d£®Z’¤c“ó I’$I’$éØðÒKpùåðå/˜Á@êF±ÄRL19äpPCMÐ%I’$I’$I}Zˆ³˜E1ÅÔRË*V1—¹ÔQÇ Ü@9ä’Ëlf³„%lg{Ð%K’tÌ0d I’$I’¤¯¤.¾ø@È ¨¢¢‚®HÐP†ò"/M4S™Êö]’$I’$I’Ô/ÄC>ùÌcËYÎ6¶±œåRHeÌ`#IÌg>%”°ŸýA—-IÒ€eÈ@’$I’$IÛÊ•Ó¦ÁÃC´‡ÄÔsRIeËXÇ:f0ƒ&š‚.I’$I’$Iêw3˜ÉLæNîd«ØÌfžäIòɧ˜b¦0…Œ` SXÈBÊ(£…– Ë–$iÀðUI’$I’$ \øœ>|á ðË_ AAW¤c@.¹,e)+XÁõ\t9’$I’$IR¿—J*…²ˆETPA9åü”Ÿ"ĽÜK¤“Ît¦SDïònÐ%K’Ô¯Å]€$I’$I’Ô#þüçá‚sÎ'ž0` ^5Žq,f1—p £ÍmÜtI’$I’$IÒ€:xºš«¨ ‚’ƒ§[¹•ì “L&0ÉLæ |l²®Z’¤þÙ $I’$I’4ðlØp`ƒÜ\(.†ØØÈ]·Ür ÔÖÖrÉ%—0dÈFŽ÷ÞÛáa^~ùe&L˜@RRÆ cÚ´iüùÏîÍ-Q?6•©<ÀÜÎí<Â#‡]î™gž!**Š7ß|³Ã}çw‘ëo¿ý6]tÉÉÉ$&&rÖYgñÛßþ¶Ý:ëׯçŠ+® 33“¡C‡2nÜ8~õ«_u߆I’$I’éH:’!f1‹bŠ©¥–U¬b.s©£Ž¸QŒ"—\f3›%,a;«Õã’¤þÀ$I’$I’–­[á¼ó`Èøïÿ†Áƒ;,ÒÚÚÊM7ÝÄÍ7ß̆ ¸ñÆ™;w.¿ÿýï#˼üòË|þóŸ'??ŸŠŠ ÊÊʨ¯¯güøñTVVöæ©›Å,¾Á7˜Ã^äÅN—ùâ¿Hff&?ûÙÏÚݾnÝ:^~ùe®»î:Þzë-Î<óLLYY6làüóÏgÊ”)”••EÖ›>}:555”––RSSÃ>ÈsÏ=ÇæÍ›{nC%I’$IêÓ‘Ô1ÄO>ó˜Çr–³m,g9…RF3˜A )PÀ|æSB ûÙßkõyûl† Æ­·ÞÊÇ>ö1{ì±È2·ÝvyyyÜsÏ=dddp '°xñböíÛÇ~ðƒ^Ú  XÀ•\É¥\Êë¼Þáþ˜˜®½öZþë¿þ‹}ûöEnä‘GHJJâŠ+®àÖ[oeôèÑüâ¿  1bÄn¿ývÎ8ã ¾ûÝïÐØØÈo¼ÁÌ™3ÉÍÍ%11‘ÓO?_þò—¤§§÷ÎK’$I’ÔC<¦#éÃÌ`&3™;¹“U¬b3›y’'É'ŸbŠ™ÂF0‚)La! )£ŒZz¬J’úC’$I’$Išš °ÞJJ`ôèÃ.:hÐ ¦L™Òî¶SN9…µk×°oß>þð‡?0uêÔvˤ¤¤0~üxV¬XÑÝÕk‹"ŠE,âÎaÓø+í°Ìu×]ÇÎ;yúé§hiiá±Çã²Ë.cèС444ðꫯ2uêTbbbÚ­{î¹çRZZ @ll,'Ÿ|2ßÿþ÷ùÕ¯~ÅöíÛ{~%I’$Iê%Ó‘ÔRI¥B±ˆ *(§œóc2Éä^2È`:Ó)¢ˆ *º½J’ú:C’$I’$IæÎ…+àùçá䓸hJJJ‡f†ÊŽ;ؾ};---¤¥¥uX7==­[·v[Ù:6ÄK1ÅäÃ\@ 5íî3f çw^dŠôåË—SYY™½®®ŽÆÆFîºë.¢¢¢Ú¿ûÝï²mÛ¶Èc=û쳜tÒI\}õÕ¤¤¤pæ™g²xñâÞÛXI’$I’zˆÇt$õ„!f1‹_ð 6°rÊYÀ¾Î×É%—,²"¡ƒ løÈÏéñ@IR_gÈ@’$I’$IýßÝwÃÂãøqG]<**êˆ÷>œèèh¶lÙÒᾚšRRR>t©:v e(/ò"ÑD3•©ìaO»ûgϞ͊+(//çá‡æÓŸþ4ãþ>6ŒAƒñío›ÖÖÖç––¿Oß~òÉ'óüóÏSWWDzeËÈÎÎæòË/ç…^èÕí•$I’$©»yLGRo‡Š)f+[YÅ*æ2—:ê¸Å(rÉe6³YÂv°ãC=Ç%I}™!I’$I’$õo/¼·Þ Â?þc·ôw¾óÞ~ûm¾öµ¯±yóf***¸üòˉåÖ[oíÖçÒ±%—\–²”¬àz®Üõ×^KQQ111Ìœ9³Ýz?úÑxÿý÷™9s&ï½÷õõõüå/á?ÿó?ù·û7*++™6m%%%lݺ•]»vQTTDCC'NìÕí”$I’$)Ó‘Ô“3˜ÉLæNîd«ØÄ&žäIòÉçWüŠ)La#˜Â²2Êh¡¥ÓÇòx $©/3d I’$I’¤þiýz˜:&L€ìö‡Ÿ2e Ë–-ãøÇ<§žz*qqq¬\¹’1cÆtûóéØ2Žq,f1ò( X¹ý+_ù Ó§OgØ°aíÖùô§?ÍÿøGÎ=÷\FŒÁÅ_LUUUäCÅÑ£G3{öl~øÃròÉ'3jÔ(~þóŸóÔSOù¡¢$I’$é˜à1I½)4 )d‹XÃÊ)çÇü˜d’¹‹»( € 2˜ÎtŠ(b kÚ­ïñ@IR_ÕÚÚÚt’´éL ˜â€+QU\\ÌŒ3ðߪ¤£¨¨¨ÝÔ—’$© ö탳φúzxí58+’>”"Š˜Ãæa®åZ^}õU&MšÄïÿ{Î8㌠˓¤^ç>²$õ ¾KêS¦ø<Ÿb?Ï—ÔÿTPAÉÁÓK¼ÄNv"ÄxÆ3 $¿šÌôIÓ=(Iê ·öͧ|¿]|Èûí˜fI’$I’$©»ýË¿@y9üá Ô¯Íb•T2‡9 Ù:„óÇœsÎ9~ (I’$I’$ !BÌ:xj¢‰·x+:¸aë 4|³„sxôŒGYÏzÎã<†1ìè,IR‹ºI’$I’$é¹÷^xì1xâ 8á„ «‘>²ïò]FNÉŒÌìŽÚÍ£>tI’$I’$I’ºY 1ä“Ï<æÑ:¹•ÖÌVÆFåË~™2ʘÁ F2’ ˜Ï|J(¡† Ë–$£œÉ@’$I’$IýÇÊ•ðõ¯Ã‚pá…AW#u‹(¢XW²ŽiL£Œ2šiº$I’$I’$I=¨¤¤¤Ãm5Ôð~C %üŠ_±… f0gr&“žNçt¢ˆ  bIұƙ $I’$I’Ô?TWCaápÁüùAW#u«Xbyš§ â. †š K’$I’$I’Ô‹ÒH£B±ˆ5¬¡œrîæn’Iæ.ÒIg:Ó)¢ˆ5¬ ºdIÒfÈ@’$I’$I}_c#üã?ˆðøãåHMx’Hb)K‰&š©Le{‚.I’$I’$IR@B„˜Å,Š)¦†Þá°€¯óuB„È%—«¹š"Š¨¦:àŠ%I‰!I’$I’$õ}ßü&¼ó,YC†]ÔcRIeËXÇ:f0ƒ&š‚.I’$I’$IRÀ¢‰&¼Hè`+[YÅ*f1‹län ›lrÉe6³YÂv°#è²%Iý˜!I’$I’$õmË–Á~?ù œrJÐÕH=.—\–²”¬àz®ºI’$I’$I}L 1ä“Ï<汜ålcËYN!…”QÆ f0’‘PÀ|æSB 4]¶$©‰ ºI’$I’$é°6l€«¯†/ùÀ¥tŒÇ8³˜K¸„ÑŒæ6n º$I’$I’$I}Ô`3ùà  †~Ão(¡„_ñ+²Á æLÎŒ,w:§ETÀ•K’ú*g2$I’$IRßÔÜ| Xœ ÷Þt5R¯›ÊTànçvá‘ Ë‘$I’$I’ÔO¤‘F!…,bkXC9åÜÍÝ$“Ì]ÜE¤“Ît¦SDkXtÉ’¤>Æ™ $I’$I’Ô7Ýqüþ÷ðÆ0thÐÕH˜Å,*©dsÈ"‹ó9?è’$I’$I’$õ3!BÌ:xj¡…7y“RJYÉJnáv±‹!&3™ñŒg2“É"+è²%I2d I’$I’¤¾ç÷¿‡ïî¿>ùÉ «‘õ]¾K5Õ\Ê¥¼ÌËœÁA—$I’$I’$©ŸŠ&šüƒ§¹Ì¥‰&Þâ-Jžã1hˆ„&3™ÏóyŽã¸ K—$õ"C’$I’$Iê[öî…k®I“`öì «‘E‹XÄF62i¬d%'rbÐeI’$I’$Ibˆ‰„æ1=ìá÷ü>:xˆ‡Ä >ͧ#¡ƒs8‡8â‚.]’Ôƒ H’$I’$©o™7jj ¤¢¢‚®Fêb‰åižf“¸€ x×H#-è²$I’$I’$ 0ƒ ÔPÃoø %”°˜Å,d!ƒÌ™œYîtN' çKÒ@bÈ@’$I’$I}Ç+¯ÀÀãCNNÐÕH}JI,e)ãÏT¦ò*¯2˜ÁA—%I’$IR÷yã xë­ö·UT¸,*jû§? ŸýlïÔ%IÇ°4Ò(2C’$I’$Iêvî„k¯…iÓ`æÌ «‘ú¤TRYÆ2Îâ,f0ƒgy–óJ’$I’Š­[aöl4¢£ÜÖÚzàò_E¾ŒA IDATÿõÀeK 47à /S£$ãB„˜uðÔB oò&¥”²’•ÜÂ-ìb!B‘Y&1‰R‚.[’ôE]€$I’$I’À-·@}=<ôPЕH}Z.¹,e)+XÁõ\ßé2¿àlbS/W&I’$IÒGtÞyœ| DÐØxàÜÔtà¾ÞÜ Ã†Á”)AW+IǼh¢É'Ÿ¹Ì¥˜b¶±U¬b³¨ ‚™Ìd$#É%—ÙÌf KØÉΠ˖$u!I’$I’$ïw¿ƒ‡†ûîƒÔÔ «‘ú¼qŒc1‹y”GYÀ‚v÷}ïq ×ð0T$I’$IRL ̘qq‡_&6®¸âÀ¥$©O‰!†|ò™Ç<–³œmlc9Ë)¤2ʘÁ RH¡€æ3ŸJh !è²%I0d I’$I’¤`54Àœ9Fª›>=èj¤~c*Sù ?ávnç¡™f¾ÊW¹Ûh¥•û¹Ÿfšƒ.S’$I’¤æòË/:œÆÆËH’ú¼! a2“¹“;YÅ*6²‘_òKòÉg1‹™ÂF0‚)La! )£ŒVZƒ.[’„!I’$I’$í‡?„5kà'? º©ß¹Žëøß`s˜ÌdŠ(Š|WC /ðBÀJ’$I’ô}6deþþŒ ?¾÷ê‘$u›tÒ)¤E,b-k)§œ»¹›d’ù? €2È`:Ó)¢ˆµ¬ ºdI:f2$I’$IRpÖ­ƒï}n¿B¡ «‘ú¥›¹™QŒ¢”RZh‰ÜM4÷q_€•I’$I’ô!DEÁÌ™×ñ¾¸8¸újˆ¶åI’‚!f1‹bŠÙÂV±Š›¹™zê¹…[ø#—\f3›%,a+[ƒ.Y’Ž¾ã–$I’$IRpþå_àøãáßþ-èJ¤~i-kù,Ÿ¥Š*šhjw_3ͼÌËü•¿T$I’$IÒå—CCCÇÛÜ'Ip¢‰&Ÿ|æ1¥,eÛXÅ*f1‹ *˜ÉLÒH#¼Hè`';»å¹1A’:2d I’$I’¤`<ý4ü÷ÃBllÐÕHýÎÛ¼Ígù,•TÒHc§ËÄÃC<ÔË•I’$I’ôvœpBÇÛC!8õÔÞ¯G’Ôëbˆ‰„–³œmlã%^â‹|‘2ʘÁ RH¡€æ3ŸJh “€ZüÿÄ…\ÈF6vóVHRÿeÈ@’$I’$I½¯¡æχ+¯„ ‚®FêwÖ²– L †šÃ i¤ˆ"ö±¯«“$I’$©\uUû)bcášk+G’¬! a2“¹“;YÅ*6²‘_òKòÉg1‹™ÂF0‚)La! )£ŒVZú¸ûØÇk¼Æ‹¼ÈÇù8Oòd/l$õ}† $I’$I’Ôûª`Á‚ +‘ú¥ã9ž¬à\Î`ƒ»ì.v±„%½Uš$I’$IÝcæLhjúûõÆF¸ì²àê‘$õ)é¤SH!‹XÄZÖRN9ws7É$ó~@dÁt¦SDkYÛéã¬d% 4ÐJ+;ÙÉLfrPMuïn$õ1† $I’$I’Ô»êêà{߃›o†Ñ£ƒ®Fê·Nã4V°‚å,çDN$šh¢ˆêtÙ{¹·—«“$I’$é#ÊÍ…O¢¢œO=N<1èª$I}Tˆ³˜E1ÅÔPÃ*Vq37SGs™ËÇø¹ä2›Ù,a [Ù À˼Lq‘Çi¥•—y™<òxš§ƒÚI œ!I’$I’$õ®ï|¢£áÖ[ƒ®D&3™wy—Gy”Œ †˜v÷·ÐÂ*Vñ¿üo@J’$I’ô!]}5 tà|õÕAW#Iê'1ˆ|ò™Ç<–³œ:ê(¡„ÌàOü‰Ë¹œ4Ò( €bŠi ¡Ýú4²“|éà©–Ú€¶D’‚cÈ@’$I’$I½§¢~úSøö·aØ° «‘Œh¢¹š«YÇ:°€D‰%6r,±,bQ€J’$I’ô!\v´´@s3LŸt5’¤~*þà{|7xƒZjyŠ§8Ó¨ ¢ÓuZhà×üš“9™gy¶7K–¤ÀÅ}I’$I’$©›|ó› Áu×]‰4 f0ó˜Ç\Á7ø¿ä—ÄC#<ÎãÜÅ]ÇqG|ŒÝ»w³~ýz6nÜȆ ¨®®¦ººšªª*6mÚÄúõëÙ´iÉÉÉddd““Cff&ÙÙÙdee‘••Å¨Q£ÈÌÌ$==½—¶\’$I’ÔŸìß¿ŸÚÚZjjjؼy3[¶laË–-‘ëµµµlÙ²…Í›7ó8@TWM˜@zz:©©©Œ9’ôôtÒÒÒHMM%555r}äÈ‘ÄÇǽ‰’¤>l8ù„Kˆ"ŠŸñ³#.ÛH#ÛÙÎ%\Â¥\JEŒ`D׸}ûvZ[[Ù±c---ìÚµ‹¦¦&öìÙCCC{÷îeÿþýìÛ·úúzسgìÞ½€ææfvîÜyÌðº---ìر#r_øñZ[[Ù¾}{‡šêëëÙ·o_—êo[ÇÑDEE1|øð.- 0tèPbbÚ·?ÇÆÆ2dÈN—‰‰‰aèС‘û† BlìAz ÄqÇסŽáÇÅ°aÈŽŽŽ<^xݤ¤$âããIHH 11‘øøx’’’º¼ R`È@’$I’$I½cõjX²Š‹!ÆÃRROÊ!‡'x‚›¹™¯ñ5~Ëo©§ž{ëîeÒ{“Ž hûÁO\\™™™Œ5Š¬¬,>ó™ÏpñÅ“‘‘ÁöíÛ©®®fÆ ¬Y³†•+WRUUÅ®]»Ú­Ÿ‘‘ dgg“yÌðeÛ$I’$IýOsss$(°eË6mÚÔ.(PSS¹¾iÓ¦v ñññ¤¦¦’––Fzz:#GŽä¤“N"==è?ÿ™Và«ÿx$°yófþïÿþšš¶lÙiŠ ;î¸ãÈÈȈ¾¾/|4hP/~·$I}Å˼L,±4ÐpÄå³<Çs¬d%?ççœÇy444°cÇvíÚÅÎ;©¯¯gÏž=lß¾úúzêëë©««cïÞ½Ô×׳cÇöìÙÃÞ½{Ùµk»ví¢¾¾žÝ»w³cÇš››;mî?’¸¸8i¤oÛ4œœù:ÜÍÇ>ö1¢¢¢:Üo²o+::ša`–êãŽ;®KÿcÃA‰®hjjjw :lÿþýìÝ»7r=Îèì¾ 6Dî ‡3àï¡Œ¶!‹ººº.Õ;$$$””ÄðáÃIJJ"11‘aÆ1dÈ:t(C‡%))‰Áƒ3lØ0Û­“””Ä°aÃÚý ¥Þ䧹’$I’$Iêwܧœ—\t%Ò€S__ÏÆ©¨¨ ººš7F.›«›É›Á¦y›øVý·øÖøo>\ÊÌÌŒÌ<0nܸÈõðå˜1c>p£Å¾}û"á…¶uTWW³zõjJJJX·n]äƒ8ðVÛç=ô2 ‘““]J’$I’ÔóÂûšÕÕÕÔÕÕQWWwØë555477·[?¼ß™œœLrr2yyyí®‡÷ûÂËÚÈq°¹oüìÂMœ‡«oãƬ^½š7²~ýzÛ­ßv¿´³úÚ^OOO7” IÄó-ÏÓ}ä€@ts4´@st3›mâó­Ÿ'æ‘š¾ÖûÝ¿7í'''“˜˜Hbbb»æñÔÔÔ ç111‘æþpƒ~xäüÁƒGbb" BêYáÐÂÎ;innf÷îÝ466Ff€Ïò4,©¯¯gçÎlÚ´‰½{÷²{÷îvÁ”¶¡ˆÎ 6¬Ãù¸ãŽ‹|œœÜé2Æ #%%ÅYô¡2$I’$IRÏ{÷]xúé³DG]ÔoìÛ·mÛ¶6@P]]Meee»ÙmØÏÏÏç‹Y_$íµ4þô™?1ù¯“™”9‰Áƒ÷HÍ „B!B¡Ð—«««ët{6nÜHYYÏ?ÿ<ëÖ­kפÒ6ÑY!33“ŒŒ ¢}‘$I’¤ŽÖˆßözWñèÖFü.ŒÞnÞ ïMW«W¯¦ººš-[¶ÐÔÔtHIÉ] $$''»Ÿ*I½ ¥¥…­[·vzÞ²eKç÷%l¥imSÇk…èÑ Ú9ˆøñ$íLbÈž! kÆð¦á¤´¦•JzF:'.>‘¬¤,†Úa$úÆçÔ/…gpèÉYÂ…ðLáðA]];vìhwÞ¹s';vì`óæÍ–9ô} x¯”’’BJJ #GŽdäÈ‘‘ëmÏáûFŽÉСC{l[Õ?2$I’$IRϻ뮳üã?]‰Ôg´m²?\€`Ó¦M´¶¶FÖINNŽ4Ó‡B!ÆßiÃ}ж!åp¨­­í4ˆž¡¼¼¼Ýâqqq¤¤¤1ˆ0jÔ¨4­·$I’$õEá`zW‚UUUìܹ³Ýú âC¡P§ ómv¹ÄÄÄ.äÃÂûð‡ûþVTTPZZJ]]›7on7q||<#FŒèR !++«G›%©?ijj¢¦¦†Í›7³qãFjjjظq#›7ofóæÍ‘ XMM [·ní°þ!C:4TçääDš©›Æ4Qùn%£F‘Íè¤Ñ?äx2c3aÎ9½¾Ù:†ÅÇÇÿ‘ß „à ;vì`ûöí6µµµÔÖÖòî»ï¶»½¡¡ý¬ácíiiidee‘––FFFínKOO'%%å#Õ¬¾É$I’$I’zVu5<ù$<ø 8rŽmGC<\€ ²²²ÝhBáÑ ÃMðá‘ Û6È3¦ûFì'âââÈÊÊ:êH”‡~ÏÛ^VTTPRRÒaÎCg|84ˆ0›h$I’$õ}m›Ù»2ã@[áÐ@ÛÆõP(Ôi#{vv6ñññmeÿþ¾vÕÑ~Žeeeèçx¸@‚?GIýQKK ›6mbýúõTUUQUUEee%555lÚ´‰M›6QSSCMMM»õ’’’Ú58ò“Ÿ$55•´´4RSS;ŒÌî룎UƒfðàÁx@¢]»vu˜ý£¶¶¶]Ðçÿþïÿؼy3555í>㈋‹#---2£Uzz:YYYŒ5ŠQ£F1zôhrrrü§Ÿ1d I’$I’¤žuß}¦µ¿üò +‘>’ýû÷³uëÖ#²í¨¡Pˆüüüí£GvÚᨫ#P¶=âÐË•+WR]]ÝaÄÉäää#233ÉÌÌtúsI’$I:Úøm¯weüüü|GÀï>H(¡+3RTTT| )wÝ0½¤Þ°eË–Hx`ݺu¾®®®ŽŒ˜EFF£F"##ƒ1cÆpæ™g¶kV˜>dÈ€·L؆ÊСC9þøãºlkkkd&‘ð #555lØ°-[¶PYYÉ믿NUU;vìh÷999Œ=:@3fL»0BRRRn¥>C’$I’$Iê9û÷ÃCÁM7ABBÐÕH‡ÕYóù¡‚M›6ÑÚÚY§móy(büøñ6Ÿ÷-áÆŠ¼¼¼Ã.ÓY˜¤íïDYY6lh÷ÁH¸ñçHA„œœŽ;î¸ÞØLI’$I=¨íLjG lÙ²¥Ý¯ð÷}ÉðþIx²³ÆðŒŒ ¢££ÚRõ–ðL{YYYGÜ_ «¯¯?j`¥¢¢‚矾ì~mŸïh„psï±6«¢¤£khh`íÚµ”——S^^NEEE»¯ëëë#˦¦¦Fš‡O=õT¦NJvvv¤©8;;›¸¸¸·FÒ‡EZZiii|âŸ8â²»ví¢²²²Ýì%ëÖ­cíÚµ”––RYYÉÞ½{#Ë9’ÜÜ\B¡¹¹¹í¾ÎÎÎîéMS† $I’$I’Ôsž}vì€k¯ º£Ú6.@PYYÙ®é£í‡íYYYLž<¹ÃHö£G&&ÆëQ|||¤¹#??ÿ°ËéwëÝwߥ¤¤ä¨¿[‡üÝ’$I’z_W¶Ã×»Ú°——gözLbb"‰‰‰GÝo ëJ0fõêÕTWWSSSCsss»õ Æ) à` ÒÀ±ÿ~Þ{ï=Þÿý!‚õë×GfÞiÛ |É%—››¡<''‡ÄÄÄ€·DRІJ^^ÞÔ[·neÆ TVV²víÚÈëÎ3Ï<Ó.¼”Ð!€››ËÇ?þqŽ?þx߇t3?©$I’$IRÏyôQ8ÿ|ÈÊ º 0 ÔÖÖ1@p´Ñæóòòøâ¿èhóúP …B„B¡#.w¤Y2JJJº4KÆ¡AgÉ$I’oß¾}lÛ¶­KÁªª*vîÜÙný„„„Ó¡P¨ÓÆꜜbccÚR©kººÿÞ=ÜßMEE¥¥¥ÔÕÕ±yóæH£1üýØKW ÙÙÙ >¼§6[R566òþûï³zõjÞ}÷ÝÈå_þò—Hè(ü¿0 qÎ9ç——iðõïXRwHII!%%…O}êSÞ~ÒöüÎ;ïðë_ÿš5kÖÐÚÚJ\\'œpyyyŒ;6ryÊ)§8;؇dÈ@’$I’$I=£ª JJ ¸8èJÔÏtÖ”}hsö¡b·mÊÎÌÌ$??¿CsvFF†’ÕëÚŽdz8û÷ïgëÖ­fÙ¨®®Ž4o¬_¿ž]»vEÖiÛøt¸ ˜1c2dHol¦$I’Ô£Ú6=weƶÂïÛ68‡B¡N  žKßíŠðþì‘þËÊÊ"_×ÕÕµ[¿³¿Ïήgee‘M|||Ol²tÌøÛßþFYYo½õï½÷ï¼ókÖ¬¡¹¹™ØØXN:é$ÆŽKaaa¤9÷ÄO$...èÒ%ã’““ÉÏÏït§íÛ·óÞ{ïµ J=òÈ#¬_¿€ÁƒsÊ)§0vìXÆŽËi§FAA#FŒèíÍèw H’$I’$©g<ú( _øBЕ¨¨¯¯ï48Ð6@°~ýz#ë$$$´kœ?~|‡¦jGŽTOVVYYY~Hþ:4ˆnÚxþù穬¬¤©©)²NøoèpA„¬¬,ÆŒàAƒzcS%I’$ ýþáÑ‚])=??ÿˆÊ’zFÛýÙ®¨¯¯?b !¶¯««ë¶‡Îgé,™™IjjªÇ‹tL«¬¬dÕªUíÎuuuÄÄÄpòÉ'3vìX®¼òÊÈhß'žx¢3’ú¥áÇsæ™græ™g¶»}ÇŽ‘@Uøò•W^¡ªª €ÜÜ\ "çÓO?ÝÀñ! H’$I’$©ûµ¶ÂÏ_þ28ÂØ€×ÐÐ@mmíååålß¾=²N\\)))íF‘<4@ítÛR‰‰‰‘©é$ŸúTЕè0ŽÔ`¾ûl®¾új¾ô¥/ ˜¶!I’$I’$u¯çžƒO„“Oº’~#<Ú‘ëׯg×®]‘uÚ~Ð …ÈÏÏïЈ;f̆ ÒëÛSUUENNN»Û¢££6lcÇŽeÚ´iÌ™3§Ïd-))aÊ”)¼ýöÛ|âŸàþûïç†nࡇâŸÿùŸ{½®ùóç³páB–-[ÆùçŸß­ßÙv÷”¾ðýü(zúgÑ™‡~˜ë®»€Ÿþô§Ì™3§ÛûÍ7ßä®»îâ7¿ù {öìaìر|ó›ßdêÔ©Ýöm%&& ……BG\.<"og„•+WR]]Ý¡©.ÜPw¸ Bff&DGG÷ȶI’$u&Ü`Ú•àÀ–-[ÚÆ¡}à2<ûÜøñã;m4í ïuÜì^î öO---´¶¶ÒÚÚÚ-ëîçôaŸçp¿W¥îîxþžÔÛû¾a‰‰‰$&&’••uÄÙÃ:ûŸqèõÕ«WS]]Mmmm»€>tüŸq¸@Brr2ééé8ÒÇ´¶¶òâ‹/rÏ=÷ð?ÿó?dffrÅWpõÕWóÉO~2èò:(++cÁ‚üñdË–-dgg3yòdfÎœÉ9çœÓå Ä@ú_דºûø`O|_>Êcñ¿¡«‚¨­/~?ÊÊʸñÆùÓŸþÄÞ½{ÉÎΦªª*è²5xð`.½ôR.½ôRY¶l?þ8×_=7ÝtÿôOÿÄ 7ÜÐïgœ1d I’$I’¤îµt)\rIÐUô5Ë$Ø´iS»qÛ6ˆ›H dfföÙQ«FEkk+×\s O=õ»w惡¹™M›6ñÒK/ñío›Ÿüä',]º´O~HÖÖ¿þë¿rÍ5×0tèÐÀj¸óÎ;¹æšk8å”S«¡»ô…ïçGÄÏâŸÿùŸ¹òÊ+ILLìöÇ¾à‚ ˜8q"eeeÄÄÄð­o}‹iÓ¦ñ /pÁtûóuU¸âHM ÔÖÖvD¨¨¨ ¬¬Œ 6°cÇŽÈ:ñññŒ1âˆA„œœœ>×ô&I’úŽÃJÝYChUU íÖï¬4<ÛÜ¡ ¡iiiÄÄô¯–÷»—û‚ýÓÙgŸÍ¶mÛz|Ýû<‡û½ú(uwÇó÷¤¾ºï{¨¶¡„#Ívèì7ý? ‡jjjhnnn·~ÛÙoŽH/×WCË–-cþüù¼ýöÛüÃ?üÏ=÷^xaŸ ‚¼ù曜uÖY|å+_áµ×^###ƒêêjî½÷^>÷¹ÏñÇ?þ‘‚‚‚.=Ö@ú_דzòø`_З‚¨­/~?fΜɧ>õ)^zé%Ö¯_ÏEÎdÞNll,Ó¦McÚ´iÔÕÕñ³ŸýŒx€ûï¿Ÿ+®¸‚ 0f̘ ËüPúש$I’$I’ú¶Õ«áý÷á8ÀØöƒ¼Ã*++ÛDÙ¶¡$++‹É“'wy{ôèÑý®‘¤+ Dvv6×^{-S§Neܸq\xá…¬^½ú˜h2ú¢¤¤$~øa À}÷ÝÇý×ñÐCõ©F‹ÎÄÅÅ‘••uÔæ‹#½V¿ûî»”””õµúÐ Â@~­–$éX³oß>¶mÛvÔ‘£ëêê:¡ý smgèl$霜bccÚÒà¸/()hýyß÷Hº:[`Xx ”ÃýŸ«¨¨ ´´”ººº¢„Cû½žÍðáÃ{j³”õë×sÝu×ñÒK/qÑEñÄOôùÀ£>J||<á‰²Ë IDAT÷ßd¥ã?ž»ï¾›’’’€«“ÔÝöíÛÇ_þòn¾ùf† Â)§œÂûï¿tY}Vrr2·Ür _ûÚ×(..æöÛoçä“Oæßøÿïÿý¿~wLÙy%I’$I’Ô}ž}ÒÒ೟ º’mÿþýTWWSVVÆÒ¥K)**âŽ;î`öìÙL™2…¼¼<†NRR¹¹¹œ}öÙ\uÕU,\¸’’êëëÉËËcÖ¬Yüìg?cùòå¼óÎ;ìرƒúúzÊËË)--¥¸¸˜;３¹sçRXXÈ„ …Býî㇑––Æ÷¾÷=ªªªxðÁÛÝ÷ÒK/qÆg˜˜HJJ W]u7nìðÏ?ÿ<$$$žžÎœ9sعsgäþòòr.ºè"FŽÉСC¹øâ‹yýõ×[Óüùó™2e Ÿüä'‰ŠŠâøão·LSS7Þx#Æ #;;›ï|ç;¸®Ãi»Ý999|éK_â7Þ8ìò7ÝtQQQDEEñÎ;ïðÔSOEn{â‰'ºô½8Úvi{î¿ÿþÈóÝÿý|õ«_eĈDEEqÙe—u›öý¬­­åë_ÿ:'œp œzê©<÷Üsçƒþ¬?ÈcGûÙç|à3f IIILœ8‘¿þõ¯ø9»ò󩨨ˆ4YÄÄĘ˜ÈÖ­[?Ò÷¢/ 7]L˜0ÂÂBæÎËwÞIqq1¥¥¥”——ÓØØȶmÛxçwX¾|9‹-bÖ¬YäååQ__OII .dÆŒœ}öÙäææˈ#ÈËËcÊ”)Ìž=›;¢¢"–.]JYYÕÕÕíš2$IR飼«ãÝwߥ´´”%K–pÏ=÷pÇw0wî\¦OŸÎ„ ÈËË#++‹ÄÄD²³³ùÄ'>ùŸ¾páB–,YBEEÉÉÉäççSøÿÙ»óø¨ê{ÿã¯ì™$d!É$3Ù`Ød1¸”ª`QQÛZ\‹]´Xqëm{¥¿Öþ W–¶öªÕÚÒÖöZ­W±.,UÜA¯µ ^…HÁD!É$“„L {r~àœÎLf&“dx?ç1ÉÌœåó=39ç|Ï|?ßïòåüô§?eãÆl۶ͧŽçt:Ù±c›6mâü#÷ß?«W¯æÚk¯åâ‹/6ëz'c‚?ÕûR]pôÕ½Ë÷ë_ÿºßÏ>ؼ¿ûÝï´¼@óûœM;mNܵµµækþ÷/}éK漃ýî{ûõ¯í³ü'žx¢ßå9rÄgž3Ï<€O>ùÄçu89ê¾áðŒžsöÙgsñųråJÖ®]Ëý÷ßÏÿøG¶nÝÊž={p:´µµQ]]Íîݻٶm>ú(«W¯fùòåf¢ÿÎ;yꩧX»v-—_~9 .dæÌ™ddd˜#2xêÌ×^{-·Ýv›O½yûöíTTTôõçd±iÓ&fÍšÅضmÏ=÷ܘH0€c:tuuÑÚÚÚç½÷ß¿Ï(ƒÝO„3_¨ãh8ÇxoC± ç¸î9ÏP\ ¤¿ín|á.s°Ûu0džP†âzð‘#GøÚ×¾FZZãÇçÖ[o¥££cPëè9` íîyá÷¾÷=óµ?ýéO\~ùåŒ7.`ü=øàƒæ7ÜpQQQ,]º4ìóÁþ¾OþRSS)((àᇦ««‹›o¾™´´4ŠŠŠxøá‡Ãø¤F‡˜˜®ºê*ÊÊÊX·n?ýéO™?>N§3Ò¡ Œ!""ÆòOo"ƒõä“O:¬ŠÈ`­_¿>Ò!ˆˆˆ ³Î2Œ¯=ÒQÕØØhìÞ½Ûغu«ñÈ#ëÖ­3n½õVcùòåFii©a³ÙŒèèh0ïFqq±±xñbcÅŠÆêÕ«õë×7n4vìØaTWW½½½‘.Ú¨ô•¯|ÅHNNø^ss³eœsÎ9æk›6m2¢££ïÿûF}}½ñᇧvš1yòd£¥¥Åœî¹çž3¢¢¢Œþð‡Fcc£ñÞ{ïÓ§O7Î=÷\ó³˜={¶qÅW.—Ëp»ÝÆm·Ý4­[·€ñÁø¼~øða0fÍšeüå/1ZZZŒ|ÐŒ×^{m@qâ)÷wÜa444ÕÕÕÆÕW_íï‡~hÆ–-[BÆ[__oÆ£>j¾Ö߶VîpÊãÙ6&L06lØ`9rÄxàŒ+®¸"hyÃÝž·Ýv›qÛm· ÆáÇÇ{ÌHHH0vïÞí³¼Á|Öá.;@ŸE8Û*Üu>þøã`ÜyçFcc£ñî»ïK—.5ãW¿úÕ€Ö9ÐÏçÕW_5ã[ßúV¿ÛádÔÞÞnTWW;vì06nÜh¬_¿ÞX½zµ±bÅ cñâÅFqq±‘ššês ILL4l6›QZZj,_¾Ü¸õÖ[uëÖ<òˆ±uëVc÷îÝ>û7QYDd´MûãÖÖV£¼¼Üضm›y ^³fqë­·ú‡Õç¼ët ,0ÇkÖ¬1î»ï>cÆ ƶmیݻwÕÕÕ‘.ê CuAÕKuÁ@±Þÿý}b UÎßþö·^^ yûû~zOnù}¯ü—WSScœwÞy>óýà?0÷ßß0ŒÁ÷ý×ßÞÞn\|ñÅÆ/ùKŸéú[~OOQTTd\sÍ5>ó¹\.£¤¤$h ªû½ÖÖV£ººÚçºë}÷Ýg¬Y³ÆX¹r¥±lÙ2cÁ‚†Ãá0âããû«½ëÎË–-3V¬XáSö¾ÛÙÙéâ·gžyƈ‹‹3®»î:£µµ5Òá Øo~ó0-Zd¼þúë!ÿßÃÙO öZ[8ÇÑ`ûÐ@Žçza¸ûùpÏy†òú` Á¶K¸ñ d™ƒ=6 öØÌP\>å”SŒ§Ÿ~Úhnn6žyæ#))ɸ馛¼ŽÁœÛážz–7qâDã¹çž Z†@sx¿ì|0Üï“g9³gÏ6þú׿---Æw¿û]#**ÊøêW¿jlܸÑhii1n¿ýv#66Öøä“OBÆ;Zíݻט1c†1eÊ”Õ=Gªn¾|ùrcùò¾ígÕRDÄP’?%ˆÈñM?؉ˆˆ—Ç #6Ö0žxbÄWíÝèdÆ Æ}÷ÝçÓðÓápqqq}~¼r8> MÖ­[g62)//?!~´Š¤P K Ã0233 ‡Ãa>Ÿ6mš1sæLŸivíÚeƺuëÌצNÚgºM›6€ñòË/mmm`<ùä“æû]]]†Õj o?ÜÝ+¦··×HII1î¼óΰã fÚ´iÆ)§œÒgãÇ7Ÿ¶aI8Û"X¹Ã)gÛ\ýõAËç/ÜíÈÅ_lÜpà æóÁ~Öá,;˜@ŸÅ`?û@ëœ:uª1gΟמyæ™>?"åçãv»_þò—FZZša·ÛÚÚÚÓKhþÇ$ï„6O£ŠØØØ€Ç$ï„6ï†åååFwww¤‹6"TG†sìiˆèIÜóN_±b…±lÙ23 Üÿ˜é_—[¶l™±råJ3iÀ;‘¯ººÚèéé¶rHpª ª.ÈXª Šµ»»ÛHJJê7ÖPIý-ïx“ Â-_8IþÞxã #&&ÆøÅ/~a¾6Øï¾÷ú[[[¥K—\o8Ë¿ûî» ‹Åb455™Óüìg?3î½÷Þ>ËSÝwôt.à}=×û\Àÿšn sïz´ÿ¹Àh«K×ÔÔiiiÆ7Þ8f;®éìì4¾øÅ/šŸ‡Íf3V­Ze¼ýöÛ}¦ çÿx°×ÚÂ9Ž$É p¯†»ŸÈ9ÏP] $Øv 7¾,3þŽMÇ{l×@¯ßvÛm>¯ßrË-F\\œQUU5 u ô0Ôöh’Á¿ÿû¿¸ ý%; ÷ûäYÎu×]g¾VUUe>‰„N§ÓŒÇiiiC§ ŽaæßUUUüóŸÿ䦛nò™fÎœ9¤¦¦òÒK/±zõjªªªØ·o7ß|³Ïtgœq/¿ü2çž{.§vÿçÿü¢¢¢X¶l‹—Ëu\ñzYEvv6uuufüáÄå/X¹SRR8tèÐqÅ ˜˜8¨m1ÐòÌš5kÀ±…ÚžÁŒ?ž}ûö™Ï[¾p–®Á~öÖÙØØrYƒ]gŸÏ·¾õ-6oÞÌÕW_ÍÚµk±Z­!§—Ð, ‡‡Ãr:·ÛmË***|Žoeee8Njkk}ö•æ±Ìsœ ô(""2’ÚÚÚp»Ý¸ÝnóXæÿ·çyUU>ó'&&’‘‘aË222())1ÿö~Ïjµ«fcꂪ zŒöº w¬111deeõk(C½¼p ¶®ë­©©‰/ùËœþùÜrË-ÀñÕ…=Ž=ÊE]„Õjåúë¯÷y/Üå_ýõüèG?â‰'žà†nàÉ'ŸdëÖ­}Ö§ºïèa±X°X,ØívJKKû¾­­-ày…÷9ÇÎ;q»ÝÔÕÕÑÓÓã3FFFÀó ÿ¿=ÓEEE WÑy衇HKKãç?ÿù°®g8ÅÅÅñôÓOóÚk¯ñðóqãFzè!zè!Î?ÿ|ž|òIÒÓÓëÙß|S§NÖã¨Ç@÷¡¡öóážó ×õÁþ„ßPµ]‡âØ0qâÿ,\¸x€wß}—¼¼¼°Ö1ÐsÀPÛc0æÍ›7à2ô'Ðùà`¾O3gÎ4ÿÎÎÎîóZNNõõõƒŠs4°Z­<ôÐC,^¼˜²²2Š‹‹#R¿Tû‘¡ñÆ0u* ¢qãÇÌ?þñÞyçöîÝKUUµµµ!ž2e ¥¥¥¬X±‚)S¦PPP@nn®yñQF·ææfÜn7³gÏ ¡¡8vÑÝ_ff¦ù¾çñÁäÁì3íÁƒxᅸ뮻øÖ·¾Å—¿üe/^Ìÿý¿ÿ·Ï‘’’âó<::šÞÞÞÅå/T¹‡Ê`¶Å@Ëc±XW¨í PVVÆwÜÁ[o½E]]ÙiÎœ9>ó ¦|á.;án«pÖYSSôý>ø?ŽÏçÁäÊ+¯ìw::ž %%%A§imm¥²²’ÚÚZÊËËÙµkÿûßÙºu+çIII!??›ÍƤI“(--å´ÓNcÖ¬YÄÅÅ WqDDä$µ{÷nŸÆT2{öl,X@qq1yyyX­V²³³ÉÎÎVÒÀIFuA|æS]pt×ýc‹‹ó‰u †zyþ†²®ëí›ßü&íííü×ý—ùÚ`¿ûÞn¹åfϞ͟ÿügþþ÷¿û|vá.?;;›/}éKüþ÷¿ç†nàí·ßföìÙ¤§§\§ê¾cS¸ÉüÝÝÝÔ××S__O]]ÕÕÕ”••ñá‡ò¿ÿû¿¼ùæ›!çÿàƒ|µµ÷ߟ… jÿ9Ú|ö³Ÿå³Ÿý,¼ð Üwß}¼øâ‹Üu×]ÜsÏ=Ç}Œ 5ßpG‡bj?î9Ïp]ìO¸ñ Ô@·ëP†"Ž@222|žgffàt:Ã^Ç@¿»¡¶Ç`¤¦¦†,Ã`ÚŸ æûäýÿôµ¡ *+++©©©¡ºº§Ó‰Ó餺ºšššöïßÏþýûyâ‰'HNN¦  À¥Àf³‘——G^^žùZnn.ñññC\h¬7b^x!YYYÀ±žÜý:tˆI“&ùLwûí·ó“Ÿü$èò322øùÏÎÏþsÞ~ûm~ðƒð™Ï|†²²2sYC)ܸ‚ͨÜýñ\Tïêê2_kiié3Ý`¶Å`Ë3TºººX¼x1yyy¼öÚkL™2…˜˜¾úÕ¯òÞ{ïùL;Ðò dÙág[…»N›Íôý>477x2¶¸\.jkk©ªª2užã^ Ä»¨¨(rss±ÛíØívòòòÌG›Í¦Ä;1<üðÃÔÖÖR__OCCuuu¸\.³ßÁƒ9xð ›6mŽ54±Z­dee‘m³<÷ÜÜ\󽬬,ó¼WN ª úΧº ¯±Tm†«|<òO>ù$ýë_}zþŠïÊwÜÁ7¾ñ Î<óL®½öZvíÚERRÒ€—ÓM71þ|ÊÊÊøýïϪU«Œ ½½½444˜çžsÏÝÿœÄ?gžsäääô9')((ÖøóóóÙ¹sç°®c¤%$$pÉ%—pá…âp8xçw€ã?F†š¯ªª Üq4‘8F„{ΩëƒáÆ7ƒÙ®Cul8Þ89|ø°ÏsÏÈv»=ìu ô0Ôö€ðÏ =ÜnwÐ2 ¥áø>(‹-bÉ’%|ðÁœyæ™üö·¿eÒ¤Iìر#èìãùÜÉëŠ+®8ß´iÓxýõ×}^w:8ª««ÍÞ„üy×ÖÖš¯½ÿþû>ÓÔÖÖö»-•{°å*ÔÔÔðío›éÓ§›¯û÷ÜNù»ìp…³­æÌ™Ö:ÇÏÔ©SÙ¾}»Ïëï¾ûî€×9ÏÇ»7JZíííf²@ ã•ÓéäÀ=zÔœ'11Ñ縴hÑ"Ÿç‡ƒ‚‚J ""£BZZ_ÿú×CNÓÖÖ†Ûí6}n·Ûçyee%ï¼óN§“úúzº»»}æ÷>6fddøüíÿ<77Wõ¹QLuAßùTìk,ÕGJ¸ßÏá(_yy9·Ür ·Þz+\pùúÌ™3Ù½{÷qWqqq<ú裔––rûí·›=Sä»xÖYg1wî\|ðA8À©§žp}ªûŽ^ý+xÿιBqqñ¨=W¸òÊ+y衇xꩧX¾|yÄâ8wÜqQQQÜyç>¯ÇÆÆg¿ŽçÎ|áG#¹õî9ÏH\ vN|ÁZæ`¶ëPŽ7Ž@vìØáó»mÛ6âââ(-- {= µ= ¼óBoï¼óW]uUÀ2 ¥ãý>¨ Ãà»ßý.“'OæôÓOt8aQ’ˆˆˆˆˆˆˆˆ¿·ß†ŽŽAd0ž‰JJJ‚NÓÙÙICCCІeeeTTTøôÞOfffÈD„I“&v]ëé顶¶–^xµk×ͦM›7nœ9Í=÷ÜÃ¥—^Ê~ð¾ýíoÓÐÐÀ 7ÜÀäÉ“¹ñÆÍé~þóŸsÉ%—°nÝ:®¿þzî¾ûnº»»¹ôÒKijjb÷îÝÜ{ï½\wÝuôöö²~ýz9í´Ó‚Æèé%jïÞ½äæærÊ)§°iÓ&ŸBé/®`<åþáÈ·¾õ-Ž9 7ÜÀW¾ò• J¦NŠÕj塇âŒ3Π¾¾žGy¤Ïtým‹`åly†Â„ ÈÎÎæÑG5{A{ã7زe &LPùŽgÙáêo[EEE…½Îµk×rõÕWs×]wqÓM7QYYÉOúÓ¯3\n·›¹sçrÆgðä“Oªü'+·ÛMEEEÈÏ÷Ð÷øâp8X°`AŸÿ¡×EDDÆ:‹Å‚ÅbÁn·‡Õx¥­­-d#ÊŠ ¶oߎÛí¦®®Î'±þ•\¨a¡‚‚Íf#**j¸Š.¨.¨ºàÀŒµºàH÷û9Ôåëîîæšk®¡¨¨(hoÑCõ])..æÇ?þ1ßþö·¹ôÒKY²dÉ€—¿jÕ*V®\ðT÷iž¤`‰ÞïUUUÑÙÙé3bbbŸãvIIIÀã»Õj%6vì4<ûì³Yµj×^{-‹…eË–E:¤A¹ÿþû™>}:çŸ>iiiTWWsï½÷òÉ'Ÿð«_ýÊœn°û‰pæ ç8l:oÞ<ŸõÔ1"Üsžá¾>l»„_¸ËüóŸÿ<èí:Ç¡ú|Ÿ~úiÎ>ûl>ûÙÏòÒK/ñðóråJìv;a¯c0ç€Á¶G¸ç…¯¼ò Ï=÷çw^Ÿ2 µãù>ˆº»»¹é¦›Ø´i[¶l!!!!Ò!…Çcù§7‘ÁzòÉ' VEd°Ö¯_éDDDŽßÚµ†1aB¤£´ÖÖV£¼¼Üضm›±aÃã¾ûî3V¯^m¬X±ÂX¼x±áp8Œ¸¸80‰†Ãá0,X`,_¾Ü¸õÖ[uëÖ<òˆ±uëV£¼¼ÜèììŒtÑ"¢²²Òg[FTT”‘‘‘aÌŸ?ßøÉO~b477œwË–-Æ駟n$$$Æ5×\c8Î>Ó½ð ƙgži$$$999ÆÕW_mTVVšïoÞ¼ÙX²d‰‘™™i¤¦¦gŸ}¶ñòË/÷ûªU«Œ´´4#55ÕXµj•ñßÿýß>å¸æškú”oÒ¤IaÇŒw¹ív»ñíoÛhkk3 Ã0V¯^Ý'­[·ÅÅņÅb1-ZdìرÜîsŸû\ØÛ¿Üá”ÇÛ†ÛíYÎlÏwÞyÇX¸p¡‘’’b+W®4¾ô¥/™ÓÕÔÔ„]>á.Û_¨Ï¢¿Ï~ ëüå/i Æ™gži¼õÖ[ætgœqÆ~>FAA±|¹®yxŽ [·n5yäcݺuÆ­·Þj,_¾ÜX°`áp8Œ˜˜Ÿm›‘‘a‹/6V¬Xa¬^½Ú¸ï¾ûŒ 6Û¶m3ÊËËîîîHmLPYDdtKûãÆÆFc÷îÝƶmÛŒ7ëׯ7Ö¬Yãsü...6l6›Õ§^g³ÙŒââbŸºÝš5kŒõë×7n4¶mÛfìÞ½ÛhjjŠtQG=ÕU f,ÕúÙ{{à|¦[´hQØË 4o°Ï)Ø´á”/Ð÷*Ðòžzê©>Ÿ³ç>mÚ´°¾+üìg?óYÖM7ÝdlÙ²ÅçµÙ³gxùÕÕÕFVV–ÑÞÞp½ªûŸööv£ººÚ<Þz®Ÿ®Y³ÆX¹r¥±lÙ2óx›žžÞç;ã}¼õÔ›½¯¥zŽ·åååFGGG¤‹;ìzzzŒo|ãFtt´ñï|ÇÜï‡6þð‡?K—.5ŠŠŠŒ¸¸8###ÃX²d‰ñâ‹/ö™>Ôÿññ\k3ŒÐÇQ`Ç:ƒ½^8ÐãF¸ç|Øœ'111äÈ66›¢¢"RRR"X²‹êÈ""£Ã‰º?noo§±±1¬Þ•«««innö™ßÓ³r#$Øívòóó‰PIEDN.<ðUUUAG]¾<×9Ã9&ÖÖÖú´…ð>öwLÌÏÏ'---‚%½žzê)n¸áRRR¸ãŽ;¸îºëˆ‰‰‰tX"rÚ»w/3fÌ`Ë–-,]º4Òáœ4ùéOÊý÷ßÏ”)SxôÑG™={ö€–1RuóË/ÿ´ýìßö³cg¬ ½ví‚Õ«#Å°óü8VRRtšÎÎN‚&"ìܹ³Oc•„„Æ2¡  €ÔÔÔ‘(¦ˆÈ˜ã,À¿QDFF†¹¯u8,X° OÍf#***‚%‘¡äI ´ÛíaMßÖÖÖoãËŠŠ 6oÞLUU}ÖNB‚ÍfÃjµ«&""áºë®»HHHàÊ+¯ä`ëÖ­‘)¢ÚÚÚú$;vÕÕÕÑÓÓã3¿§£ïëŸÁŽ]ª+åË—3þ|Ö®]˪U«øÏÿüOn»í6®½öZ’““#žˆˆ ÒG}Ä<Àþð’’’¸çž{X¹r%qqq‘mÀTC‘ããt‚ËsçF:’Q!>>Þl´RZZt:ïþüöìÙÃK/½Dee%]]]æ<þ=jû'"Øív Õ0EDNþûÊ@ ¤»»ÛœÇ_¹xñbí+EDDdP, ‹»Ý2ÙÜÓ”ª7è²²2œN'õõõ>ç0à{*!!##C#≈·ß~;wß}7?úÑ(**Št8C*Ð1%Ø1&œcŠÍf£´´4à1&''G=èGH^^¿ýíoùÎw¾Ãþçòï|‡ïÿû\uÕU¬X±‚³Î:+Ò!ŠÈ÷½ï}ÏéçËÎfg IDAT‚ .àšk®á±|TòHhkkãÙgŸåÑGåÅ_¤°°5kÖ˜#ÖŒUº‚.""""""""Çg×.ˆŠ‚ñy²³X,8GÈé¼{çöodûâö©n«¦¡·Þ´^Œ‹ÝBrA2IYI,ýÛÒ>kÕ;·ˆDZGG‡ ™@PUUEKK‹9ÿ¨/%%%\|ñÅ>û¶ÂÂBÆÁ’‰ˆˆÈÉÌ;)!TÒ¹‡¯Óþ I+**ؾ};n·—ËEoo¯ÏüÞ½N‡JHð<9‘ÜqÇÜqÇ‘#lÁFÇ ”8îè8‡#`ršFÇ{¦OŸÎo~óî¾ûn~ÿûßóÈ#ð«_ýŠ)S¦°|ùr¾ð…/PZZªë¹"2`ëÖ­cݺu‘ã„ÔÖÖÆ /¼À³Ï>ËsÏ=Gkk+K—.婧žâÒK/=!øt6!""""""""Çg×.˜022"ɘd`ÐH#nÜ4~zóù;ÃMcF#î’¾ïwÐÑgy–n IíIÄ´ÄÐåî¢âWÇ¥„j¬hD5Ö‘Á •åyÍ¿‘œ§œgÿSZZÚgߤä(9Ñ„›|îá9Ï Ö(uçÎæ{µµµ†aÎëiœNBB~~>iiiÃUl‘‚'y>œÄ§ÓISS“ÏüöËþIž÷òóó‰PIe$eeeqûí·sûí·³sçN{ì1þô§?q÷ÝwSPPÀÅ_̹çžËg?ûY233#®ˆÈIgïÞ½¼òÊ+¼ôÒK¼ð ´··3þ|î¼óN®¼òJ¬Vk¤CRJ2‘ã³kÌé("®v3` ·zê馻ÏòI$Ãï6‘‰,`6l}ÞË ƒ,²ˆ‡ŽÝíÀ‹ÿZ¦w/™þ~wîÜÉæÍ›9xð Ïðêž¡Õƒ%"ØívŠŠŠNˆYD$4ÿ}H }Iee%]]]æ<ž}ˆg±`Á‚>û”ÂÂBõ°("""OƒÓp´··ÓØØ4!ÁétRQQÛí¦ººšææfŸùƒ%%¨ñ«ˆœÈúKæò~/T2—gÿXZZpß™——GzzzK*cAii)¥¥¥Ü{ï½¼ûî»<ûì³üíocýúõ†Á¬Y³8÷Üs9çœsX´h‘:‹ü1¯¾ú*¯¾ú*¯¼ò N§“qãƱhÑ"î½÷^.½ôRrrr"æ°ÑU{9>»vÁ׿é(†”AƒßÍ…‹zêÍçõÔSG 4ÐH#m´õYN2ÉŒÿô–AãO.¹Ì`†ùº÷{ž¿SI–r…ÛK¦w/äþ½—••át:ûüê¢=X"‚çQDFŸÎÎN&xËËË}z]ô Åf³QRRÂâÅ‹}Ô®ˆˆˆHäx’=퇵µµõÛ¨¶¢¢‚Í›7SUUEgggŸõ…“`³Ù°Z­J2‘á,ßß>®®®ŽžžŸù=£îyöc%%%A÷q}O†Ó©§žÊ©§žÊwÞISSo¼ñ¯¼ò [·nåÞ{ï%&&†yóæ±hÑ"N?ýtæÍ›Gaaa¤ÃSºººØ½{7;vìàí·ßæÕW_åã?Æb±°`Ánºé&Î=÷\æÍ›wÒÔgNŽRŠˆˆˆˆˆˆˆÈðhn†O>9s"IH]t™ ÞÉÁh —^ŸeŒg<Ùd“õé­ˆ"Nã4²É˜(0žñÄ36{rôþá4O¯˜***ؾ};äÈ‘#æ<þ=šzœ0aÉÉÉ#QL‘“‚wÒP°ÇO>ù„ÞÞíó<#¼V¬Xá“@››KtttK&""""CÉb±`±X°Ûí!ë‚ž¤„P½|{’Ôëëë}FÌßúa¨„„ŒŒ {Šˆ)о'ؾ¨¡¡Ág¤=è»ï±Ùl”––ÜåäähäN•ÒÓÓ¹ä’K¸ä’K¨¯¯7{ÙÞ¼y3÷Üs===X­VæÍ›çs·ÙlŽ^DdtèééaïÞ½ìرü¿÷Þ{´··“’’Bii)×^{-çœsgžy& ‘9"”d """"""""ƒ·oèjÛhÃíu«¡'ΠÏ]¸ú$ $’ˆ;6ldAEœÅYæsÏÍŽ|òÇlÂÀpñî³´´4ètžãüjjjعs'›7oæÀ>=Åù7pôXTT¤zå¤ÖÞÞŽÓé ™@pàÀŽ=jÎãŸèSZZêóÜápPPP@\\\K&""""cwRB¨:¡‡oâþ „=Éên·—Ëå“ ¾½‰‡JHð<‘±!Ø(*ÂEÅápLbÒ(*r¢ÊÎÎæòË/çòË/àÈ‘#ìÚµËl4ûÄOpçwbyyyÌ›79sæP\\Lqq1S§N%>^×¾EäÄÕÔÔDYY{öìaÏž=¼ûî»ìÚµ‹#GŽ˜˜Èœ9s˜7oßüæ7™7oÓ§O×ïOŸÒ™“ˆˆˆˆˆˆˆˆ ÞþýC0ôrݧ·ÚOoõÔãÄiŽ:PO=.\4Ð@+­>ó&’hŽ0CYd1yd‘…õÓ[–ßMF†ÅbÁápàp8BNçiT¨¡´§÷Ëššsúøøx233C&"Lš4‰ôôôá.¢È õÿàýèáÿÿàp8X°`þDDDDdÔ·nèá‘+XÃã;wšÏkkk1 Ãœ×Óè8œ„„‚‚RSS‡«Ø"'ŽŽ:Vâ€Ó餩©Égþ@ÿ¿‡£Ïÿ¯Ýn'??_ £EHIIaáÂ…,\¸Ð|­¹¹Ù§·îÇœŠŠ zzzˆ‹‹còäÉ”””0cÆ JJJ(..fÚ´iú‘1Å3šÚž={(++3ïÕÕÕÀ±ýãŒ3˜;w.+V¬`Þ¼yÌœ9Sî„ $¼ýûaâDÒ ˜7µÔRG™0àÂE 5ÔQG 5¸pQGÝt›óÅO6ÙØ°‘K.V¬Sl&xnž‚RFªÄ2L222(-- Ùf¨žÛ=½^ö×s»ÿ£zn—‘jdÏc#{”””hd9áy‡£½½ÆÆÆ ½Ÿ{êŒn·›êêjš››}æÔz°çª?ÊÉÈ;駿Äÿ¤Ÿ„„ÆïóTZZð,//OÉñ"Ã$--óÎ;óÎ;Ï|­££ƒ½{÷òᇲ{÷n>üðC6lØ@yy9ÝÝÝÄÆÆ2yòd¦OŸÎ¤I“|î………:ŠHD>|˜ŠŠ ÊËËÍûG}DYY™Ù1ϸqã˜>}:3gÎdÉ’%fòTQQQQQ.ÁØ¢$0ק·ªäש½1*þƒZj©¦.3¡ ƒsžb̤;v¬X™ÉLlØÈ&;vrÈ1Gñ—˜˜ö¨Ápïܹ“Í›7óÉ'ŸÐÛÛkÎãßÛ?Áf³‘››KttôpSÆ O£¦P •••>|ØœÇ?¦´´´Ï÷¯¨¨ˆ”%Q‰ˆˆˆˆ„â9·¶ÛíaMßÖÖ2!Á3²ØæÍ›©ªª¢³³3àúúKH°ÙlX­VbƒtÌ )žøpêêê|áá_×P<ßwÏhzþl6›ó‰ŒR Ìž=›Ù³gû¼ÞÙÙi&ìÙ³‡}ûöñúë¯óûßÿ·Û @ll,………}’<÷äääHIDN.—Ë'‰Àû^WW@TTùùùæ~géҥ̜9“3fPTTáœ8T“Ó!QC ÕTSK-UT™.\æc'ŸþÀþ]HîŒ#ŸÇÉ!‡ ˜Ìd3‰ ‡lØ°b%›l¢Qm~ž²KJJ‚NÓÙÙICCCÀD„ŠŠ vî܉Ó餩©ÉœÇÓû^¨D„üü|ÒÒÒF¢˜2B¼“V‚%ø÷Ôèida³ÙÌƾ7"""""2ò, ‹»Ý²ÞèáIJ–àv»)++ÃétR__Oww·ÏüþI ¡”Ü.ƒè;ì;ÛÐÐ@WW—Ïüg<£èùGsrr4’žÈ .>>žY³f1kÖ¬>ïyñ¼ï}ô/¾ø¢Ï蜞ýŠçú˜çoÏóÂÂBÆ7ÒE‘Q Ðh¿Þïß¿Ÿ––àØþ(??‡ÃÁ)§œÂ¥—^jîK¦OŸ®„¦ $‘“@=ÔRK%•TQE5Õä ÕT›Ïk©¥vs ìرa#<Îà ¾ÈÉ%—<òÈ%—ü¢Œ[}¬ZÁÒ‰ \||¼ÙÓeiiiÐé¼{÷óODسg/½ô•••>?Ðû÷P`·Û),,To–æÿÙJ 8xð O!ÿH=.¼?碢"5¸9x'%„ª?zø÷ïß𻢢‚íÛ·ãv»q¹\>£ìo/ñጘ 'ž`£mJ¨®®¦££ÃgþÄÄÄ>߇Ã0Ù%;;›¸¸¸•TDÆšŒŒ JKKÛÛÛùøã)//§²²’ªª**++9xð o½õUUU>û+«ÕJ~~>ùùùùŒd·Û±Z­X­V]g#ÚÚÚp¹\ÔÔÔPWW‡ÓéÄårqðàAªªª¨ªªâàÁƒ=zÔœ'==ÝÜäååqúé§STTDaa!‡ƒüü|%àF˜~Åã< ÞIþ×RK7ÇÊFu,A€|òÈ£”R–±ÌL(°zË #ôŠÝn8è†)SF ”"‘a±XÌžqBñîíÞ¿Ñú›o¾i^P÷n<âi8,Áf³a³ÙˆŠŠîbžP:::8tèPÈ‚ªª*³7$ð¥ÂápPZZÚ'QD=¬‰ˆˆˆˆH8­Gzxê“Á”ïܹÓç¹7Ocòp HMMŽ"K?<õÔpüGU„ÀŸ³Ãáèó9Ûívòóó‰PIEäd–˜˜ÈŒ3˜1cFÐi\.—OccO2®]»Ø´i.—‹¶¶6súèèh¬V+ÙÙÙØívrrrÈÉÉ1“l6999X­V233ÕYdˆµ··sèÐ!jkk©­­5“êêꨭ­¥¦¦†úúzœN§Ïõvø×ïùùù0þ| ÉËË£  €ÂÂBRRR"T2 —’ DDDDDDDDF¹fš9À>ùôv€TRiŽF, Ÿ|æ1ÏóyòÈ£BòÉÇŽx†àÇÆ?>ö8qâñ/KdŒóüØ_RRtšþ¿ïܹ3dã÷`‰'Sã÷PÉž×B%s8,XÐ'@É""""")žúd8ÚÛÛill ÙP½¢¢·Ûݧ~ {¸ö¼  @=܇àÒ_â@mm-†a˜ózêúÞÛ»´´4àç‘——GzzzK*"2t½Ù° MA8ªªŽ=æåÌúDƸ„„ìv»Ùx ˜¶¶¶ è÷ìÙÃK/½ÄÁƒéîî6çILL ˜€àýXXXHlìè¼$ì_æ@e¯¬¬¤««ËœÇSfOù/^<¦Ê,"""""2PžzÝn™äîáßP3Pc²ŠŠ 6oÞܧÎå½¾þ<=Jåú—w½´¿Äºº:zzz|æ÷$¸{¶‹'É=ÐvS¢»ˆHp‹‹ÅÒï5T€úúzêêꨫ«£¡¡††:dÞÙ¿?‡¢¡¡·ÛÝg)))dff’žžNZZZÀ{zz:ßKJJ®M!⣩©‰æææ>wï×›ššúL×ÔÔÄ¡C‡ú$åÄÅÅ‘™™ÉøñãÉÌÌ$33“œœJJJÈÊÊ2_ó¼n³ÙHNNŽPé%RÆîÙ½ˆˆˆˆˆˆˆÈÑ@å”û$x'å(pl6&0"Š¸€ (òºM`IŒ¢ ÖN'ŒK¤#9¡X,‡#ätÞ½úû÷æ_VV†ÓéìÓÓ§¡C°DÏãPéì줡¡!dAuu5ÍÍÍæ<þ£7”””°xñbŸ$Šüü|ÒÒÒ†,N‘Ñ@jBà¤ÿ†öžúf}}½Oò;ômh*A!77—èèèá*zX žç•••tÍì‚ï+ÖÀ %%%æsï÷rrrˆ‰‰¶²ˆˆH`ÙÙÙdgg‡•xÐÓÓã“„àŒàßX»²²²OÃmïë¬qqq¤§§3nÜ8RSS±X,$''“––†Åb!))Éçïôôtóøœ‘‘aþžžNRR $''?BIÉ;rä]]]>|˜¶¶6Ž9BKK mmm=z”ææfÚÚÚhmm¥©©‰öövóﶶ6óÆ3MsssÀ€øøxŸ™ôôtÒÓÓÉÏϧ¤¤Ä|Ý;a ++‹¬¬,RSSGxËÈX¤$‘!àÆÍöPF^·øˆfþÕx6ƒ ŸÞ–± lØ°cg:ÓIf õR] CØYDÆ»‘C0­­­TWW›&jkk©ªª2¶nÝŠÓ餣£Ãœ'))‰‚‚rss)((Àf³‘——gö–™ŸŸOnn®Ù£ºº§Ó‰Ó餪ªŠÚÚZ*++©©©¡¾¾Þ\ntt4999ær&OžÌÂ… }Ö•››KVVÖ°n7 l I ===444P__O}}=µµµÔ××ÓÐÐ@]].—‹={öðúë¯S[[ë“\Ç̳²²°Z­äääMVV999X­V³á¨ç9`.׳NÏsO.—ËìÍÚ»ž ––fÖ9³³³ÉËËcîܹæóÆ©üpΙpùž~[Œmh7®ˆˆD\LL V«Õ<® TKKKÐÞä[ZZ8|ø0­­­fCò––jkk6oooïw}ž¤ƒÄÄD, $%%Orr2±±±Œ7Ž˜˜RSS‰ŠŠ"==8víØ#%%…¸¸8sž@ïy–ã,ÙÁO8ÒÓÓý§££ƒÖÖÖ°–ÙÜÜLoooŸ×=ûáØyJKK‹ùÞÑ£GÍžü{{{}ÎKZ[[Íó†ÎÎNŽ=Jww7‡6—ã=§Á¿'ñÄÏáÇû$`,Å“ àIbôžÆóžÿHá~"ƒ¥$‘0áå”SA…Ïc9åä ]»p™F“˜„‹YÌJVšÏ ( öDºãtB^^¤£‘’’’˜2e S¦L 9§Aˆ"BuuµÙK¥Ëå2{ër.øt–c?Œäçç› ³fÍ2G°Ùl““Clì ´9‰ÅÄÄ““CNNNXÓwvvú$$x¼öïßo& =zԜ׳—×ò’““Í…¬¬,l6sæÌ1²²²ÈÍÍ5“Âéú.à˜ YÀžçyf0c[DDDNt©©©CÖû»a455ÑÚÚJ[[ÍÍÍfCxOcuO£yO#xOrB{{;mmmfƒøööv6ˆßFùžyNvÞÉÞIž$ŒèèhÒÒÒÌÄèèhsäá´´4¢££IMM%&&†qãÆk&ex’CRRR°X,Œ7ŽqãÆa±XHII‰X™EC¿èˆˆˆˆˆˆˆˆ|ªƒö±òOóq?û)§œ:êˆ"Š<òpà`“ø Ÿ1ÿvà ‹“¨îêj%ˆœ <.N9å” Ótuuár¹¨¬¬ä¬ùó)[³†˜«®"??Ÿää14 ‹ˆˆˆˆˆˆŒ¸øøxòòòÈ óZRkk+ ÔÔÔPøÝïpðž{°Ùldee‘””4ä1:pðoñy>Ïð,ϲˆEC¾‘¨¨(s¤ÚHjii¡§§蛀àýž·`# ø •ÐPVVÆO~òÖ¯_Obb"@ØÛÂ3¢ƒ?‹Åb.Ëy}ADþEI"""""""rR10¨¤’}ŸÞþùémû8Àzé%šhŠ(b*S™Ç<®à &}z›ÈDIìE'ƒêj8í´HG!"#$..Žüü|òóó(..†iÓ"•ˆˆˆˆˆˆœˆ’’’(,,¤°°l6lgœ1ìëÏx¶²•¯òU>Ççx˜‡¹†k†}½"""‘0T#3 ÔŽ;X»v- ,0G‘ÑGI"""""""rBj¡ÅL$ØË^Ÿ¤‚VZÈ ƒ©Le:ÓYÄ"¦zÝHˆp Æ€ÚZóG^‘A <Îãüˆ±‚ìg?kYé°DDDNV«€ºº:%ˆŒbJ2‘1­“Nö³Ÿ2ÊØÃóq/{饗Xb)¤²oòM^7¤Þ^p»!33Ò‘ˆˆˆˆˆˆˆˆˆ ©(¢XËZòÈc«¨¤’_ók∋th"""cžw’ˆŒ^J2‘1¡“NþÉ?ÙÃ>àÊ(ã>àc>¦—^H`3(¦˜¬ ˜bf0ƒ‰L$V—@†^K ôôÀøñ‘ŽDDDDDDDDDdX|ƒo`ÅÊÕ\M%•ü™?“Jj¤ÃÓIMMÅårE: A¿°‹ˆˆˆˆˆˆÈ¨ÒK/å”ó>ﳇ=ìf7{ØÃ~öÓE±Ä2…)Ìd&×r-%”0“™Lb’’ F’Û}ì1##²qˆˆˆˆˆˆˆˆˆ £K¹”×x‹¹˜³9›çyž "–ˆˆÈ˜fµZ5’È(§_ÞEDDDDDD$bzèa/{ÙÉNÊ(c{øþ‡CÀ†JXÌbV³šJ(¦ –G.46{ÔH""""""""r‚;Óøþ‡‹¸ˆ39“çyž9̉tX"""cVNNõõõ‘CDBP’ˆˆˆˆˆˆˆŒˆ:ø€x—wÙÅ.Þå]>àÚh#NáNåTþÿ¹Ìe&3I")ÒaK0É@DDDDDDDDN"™Èv¶óy>Ï9œÃs<Ç"E:,‘1Éjµâr¹"†ˆ„ $r­´²ëÓ›'©`{袋R˜ÍlÎà näFæ2—bŠ‰#.ÒaË@¸Ý ii‘ŽDDDDDDDDDdDŒg<[ÙÊ—ù2KYÊc<Æe\é°DDDÆ«ÕÊþýû#†ˆ„ $9nNœìd'oò&ÛÙÎvÐA©¤r §ð>ÿño”RÊt¦CL¤C–øðCøÿ€ÔÔcÏãã¡¡&O>öºÅrìõädÈ˃/|!r±Šˆˆˆˆˆˆˆˆ £ØÀþg9Ëù?ã;|'Òa‰ˆˆŒ)V«•7ß|3ÒaˆHJ2‘i§ìàmÞæ-ÞâmÞ¦†âˆc.s9“3¹™›9ƒ3˜ÈÄH‡+CÁá€M› ­ bb *êØ`ݺc†ð½ï)É@DDDDDDDDNhQDq÷P@ÿÆ¿qƒÜÇ}DéÐDDDÆ«ÕJ]]]¤Ã‘”d """""""!ùR°“´ÓN9œÆi¬d%gs6ó™OI‘W†CB\z)<õtu…žö+_™˜DDDDDDDDD"ì6n#ƒ ®çzšhâwüŽ8â"–ˆˆÈ¨gµZihh »»›ØX5eôŸ)""""""">ª©æåOo¯ð UTK,³˜Å|æs#7rgáÀéPe$}éKðßÿüýèh(-…éÓG.&‘»–k±aã2.£†žæiÆ1.Òa‰ˆˆŒjV«•ÞÞ^:DNNN¤Ã‘”d """"""r’sãæU^å^áe^f/{I ³8‹¸…,dóH&9Ò¡J$-]zlDƒööÀïGEÁõ×lL"""""""""£À–ð2/sqçñ<Ï“Mv¤Ãµ<‰uuuJ2¥”d """"""r’i£7y“ílçMÞäu^§‡æ2—ó9Ÿÿà?øŸ#•ÔH‡*£‰Å^7Bwwß÷cbàòËG>.‘Qà4Nãø–²”³8‹¿ñ7&39Òa‰ˆˆŒJV«8–d "£“’ DDDDDDDNpïñ导Ì˼Å[tÐÁt¦sçq#7rçAF¤C•ÑnùrxöÙ¾¯ÇÅÁe—AzúÈÇ$""""""""2JLbÛØÆ…\ÈB²…-ÌaN¤ÃuÆO\\œ’ DF1%ˆˆˆˆˆˆˆœ€Úhãe^f3›yžç©¢ 6–°„ßðÎã<òÈ‹t˜2Ö,[±±ÐÕåûzW|ík‘‰IDDDDDDDDdÉ%—7xƒË¸Œ…,äižæ|ÎtX"""£JTTYYY¸\®H‡""A(É@DDDDDDäÑH#ÙÈ3<ÃK¼D;í”RÊõ\Ï2–q*§ET¤Ã”±,%–,^€žž½ž“çž¹¸DDDDDDDDDF‘RØÄ&¾ÂW¸˜‹y„G¸’+#–ˆˆÈ¨bµZ©¯¯t"„’ DDDDDDDÆ°xŽçxš§y™—‰&šó9Ÿ_ð .â"lØ"¢œh.¿þö·=‹ƒo|bb"“ˆˆˆˆˆˆˆˆÈ(O<âOüÿÆ5\C ÜÌÍ‘KDDdÔÈÉÉ¡®®.ÒaˆHJ2cŽr”gy–ÇxŒ—y™8âXÊRþÀ¸˜‹I%5Ò!ʉìÒK!:û›øX IDATz{=ïê‚/9²1‰ˆˆˆˆˆˆˆˆŒBÑDs?÷cÃÆ­ÜÊ!±†5‘KDDdT°Z­¸\®H‡!"A(É@DDDDDDd 襗Wy•?òGžá:èà.à1ã"."…”H‡('‹ôtøÌgàõ×Á0à´Ó`Ú´HG%""""""""2j}ïaÇÎu\G=õü‚_Mt¤Ã‰(«ÕÊþýû#†ˆ¡$‘Q¬†~Çïø-¿¥’JNçt~̹’+É"+ÒáÉÉêŠ+Ž%DEÁ7¾éhDDDDDDDDDF½k¹–TR¹Š«h¢‰?ð∋tX"""“M]]]¤Ã‘ ”+""""""2ʼÂ+,g9Eñ ~Á•\ɇ|Èßù;7s³ $²¾øÅc£ÄÄдx1n··ÛMooo¤#µ>Ïçyžçù á2.£¶H‡$""1999¸\®H‡!"Ah$‘Q¢ƒåQîå^Ê(ã,Îâaf9ËI$1ÒáÉ(uôèQš››ijjò¹9r„¦¦&zzzhnn¦³³“£GÒÚÚJGG---twwÓÔÔDww7‡¦££ƒÖÖÖ>ë8|ø0ÝÝÝ>¯m>îìdÅ„ !ã‹eܸq}^ONN&>>žqãÆKFF111¤¦¦’@RR’9MZZ±±±¤¥¥‘’’BzzzŸ{RRÒñlF‘q.çò2/s᧷¿ðRItX"""#ÎjµÒÚÚÊÑ£GINNŽt8"âGI"""""""Ö@¿âWü’_âÆÍÕ\ÍŸøs˜éÐd„¹Ýn\.õõõÔÖÖâr¹¨««£®®ŽÆÆFÜnwŸ„‚®®®>ˉ‹‹#%%ŧá~||<ÉÉÉ$%%‘@^^^ŸÆýÑÑѤ¥¥õYžÅb!1Ñ7Ñ%i˦OŸÎÖSO ··—æææ>ó¶··ÓÖÖ·G¶ææfz{{&;´´´P[[ËÑ£Géììô™æÈ‘#AË(ù 33«ÕJvv6¹¹¹ää䘧§§‡ýÙˆˆˆˆˆˆˆˆˆ •Ó9×yó9Ÿó8-lÑèµ""rÒ±Z­¸\.G„£J2‰rʹ‡{ø#$‘DnànálØ"š ±¦¦&*++9pऺº§ÓI}}=.—‹ÚÚZêëëéèè0牊Š2Ç[­V233™8qbÀ†ôiii>ÏG¤·—sÏ…Œ ˆŽþuù9zôhŸ‘æÐÔÔÄ?ÿùO¶oßN]]õõõ†a.'!!¡OòÝn'??Ÿ‚‚&L˜@AAAÀÄ ‘ãQB ÛÙΖð>˼H>ù‘KDDdÄx’ êêê”d 2 )É@DDDDDDd„•QÆù1Oð…ò~Â×øÉhбª²²’O>ù„O>ù„ÊÊJóîyÞÒÒbN;~üxòóóÉËË#;;›3f`³ÙÈÎÎ&''‡ÜÜ\3¹ &&&‚¥êGffÄVœœLrr2yyyš¯§§ÇL6¨©©1ÿöNøعs'UUU¸Ýns¾ÔÔT )**¢  €ÂÂB (**bâĉäçëÇ_¸‰LdÛøŸãlÎf+[™Â”H‡%""2"rrr€cI"2ú(É@DDDDDDd„¼ÏûÜÃ=<ÎãLcó0Ws5±ªž n·›ŠŠ ó¾gÏÊÊÊØ·o‡ >>žÌÌLìv;‡ƒeË–a³ÙÌç“'OV¯øƒÍfÃf³1kÖ¬Ó¶··ãt:©¨¨ÀétRSSCEE}ôo¼ñàèÑ£À±òòò(..¦¤¤‡ÃÃá ¤¤›M#“ˆˆˆˆˆˆˆˆHp6l¼Æk\ÄE,d!/ð³™é°DDD†]bb"©©©J2¥ÔŠADDDDDDd˜í`kXö0—¹l`ŸçóDéÐ$§ÓÉ|Àÿþïÿ²gÏöíÛǾ}ûhllŽ]ðœ2e S¦LaÉ’%¬ZµŠ©S§2iÒ$rss‰ŠŠŠp d($&&šÉ†AMM åååìÛ·ýû÷³ÿ~žþyöïßOGG™™™L™2…iÓ¦Q\\ÌìÙ³™5k–’DDDDDDDDÄ4žñüöî<®Š²ÿÿøëÄ"(‚DE Å 5HSKpE4÷-ÍR)óke¥V–¦ÞÝæRz‹æRÝ.-jiY¸£©¹àî[š†l®((ÂùýÑíùIjá:€ï§óf®¹®÷ 9çÌg®Õ¬æYž¥ Mˆ"Š40:–ˆˆÈg6›IJJ2:†ˆÜ‚Š DDDDDDDÝìf#ø‘©G=¢ˆ¢%-1¡›Ðóƒ«W¯²ÿ~öìÙÞ={ؽ{7»wï&55oooüýý©_¿>Ï=÷~~~TªT oooŠQÈ£Îd2QºtiJ—.ÍSO=•k]NN'OžäèÑ£Ö"•#GŽ°fÍððð 00ÐZt@µjÕxì±ÇŒØ1˜3ÎDEwºÓŒf,b-hat,‘Êl6“’’bt ¹ˆˆˆˆˆˆˆÜg9È¿ù7_ó5Õ¨ÆБŽ*.0Pvv6ûöícóæÍÄÄÄ°k×.>LVVøûûH›6m 00777£cKU¤H|||ðññ!444׺3gÎk-cݺuL:•ÌÌLììì¨R¥ µk×&88˜àà`üýý±±±1hODDDDDDDDäa²Çž…,¤ýhK[¾äK:ÑÉèX"""ŒÙl&99Ùè"r *2¹Oâˆã=Þã+¾¢ÕXÈBžåY 55•˜˜bbbؼy3Û·o'==êׯOXXÇ' ???lmõ‰<nnn<óÌ3<óÌ3Öe×®]ãÈ‘#Ö5¶oßÎ[o½ÅÅ‹qqq¡nݺDPP%K–4pDDDDDDDDäA²Á†Ïø 'œèF7.r‘xÁèX"""„§§'G5:†ˆÜ‚>A¹Gg8Ç|H$‘xãÍ—|IºP„"FG{d$&&²zõjÖ¬YÖ-[8zô(&“‰Ê•+D×®] ¦Zµjz*¼ä;¶¶¶T«VjժѵkWàÏÙ7öïßÏ–-[زe ‹-âÃ?Äb±P©R%‚ƒƒiÚ´)!!!xyy¼""""""""r?¡‘DâŠ+}éK&™ `€Ñ±DDDî;³Ù̦M›ŒŽ!"· "‘»t•«Ìf6ÃN9|À¼ÆkØcot´B/==uëÖMtt4û÷ïÇÞÞžàà`ºvíj}â»›››ÑQEîŠ À™3g¬3tüòË/ôë×+W®àïïOhh(!!!4nÜgggƒÓ‹ˆˆˆˆˆˆˆÈý0†1¸àÂ@r…+¼ÎëFG¹¯<<£"‘<8ËYF2’©L¥µØÈF‚ 6:V¡“““ÃæÍ›ùöÛoY´h§NâñÇ'<<œN:Ñ AÝ$-ržžžtïÞîÝ»°ÿ~¾ýö[,XÀ„ ððð E‹têÔ‰–-[bk«·EDDDDDDD Š—x [l‰  >â#£#‰ˆˆÜ³ë…III*2ÉgôI¢ˆˆˆˆˆˆÈßÈ!‡/ù’7y[l™Æ4úÒ—"1:Z¡òÛo¿1kÖ,æÌ™CRRþþþôíÛ—:P£F £ã‰Hþþþøûû3räHöìÙâE‹X´hóæÍ£T©RôîÝ›~ýúQ¡B££ŠˆˆˆˆˆˆˆHô¥/E)J/z‘N:‘DbBe‘‚ëzaArr²ÁIDä¯Td """"""rëYÏ«¼Êð2/3šÑ£˜Ñ± «W¯òÃ?0sæLÖ®]K™2ex饗èÚµ+UªT1:žH¡@@@|ð‡â›o¾áóÏ?gܸq4mÚ”ˆˆÚ¶m‹ÑQEDDDDDDDäot£6ØГžd“Í4¦é¡8""R`¹»»cgg§"‘|HÿÂù‹Sœ¢½xš§ñÀƒXb™ÌdÜ'¿ÿþ;Æ ÃÛÛ›îÝ»ãèèÈ’%K8qâ#GŽÌ·;wîäÙgŸ¥lÙ²ØÛÛãëëKÿþýY¿~=‹ÅÚ.::“Éľ}ûHŽÝAðÙgŸa2™0™LLŸ>ý¡;lØ0ë¸+V¬xhãÞoUªTáƒ> ..Ž~ø{{{ºv튷·7o¿ý6'Nœ0:¢ˆˆˆˆˆˆˆˆüÎtf1‹™Íl"ˆ ‡£#‰ˆˆÜ“ÉDÉ’%Ud ’©È@DDDDDDä2Éd c¨D%bˆá'~b5«©F5££ ûöí£W¯^TªT‰¹sçÒ§O~ûí7~úé'Ú´iƒÑoë×_åÉ'ŸÄËË‹Í›7sñâEÖ®]‹³³3Mš4açÎFG|¤ôíÛ—ŒŒŒ‡>îرc9xðàC÷A±±±!<<œ¨¨(Nž<É믿Î7ß|C… hÓ¦ »ví2:¢ˆˆˆˆˆˆˆˆÜFa,f1_ò%=éÉ5®IDD䮘ÍfˆäC*2~ájQ‹øˆ÷yŸ}ì£5­ŽU(:tˆ:@ll,³gÏæäÉ“Œ;£ãåÉÿû_ìí퉌Œ¤\¹r<öØcøøøðñÇS£F £ã‰Ü³Ò¥K3tèP~ûí7æÌ™Ãï¿ÿN:uèÔ©‡6:žˆˆˆˆˆˆˆˆÜB+ZñÃÿþô YdIDD䎩È@$R‘ˆˆˆˆˆˆ<Ò.pWy•&4Á_ö³Ÿ¡ å13:Z—œœLÿþý©Q£Gå‡~`÷îÝôèÑ[[[£ãÝ‘ŒŒ ²²²¸|ùòMëöìÙC:u6l¡¡¡Ô¨Q“Éd-¤HMMå­·Þ¢bÅŠ888P³fM–,Y’«¯ÈÈHL&&“‰ÈÈH^~ùeÜÜÜ0™LtíÚõoû¿•¼Œ pìØ1Ú¶mKÉ’%qqq¡]»vÄÄÄܶßÊy]TTuêÔÁÁÁOOO^zé%.\¸pÇù¦NJùòåqrrâ駟æèÑ£·Í÷W+W®$((GGGÜÝÝyî¹çHLL¼åþLŸ>AƒQ¼xqÊ”)èQ£nÛozzºu;“ÉDPP'NœÈµ¼ ±µµ¥gÏžìÝ»—Å‹sèÐ!ªW¯NDD„ÞàɇšÓœ¬`9ËiO{®pÅèH"""wDE"ù“Š DDDDDD䑵”¥T§:_ò%Ÿò)KYJ9Ê«ÀËÉÉaÆŒT©R…+VðÙgŸKxxx»áúºzõê‘™™IXX6lÀb±Ü²ÝرcY½z5{÷îÅb±pâÄ ÆŒCVV[·nµÞ\ߥKöïßoÝ~àÀ\¼x€‰'òÌ3ÏÏ”)Sþ±ÿ[ɢ:tÀÑÑ‘!!!·í÷Ÿr,Y²„ððpZµjEbb"«V­býúõ<ûì³Öã—×|ß|ó ¤_¿~œ:uŠ?þ˜!C†Ü6ߢ¢¢hÕªM›6%>>žM›6qøða5jd݇÷çÓO?%$$„„„†ʈ#X¿~ý-ûvvv&;;›òåËÓ£Gka†IIIøûû““““§œùÉd¢]»vÄÆÆ2kÖ,–-[FÕªU™5kVÝ'‘ªXÎr6°ö´'“L£#‰ˆˆä™§§§Š Dò!ˆˆˆˆˆˆÈ#'‰$zÑ‹0Â"ˆÃ¦?ýŽU(œ&MšÄ¤I“pwwÇÙÙ™=zЬY³\7æß($$„N:Q´hQÈüùóï8w^ÆÌÌÌd÷îÝ´oß³ÙŒ««+&L hÑ¢yãv9‡ ‚¿¿?£F¢D‰2~üxÖ®]ËÏ?ÿ|GÇdäÈ‘Ô¬Y“áÇS¢D jÕªEÿþyûoöÍ7ߤZµjüë_ÿ¢dÉ’T©R…™3gòÛo¿1mÚ´›ÚשS‡ððp\\\xå•Wprrâ—_~¹mÿEŠ!""‚Å‹sþüyëò¹sçÒ·oß[Xs Ï?ÿ<¤OŸ> 0ÀZT"""""""""ùG°–µÄC;Ú‘A†Ñ‘DDDòÄÃä¤$£cˆÈ_ìOøEDDDDDDî€ s™‹?þüÂ/¬` YHIJ­PX¼x1œ9s†íÛ·3~üø<ߨžßÙÙÙ±hÑ"~þùgzöìÉ¥K—˜6mAAA4oÞœsçÎÝU¿nnn9rä–ëî%ržÇtpp nݺ¼ýöÛ|ûí·ddd`kk›ç7so•3!!#GŽÐ¤I“\ËëׯÀš5kòœïÌ™39r„† Þ²¯¿“ÀáÇiܸq®å5kÖ¤X±bDGGß´M5¬_ÛØØP²dÉ|zNß¾}ÉÉÉÉU²`ÁžþùÌXP8;;3a¶oßNJJ |ÿý÷FÇ‘<Á¬b;ØAa\æ²Ñ‘DDDþ‘ÙlV‘H>¤"y$ç8ÍhFúÐìe/Íint¬Bãßÿþ7;v¤K—.lß¾ÀÀ@£#=Mš4aÞ¼y$''³dÉž~úiV­ZŘ1cþqÛо}{J•*E‘"E0™LÌ™3‡³gÏÞ²½££ã=çÍë˜+W®¤]»v¼öÚk¸ººÒºuë<ÏÒp«œ©©©DFFb2™¬/³Ù ü9ãE^ó%&&Üè¯ßGEEå+,,Ìšã¯mÜÝÝ­ëoäììœë{;;;rrrþöxxxбcG¾øâ bbb ÄÕÕõo·+ˆj֬Ɏ;èر#:t`ìرFG‘<Á¬e-{ÙK8ášÑ@DDò=OOO._¾Ì¥K—ŒŽ""7P‘ˆˆˆˆˆˆjÙd3ŽqT§:É$C 3˜3Îÿ¼±äÉ Aƒxï½÷˜2e Ó§O¿/7Ççwööö„‡‡³jÕ*¼½½Ù¶mÛ߶ÏÊÊ"$$„øøxÖ­[GVV‹…Þ½{c±XHÆ;³D‰Lœ8‘S§N±~ýz233iԨǎ»«±K–üsv!C†`±XnzÍ›7/Ïù¼¼¼€?g4¸Ñùóçs}–kŒ¨¨(kŽ¿n ––f]?¼òÊ+lÛ¶ðÅ_0`À€ûÖw~ãèèÈÌ™3™4iÇçµ×^3:’ˆˆˆˆˆˆˆˆÜ €¢‰&–XÚÒ–L2Ž$""r[×RõO3K‹ÈÃ¥")´~ã7žâ)F0‚á g;¨K]£c*ï½÷Ÿ~ú) ,à•W^1:Î3|øpÞ{ï½›–ÛÚÚbgg‡»»»uY‘"7¿ÝrüøqéÒ¥ UªTÁÆÆ€+W®Üq–[õ+yóôéÓÔ¨QÃú}PP³fÍâêÕ«ìرãŽó”-[–*Uª°}ûö›Ö²`Á‚<çsss£R¥Jlܸ1×ò]»vå)GåÊ•Y·n]®å±±±\¸p;Ü³Û ¦V­ZDFFGíÚµï[ßùÕ Aƒøú믉ŒŒäý÷ß7:ŽˆˆˆˆˆˆˆˆÜàz¡ÁNvÒŽv*4‘|KE"ù“Š DDDDDD¤PšË\jQ‹‹\d [x‡w°ÃÎèX…ÊâÅ‹ù׿þŬY³èСƒÑq¸É“'óÕW_‘’’ÂÕ«Wùý÷ß4h'Nœàå—_¶¶»þäýC‡‘ššŠ——©©©xxx0oÞ<8@ff&«V­bùòåwœãVýߪÀÇÇ'ÏcîÛ·O>ù„ .pîÜ9f̘ƒƒuëÞ}QÎĉÙ°acÇŽ%55•ÔÔT̵k×hÛ¶íå9r$±±±Œ3†³gϲgÏÆ—§&LààÁƒ¼û¥¥qøða"""¨X±b®ŸÛý0`À¦OŸNÏž=ïk¿ùYçΙ9s&cÆŒá‡~0:ŽˆˆˆˆˆˆˆˆÜ &5‰&šml£=í¹Â?ôDDDäA»^d””dp¹‘Š DDDDDD¤PI"‰pÂéC^àv°ƒšÔ4:V¡“””Dß¾}‰ˆˆàùçŸ7:Î7lØ0þóŸÿðå—_R·n]œyâ‰'8tè+V¬ Y³fÖ¶U«VeÀ€ôíÛ— *о}{4hÀÒ¥K)^¼8õëקR¥J,Z´ˆÐÐPvïÞÉdâôéÓÌŸ?úõë‡Édâܹs¹²Üªÿ:uêÜ”ÙÞÞ>Oc–*UŠ¨¨(–/_Ž¯¯/åË—góæÍ,]º__ß[¼älժ˖-cÉ’%”-[–êÕ«“””ÄÊ•+qppÈs>€nݺ1uêTfÍš…——|øᇼüòËÝögÆÒ¥K‰ŽŽ¦L™2ãççdž (V¬Ø-÷§gÏž$$$`2™8vìS§N¥bÅŠ 6ŒªU«вeË›Š Zµj…»»;;w¾mžÂè…^ oß¾¼øâ‹zʈˆˆˆˆˆˆH>S‹Z,cÙH7º‘E–Ñ‘DDDrqttÄÅÅEŸ1ˆä3&‹Åb1:„ˆˆÑ:óç Yhp)¨.\H—.]ÐÿVEänÌœ9“þýûC¤PXÄ"^â%œqfshD#£#ݵï¿ÿžöíÛ³k×.jÕª•k]³fÍ8sæŒõéõ{÷îeøðálØ°ÌÌLjÕªÅرciÔèÿï||AAAŒ=€¬¬,¶nÝJ=¨P¡ŽŽŽÔ®]›¯¿þOOÏ;ζbÅ ®]»F×®]ïÏΊpC† ! €R¾|y£ã¦[·ndee±bÅ ££ÈC–œœL=xꩧ(^¼8C† áñÇgöìÙÖ6ÇÇßߟɓ'SªT)*V¬ÈüùóÉÌÌdܸqÆ…‘E× ÷¦! ùžï‰"Š¾ô%‡£#‰ˆˆ*2ÉTd """"""R&™ cÍhF]겟ý„nt¬ûª_¿~\¸pE‹““ÃìÙ³éÚµ+...\½z•Ÿþ™°°0lmsOmݸqc6nÜ€•+Wæßÿþ7 ,àܹs÷”k×®]Ô¨Q77·{êG¤0>|8‹…³gÏ2hÐ £ãÊÝÝêÕ«³k×.££ÈCfccChhh®eU«Våĉdff²mÛ6ÂÂÂrµqww§Aƒ¬[·î!%‘‚N× ÷.„–°„,P¡ˆˆä*2ÉTd """"""ÎP‡:Lg:_ð‹XDIJë¾+_¾<Íš5ãóÏ?`õêÕœŠHáqþüyŠ+ft yÈL&Óß®wuu¥H‘"¤¤¤Ü´.99ww÷MDDDDDD ]ƒÞ?ÍiÎ÷|Ï—|I*4Cyxx””dt ¹Š DDDDDD¤@È ƒþôçE^¤/}YÏz|ð1:Ö†——ãÆãǤÿþÖu4iÒ„%K–§þœœœhÖ¬ .ÄÞÞž­[·Þq¦ÇœC‡å*b[ûì³Ï¬³KLŸ>ýÉdbß¾}Û.!!á¦Ù/®¿Ê–-KïÞ½9}úôÏ›Ÿ:uŠ_|‘òåËcooo=>Õ«W7:Z¾•Í‘#GxüñÇŽ"ùŒƒƒõêÕ»©°íÌ™3lÚ´‰Æ”LDDDDDD ]ƒÞ™–´äk¾f6³Ì`£ãˆˆÈ#Ìl6“–––çÏÒ…$;väðáì^½šôôt, /¾ø¢Ñ±òµM›6‘’’BË–-Ž"ùШQ£Ø»w/¯¿þ:III?~œnݺaggÇ!CŒŽ'""""""…ˆ®AïL{Úó5_I$oð†ÑqDDäåééIvv6iiiFG‘ÿQ‘ˆˆˆˆˆˆäks™K]ê↱ÄÒšÖFGzè®ßØܹsgŠ/žk]`` Û·o qãƸ¹¹Ñ®];¬EåÊ•#""‚ &P¹reÊ–-Ëœ9søî»ïîªÈ jÕªÔ«W>úè÷Lò'''Ú¶mËsÏ=ǶmÛøõ×_Ždˆ+W®°mÛ6zôèA¥J•°³³3:RðÑGDåÊ•Ž"ùPhh(Ë—/gÛ¶møøøP³fM{ì16mÚDùòåŽ'""""""…ˆ®Aï\G:ò_ñþÃ[¼ety™Íf’““ N""שÈ@DDDDDDò¥‹\¤;ÝyžçéK_¢‰¦ eŒŽeˆßÿ€ˆˆˆ[®¯R¥ óçÏ'99™ŒŒ <È„ (]º´µMëÖ­Y±b©©©œ?ž-[¶Ð¾}û»Î4zôh¢¢¢X±bÅ]÷‘¤¦¦òÖ[oQ±bE¨Y³&K–,ÉÕ&22“É„Édbúôé 4ˆâÅ‹S¦LFuSŸS§N¥|ùò899ñôÓOsôèÑþü›~‡…††²iÓ&222¸pá?ýôÕªU{XQEDDDDD¤€Ó5èƒÕ™ÎÌbó1£¸ù}O‘IE"ùŠ DDDDDD$ßÙÅ.jS›h¢YÆ2&3;Í'‰§¥¥ñÎ;ïШQ£|ur³fÍèÙ³'½{÷æ?þ0:Î]3f YYYlݺÕz³x—.]Ø¿¿µÍÀ¹xñ"Ÿ~ú)!!!$$$0tèPFŒÁúõë­m¿ùæH¿~ý8uêüqž§`ïСŽŽŽ8p€øøx||| ±®_²d ááá´jÕŠÄÄDV­ZÅúõëyöÙg±X,Œ;–Õ«W°wï^, 'Nœ¸«ãr}»‹þi|øó†ÿV­ZJBB[·nÅÞÞž¦M›Zû¹Þ¦iÓ¦ÄÇdziÓ&>L£F¸xñ"ÎÎÎ\»v2eÊУGëÄ>>>$%%Q¹rerrrò”éÆŸÝĉyæ™gˆgÊ”)·Ý÷·™5k‹…¨¨¨Û¶ÏË9¹ÏÄÄD¦OŸÎСC?Ï«¿~^œ:uŠ>}úЫW¯\笈ˆˆˆˆˆˆˆ,Ïó<Ó™ÎHF2‰IFÇ‘Gˆ»»;¶¶¶$%%EDþGE""""""’¯La Oò$>ø°—½´ …Ñ‘ ‚——&“‰ÿþ÷¿FǹɴiÓpww§yó椥¥ç®Lš4‰I“&áî³3=zô Y³f·½ ½N:„‡‡ãââÂ+¯¼‚““¿üò‹uýÈ‘#©Y³&ǧD‰ÔªU‹þýûÿcŽÌÌLvïÞMûöí1›Í¸ºº2aŠ-jm3dÈüýý5j%J” 00ñãdzvíZ~þùç{?ÀåË—ùñÇ™7oíÚµ£víÚw4þ›o¾‰¿¿?£GÆÝÝÒ¥K3cÆ ìíí­ý¼ùæ›T«Výë_”,Y’*Uª0sæL~ûí7¦M›€ ½zõbñâÅœ?ÞºíܹséÑ£‡u&ƒ;9&!!!têÔ‰¢E‹2pà@æÏŸ_ŽY^Ï¡¿ž 6ì¾d0Rjj*Í›7§dÉ’L:Õè8""""""""rúÑÉLf0ƒ™Å,£ãˆˆÈ#Âd2Q²dIÍd ’¨È@DDDDDDò…K\¢;Ýy×ÎpV²O<Že¨èèh®^½ÊæÍ›ñõõ5:ÎM\\\Xµjéé鄆†è näææÆ‘#Gn¹®FÖ¯mllr½ÙyæÌŽ9BÆ smS¿~ýÓÁÁºuëòöÛoóí·ß’‘‘‘ëi- 9r„&MšÜ²ï5kÖäyÿþêÒ¥K˜L&L&E‹¥[·nLž<™E‹YÛäeü„„ëŒ7rvv¶¡\oÓ¸qã\mjÖ¬I±bňŽŽ¶.ëÓ§¹ŠæÌ™CïÞ½óœéFy=$÷ì¯çÐíÎ:uê<´L©S§hÚ´)—/_fÕªU8;;IDDDDDDDDîƒÿãÿÎp^æe°Àè8""òˆ0›Í¤¤¤CDþGE""""""b¸ßø`‚YÅ*–³œá §ˆ.Y „²e˲víZ233 "66ÖèHwäÀ´oßžR¥JQ¤HL&sæÌáìÙ³·lÿ×›¨íììÈÉÉ 11øóóýõûÛY¹r%íÚµãµ×^ÃÕÕ•Ö­[³uëVàϧÅDFFZ L&f³€“'OæqoV´hQ, 999=z”Ê•+3bĈ\³Säeüëmþnÿ®»»»u=€ŸŸ 6ä‹/¾ &&///Ê•+—çL7rtt¼ƒ£’wy9‡nwn+Vìdz~ýõW‚‚‚¸vík×®¥lÙ²FG‘ûh£x×xŽçXÆ2£ãˆˆÈ#ÀÓÓÓú.1žîØC-cõ¨‡-¶lg;¡„IîÐã?ΦM›¨T©ÁÁÁüç?ÿÁb±ëeeeB||<ëÖ­#++ ‹ÅBïÞ½ï*¿——ðçSëotþüù>>xxx0oÞ<8@ff&«V­bùòåwÝçÈ‘#‰e̘1œ={–={ö0nܸ>ùä.\¸À¹sç˜1cÔ­[€‰'²aÃÆŽKjj*©©© <˜k׮Ѷm[àÿ?1ÿСC¤¦¦âååÅŽwþÁ߈#°±±aðàÁÖey„ 8p€÷Þ{´´4âââxá…èÝ»7îîîÖ6äÝwß%--ÇAÅŠyùå—såpvv¦S§NLŸ>°°0r­ÏK¦éNΡ¿ž±±±Ì˜1ãg¼._¾Ì|@@@)))lÚ´‰wÞyÇZT!""""""""…— 6Ìc h@KZr€FG‘BJE"ù‹Š DDDDDDä¡:Ä!‚b+[ÙÀ†2ÔèHrŸÕ¯_Ÿ;w2zôhÆŸŸ3fÌàÚµkFGËÅÞÞž¥K—R¼xqêׯO¥J•X´h¡¡¡ìÞ½“ÉÄéÓ§™?>...ôëמ={’€ÉdâرcL:•Š+Э[7¦NʬY³ðòò"""‚?ü€—_~™   [f)UªQQQ,_¾___Ê—/ÏæÍ›Yºt)¾¾¾´jÕŠeË–±dÉÊ–-KõêÕIJJbåÊ•Ö›ï«V­Ê€èÛ·/*T }ûöÔ©Sç¦ñ®çŸ3g—.]Âd2Ñ·o_ëúråÊÑ¿¢££1™LLš4)O㇅…±téRV­ZE™2exòÉ'ñ÷÷gòäÉÖ¾¯·‰ŽŽ¦L™2ãççdž (V¬ØMY_xá, }úô¹iÝ?eúëÏÎd2qîܹ¿=/"##smÓ¤IlmmùüóÏÙ¿?&“‰;väù‚ÜçFéÒ¥4hcÆŒÀÎÎîo3%++‹éÓ§S±bE&NœÈèÑ£Ù¹s'õêÕ3:šˆˆˆˆˆˆˆˆÏ7Ú9ÎÑ„&d’ÉF6R’’FG‘B$**Š6mÚžžNÑ¢EŽ#b¸‡umÞùÿÞ^ø—ok&yà²ÉæMÞ¤Ýx‘YËZ<"Ìf3S¦LáàÁƒ4mÚ”W_}___þýï[Ÿð.ò(øì³ÏhÓ¦ ñññ\½z•ƒòꫯR³fM‚ƒƒŽ@bb"~ø!¾¾¾¼þúë4k֌Ç3yòdˆˆˆˆˆˆˆˆ®¸²ŠUäCKZ’NºÑ‘DD¤¹þYDrr²ÁIDTd """"""ØyÎÓ†6Lcó˜Çd&c‡Ñ±ä!óõõeæÌ™?~œîÝ»3aÂÊ•+GÇŽYµj999FGy ºwïNÆ iݺ5%J” E‹T¬X‘eË–aggÜïÄœœV®\I‡(_¾<'N¤gÏž?~œO?ýò‰ˆˆˆˆˆˆˆHþcÆÌr–“@miË®IDD OOO@E"ù…Š DDDDDDä9ÎqžäIb‰eëèA£#‰ÁJ—.Íøñãùã?øꫯ8þ<-Z´ |ùò 6Œøøx£#Š<NNN :”={öpéÒ%âââøüóÏñò2fV—Ó§OóÑGáççG‹-ˆ‹‹#22’øøxÆgX.Éÿ*P•¬d»xžçÉA‘‘{§™ DòˆˆˆˆˆˆÈ±‰MŒvÄC=êIò{{{:uêÄêÕ«Ù·o:t`æÌ™T¨PæÍ›3sæL½(rŸ%''3cÆ š5k†··7ãÆ#<<œ°cÇú÷ï“““Ñ1EDDDDDDD¤ €ïùžø 4:ŽˆˆŽŽŽ¸¸¸””dtAE""""""ò|Á<Ã3<ÅSlbå(gt$ÉǪU«Æ¤I“8uê³gÏÆÙÙ™×^{Ò¥KóôÓOÉüatL‘éÔ©SL™2…&MšPºtiL±bŘ;w.§Nâ“O>¡jÕªFÇ‘¨ M˜Ï|f2“ñ/£ãˆˆH!`6›õ 2‘|BE""""""rßd“Í0†Ñ—¾¼Îë,d!E)jt,) éÞ½;‹-"%%…ùóçSªT)Þyç¼½½iРãÇ'66‹Åbt\‘|)''‡_ý•qãÆñä“Oâííͻﾋ——óçÏ'%%…ï¾ûŽnݺáàà`t\)àÚÒ–©Le8Ù£㈈Hg6›III1:†ˆ¶F‘Â!tzЃ•¬d6³éE/£#IV´hQ:vìHÇŽÉÌÌdÕªU,Z´ˆqãÆ1dÈÌf3M›6%$$„ÐÐP¼½½Ž,b˜“'OÍêÕ«Y³f )))xxxЪU+Þ~ûmBCCUP """"""""Lœæ4¯ó:å)O8áFG‘ÊÓÓ“¤¤$£cˆ*2‘û  'žxV³š§xÊèHRˆ888Nxx8999ÄÆÆMtt4$##ƒÊ•+JHH 6ÄÝÝÝèØ"Ljj*›6mbõêÕDGGsøðaiØ°!o½õ!!!R¤ˆ&1‘‡c#øƒ?èF7Ö²–úÔ7:’ˆˆ@f³™cÇŽCDP‘ˆˆˆˆˆˆÜ£l¤=í)Cv± oôDyypŠ)BíÚµ©]»6C† áÚµkÄÄÄEtt4Ó¦M#''///6lHƒ xâ‰'¨W¯=ö˜ÑñEîXvv6‡bçÎlÚ´‰7rðàA, ¾¾¾„……IÆ 5[ˆˆˆˆˆˆˆˆjÓH"‰0ÂØÌfüð3:’ˆˆ0f³™-[¶CDP‘ˆˆˆˆˆˆÜƒ, 7½iIKæ1gœŽ$[[[6lHÆ 8sæ [¶l!&&†˜˜Þÿ}.\¸€³³3uêÔáÉ'Ÿ$((ˆ:uêàååepz‘›%&&²}ûvbbbزe Û·oçÒ¥K+VŒzõêÑ¡CêׯOpp0nnnFDZ²Á†¯ùš¦4¥%-ÙÌf̘Ž%""ˆÙl&99Ùè"‚Š DDDDDDä.Mf2ƒÌ@ò ŸP„"FGÁÍÍÖ­[Óºukë²ãdzqãFvîÜÉÊ•+;v,999”(Q‚jÕªñÄOàïïoýÚÑÑÑÀ=GEVVGŽáÀìß¿Ÿ;w²cÇNŸ> €¯¯/ 4 ]»v4lØZµjQ¤ˆ~ÏŠˆˆˆˆˆˆˆHþæ„?ò#Oò$mhÃÏüŒNFÇ‘Âl6“ššJvv6666FÇy¤©È@DDDDDDîH6ټʫLcïó>#it$‘¿åë닯¯/½zõàܹsìÚµ‹={ö°gÏ6nÜÈŒ3¸rå vvvT©R…€€ëËÏϽ‘)wåÚµkÄÅÅqäÈöîÝk=ï:DVVøûûÈ°aàvíÚ/^Üèè"""""""""wÅ–³œ'y’®tå{¾Ç½¿*""ÿÌl6“Í™3gððð0:ŽÈ#ME""""""’g—¹L7º±’•|Ã7t¡‹Ñ‘D«+Ï<ó Ï<óŒuÙµk×8yò¤õ‰ò;wîdÆŒüþûïX,ìììðöö¶,T«V |}}ñññÑæ…³gÏrüøqöïßÏ8~ü8ÇçàÁƒ\¾|///üýýyúé§yóÍ7ñ÷÷§zõêØÛÛœ^DDDDDDDDäþªHE±ˆf4c ù”OŽ$""€Ùl ))IE"S‘ˆˆˆˆˆˆäIi´¥-9ÈjVóOIä¾±µµµ´iÓƺüܹs=z”#GŽX_;wîä›o¾áâÅ‹¸¸¸àççG… ðöö¦|ùò”+WŽråÊáíí­7@ ‰äädâãã‰'..Ž¸¸8âãã9vìG%==€bÅŠáççG¥J• cðàÁÖï5;ˆˆˆˆˆˆˆˆmmçèèHùòåñöö¶øøø`6›)Uªf³³ÙŒÝÃÜ5ùŸ«W¯’’’Brr2§OŸ&))‰¸¸8Nže6›Ud ’¨È@DDDDDDni*SÄ ^á>ál°1:’H¡âââ‚‹‹ •*UúǶ™™™7ÝLþüù[ÞlòäIöìÙ“ë¦ük×®qñâÅ;Êw½ áFEŠ¡xñâ·lo±X€?Ÿ0pþüyrrrrµ¹^8p'\\\°µµ¥D‰ÖˆëE¾¾¾·,®¸þ*^¼8%J”ÀÁÁáŽÆ‘û˾ækЀpÂù…_pæá>ôDDD ˆä*2‘›|ÄG¼ÍÛ¼ÏûŒd¤ÑqDyxyyáååuOýdff’‘‘‘kv€k×®qáÂë ×]os£¿¶¹ÑÈ‘#iݺ5uëÖ hÑ¢<öØc¹Úüu¶„ëmn5£‚£££ŠDDDDDDDDD ‘bã'~¢>õéB~äG=àHDDnâéé©"‘|@E""""""beÁ¼ÁøÓ™NúIDî#(Q¢Ä}ï{Ê”)4lØ×^{í¾÷-"""""""""…ƒ>,f1MiÊ0†1žñFG‘|Æl6sìØ1£cˆ<òTd """"""\å*½éÍ÷|Ï7|C':ID gggÒÓÓŽ!"""""""""ù\0‡9t£¨ÀK¼dt$ÉG<<<4“H> "á—èHG6²‘Ÿø‰PBŽ$"Œ³³3—.]2:†ˆˆˆˆˆˆˆˆˆ]ÖV†0 IDATèÂ>ö1ˆAøáGSšIDDò OOOˆäEŒ """"""Æ:ËYšÑŒml#šhˆÈ]ÑL"""""""""r'F1Šgy–Ntâ(GŽ#""ù„Ùl&==Ë—/E䑦"‘GØüAcsŠSlf3õ©ot$) œ¹xñ¢Ñ1DDDDDDDDD¤€0ab6³ñÅ—v´ã"zQDDþ,24›ˆÁTd """""òˆ:Ä!‚ &›l6²‘ÊT6:’ˆ`šÉ@DDDDDDDDDî”#ŽüÈœãÏñ9äIDD v½È ))Éà$"6ˆˆˆˆˆˆ<‚v±‹§xŠ²”å~¡,eŽ$"œ‹ËÿcïÞ㢪ó?Ž¿¹_DgT¼[â­ÅZ˲Z±_mQí¦¹¥–¥¡ífV[iw»ýÄê—º^S«Õ²M³{i%]LºXR™—ÕLò **7åv~¸Ì20À Â^O<À3ß9ó9sf†ó=|ßçFÈ€Û¢­7ô†Öj­žÐf—0™Íf“ÄL€Ù@ óµ¾Ö0 Ó9:Gë´NmÕÖì’xf2P_èÍÑ=¡'ô†Þ0»€‰‚‚‚FÈ0™¯Ù|˜´2/ó´žÖCzHó5_÷è³ËÐB2ÐР7õ¦2”¡$%™]ÀCl6›ÊÊÊtôèQ³KZ,BàEfj¦Ñ#š«¹º]·›]€¤"d——gr%¼IguÖ2-Ó¿ô/-ѳËx€Õj•$egg›\ Ðr2/1S3õ€Ð<ÍÓßô7³ËÐÂ0“€Ær¥®Ôz@“5YiJ3»@#«dee™\ Ðr2/P9`ðWýÕìr´@„ 4¦'ô„.ÖźN×鈎˜] EFFÊ××—™ ùš]pÍ‘#GTZZjv€&è¹Ðçô!ÿ§'f躢ë”%®æÀ5¾¾¾ŠŒŒluËÇÇÇ´ÇË€gÙ$?~\'¹Š@³R1P€æÈG>zE¯èwúnÑ-zWïÊ"‹Ùe"## &âìÍDrr².¼ðB³Ë41ÿ:ë_Ze[¥‰›'ê¬=gé[}kvIš‘ÔÔT=ûì³ ².‹Å¢ÓB/žu¤]»véà·{4»wïÖÅ_¬øøx³K Þ¬²j¥Vê]¢Yš¥{tÙ%‰Íf#d˜ˆÍDÏž=uÍ5ט]  yTê ½¡—ô’Æ ' 0»"ÍMV_<,,Lyyy ºNWq¼ xÞ Aƒ4ˆ÷@³‘––fv 4ˆ!¢'ô„¦išÎÿÏ?€÷±Z­„ ù˜]À}ëa=­§O 4Îìr@’jÚLZŽ©šªKu©ÆhŒŽë¸Ùå!À\„  ™yBOh†fè%½¤›u³Ùå€]hh¨ Ì.€—ó‘^Ñ+*R‘nÓmf—h„ s2€fäy=¯éš®ùšOÀ@“ÃL<Å*«^ÓkzKoi‰–˜] Y­Veee™]Ðb2€fbŽæè^Ý«¹š«Išdv9P !žt‰.ÑTMÕºS›µÙìr ˆ™ s2Ðh>úè#Y,Íž=ÛÔ:6mÚ$‹Å¢éÓ§»Ô¾©Ô]ÙR-Õݺ[34CÓß$¹¿]ÐØBCC•——gvœ±¦Ò'ð†¾Œ3ôe4¤Çõ¸inÔ*T¡ÙåˆÕjU~~¾ ùlÌ@È@‹”šš*‹Å¢§žzÊìRêô²^ÖDMÔ“zRS5Õ”šÓóÀ<Ìd@ããØÜ=<_€÷ó•¯þ¥)KYš¢)f—h 6›M’˜Í0 !h–k¹&h‚Ñ#zH™]ÔŠ3tTG-Ó2½¨µB+Ì.ЬV«$B€Y@µZ«5^ãu—îÒtM7»¨SXX!¦¸RWêÝ¡Iš¤Úiv9€3T1“AVV–É•-! }ô‘,‹fÏž­O?ýT\p‚ƒƒeµZuÛm·)''§Æöëׯ×Å_¬°°0 4H’TTT¤Ç\gŸ}¶Õ¦M 6LüqµÇ6 Cÿüç?5tèP…‡‡+,,Lçž{®–,Y¢ÒÒR‡v/½ô’.¸à………)((H Ðüùóe†½]YY™æΫøøxEDD(<<\ƒ ÒóÏ?¯ÂÂB·Û¹ëÛo¿Õ%—\¢EFFêæ›oÖÑ£Gn·+Û#I_~ù¥F­=z( @íÛ·Wbb¢¾úê«ZkyꩧtÑEI’yäY,û—»uóÍ7²X,ºãŽ;œ>ÖªU«d±XôÜsÏÕùUö¶ñ¶F•’m•M/†¿Xãþ¯jéÒ¥²X,Z½zu·½óÎ;öe®ìoWž/W÷[]ïÍ[HH!p}3ëk¸{Ü[ùùsµoR¾Œ{}W__UїОճê¥^­Ñ*V±ÙåÎ@PPBCC™É0‰¯ÙÍÁ×_­{ï½Weee’NàYºt©RSSõý÷ß+44´Zûûî»Ï>€¢¼¼\ÅÅÅ>|¸Ã ‘S§Né³Ï>Ó矮 hÒ¤I’Nr¸á†´råJ‡õnÚ´I›6mR×®]• Ã04vìX­Xá8íëÏ?ÿ¬;î¸C›7oÖâÅ‹%I<ð€ž}öY‡viiiJKK“¿¿¿}`‰«íÜñý÷ßkÚ´i:uê”$©°°PË—/מ={´~ýz{;w¶çСCºøâ‹Ú>|X|ð>úè#}úé§:t¨Ûµº[÷ù矯sÏ=WË—/Wrrrµ×‚ ¢ &¸ü¸ëŒuQ:Bå/–ëà_Jÿ×Ruÿ7„†Øßîì· ÎÞ#š¿ÐÐPååå™]4+ô5ά¯á.Wû&/sš«¯¯†@_€3 Ðëz]ñŠ×CzHÏêÙºïh²¬V+!À$Ìd¸à7ÞÐرcµk×.åççëË/¿T¿~ý´cÇÍœ9Óiû›nºI;wîTii©~øáÍ›7O_}õ•:wî¬÷ß_Ç×¾}û4}útY,Ý}÷Ý:tè$饗^ÒÊ•+©E‹iß¾}ÊÏÏ×÷߯ &ÈÏÏO’ôꫯjÅŠêׯŸÖ¬Y£#GŽ(??_ëׯ׀´dÉ}óÍ7’¤wÞyG!!!zóÍ7uìØ1觟~Ò½÷Þë0˜ÄÕvîxíµ×tË-·h×®]*,,Tjjª:wî¬/¿üR›7o¶·sg{,‹†®÷ß_û÷ïWqq±²²²´jÕ*(99¹Æz~øamØ°A’ôä“OÊ0 ûW}ê¾óÎ;•——§W^yÅáþÛ·o×úõëuÓM7)<<Ü¥çj£6*±4Qåï•+ò±H-ZXóþo®ìﺞ/wö[gïÍ_XX3€›èkœY_Ã]®ã»Ûž¾Ìi®¾¾}5é©žú‡þ¡çõ¼R”bv9€3`³ÙªÍú À3˜ÉpÁy秗^zI‹E’tÑEéwÞÑYg¥Õ«WëÉ'Ÿth?xð`-]ºÔÞ^:=A’V®\©ÁƒK’Z·n­Ç{LZ¼x±Þ{ï=%%%iÙ²e’¤×_Ýá*ƒ Ò Aƒìÿùå—ÕªU+}üñÇŠŠŠ²/:t¨^{í5ÅÅÅéÝwßÕù矯Ž;J’®¾újùúžî 0@ p¨ÝÕvî¸ì²Ë´páBûÿ‡ ¢ûï¿ß~eÈŠu»³=6›M3fÌÐÌ™35qâDeggÛ¯$)I[¶l©w½îÖ}ýõ×ë¾ûîÓ‚ tûí·ÛÛ/X°@’4yòd—o‹¶èú£B¾ Ñ©Néõ5µïÿ†ÐûÛýVÁÙ{@󬢢"†Áû\D_ãÌúîrõßÝö-½/SÁÕ×WC / 6ã4NŸèÑmÖfÙd3»$@=X­Veee™]Ð"1“à‚Ë.»¬Ú‚nݺ©W¯^Ú½{wµö ÕÚÿú믊ŒŒ´ú©ìª«®²·‘¤;v(""ÂaP†3Û¶mSYY™:uê$___µjÕJ>>>òññQ\\œ$iß¾}’¤Y³f©¼¼\=zôÐĉµ`ÁýøãÕÖéj;w\rÉ%Õ–uëÖM’”——W¯íùúë¯uÁè7ÞPFF†Ã I***:£šÝ©Ûßß_“&MÒÖ­[õå—_J’òóóõÊ+¯è²Ë.ÓÙgŸ]çcýª_õ?ú ÐùüÅG¡uïÿ†ÐûÛýVÁÙ{@ó,Ã0ä3Z úgÖ×p—«Çøî¶oÉ}™Ê\}}5ú2ê²@ ¨@ݪ[eȨû€&Çjµ*;;Ûì2€‰Ð"##.oèååå’¤²²2•••©¼¼\†aÈ0þ{¢´¸¸XÒé+:îرCË—/W×®]µaÃ]~ù劋‹s¸R¦«íÜTmYÅsQ¹Vw¶'99YÅÅÅzì±Çô믿ª¨¨ÈÞ¾wïÞõª³¾uKÒ¤I“äïïo¿âç+¯¼¢'NhÊ”)u>ÎÐp W'uÒ»zW–SõøøœîæU<—•9¬ÔûÛýV¡¦÷€æ-88X’TXXhr%à½èkœæîqowŽñÝißRû2 ‰¾ €†®p½ªWõ±>Öb-6»@=2ÌCÈpÁ'Ÿ|Rm Fzzº~ùåuïÞÝ¥uôèÑC‡Öwß}Wí¶5kÖØÛHÒYg¥ÜÜ\}ú駵®ó¬³ÎRpp°Ž;fQõkõêÕöö¾¾¾:t¨¦M›¦ýë_úí·ßtâÄ ?Þa½®¶khîlOzzºl6›¦OŸ®îÝ»+00P‹E»wïÖ®]»ê|¬Š,U¯Z_6›M£FÒ[o½¥C‡iáÂ…êÙ³§®¸âŠZï—£]¦ËªP­Ñ…)ÌåýïŒÕj•$ýöÛoÕnûì³ÏœÞÇ•ý]Ûóåîë€÷"dQ¿¾F}Ž{SKìË8C_@Ss¡.Ô4MÓ]ºK[T¿‹iÌCÈ0!Àß}÷Ư_ýUJMMÕŸþô'•””hĈ.­cäÈ‘’¤Q£FiÍš5:qâ„8 'Ÿ|R‹/V@@€®¾újIÒÍ7ß,Iºá†´dÉ8p@JKKSRR’Ö¯_/I?~¼ • >ø@999*..ÖÞ½{õá‡êºë®³î¸à‚ ´hÑ"mß¾]EEE:~ü¸>úè#9rDéééö:]m×ÜÙžÎ;+;;[óæÍÓñãÇuüøq­Y³Füã^ù²ª¶mÛJ’6lØ #GŽ4HýS¦LQII‰n½õVmÙ²E“'O®õŠ²ÇtL—é2•¨DŸèEêô1]ÝÿÎôéÓG’4{öl}ñÅ***Òž={ô÷¿ÿ]ï¼óNµö®îïÚž/wöïFÈÜG_£~} w{ëËÔ„¾ €¦hº¦k êFݨ“:iv97X­Våää¸tÞ @Ãò5» 91b„–/_®—_~ÙaùYg¥©S§º´Ž;î¸Co¾ù¦¾þúk]yå•ÕnŸ={¶:tè Iºå–[ôÑGiõêÕJJJªÖöú믗tzÇúõëõÏþS‰‰‰N÷¶Ûn“$ýðÃúæ›ojmãN»ÆàÎöLœ8Qk×®ÕäÉ“5yòdûíçœsŽúöí«ÌÌÌZ«gÏžŠ‰‰ÑgŸ}¦víÚÙ—W½Š¬;âããuÁhíÚµjݺµÆWcÛ“:©ktë°6hƒ¢e¿ÍÕýïL·nÝôç?ÿYo½õ–.½ôRûr___;V¯¼òŠC{W÷wmÏ—;û €w#dQ½+Ü=îml-©/Sú2š"_ùj…V裇õ°žÓsf—p‘ÕjUYY™Ž9¢öíÛ›]Т0“à‚!C†híÚµ:ï¼ó¤víÚiüøñúòË/êÒ:üýý•’’¢Ç{L½{÷–¿¿¿ÂÂÂt饗jíÚµš4i’½­V­Z¥Å‹kðàÁ QëÖ­uÞyçiéÒ¥ºä’K$I‹E/¿ü²V®\©„„EDDÈßß_ݺuÓµ×^«·ß~[ ’¤7êoû›úôéc߆!C†héÒ¥š5k–ý±]m×ÜÙžk®¹F+V¬Pÿþý¤¨¨(Mœ8QŸ~ú©ê|¬V­ZiõêÕºð Ò`ÛP±o½õV………9mS®rÕXmÖf} ÔE]nwuÿ×äÅ_Ô­·ÞªÈÈHêüóÏWJJŠ†Z­­«û»¶çËýÀ»2÷Ñר_ÃãÞÆÖRú2u¡/ ©ê¦nš£9z^ÏkÖ˜]ÀE6›M’”mr%@Ëc1ÎäÒ6à%®×é+h­Ò*“+AsµjÕ*5ꌮW—Å‹;½€ÆõÑGéŠ+®Ð¬Y³t×]w™]š»ï¾[sæÌÑ®]»Ô½{w§m¦hŠk±>Ñ'ºHy¸Bø¯†>Æ]¾¾¾ºüòËuêÔ)M:µZ›Ÿ~úI–›,zÔxTew–i„eD‹Þxùúú2Ô } Ïs¹/Ã~à%®åZ®Oô‰k±Ùåêàãã£ÈÈHeee™] Ðâ2€­çŸ^W\qEµÛ6´Þ ½$é Is=^xLpp0!š™Úú2àm†j¨î×ýú»þ®]Úev9€:X­Våää˜]Ðâøš]Д]~ùå2 ¦J­làÀ<'ULŸ>]Ó§O¯ñöïôèö€nÓmZüØbé1ÏÕžFÈ\C_£:úžWW_Fb¿ðNOè ¥(E7è}£oä'?³KÔÀf³);;Ûì2€‡™  ýª_•¨D]ªKµ@ Ì.!M¯|µLË´]Û5C3Ì.P «Õª¬¬,³ËZBÐH2”¡á®®êª×õº|™L@ ¬¢¢"³Ë€Z­³õ´žÖ“zRßé;³ËÔÀjµ2“`F¸@#8¡ºRW*XÁZ£5 QˆÙ%€G0“à…žxB:xÐqYLŒôê«Ò§Ÿ:.üq©CÏÕð.~(½÷žã²ß~;ý}âDÇåW_-]y¥gê‚׺Kwé}¢›u³Ò”¦`›]  B€9@=ý¨Õ_ýÕJ­–«X×é:e+[_ëkµU[“*Ï#dx¡ü|iñbÉ×W²Xþ»ü£þûsY™--ZäùúÞ£}ûÓ}ÐV­$ÇÛ6o>ý½¼üt?tüxÏׯc‘EKµTýÕ_Ó4MÿÐ?Ì. P!À>u783Fct•®R¾òíËÊU®1£ïõ½Öhbkb…ày„ /tà §¿—–J%%ο|}¥›nr !à®óΓbcO‡jꃖ•I:Içžkvµð1ŠÑ?ôÍÓ<­ÕZ³ËTaµZ•——Çߟ#dõð¾Óvm×:­Ó` ÖA”$Ý£{ôþþ Г«Ï#dx¡sΑzô¨½MqñÃœ‰›n’üüj¾ÝÏOºå‚îhP£5Z×ëzMÐѳËTbµZ%‰Ù #dõ°DKä'?•©L;µS5PÓß4Wsõª^ÕEºÈìÀ„ /5vlíž­ -B¸Âõ¢^ÔÛz[+´ÂìrÿÁL€9€›^×ë:©“ËÊþóo±k¦fšT˜ॺw— p>À£´TúË_<_À{Ýt“ÔªUõåÝÑȆk¸þª¿êÝ¡ýÚov9þÛShh(!ÀÀ›^Ð N—2T®r= ”¤$•ª†«ý€#dx1g<,iÐ ©kWsjx§nÊʪ//-•®¿Þóõ EyFÏÈ&›Æk¼ f—ÐéÙ ²²²Ì.hQ€¶k»¾×÷*Wym Z¢%¥Qœ|Ðâ2¼Õ_þ"•W9jÕêtø€†Ô©“4x°äSih“Ïée±±æÕ…!XÁZ¡úB_Ôxá)€gY­Våää˜]Т27¼¨å'¿o÷ùO7ë2]¦š!‹,ž* šf2¼XT”4dˆãòriäHójx¯±cOÏ WÁLJ ;<&^ñú»þ®{u¯~Õ¯f—-žÕjUvv¶Ùe- !pQ±Šõ’^R‰JœÞî#uVg½¯÷õ±>V/õòp…`>B€—;ö¿?·j%]r‰d³™VÀ‹U µ†ôç?›S Z¤éš®îê®q§2•™]´h6›MYYYf—´(„ ÀEoëm×ñjËýä§ é=¢Ú¡«t• Õ@Ó@Èðr#FœT¨: !µk'%$œî‡¶juúg«ÕìªÐ‚(@Ë´Lßë{ÍÕ\³Ë€™ Ï#d.Z¬Åj¥ÿªó•¯,²h”Fé7ý¦éš®˜X!˜/88XÅÅÅ*--5»!"BúŸÿ‘,ÉÇGúÓŸÌ®àÍÆŒ9=ƒaœþð°¨õ ÐÚ®íf—-Vûöí ækvÐXN:¥œœegg+$$DÑÑÑ «×ºök¿¾Ð*W¹|ä#C†ú«¿æk¾kpWÍWpp°$©°°P­[·6¹áDb¢Zð /½Tþ!!œd¸äøñã*//WAAŠ‹‹¶9vì˜ Ã°ÿߧK ðõ•EÒO±±*OK³ßf±Xît=þþþ Q«V­8G…3öÒ‡úPã5^©Ju¸ À3l6›rrrT^^.®¯xÿЬ+;;[‡RVV–²³³•‘‘¡ììlûÏ999:tèrss«Ý?((H6›MQQQjß¾½¢¢¢d³ÙdµZ«ýj¿ßR-µ ¬²j–fi”ùºBó IDATFÉ"‹'7šíÝ»×¾ÌRT¤lIã>ùDo*&&F;wV×®]«Î;+66Öþhöf\tâÄ =zT¹¹¹*((P~~¾òòòtìØ1ûÿóóó•››«üü|Ø——””¨¨¨H'OžTII‰òóóe†Ž;vF5½ñŸï#‡=£õ„‡‡Ëb±(44T~~~ R`` üüüªû÷ˆˆ…††Ú¿Ú´i£°°0ûÿÃÃÃYï ¡yñ•¯–i™~§ßé9=§©šjvIÐâX­V•••éèÑ£j×®Ùå-!¦+))Qvv¶²²²tèÐ!egg+33SYYYÊÉÉÑÁƒ•““£¬¬,9rÄá¾!!!á€~ýú©}ûöö ÕjUûöí•ŸŸ¯ÌÌLûãdff*''G»víRjjª²³³•““ã°îàà`uèÐA¶h›~|÷GµjÝJ¦^¨k¹VÖ@}ÝþkuèÐA:tPHHˆ'Ÿ2h²*‡ 4=ÅÅÅÚ¿¿=4P5L°ÿ~•””H:}ÕÇŽ;ÚÿÿýïÕ¥KÅÆƪdñbÝ7i’Ffe9¬ëÇÔÞ½{•——gÌ:8 T¬‹@4ŽÂÂBeddØ/Ø“­£GêÈ‘#ß+ÿ\Ñ'¬ªM›6öøaaa ·Ê·Ùl V@@€Ó*÷‡……É×××>¸ß™ùûû;.[·N²X´;!Áayqq± œ®çäÉ“***rv8qâ„ÊÊÊì3*œ:uJ………ÊËËS^^ž233uüøqåååÙÇwú8~~~ŠŒŒTÛ¶mÕ¶mÛj?GFFª}ûö²Z­êСƒ¢¢¢ìçÏмœ­³õØþýQT?õ3»$hQ¬V«$)++‹à!„ 4šÜÜ\edd(77W™™™ÊÈÈ°¯¼,;;[eeeöû*""BÑÑÑŠŠŠRçÎ5xð`EEE9,ŽŽVDD„Ëõ 8°^5§¶NÕþ}ûÕk^/ÿ鸞Éx¦Îškú9&&¦Æé›À2ÌuêÔ){Cnnnµ°FÅ÷C‡iÛ¶m:zô¨>\m&‡ÐÐPÅÄÄØgUîСƒ¬V«bbbeç™´¥¨Éýº_êCݬ›µQå'?³K€£"d­¸¸8“«ZBÜâl¾³AMƒð+¸‹‹«q@¾Y*j¬lœÆ©ÚHKÛV<Ξ‡ÜÜ\mß¾ÝþÇòòrûýj $T^Ö±cGµiÓÆ› †иj¼_qLRÛàý„„‡ü]»v•Åb©_!uÌ>d\gj C¤¤¤Ô†¨>¨øêܹ³|}9Õ À{;vL¿ýö›öíÛç0û\Å÷ììl{Ûàà`û@u›Í¦Ø¬WÄîç×LD›.¨V¯ÀEÅlΙ™™!ììledd(--M‡Rff¦Ã95›Í¦Î;W›066V]»våï&ð‘–j©ÎÑ9š¡zTš]´íÚµ“¯¯¯Ãq€ÆÅ_^¸ÈÉÉQii©ý~jÛ¶­KÁ¨¨¨úb3Y9?Q_H¨ëJ ÅÅÅ:|øp­ÏoZZšrssë $Ô4SB§NìSO€™g&77·ÆYvïÞípõȈûÀû¸¸8%&&ÚÞwïÞ½IÏžPk¡¤¤D999NiiiÚ³gýsÆÏÏOíÚµ«>¨xnºté¢OnÔ©¸¸XPzzº¶mÛ¦íÛ·Ûû=•ä•û~¿ûÝï4bĈj³À¡ióóóSLLŒbbbêlëìâúùçŸõÎ;ïhÏž=ö¿!T¼6ºuë¦>}ú(..NݺuSïÞ½ÚØ›ÕbõVoý¯þW÷ë~%*Qçè³K€ÁÇÇG‘‘‘„ "dx©Šѵ]i?33Sûöí«38Э[7§ƒÜ›spÀ“üýý­èèè: §NÒ‘#Gj $¤§§+55U¹¹¹:tèýM’ë„Î;›?½5¯EȨ]åAÕ0ÁÎ;•ŸŸoo[y°@BB‚’’’ì¿ã½}À€ŸŸŸýø)>>Þi›ªÏeÅó˜’’¢]»véĉö¶•ŸËª³!ôèу«?h4§NÒ¶mÛ´eËýüóÏÚ²e‹~ùåíÛ·O†aÈÇÇG;wV¯^½Ô»wo%&&ªW¯^êÚµ«bcchö&ÀƒêšðäÉ“Ú»w¯ÒÓÓõË/¿è—_~Ñ®]»´lÙ2íÛ·OåååòññQ§NÔ«W/õïß_ýúõSÿþýÕ§O4‘Y!š»;u§ÞÒ[ºU·ê;}'?5ÓYB ™±Z­„ "d4#®öï߯’’ûýüýýYcp òò:ÈÇÇÇÄ­lÙÎ(PùµPW ¡¦BåBÇŽåïïßØ› À‹Ëb±2@‹Tù*”Ug!HOOwwúûû«cÇŽöß¹!‚Š;w–¯/§ok¡øøøZCÎöÃW_}e_^y]UÕC „k¸"33S?üðƒ¶l٢͛7kË–-Ú¹s§JKK¤¸¸8õïß_ êÙ³§zõê¥=z0ð. TïÞ½Õ»wo]qÅ·:uJ»víÒ®]»ôË/¿hçÎúâ‹/4þ|òÑ2-S?õÓ ÍУzÔì’ E dx¥LV98PÓ`qgÁÉq0Á–Ç@ÂÉ“'uôèÑÃ)• •ÜI¸ÇÇÇG„ à•NžúHÿ÷ÿ' >>žÜ<À+lÛ¶Mo¾ù¦Þzë-mÞ¼YíÚµÓUW]¥‰'jØ°aêСƒÙ%¦ Sbb¢%I™™™öÙ=žyæM:UÔu×]§?ÿùÏêÓ§e°¿O÷ém½­ñ¯T¥ª•8ïÁf³)//OEEE 2»Àë1šÍB]W@¯XvðàA?~Üá¾U¯€^Sp S§Nòóó3i Tæn ¡¦0QCl6ƒQ70“꣦ÁÖÇ{µ ¶NHHplݵkWY,“·¨[PPýuìLmáš”””ZÃ5U „k€ÿÊÌÌÔÒ¥KõÚk¯iÇŽŠŽŽÖµ×^«çŸ^_|1ÇÎ@ ¢¢¢4fÌ3F¥¥¥Z¿~½Þ|óM-X°@<òˆÎ>ûl=ZãÇoQ_ùêE½¨A¤¹š«»t—Ù%€W²Z­’¤ììlÅÆÆš\ àýÀ4µ*ÿœ‘‘¡cÇŽ9Ü·rp ""Bqqq‚ -G]ƒó*s%™™©¨¸¸Øá¾Î ΂ €ÓïË‚‚³Ë@“››[ã,»wïv8Þ‹ˆˆ°”Ž‹‹Sbb¢ý³¾{÷î 7qKÏ ¨õ8§¤¤D999N:iiiÚ³g=ôåçç§víÚÕ8B—.]âÉÍ<Ê0 }þùçZ¸p¡Þ}÷]…‡‡k̘1Zºt©Î?ÿ|fÜäëë«aÆiØ°aš7ož¾þúk½ù曚={¶üq]{íµºýöÛuÉ%—´ˆ‹AôS?=¨õÒUºJ=ÔÃì’Àë2<‹ÔÉ“'uôèÑ:ƒ™™™ÊÍÍu¸oÕà@·nÝ4dÈjz;vì(“¶@sÖ„Šïiiiúàƒ´ÿ~•””8Ü·"à,„Pùç:ðÇKx¥ÀÀ@:uÊì2àa•CUÃ;wîT~~¾½mű^Å,IIIöÏÍÞ½{+44ÔÄ-š???EGG×:ûSÕ÷fÅû2%%E»víÒ‰'ìm+¿7«Î†Ð³gOµnÝÚS›4˜’’-]ºTsæÌÑÎ;5dȽüòË1b„Ì.ð >>>ºð uá…jÆŒZ½zµ.\¨?üá:묳4yòd?Þëßsé!½§÷4Aô¹>—EÞ®O²Ùl’¤¬¬,“+ZB¨SEpÀÙ [g!‚Êh®2žžN -J`` Nž¨JˆŠŠjW§@ó`†V®\©‡~XÐ-·Ü¢U«V©ÿþf—xµÀÀ@3FcÆŒÑæÍ›µpáBýýï×ÿþïÿꡇÒm·Ýæµç†|å«õ¢ÎÓyZ¨…ú«þjvIàU‚ƒƒ¢ììl³KZïf—Ó"DDD(""BqqqNo¯éó!==])))µ~>T $ðùOùüóÏuß}÷éÇÔرcõøã+66Öì²ðgÒ÷sç¾ ÝǤÏê¾hÑ¢EzôÑG5sæLÝ}÷Ýš?¾æΫK/½ÔìòÅ@ Ô}ºOÓ4MWêJÅŠÏhH6›à!„ ¼ˆ³à@M!‚ú:uê¤Ö­[›¸•5;xð :w¤$-\¸°ÑïÀêÔ©“òY³fé®»îrÚ>%%EÇ×–-[Ô·o_IÒ¼yó4yòdIÒ’%K4a„Æ-º…KKKÓwÞ©Ÿ~úI………Š‰‰Ñªµ3c_M›6M3gΔ$­]»V—_~yƒ®ß™¥K—ê¶Ûn“$-\¸P“&Mj´Çjìísgý?þø£ž}öY­_¿^êÓ§|ðA]uÕU ZSMÜ $T\Y¸¦Ïu hªuøða³Ë@%5]©¼r˜ BÕ+•'$$4©+•sÌWÇ”5ýî=Óí///—a}•†à¬.³œÉ1˜Ç1µ»Ô6ÓIJJJ­3T ™Np¦ uÿý÷kÁ‚úãÿ¨Í›7{¬à¬ÿX!&&FÆ ÓÌ™3Õ¡CÔãWÏ7”3éû9»oM}¾ú>NM}ÞÆ곺úøÍYtt´æÌ™£É“'ëž{îÑ°aÃ4yòd%''+((ÈìòÜ£zToëmMÒ$­ÕZ³Ë¯bµZ ÂÙz€&®¸¸X‡®q€i埳²²ìW”ª¢¢¢_m€iÇŽÕ¦M·²a,_¾\åååzýõ×5kÖ,6êãuìØQ†ahܸqZ½zu½®Ø|ÇwhܸqÌøà!£GVÿþýõñÇkÿþýºæšk\¾ocï«ääd7NgŸ}v£¬ß™ &h̘19ßØÛçÎú¯¸â ]zé¥JKK“¯¯¯yä]}õÕúðÃuÅW4J}õUqeaWÔHؾ}»222”­²²2ûý*~WÔB €ú ÔÉ“'Í.£EÉÍÍ­q‚]»véĉö¶ö½C† qèÛ³gÏ&(­À1Ÿç×ïŒ;¿{ݩ.ÒÑ£G¢Ä&ëLŽÁšâqŒ¿¿ʪŸQŸS)))Ú¹s§ÃûªògTÕz÷î­ÐÐPOlš¡C‡)11Q»wïÖŠ+tà 7xôñkê?jݺuºñƵcÇ}óÍ7Mn†¯39w^gÒ÷sç¾õ}œšú¼žê³šñ÷OéÑ£‡Þ{ï=­X±BwÜq‡6nܨwß}W6›ÍìÒT€ô’^Ò Ñ«zUc4Æì’Àk2<‡€I*_MöL‚Š‹‹«64&&Fááá&n¡çýóŸÿÔÈ‘#õÆoèí·ßöøд>Þéí!„ª³!¤¤¤h÷îÝ:vì˜Ãº**‡ºwïÞâúÁ8-++K]t‘|||ôÝwß©Gf—d¬k®¹FcÇŽÕ /¼ ü±Æcc3pîf=z´Î;ï<]y啺袋”šš*«ÕjvY j°ëvÝ®»t—.Óe²Ê»¶ÌbµZµwï^³ËZ„¦hærssµmÛ6¥¦¦ê7ÞÐœ9s4mÚ4ÝtÓMJLLÔ Aƒ-___µmÛV}ûöÕðáÃ5qâDÍœ9S)))ÊÍÍUDD„4uêT½üòËZ·n¶nݪ£Gª¨¨HÚ´i“Þÿ}-_¾\ÉÉÉš2eŠnºé&%$$(..®Å ¬øꫯ¨äädIÒË/¿\­Í¼yód±Xd±X´hÑ"Ýyçjݺµ:uê¤_|Q%%%ºãŽ;Ô¦MÅÆÆêÅ_l°ú¦M›¦áÇK’úõë'‹Å¢.]º8´)--ÕwÞ©6mÚ(&&FO<ñDµõ|ðÁ4he³Ù4iÒ$‡+×äã?ÖàÁƒ¤ÈÈH;V™™™öÛìÏÅbQii©$é駟VŸ>}ìí~øa{›§žzÊésZßú+¯kÞ¼yºýöÛÕ¶m[Y,ýå/9£í›7ožýJ±'N”Åb©qza³÷•3u­ëðáúï¾ûÔ£GjàÀz÷Ýw®kþüùŠUpp°.½ôRíÚµËíÇ<“}åL]ûÏY»N:iĈÚ¸qcë]´h‘Ãëúõ×_—$¥§§ÛæI’¯¯¯‚‚‚täÈ·ko®*j^x¡FŽ©)S¦(99YË—/×û￯M›6)##C¥¥¥:zô¨¶nݪuëÖé…^ÐÔ©S• ˆˆåææ*%%E3gÎÔ-·Ü¢áÇ«oß¾Š‰‰QPP¢££5hÐ %&&ꦛnÒ´iÓ4gÎ-_¾\)))Ú¶m›Ã AxBî9uê”}0íâÅ‹5}útMœ8QÇW÷îݤ˜˜ 4HcÇŽÕâÅ‹•žž®¨¨(%&&jÖ¬YZ·nvïÞíp¬¸jÕ*%''+))É~ŒØÜóÕ­!ÖïαECnå}·téR·jÞ½{·®¹æµk×NaaaºöÚkõí·ßº\—³\9¾uõ8¦!ŽÁ$ï=Ž©!$&&*))IÉÉÉZµj•6mÚ¤ÜÜ\j÷îÝZ·n’““•    mÛ¶M‹/Ö¨Q£4hÐ EDD(((HÝ»wwèk¿ñÆJMMUzzº Ã0{sÐÀJKK•˜˜(¥¦¦6©€AeýʃâÜé¹r~ÔÝþd]çÎ]9÷îî¹bg}?WûbÎî[SŸ¯¦>f}û¼ÎÖwèÐ!‡sЕ¿FŒQïýâLMç»k[~~¾Ã},IÚ³gÃr³ôìÙS©©©’¤ÄÄDûߦ¼É ÍP°‚uî1»ð6›™ O1ÆÈÿüêkåÊ•FcÿZ}á…uý¨ÙÑ£G­[·6l0V­ZeÌž=Ûxì±ÇŒ¤¤$㪫®2âã㨨(£U«V†$ûW`` eôéÓÇHHH0ÆŽkL:Õ˜={¶±lÙ2cݺuÆÖ­[ƒš½‰^a„ Æ3ÏW÷_E»‡~Ø8|ø°qðàAãÆotxïU]ÿÉ“'ÄÄDcþüù5Öd†ñù矒Œ»îº«Öv¨[ÅïÌuëÖË–-3fÏžmL:Õ;v¬ÃïL§¿3ããã«®ºÊáwæªU«Œ 6[·n5Ž;fö&zÆ:Æ|â‰'ŒÞ½{7ʺkÒ”— Ý»wëÖ­3^xácêÔ©ÆÈ‘#!C†ݺu3,‹Ãû¡[·nFBB‚‘””d$''Ûß»wï®õ÷HKÀ1_Íû¿!×ïê±…³ßígºýÎŽÙ\©yÀ€ƨQ£Œ¬¬,#77ט2eŠÃþª©.WŸKgÇ·®Ç4ä1XUÇœvòäI‡ÏÙŠ>{BB‚Ñ­[7‡þz@@€ýs¶â˜ã…^0Ö­[gìÞ½Û(--5{sh4›6m26mÚäRÛ¦ÜǪzæ™gŒ   ã—_~1»Ã0jî?Nœ8Ñd¤¥¥Ù—¹Úÿråüè™ö'«ž;w÷Ü»;çNi`Ê IDATŠ=¦«}1g÷­«/Z¹í™ôy«®/33Ó6l˜Ãýzè!#00ÐøùçŸ Ãh¸¿Ôt¾»®õ—••±±±Æèѣ•eÄÅÅ5‰sM;vì0çž{ÎìRÅc!CÆ{Æ{f—^aöìÙF‡Ì.ðOõÍGŽiŒY}ü,!0àÌ2h~\ øúú: ‚ p98ÐNʵFxx¸qàÀÃ0 cþüù†$㩧žªÖ¶âðøñãíË8`Hr8Áš‘‘aH2^{íµ:¿¡œÝzë­öeåååFhh¨ñä“OÚ—õêÕËèÛ·¯Ã}ßÿ}C’ñ駟Öø¸½{÷®v¿üÑd$''Û—­X±Âd¤§§†qúäxÅÀÇY³fÙÛþùágõ—––ÁÁÁn×_±® &Ô¸=õݾ† 4Ö¾röG‹ú®+11ј8q¢Ã²^½ztXöÖ[oUàÖXûÊÙö¹ºÿz÷î]-“——g´mÛÖéú Ë/¿¼Öý››kÌŸ?ßhÓ¦m:tÈåmÁ™9uê”qðàÁ FŸ>}Î(püøq³7³Yh¬cÌgžyƈm”u×ÄÌãå£G›6m²WV„âããððp‡×pDD„ýõ[9D°iÓM×´mh8æ«ýwc¯ßÙ±E}BuÕç옭®š‹ŠŠ IÆÊ•+í·—””V«µÎºœqõøÖÕ㘆<«ÀqŒ{Š‹‹ƒÚ?«“““í!„>}úÁÁÁöÏi???û±ÆÈ‘#B[·n5 ÌÞê¼Qii©Ñ±cGcÚ´if—bWµÿXðÿìÝy\Ôõ¾Çñ°"#2,²‰)*¹jF.¥h™šuÊêXi§…Ê6óž4•Ö­nuê”iç^ÍÊŽÕIÓ{*m•5—:Z–Z.G•=EdùÝ?¼3‡Q~0¼Ÿ>æ!üæ7¿ù|g˜ßü–ïû÷=~ÜøðömÛW_}õiïjÿ«!ÇGÏu²æ±Ô3=ö~&ÇŠë œn_ì\C®4tŸ÷tË[³fáíím¼òÊ+Žiq¾ ¾ãÝ Yþ3Ïژǥë{ÎúŽ½ŸË±âêêÛs—ÆØç=rä7ß|3#GŽäþûïç}©ïxwC—ÇwðÄOðÞ{ïq×]w°dÉV­Zuö nd7Þx#Ï?ÿ<;wî¤Gf—Óèæ2—žôäQesÌ.GD¤E §²²’ÇÓ±cG³Ëñh ˆˆˆˆG9vì#GŽ¬óþ¶mÛÒ¿z÷îMïÞ½éÛ·/ݺu#,,Ìé`¯´,o¾ù&·Ür‹ã÷ÐÐP®¸â V¬XÁÚµk‰ÅbaÖ¬Y=z”>úˆ±cÇ6¨~‹ÅrÖõ¸|Žsi_crÇ{åJC—õË/¿ð裲~ýzòóó1 8uÂÇ.''¨ýZÕüÝ]ï•+ }ÿê›Ï•ûï¿Ÿ>}ú°lÙ2¾ûî»Zøª›7o^³;)Îüýý‰‰‰!&&¦ÞùŠ‹‹ÉÈÈ`ëÖ­lÙ²…Ÿ~ú‰­[·²cÇvìØáò1 ,àÎ;ïtGÙ­^K†ÁŽ;X½z5kÖ¬aýúõ8pÀå6——‰‰‰ 2„¡C‡’’’Bll¬c;B—¶ù¶ÑËoȶÅÙª¯>WZóçŸÎSO=Å”)S¸ùæ›IMMåñǯw;àLkuµ}{ºíwmƒÙçÕvLãhÓ¦ :u¢S§N\rÉ%µî?|ø0›6mbÍš5¬Y³†ï¿ÿž¬¬,Ö­[Wk^«ÕJ¿~ý>>Ìœ9“O?ý”?ü矾ÉêoèrÚ¾¦Ð˜mmȲÊËËIMM¥S§N|óÍ7tíÚooon½õV¶lÙâ˜/22¨ýZ=zÔmõŸNCß¿úæsåÑGåÎ;ïdàÀLœ8‘üQW“m¡Ž?NNNyyy••EAAc4ƒüü|rssÉËËãĉN Ãf³Ñ³gOÇhÕ¿Ï###MYG´-9dàååE=èÑ£wß}7'OžäÀŽÑ j^ézÑ¢E,X°8uâ;&&ÆqeëêWºŽ‹‹#&&___3›Ø"i›¯é–ßÐm‹¦ÒÐ×Äjµòâ‹/òâ‹/²qãFfÎœÉ!Cøå—_ܲ¾oèvLsÜ“ÚrrrœÖëöõýÞ½{Ù¿?ÇŽsÌÉ\à4z}ƒ¸¸8ÚµkgbKDDDDZûhwÍòü——çwï¼ó½zõbæÌ™Žc ÝÿjÈñÑÆÞ§hŽÇÞƒ»öyßzë-–,YÂ'Ÿ|â4Ò\c¼/õï>“åß{､¤¤ðË/¿ðÆo0yò䳪Ç]öìÙü;là‰&2‘wy—;¸ƒ-lÁ?³Ki‘ª‡ zöìir5"žM!ñ(mÚ´qt ;’’rrrÈÎΦ°°Ðñ³ýÿýë_äääpàÀ§@œêÀITT‘‘‘X­V—?GDDèJ¶n¶hÑ"&L˜ÀÛo¿í4ýøñãÄÇdztéR^yåMªðßÎåo!::šîÝ»óÏþ³Ö}}úôáOú7ÜpƒËÇ%&&òÍ7ß8Mß²e EEE¤¦¦:M7n<ð=ô#FŒàüóÏ'!!¿ÿýïìÛ·¤¤¤&«¿!Ë=“ö5”ïÕÙ.«oß¾äää0uêTºwï¬¬Ìiþ:Э[7¾ýö[§é?üðƒÛê?†¾öùV¯^í4_vv6 deeꘞ€ÅbañâÅ$''3mÚ4—Wqª~E)i:¥¥¥>|ØåwpÍïãÂÂB§Çúûû;}ßöêÕ‹‘#GÖúŽ‰‰Áb±˜ÔBSïUYYUUU±-äëëK—.]ê=^XXHFF†ãfÿ;^¹r%»w令¨È1¯Õj%!!„„Ç6¥ý÷®]»ÜÍjQ´Í×8ßùmQ—ÆþÌ7¤æ¡C‡2bĶnÝ ÀÀyíµ×èÒ¥ ›6m¢K—.n©«!Û1îÚÓvÌ™©¹Ž®¾žÞ¹s'ÅÅÅŽy«¯£ÇŒã´®NLL¬uUU1×yçG·nÝxë­·Îi$3wëÑ£“&Mâ7Þàþûï§W¯^ddd4hÿ«!ÇG{Ò]ÇÞÝ¥¡û| }ÍÏÄž={¸ÿþûyà5j”cúùçŸÏ¶mÛÎù}©ïx÷™¼ï_|1\póæÍ#33“ /¼ð¬Ûì‹-¢{÷îœwÞyf—âVó™ÏùœÏ³<Ë,f™]ŽˆH‹Š··7yyyf—"âñ2‘V+ ÀÑqàtNHÈÈÈ`åÊ• $41Ã0X¼x1+V¬¨u_`` =ô3gÎdÙ²eLš4É„ Ù¯bºcÇ"""èÕ«+V¬p:]Ÿ_|‘«®ºŠgŸ}–;gžy†ŠŠ ÆWçã^xáÆÇÌ™3™:u*¿ýöwÝuçw÷ÜsÓ¼±±±ôíÛ—÷ߟ/¿üÒ1}ܸq¼ôÒKL:õL›}ÎõŸÎ™´¯¡Ìz¯ÎfY^^^„……±xñb®¼òJX³f Ÿ~ú)ñññNËš={6&L੧žâÞ{ïåÀ.G¦p×{åJCß?û|=öS¦L¡¸¸˜»îº‹I“&9 ªëÙ³'ÿõ_ÿÅÔ©S7nœ#8§:š]pÁ\tÑE,Y²¤QÛÔÙƒ®¾#]…ª«HHHà’K.qúŽŒŠŠ"::ZWoAüýýS'hL®¦iX­V’““INNvyaa¡ã³Q½“ëºuëÓ«/«fø z(!22//¯¦jšé´Í׸ßùmáʹ¶ÿlj>räÛ¶m㥗^âöÛo§ªªŠùóçãïïOÿþýë­«_¿~g]WC·c{LÛ1ÎNž<ÉÁƒ‚Õ׳û÷梨¢8‹ŽŽv¬cSSSIKKs¬ccccññÑi‘–ÄËË‹iÓ¦q÷Ýw3iÒ¤f4˜5ko¿ý6S§NeÕªUÄÇÇ7xÿ«!ÇG{ÒÇÞÝ¥¡û¢gòš7DEE7ÝtqqquŽ$ÐXïK]Ç»Ïdù“'O&--·ÞzëŒÛêN6l`áÂ…¼öÚkf—âvqÄñ$Oòpבę_ØKD¤µóöö&44”üü|³Kñ|†ˆˆãÿÿŸÈÙZ²d‰áî¯Õùóç»uùÒxNœ8aìÙ³ÇX»v­±téRãå—_6fÍše¤¥¥cÆŒ1.¹ä#!!Áðññ1ÇÍÏÏψŒŒ4zöìi¤¦¦·Ür‹ñÀÏ>û¬ñÖ[o«V­2¶mÛfdeeUUUf7Ót999N¯ß?þètÿôéÓî¿üòË¿ÿýïNÓnºé&cÕªUNÓ®½öZã믿vš6|øp—58pÀi>Àx饗ê­{òäÉFûöíàà`còäÉ.kª¹Ü.]º8ÿùçŸ4üüüŒððpc„ ÆNûz}úé§Æ€ ???Ãjµ7Ýt“‘írÞY³f!!!Fyy¹cÚ7ß|cÆ7ß|ã4ocÖ_sY€QXXxÚ¶5¤}sçέµìµk×ֻ̦~¯jþÍÞtÓM ^Ö÷ßo <Ø 2bbbŒ´´4ãºë®s,+''Ç1﫯¾jÄÆÆ~~~ÆÀõë×;æ»è¢‹ôœgó^Õ×¾†þ}VŸ/**ʘ:uªQRRb†aüùÏvZþ½÷Þk|úé§NÓúôéãXÖáǘ˜cüxmÖ¥´´ÔÈÊÊ2¶mÛf¬ZµÊxë­·ŒgŸ}ÖxàŒ[n¹ÅHMM5zöìiDFF^^^N¯µ¿¿¿i$''cÆŒ1ÒÒÒŒY³f/¿ü²±téRcíڵƶmÛŒ¢¢"³›Ùê¹kó‹/¾0ãðáÃnY¾+-}{¹¤¤Äسg±jÕ*cþüùÆôéÓŸµ„„£M›6NŸ±„„#55Õñùš?¾±jÕ*cÏž=FEE…ÙÍi4ÚækØvDc/¿!Ûõ}·Ÿmûkn³ :´Á5¯\¹Ò1b„jƒ 2¾üòËzß—Æx-ºÓÛ`v­m;æ\ÖÕ÷§÷ìÙcTVVšÝSmÚ´ÉØ´iSƒæméûXÒºTUUW\q…aìÞ½Û´:\í?Þ~ûíNóÜwß}Nû–grl·¾ã£vgº?yºcç§Ûç9›cÅ®öýc¿±æ>_]óží>¯«å½ÿþûµÚo¿%&&žõûr¦Ç»ºü¬¬,£cÇŽFiii½ïQSÚµk—nŒ=ºÕœÿ¬0*Œ‹Œ‹Œ‹‹JCû©""g£W¯^Æc=fv"n×TûæãÇwyÎÁË0 ‘Vîz®`)KM®DZª¥K—rà 7àίÕ ––æ¶å‹9ìW´uu•çê?W¿ò"€ŸŸ:t¨sT„êÓZÛ•nED¤neee:t¨ÎïêÓrss¶müýýëü®©þsLL ÁÁÁ&¶R΄»¶1׬YÃСCÉÎÎv\IÎÝ<}{Ù~¥nWWé>Ý•ºkŽ† +u‹HsR×H/ÕG&°;ÝH/QQQ&¶DDD¤ùÛ¼y3@#°UçéûXâyŽ;Fjj*ûöíãÃ?dàÀf—$ÒìÌ;—ƒÖ9êBS[·n×\s ¤§§dvIMf+[I&™9Ìáš×È ""-Ajj*]ºtaþüùf—"âVMµo~ýõÿßv©sÿYM1‘ÕjÅjµ6hÞÓ~ùå²³³ÉÏϧ²²Òñ8W„º:†* "ÒòœNœ8€Åb¡cÇŽŽuJjj*iiiŽuMçÎiÛ¶­É-‘–,00>ø€×^{‡zˆ÷ߟٳg3~üx³K1Õ´iÓxæ™gxâ‰'ˆ‹‹3µ–ï¾ûŽ|Ÿ~ú‰'Ÿ|’‡~˜6mÚ˜Z“Yf1‹÷yŸ‡y˜wxÇìrDDZ›ÍÆ?ÿùO³Ëñx ˆˆˆˆx æHP'V‘;Ý:×þsCÖ¹®‚QQQ ^ÿ‹¸“BÍÓéB%%%.; oß¾ôôtöîÝ‹aÀ©÷¸fø z(!>>¾Õž ñ$eeedee¹… ##ƒÌÌLÇ6‹ŸŸ:ur¬ ’““ÖqqqºŠ¬ˆˆˆˆ¸——iii :”3fpýõ×s饗òÜsÏ1`À³Ëir>ú(>ú¨Ùeðã?2{öl>úè#.¿ür¶lÙBbb¢Ùe™ª-my•W¹’+¹‰›¸’+Í.ID¤ÅÐH"MC!‘Vîl u]UÛHhÈUµ«wˆµO‹ŽŽ¦}ûöîjªˆˆ[œkp ú:ÑUp@£ÇHK¤AËàè ìJ}ÓÓÓëíl\3 ÎÆ"ÍC]á"û绾pQjjªÓg¼sçÎxyy™Ü"‘Sùßÿý_Ö¯_ÏôéÓ8p £FbòäÉŒ5JÁx‘&PYYɧŸ~Ê_ÿúW>ûì3.¼ðB>þøc®¼RéíF1Šßñ;äA†1 üÍ.ID¤E°Ùläåå™]†ˆÇSÈ@DDDDÌÞ6))©ÞùNž<Éo¿ýV+„И„˜˜‚ƒƒÝÝdi¥\\… ¨¨¨p<ÎÏÏ:4(8©Žxâ±2ðL~~~õ†ÊËË)((pÙayóæÍìÛ·'N`±Xèرc£!ÄÇÇØ”ÍñH………uŽB°gÏŽ9â˜×jµ:>ƒIIIŒ;Öñ¹ìÒ¥‹F¨‘)%%…µkײråJ^~ùeÆŽK\\iiiÜvÛm„‡‡›]¢ˆÇÉÍÍåõ×_çµ×^cÿþý >œ?ü1cÆ蘸 ó˜Gzð Ïð$Oš]ŽˆH‹`³Ù8vì%%%˜]ŽˆÇRÈ@DDDD¯¯/QQQDEE6PVVÆ¡C‡êíÈ»yóf ÉÍÍu\Ej\¬V+±±±´k×ÎÝÍ‘fÎ~¥ÞúFdÉÉÉaÿþý§ $$$¸\÷(8 rŠý€nYY™É•HS²X,ŽmÀääd—óTïð\½Ószz:»w令¨È1¯½Ã³«ÑºvíªÀ©µ?SÕ?W;w¸Ø1oõÏTjj*iiiŽÏVbb"AAA&¶DDDDDĽƌØ1cصkóçÏç…^`öìÙŒ3†ë®»ŽÑ£Gk?Sä=z”•+W²|ùrV®\Ipp0·Þz+wÝu]»v5»¼f-’Hf3›éLç÷üžô0»$‘fÏf³ŸŸO\\œÉÕˆx.… DDDDÄT~~~çH¨Þ98##ƒo¿ý¶Î@B]!„ê?ÇÅÅ©sH ÒÐàÀ(//w<Î××—ÐÐÐ:ƒÕ§GDDhøt‘3d±X€S£‰TgµZINN®7„Pójë¬[·Î1½ú²j†ª‡ü’–îäÉ“úè#–/_Nzz:†a0lØ0^ýuÆïíTNï~îg1‹¹›»ù†oðBÇ´DDêcJ!÷ÒYi1Î$PZZÊáÇH¨ÞY HˆŠŠ"::___w6Y¤Uª¨ë3ì*8ÎN1ŸŸ ‘ äÌY­V¬VkÛz¥¥¥dgg׺j{FFéééìÛ·ªª*àßÛtu†¯ï1U]Ïö@A}Ï©©©NÓú{93þþþLœ8‘‰'RXXèè(}Ï=÷––ÆàÁƒIMM%55• /¼ooo³K1]ee%›7o&==ôôtÖ®]‹#GŽdÁ‚Œ;«Õjv™-’7ÞÌg>Èßø“˜dvI""ÍZõˆ¸B""""â‘ìpÎ$P×UÐí„œœ ]>«Bõi111Ž«:‹´Fuj†œÄÄD³Ki2;vì ==/¿ü’o¾ù†#GŽÉðáÃY¼x1£G¦]»vf—éúÑ{¸‡ÿà?Íh:ÒÑì’DDš­¶mÛH^^žÙ¥ˆx4%‘V¯z !99¹ÞyKJJêì ]=Í‘#G\> â)38œœ¬à€ˆ‡ñõõUÈ@šœ¯¯o½!À±ÍV³Ãwzz:;w¸Ø1¯Õj­>°o¿uëÖM'Ñ[¹šKÕ»w令¨È1oõ¿¥K.¹Ä)ØÒµkW‚ƒƒMl‰ˆˆˆˆˆØóûßÿžßÿþ÷lß¾ÝÑÁú‘GáرcDEEqÑE1pà@Hrr2&W.rîŽ?ΦM›Ø¸q#7nä»ï¾#''‡àà`.½ôRžxâ †~Ú‹{ÉÙ{š§ùÿàa! Í.GD¤Y³ÙlÉ@ÄÍ29œs !''‡íÛ·óí·ß’••ÅÑ£G[=àªãµýg]aW[Í¿Û¤ALc IDATºBàرcNup¨±Ùlú»i2æÊjµ’œœ\ç¶\õŽãÕ;§§§³gϧ ©½ã¸«ÑºtéBHHHS5KYyy9uŽB°oß>Nœ8€Åb¡cÇŽN£¤¥¥9þ6âããÕáHDDDD¤…JJJ"))‰|ŠŠ ¾ÿþ{Ö®]ËÆyùå—ÉÉÉÁÇLJóÏ?ß)t˜˜¨‹ I³V^^ÎÎ;Ù´iß}÷6l`ûöíTTT8‚4S¦LaðàÁôïß_Çô›H0Áü…¿p#7r7q—™]’ˆH³e³Ù(((0» ¦-@79—@BÍŽÝ›7o¦°°ƒ:]jwì®k¤uìn½hÈßWÍà€/"R___ÊÊÊÌ.CäŒ.„`ɧfÇsûÕ-÷îÝ‹aÀ¿ƒ£u†O›6mš²yòÿÊÊÊÈÊÊr(ÉÈÈ 33“ÊÊJüüüèÔ©“ã½KNNvz?ãââ4ò’ˆˆˆˆH+àããCJJ )))ŽiÙÙÙlÞ¼™Í›7³nÝ:þö·¿qâÄ , ]»v%))‰ž={’œœLRRR½#Kaa!Û·ogóæÍüòË/lß¾~ø’’, ½{÷fèСL:Õñ·*湞ëYÌbîç~~à|ñ5»$‘f)<<œ¼¼<³Ëñhê"""""Ò œi ¡®NâÕ ¹Ò|]„ððpu”jæN7R†}ZCFÊPp@DÜÁÏÏO#ˆG pt.w¥¾Îëéééõv^¯HPçõ³WWÄþ~ÔIMMuzO:w——É-‘æ(**Š¨¨(ÆŽ œº:ü¯¿þÊÖ­[ùùçŸùùçŸY¸p!O<ñaaaôîÝ›ÄÄDºuëF·nÝèÚµ+ñññ:+礢¢‚}ûö±{÷nvîÜÉ®]»Øµk?ýô¿ýö:u¢W¯^ 4ˆ{^½zÑ£GºÑ ÍcI$ñ"/2ƒf—#"Ò,Ùl6233Í.CÄ£iEDDDD¤…9]ǶêHÈÉÉáàÁƒµ:‚º $¸ &(Ðxê Tÿ9;;›#GŽ8=¶zpÀjµ’””äò=‹‰‰Ñ i¾¾¾ H«äççWï¶Zyy9.;ÀoÞ¼™}ûöqâÄ , ;v¬s4„øøx›²yÍFaaa£ìÙ³Çi[Éjµ:^³¤¤$ÆŽëx»téBHHˆ‰-Ob¿*|ïÞ½¹é¦›Ó:ÄÏ?ÿÌÖ­[·åË—;®¾k±XHHHp téÒ…ØØXbccñ÷÷7«IÒŒ”––²ÿ~öïßÏž={ؽ{7»víbçÎìÝ»—òòràÔ•ILL䪫®¢W¯^ôéÓ‡:˜Üi¨8âøâ?ùOnàÐ((""5Ùl6þùÏš]†ˆGSÈ@DDDDă5F ÁþÿæÍ›Y¹r%p¨¶³\…ªÿA›6mÜÕÜf©´´”ÇŸ68““Caa¡Óck¸ä’K‘A!×,‹ãJ—u`U½}õNôéééìÞ½›¢¢"ǼömW£!tíÚ•ààà¦jZ£ªùddd—•ÅÁÜ\vîÜIqq±cÞê¯Ajj*iiiŽ×"11‘   [""""""¡¡¡\vÙe\vÙeNÓ=ÊîÝ»®>¿zõj.\è4BmDD„#pK\\qqqÄÆÆÒ©S'l6[S7IÜ //ììlöïßOff&™™™ŽPÁþýûÉÍÍuÌÛ¾}{G e„ N•–z,@œMcKXÂd&óŸ™]ŽˆH³c³ÙÈÏÏ7» ¦ˆˆˆˆˆHÈÈÈhP ¡cçŽØl-.`¸j»«AuujŽ¯¯¯I-9w ˆœ=«ÕJrrr½!„šWïÏÈÈ`ݺuŽéÕ—U3|P=”‰——WS5 €“'OrðàA—mÈÈÈ`ÿþýTTT§Ö%ÑÑÑÜååÅaùõ×Ó!-ÍцØØX||t˜[DDDDDZ¦öíÛÓ¯_?úõëWë¾C‡9:™Wïp¾víZÞyçÇ(p*Ðn³:ÖADD„c$b{!22’Ž;*ˆÝÄŠ‹‹ùí·ßÈÎÎ&??Ÿ¬¬,òóóÉÎÎ&77—ÜÜ\rrrÈÏÏw:Ÿî” 4È)XKhh¨‰­’¦àƒ¯ò*CÂû¼ÏxÆ›]’ˆH³b³Ù((( ªªªÙö+iétöEDDDDDÎØÙìÿ+`iêR6·ßLÏ´ž. ~~~tèСÎQªOkŒ@BYY‡jPp 77Ã0mHpÀjµK»víΩN‘–ÂÏÏO!7±Z­X­V’’’\Þ_ZZê~V !##ƒôôtöíÛGUUðï혺FCˆ?ãí¬ºžß(¨ïùí£6Õzþ]»`üxy÷]>RSÏíEiæBCC å /tyII ™™™Žc×öìyyyddd°~ýzrss9tèÓã|}} ¥C‡Ž[õßíÏÛ¾}{‚ƒƒ $((ˆàà`Ú·oßê:ñUVVRTTDQQÅÅÅsìØ1Ž=Ê¡C‡8tè‡vüo¿Ù¯y|¬cÇŽ„‡‡;.¶”˜˜è‚ØG>Œ‹‹ÃßßߤKs2ˆAü?ð0‚„bvI""͆Íf£¢¢‚ÇÓ±cG³ËñH ˆˆˆˆˆˆ[Õ $d“Íïø¹äòï0víXªªª((( ??Ÿœœòòò(((pœÉËËcË–-ŽyìÓàTç´°°0¢¢¢°ÙlŽôaaaDFFJqq±cÙùùùŽ+ÙŸ£¨¨È©îàà`¢¢¢ #""‚=z0dÈÇsØøÛl6üüüšîi!4’ˆyüýýë „–––:]Óþó¯¿þÊgŸ}Fvv¶c$???Ç•íWMŒ‹‹#::šÂÂBÇc÷íÛçX^aa¡ã¹:vìèxL¯^½3f ñññŽåuèСaêÖ ¾û¦O‡ñãáþûá…Àb9ç×KDDDDD¤%  {÷îtïÞ½ÞùÊÊÊWίÙÞÞ>;;›mÛ¶qøða~ûí7Ž9RïóÑ®];Ú·oOPP´o߀¼¼¼ Âb±àïïO@@‹…   ¼¼¼ ©ÝQÚ>MÕç?räˆÓ€ìÊËË)..®5Ý>ÿ±cǨ¨¨ ¤¤„ÒÒRÇüUUU=z€£GRRRBqq1GåرcSRRRçkaµZk4bcc]8ìA`,gêÏü™¬àqç^1»‘fÃf³ŸŸ¯ˆ›(d """""MæG~äj®Æ?¾ã;zÐÃq_›6m'<<œ^½zÕ»œÊÊÊZ„ê‚}ûö±qãFÇ<Ó ƒƒAAtêÔ‰°°0ÂÃÃéÓ§6›ˆˆÂúJˆÈ¹ñõõ¥¬¬Ìì2DÄILLtyEEYYYNáûmíÚµdeR6¸Œ6Ÿ·!"(‚Î;˨Q£œ‚ñññ´mÛ¶1 ‡9s 9&O†aɈo¼çñ0~~~ÄÄÄÓàÇØ;ßqüøqŠ‹‹)**âèÑ£Ž«ùSXXHqq1'Nœ ¸¸Ã0Ø»w/EEETVVrâÄ ÊÊÊ(++ãĉŽ‘šBpp0ÞÞÞâëë‹ŸŸmÛ¶ÅÛÛ›àà`àß¡ˆèèhÚ¶mK`` V«•   ÇÍ>ªƒý÷víÚ9'ânèÀó<ÏíÜÎ&0f—$"Ò,„‡‡§B={ö4¹Ϥˆˆˆˆˆ4‰÷xÛ¸! á=Þ;§!]½½½‰ˆˆ ""‚Þ½{×;oEE> O.^ŒïÍ7ŸõsŠˆÈ™ÑH"-—#(0dÈZ÷obýéÏ/å¿hqTp«‰aÀ€S#ôïûŒÕôuˆˆˆˆˆˆx¨6mÚ`µZ±Z­MöœuRpòäIn»í6Þ|óÍÓŽv â‰&1‰Å,æ.îb3›ñQ—?BCCñöö&??ßìRD>§´jb‘¦åç秈‡ª¢ ‹‰#?uïßãÆÁèÑðàƒP^n^=""""""rNBBBÁ†ê·ððp Äf³¹œGñt^xñßü7;ÙÉ«¼jv9""Í‚··7¡¡¡ ˆ¸‘B"""""â6Ç8Æïø/ó2oò&s˜ƒ7Þf—%""M@#ˆx®J*hcöáå€X¸-:õÿˆmnM""""""""nÐn<ÌÃ<Æcä Ù刈4 6›M!7RÈ@DDDDDÜâ_ü‹‹¸ˆMlb5«¹•[Í.IDDš¯¯/eeef—!"n`ÉÀôÝĉðÏBAôí _|avE""""""""îQ%Š(¦0ÅìRDDš… DÜ«™œOò9ŸÓŸþÀF6r™]’ˆˆ41d ⹚]È gOظ†‡+®€GÊJ³«i4~øñ ¯°œå|Â'f—#"bºððp… DܨO°€Œa £Å·|K 1f—$""&ðóóSÈ@ÄC5Ë@»vð÷¿Ã¢EðÊ+0bäæš]•ˆˆˆˆˆˆˆH£ÉH®ã:àJ)5»SÙl6òòòÌ.CÄc5³³@"""""ÒR•QÆø“™ÌS<Å»¼Kf—%""&ÑH"ž«Ù† ì&N„5k`ß>è×Ö¯7»"‘FóþB.¹ü…¿˜]Šˆˆ©ÂÂÂ4’ˆ5Ó³@"""""Ò’d“ÍP†ò¿ü/ÿàLgºÙ%‰ˆˆÉ|}})++3» q{ÈÀo“+©G¿~ðÃœ —]ý«Ù‰ˆˆˆˆˆˆˆ4Šbx„Gxš§É$ÓìrDDL®ˆ)d """""çäG~äb.æ0‡ÙÈFÆ2Öì’DD¤ÐH"ž«’J d`|O> ÷ß7ß 'N˜]•ˆˆˆˆˆˆˆÈ9›Æ4¢‰fÓÌ.EDÄ46›¢¢"JJJ8~ü8{öìaÆ |òÉ'f—&â|Ì.@DDDDDZ®·y›;¹“a ã]Þ¥=íÍ.IDDš … D<—}$ƒf2ðò‚éÓ¡wo¸é&4–/‡ÎÍ®LDDDDDDDä¬ùâË\ær9—s·q9—›]’ˆˆ[•””0þ|òóóÉÍÍ%//½{÷âïïÕju]»ÿþ\yå•&V+â2‘3f`0‹Y<ÅS<ÌÃ<Ã3xãmvY""ÒŒx{{SYYiv"â-*d`7j|ÿ=üîwп?üýï0b„ÙU‰ˆˆˆˆˆˆˆœµ‘Œd,c™ÊT¶° ³Kq›€€–.]ÊÆñöö¦¢¢Âå|‹…áÇ7qu"ž©‘æ Œ2&2‘ÿâ¿x•WyŽç0‘Z|||ê<À+"-[‹ œw|÷Œ}*tðÜs`fW%"""""""rÖ^æe2È`.sÍ.EDÄí~øa è÷üSyy9Æ kªDd «WÃõ×õ×Â#@U•ÙU‰ˆˆˆˆˆˆˆœ‘k¹–+¸‚û¸ t\VD³«©Ó“<ÉR–òO𠯘]Žˆx°ÂÂBŽ9â¸ÕüýÈ‘#=z”’’GP °°òòrŠ‹‹)))¡´´”¢¢"*++Ϫ†… ²páÂFi=„Œ!!!øúúHÛ¶mñóó£}ûöcµZ qºUŸØ(5‰˜A!‘V¨B®â*¶³/ø‚A 2»$ñ0 ˆx®V2 „eËàùçáÁaëV˜7,³+©%˜`žæiîäNnçvúÐÇì’D¤…8|ø0yyyäçç“““ãôs~~>yyy:tÈ&p¥]»vNîÛ·o¿¿?QQQX,BBB#Ø;õ·k׬V+>>>´k×Îi™®F ðòòâƒ>à–[nÁÛÛÛé>{ˆ¡¦š#"TUUqôèQÇè 'Nœ ¬¬Œ¢¢"***8räH­@Ä8zô¨S¨¢¬¬¬ÖsY,GèÀf³a³ÙˆŒŒÄf³Ndd$aaaDFFN@@@ƒß'wSÈ@DDDD¤•É$“+¸‚JXÏzºÓÝì’DDÄ)d â¹ZmÈÀË ¦O‡„¸õVÈÈ€¥KÁj5»2‘ZnåV^ã5îå^Ö²/¼Ì.IDL–››Ëþýû·ÌÌL233ÉÎÎv„Nž<é˜ßÛÛ›ÍFXXQQQØl6zöìIÇŽ‚šWó·Z­µ:ü»Ó”)Sðòr½~³ÙlMVGIII£:RPP@nn.[·n%??ŸÜÜÜZ!   ÇëGll¬ãG\\AAAMÖ&iÝ2iE¶±QŒ"„¾å[¢‰6»$ñP ˆx®J*ñ¦éN5KãÇÃyçÁ¸q0p ¬\ ]»š]•ˆˆˆˆˆˆˆˆ/¼˜Ç<0€wx‡›¹Ùì’DÄÍŽ9®]»ØµkNa‚ýû÷SZZ @›6mˆˆˆ >>žØØXL§N s\aß.hÓ¦y_t¦®€AS  €¨¨¨?¦´´”‚‚²³³£DØÙ™™|üñÇìß¿ß)ŒÐ¡CGð >>ž¸¸8èÞ½; øúúº£yÒ )d """"ÒJ|Í×\Ã5\È…üƒОöf—$""L!ÏÕªG2¨î‚ àûïÿ4X¶ .»ÌìªDDDDDDDDœ\È…ÜÁü‘?2–±:G(âÊËËÙ»w/;vì`çÎŽPÁŽ;ÈÏÏÀÏÏÏ ˆeРAN¿GGG«3z3àïïOLL 111õÎWTTÄþýûÙ·oŸÓH›7ofùòådggc>>>ÄÇÇ“˜˜Hbb"ݺu£[·n$&&žQøA2i–³œ›¹™ÑŒæmÞƳKg±X€SºEij(dPMD¬^ ·ß#GÂܹp÷ÝfW%"""""""âäižfËøOþ“xÁìrDä dffòÓO?9n[·neïÞ½Žó/111tëÖ¤¤$®¹æºwïN·n݈‹‹ÃÛ»•HëA‚ƒƒ9ÿüó9ÿüó]ÞâÄ GØÄ8Y»v-¯¿þ:G ]»v$&&Ò§O§[ûö Ÿ‰k ˆˆˆˆˆx¸9Ìa*S¹ûx‰—Ô!LDDš„F2ñ\ Ôàïo¿ ]»ÂäÉðë¯ðÒKÐ̇‘Ö#”Pžâ)äAÒH£ÝÌ.IDj8yò$Û·ow lÙ²…ÂÂB¼¼¼HHH oß¾L˜0Á$èÖ­f—.Í@Û¶méÛ·/}ûö­u_nn®cÄ‹;vðÓO?ñÁpèÐ!:wîLß¾}‚;wnê&H3¤ˆˆˆˆˆ‡20˜Á žçyçqf3Ûì’DD¤QÈ@Äs)dà‚—Ìž =zÀ­·ÂÁƒ°x1´mkve""""""""ÜÉü7ÿÍù#ñ‘Ù刴zdݺulØ°õë׳eËÊËË àüóϧoß¾\{íµŽNßíÚµ3»di¡"""ˆˆˆ`èСNÓ8àhyûí·yòÉ'©ªªÂjµ2pà@.¾øb.¹ä @PPI-³(d """"âNr’?𖱌·y› L0»$ie,  ˆ'RÈ 7Ü11pÍ5’+Vœú]DDDDDDDÄdÞxó/1œá|Îç\Îåf—$ÒjTTTðã?:ëׯçÀøøøЧORRR˜2e }ûö%11ooo³K–V &&†˜˜ÆŒã˜vìØ1¶nÝÊæٰ͛a .äñÇÇÛÛ›Þ½{“’’âÄÇÇ›W¼4 … DDDDDï½ Àõ×CI‰Ù‰ˆˆˆˆˆˆH+L0³˜Å\沋]f—#ÒìÙƒ ôïߟeË–1~üx6mÚÄž={xùå—¹òÊ+ 4»TÓÅÄÄpûí·³dÉòóóY¶l:ubÆŒDGG3tèPæÍ›§ÀA  3A"""""-Ø›¼Éïø7pËYN[Úš]’ˆˆ ‘ D7»‘fcË–-Ü}÷ÝDEEñÐC‘œœÌwß}Ǿ}û˜3gƒ r\hIDÎßߟ±cÇò·¿ý¼¼<ÞyçNž<Éõ×_O÷îÝyá…8tèÙeJ5 ˆˆˆˆˆ´0•Tr7w3“™üÿóà.î2»$—4’ˆçRÈ DDÀ×_ŸÍàòËáÿ0»"i¥þÂ_ØÃ^ã5³Kir'Ožäõ×_§{÷îLš4‰˜˜¾þúk¶oß΃>HHHH“×aLš4‰ÀÀ@ ÃÀ0 Ž?Ϋ¯¾Ê²eË7nœG_äéÙgŸå×_5» 'fÔÔ_wHJJbîܹdeeñÊ+¯°uëV’““;v,7n4»¼VMg‚DDDDDZˆÃf$#YÏz¾à Æ0Æì’DDDꤑ D<—B$(>ø&M‚ñãá¯5»"i…ºÐ…û¸Gy”Ã6»‘&³bÅ ’’’¸çž{HIIaûöí,Y²„K/½ÔqŽ£9ù?öî;:ªjoãøwRHïB@B“) R¤CÞDB“¢4z¥(¢¢€ JQAA/M$H碈"M„xÁ¡$”„iï¼Ì5†N’3™<Ÿ»f™Ù§=gf.ëÌìý;ÛÙÙ™víÚñüóÏóóÏ?ëNïbu\\\èׯ¿ýöááá\¾|™ÚµkÊáÇŽW ©'HDDDD$ø‹¿¨CÎqŽ=ìáiž6:’ˆˆÈ]i&ë¥"ƒdk{³¸`æL<ÆŽý»)"""""""yl")D!Þâ-££ˆäºß~û:uêо}{j׮͟þÉÒ¥K)_¾¼ÑÑîK©R¥8uêÆ Ãd2a2™8rä«V­2·-_¾€¹sçšÛ,XÀСCñðð xñâLž<ù®Ç;v¬yÛÏ?ÿœÎ;ãæ憷·7C‡åúõëYÖŸ7o8::R»vm~üñGóöµjÕz óMHH OŸ>xxxÜöx—.]bÔ¨Q”+WGGGBBBX³fM¶ýœ8q‚víÚQ¸paÜÜÜhß¾}¶»ä¯_¿ž'Ÿ|GGGŠ-ÊÀ‰ÏµL›6m¢V­Z899ñØcÑ©S'öîÝ{Ç×cÁ‚æ×Òd2±bÅŠûþ <èûh¤¦M›²gÏ6nÜHLL ÕªUãå—_&..ÎèhŠz‚DDDDD,ÜŽP—º8ãÌøIDDäž4“ˆõJ']E9í•Wà³Ï`Ö,èÓRSN$"""""""ˆ;îLbs˜CFÇɉ‰‰¼úê«<õÔSØÚÚ²oß>–.]J@@€ÑÑHdd$ð¿bƒÙ³g³yóæ,ëtêÔ‰‹/fi}úЦMæÍ›Gff&]ºt¹ïÏÀƒ¼–¢yóæüöÛo,\¸Õ«WóÄO°jÕ*£cê ±`ûØG#QŽrlcÅ(ft$‘ûrk&ˆXŸ 2°ÅÖèÖ§gOØ°¾ù:v„¤$£‰ˆˆˆˆˆˆHÒ~T #it‘÷ûï¿S«V-/^ÌŒ3رcÕªU3:ÖIJJbíÚµ,[¶ŒöíÛ?Rþ'Ÿ|’¶mÛâææÆË/¿Œ³³3»víº¯m;uêD»vípww§C‡¼øâ‹,\¸³gÏðúë¯Âøñãñòò"88˜±cÇ>tÖÆÓ±cÇ;oöìÙÌž=\]]éÞ½;Íš5cÎœ9æ}¤¤¤pðàA:v숯¯/žžžÌ˜1ó:£G¦bÅŠLž<ÙœûÝwßeëÖ­lÛ¶-Ç3Œ9’Š+òæ›oâãヿ¿?}ô·}-’““iß¾=mÛ¶eРAýšÂ½ßGKbccCŸ>}8vìÏ=÷aaaôìÙ“„„££Y=ˆˆˆˆˆX¨­l¥ M¨E-¾ç{<ð0:’ˆˆÈ}»UdðÏ»»ˆHþ—A†f2È-¡¡°e ìÝ Á?î4%"""""""’[l±å=ÞcëØÄ&£ãˆä˜Å‹óä“OâããÑ#Gxå•WÌ}–.11“É„ÉdÂÅÅ…®]»òþûï³zõêGÚoåÊ•ÍÛÚÚR¸pabbbîkÛ'Ÿ|2Ëózõê‘ššÊþýû¹rå Ô­[÷®Û<ˆš5kÞñxwâííMDÄÿfeqtt¤FŒ7Ž¯¿þšäädìì숎Ž **Šˆˆ6lxÛcoÙ²%Ç3EEEqüøqêׯŸe=WWW._¾œmûÄÄDZµj…‡‡}ûö½ãqî×ÝÞGKåééÉûï¿Ïºuëøþûï©^½:Ç7:–UËÿRŠˆˆˆˆ0ßò--iI;Úñ ßà„“Ñ‘DDDˆÉd4“ˆ5R‘A.«Q~ü®\ÚµáÄ £‰ˆˆˆˆˆˆHјƴ¥-#AiFÇy$™™™L˜0¾}û2bĶlÙBñâÅŽõ@\\\ÈÌÌ$##ƒ?ÿü“ÀÀ@&MštÛAèÂÕÕ5Ës{{ûûîÏqwwÏòÜÇÇ€sçÎqþüyàæ€ú»mó ¼¼¼îx<€£GÒ±cGüüü°±±Ád2±dÉbcc³l·iÓ&Ú·oÏ°aÃðôô¤U«VìÝ»€K—.0wî\sQ‡ÉdÂ××€Ó§Oçx¦[Çüçku'C† ÁÁÁU«V™s?Š»½–®eË–8pêÔ©ÃîÝ»ŽdµÔ$""""baæ3ŸNt¢?ýYÂì±7:’ˆˆÈÓL"ÖKEy lYص ÜÝ¡^=8pÀèD"""""""R@Ìb'8ÁÇ|lt‘G2zôh¦M›Æ¢E‹xë­·°µµ5:ÒC3™L”+WŽÏ?ÿœèèh^{íµ,Ëoõɤ¦¦šÛâããs%Ë?ïß*xð÷÷§X±b\¹r%Ë:ÿ|þ ®]»vÇ㥦¦Ò´iSÎœ9ÃöíÛIMM%33“^½zeëŸòòòbæÌ™œ={–;v’’Býúõ9qâ… n~f233³=–-[–ã™nó~_›ñãdzvíZ‚ƒƒéÙ³'IIIY–?ègànïc~àïïÏ–-[hØ°!Ï<óŒ r‰z‚DDDDD,È4¦1ˆAŒbðo‰ˆH¾¥™ D¬—Š òˆŸìÜ •+Cýú°y³Ñ‰DDDDDDD¤(KY3˜ñŒç ?0XÄH3gÎdÖ¬Y,Y²„>}ú'ÇT¨P^½z±xñb>ln/R¤.\0·:t(W2üüóÏYžïÚµ {{{ªW¯Ž··7åË—Ï6à{ß¾}}¼_~ùåŽÇ;yò$çÏŸ',,Œ   s!Éõë׳lsáÂ*W®l~^«V->þøcnܸÁ/¿üB‰% ºmÎàà`V®\™ã™J”(A`` ;vìÈÒ~îÜ9³ÍVQ¦LìííY¶l§OŸfôèÑY–?ègànïc~áääÄW_}EóæÍiÓ¦ ¿ÿþ»Ñ‘¬Žz‚DDDDD,@&™Œd$¯ñ XÀT¦IDDä‘h&ë•NºŠ òŠ«+¬] Í›CÛ¶ðïHDDDDDDD €‰LÄ{Þâ-££ˆ<°ü‘±cÇ2}útºvíjtœ7iÒ$lmm1b„¹­|ùòøúúòá‡råÊŽ?Î’%Kråø[·nåßÿþ7×®]ãÛo¿eÑ¢Eôïßß|ü×_0eÊbcc9pà}ôÑCoõêÕ¬[·î¶Ç+UªEŠaÙ²e=z”””ÂÃÃÙ¸qc¶ý9r„÷Þ{øøxâââøè£ptt¤FÀÍ”;w2uêT.]ºÄ¥K—1biii´k×.W2͘1ƒ£G2aÂ._¾Ì©S§xá…èÕ«>>>·}=žxâ Þyç>üðC6ÿíÆ4ú¸×û˜_ØÚÚòÅ_P±bEºtéBrr²Ñ‘¬Šz‚DDDDD –N:}éËæð9Ÿ3€FGydšÉ@Äze-ùwjñ|ÇÁV¬€Þ½á¹ç`éR£‰ˆˆˆˆˆˆˆ•sÇ×y9Ì!‚£ãˆÜ·´´4z÷îM³fͲ ÂÏO¢¢¢0™L,Y²„ÄÄDL&}ûö5//Y²$ýû÷ç‡~Àd21{öløüóÏ9yò$%J”`À€Œ?€çŸžæÍ›³bÅ ÜÜÜèׯ=zô0ëĉÌ›7råÊÝ3ß„ øæ›o(^¼8/¾ø"/¼ð3gÎ4/ïÚµ+óæÍãã?Æßߟ¡C‡2eÊìííïë5;v,*Tà7Þ`Ù²eøûûg;žƒƒ6lÀÃÚ5kR¾|yV¯^Mhh(Äd2qáÂüüüX¿~=7n¤L™2°gÏ6lØ@™2ehÙ²%ß}÷kÖ¬¡D‰TªT‰èèh6mÚ„££cŽghݺ56l <<œâÅ‹S§N*V¬Èûï¿Ü,B¸uÌ-Z0xð`¾ÿþ{†Nff&Íš5#$$Ä|Ü{}ä}ÌOn{TT”ù³&9Ô©ÛɉˆÐ™Î|ÅW'‘üꫯ¾",,,WïÒºpáBú÷ïŸkûc\ç:ÝèÆ&6±ŠU4§ù½7Êaß~û-;vdÿþýT­Z5˲fÍšqåÊót‡füøñìܹ“””ªV­ÊÔ©S©_¿¾y›3gÎ0f̶mÛFBB*TàÕW_%,,,OÏ+ “ V®däÏ?³|ùrŽ9B¿~ýؼy3žžžŒ=š¡C‡fÙdË–-Lš4‰ýû÷cooOƒ ˜>}:AAA„ˆHÎËÍkÌ´´4ìííùæ›oèСC®ãït½,’w~ÛwX÷õ£¥š6 ƃwß…W_5:ˆˆˆä3¿þú+Õ«W¿çºúŽ%"–¤sçÿïÏÿJýù"y)tªQXËÚGÚWè‹°xñb^zé%Ž?N©R¥ŒŽcUŽ;F… ظqc¶ë÷ràÀªV­ÊŽ;²üYòÞ£¼–næÌ™Lš4‰“'Oâëëktœ‘WßÍït½­™ DDDDD r•«„Ê6¶N¸!mÚ´¡X±b,Z´(Kû©S§Ø²e ýúõààÁƒÔ®]~ýõWΞ=KóæÍ 5wPÂÍ/111ìÞ½›˜˜,XÀš5kˆŽŽÎÓóº“ÌÌL† ƈ#8{ö,C‡å•W^áÇ4¯³eËžyæªW¯ÎÉ“'ùõ×_INNæ駟æôéÓ¦É?4“ˆõ*Þ¦8ŽÅ Ìõ£E3fφQ£`ìX£Óˆˆˆˆˆˆˆˆ³Å–÷xu¬c›i_­?RŒ³páBºuë¦}òÉ'´iÓ†3gÎpãÆ þøã^yåBBB¨]»¶ÑñÄŠ 4È<«ä ˆˆˆˆˆ šhшÿò_¶³:Ô1,‹/¼ðŸþ9)))æöÅ‹ãììL·nÝ=z4%K–déÒ¥”)Sooo&NœH­ZµxóÍ7HMMeïÞ½tïÞ²eËâääDµjÕøâ‹/(Z´¨!ç÷O111tïÞzõêáááÁèÑ£)]º4Ÿ}ö™yñãÇ›§!ôóó£\¹r¬X±‚””¦OŸn\x‘|ÄÆææÏNšDSÄúdÚeðB@¹~´8C‡ÂgŸÁÌ™ðòË b.É%iL[Ú2’‘¤“þÐû)hý‘bŒ˜˜öíÛG§NŒŽbuÆŽK… hÑ¢=zô¸ãºÝºu£nݺ´jÕ ///š7oN¹råøî»ï°··Ï«Èrò>æGNNN´lÙ’ 6Åj¨È@DDDD$EI=êG»ØEª‰~ýúÏêÕ«›wþì³ÏèÒ¥ nnnܸqƒm۶Ѻukììì²lÛ AvïÞ €½½=¼óÎ;¬\¹’¸¸¸Ò~ R¤#""‚ŒŒ ªW¯nt«3uêT233ÍåË—ßq]gggÆŒáC‡HLLäÔ©S,Z´ˆbÅŠ™×1™Lw}Hîx÷1¿ªV­ÇŽ3:†ÕP‘ˆˆˆˆH:ÊQêQØÍnÊRÖèHЬY3ó¥›7oæôéÓæ©IcccIMMåÝwßÍöÿÍ7ßäÊ•+æ}ýûßÿ¦|ùòôìÙj׮͊+ 9¯ÛñññÉöä››W¯^ ..ŽŒŒ |}}³m[´hQ._¾œ'9EDDD,U:鸸˜ëG‹Õ®|÷¬[;Br²Ñ‰DDDDDDDÄ •§<ýéÏD&’HâCï§ õGŠ1n}F¼¼¼ N"÷ò÷î·{ˆ<¬Â… kLGR‘ˆˆˆˆHÙÇ>Ѐ2”a7»ñÇßèHY 0€íÛ·sâÄ >ù䂃ƒyꩧðððÀÖÖ–7Þxã¶_òÿ~—êÀÀ@Ö¯_Oll,7n¤xñâtíÚÕb¦¤»×<==±±±áâÅ‹Ù–ÅÄÄàãã“[ÑDDDDò… 2°Å¶À\?Z´F`ËøñGhÑâãN$"""""""Vh“H"‰™Ì|¤ýè÷$ÉM·úq5ÀØ2ÅÅÅÑ£GbbbŒŽòHz÷îÍï¿ÿnt ¹ƒ‹/R¤H£cX ˆˆˆˆˆä­l¥ M¨E-¾ç{<ð0:R6­[·¦X±bLŸ>µk×Ò¿ó2GGG6lÈš5kHOO¿¯ý9;;Ó¬Y3¾úê+Ø»wonEÏQŽŽŽ<õÔSÙ~„¼rå ÿùÏhРAÉDDDD,CØ`£ëGKQ£lÛФ ¨UDDDDDDDrXŠ0šÑ¼Ë»œçüCïG¿'InzüñDZ±±aÿþýFG‘ˆ§~ýú4jÔ___£ã<’!C†Ð¢E KPPG¥P¡B¼þúë4mÚ”„„ÒÓÓ)S¦ uëÖeùòå”*UŠèèh7nÌáÇ1™LôîÝ777æÏŸÏ›o¾É[o½Å§Ÿ~Ê+¯¼Bƒ hРëׯ§]»vüë_ÿbÆ \¿~Q£FѤIÌ™¼¼¼ båÊ•4iÒ$·_B¹‘‘‘¼õÖ[ >___£ãX Íd """"’Ã2Éd$#Ç8Þã½|S`pùòeþõ¯Q¿~ýl_ðEDDDDnI'ltýh©Š…íÛÁ×5‚?ÿ4:‘ˆˆˆˆˆˆˆX l˜Á Ö±Ž-lyàíõ{’ä6;;;>ûì36oÞÌÌ™3Ž#ÀþýûqssÃÛÛ;KûìÙ³™={6>>>¸ººÒ½{wš5kÆœ9sÌ뤤¤pðàA:v숯¯/žžžÌ˜1lll0`ß|ó W¯^5o·téRúöí›­ÈäÉ'Ÿ¤mÛ¶¸¹¹ñòË/ãììÌ®]»ÌËGŽIÅŠyóÍ7ñññÁßߟ>ú‡lçUºtiöïߟ#¯‘<š””ºwïNÉ’%yíµ×ŒŽcUTd """"’ƒÒI§/}™Ã¾äK†1ÌèH÷¥iÓ¦+V “ÉħŸ~jt±`d°©é&]?Z2oo‡âÅ¡^=8rÄèD"""""""b%шgx†QŒ"ƒŒûÞNý‘’WjÕªÅôéÓ3fŒùîöbœ .àééy_ëz{{a~îèèH57n_ý5ÉÉÉØÙÙm^§oß¾dddd™eåÊ•ôîÝ;Ûþ+W®lþÛÖÖ–Â… @TTǧ~ýúY¶quuåòåËÙöåééÉ… îë¼$÷¤¥¥Ñ¥KŽ=Ê—_~‰£££Ñ‘¬ŠŠ DDDDDrH )t +YÉZÖF˜Ñ‘îÛ?üÀ7سgeÊ”1:ŽˆˆˆˆX° 2hýCk]?Z:OOؼžxš4ƒN$"""""""Vâ]Þå‡XÉÊûÞFý‘’—†ΨQ£èÓ§ü±Ñq ´ØØXìíí³µ=z”Ž;âçç‡ &“‰%K–›e½M›6Ѿ}{† †§§'­ZµbïÞ½æåEŠ¡S§N,^¼€Ÿ~ú‰àààÛ6¸ººfynooOFÆÍb©K—.d›qáNìíí¹råÊ}­+¹#))‰gŸ}–~ø 6P±bE£#Yˆˆˆˆˆä€hIKv³›Ílæž1:’ˆˆˆˆH®È ý´œ?¸ºÂúõP¥ 4lë|yX•©LOzò/þÅu®G䶦NÊøñã0`cÇŽ%--ÍèH’——©©©YÚRSSiÚ´)gÎœaûöí¤¦¦’™™I¯^½ÈÌÌ̶ýÌ™39{ö,;vì %%…úõësâÄ ó:/¿ü2?ÿü3GeñâÅ 4èskØ¿ IDAT.\ྠRSSï» Ar^TT5bÏž=lÞ¼™:uêÉ*©'HDDDDäÅG3šñ;¿³•­Ô¦¶Ñ‘DDDDDrŠ ògç›…õëÃ3ÏÀž=F'+0…)\ä"s˜ct‘;š4iŸ~ú)|ð7æÌ™3FG*püüüˆ‹‹ËÒvòäIΟ?OXXAAAØÚÚpýzÖ¢¥ .P¹reóóZµjññÇsãÆ ~ùås{íÚµ©Zµ*sçÎåÔ©ST«Vís–(Q‚ÀÀ@vìØ‘¥ýܹs8::rùòå,íqqqøùù=ðqäÑ­[·ŽâããÙ³gµkkŒNnQOˆˆˆˆÈ#ˆ&š†4$’H¶²•BŒŽ$""""’«Td98ÀW_ÝœÍà™g`ûv£‰ˆˆˆˆˆˆH>ç?ÃÆ[¼Åe.ß{ƒôêÕ‹_ý•¸¸8*W®Ìûï¿OFF†Ñ± ŒjÕªqíÚµ,3”*UŠ"EŠ°lÙ2Ž=JJJ ááálܸ1ÛöGŽá½÷Þ#>>ž¸¸8>úè#©Q£F–õ Ä‚ èÑ£ÇCg1cGe„ \¾|™S§Nñ /ЫW/|||²¬û×_=T1ƒ<¼¸¸8 @»víhÙ²%ûöíãñÇ7:–USOˆˆˆˆÈC:ÍiêSŸxâÙÅ.*RÑèH"""""¹.täGðõ×Т´n­Bydc‹#Ž¼Å[FG¹« *ðÓO?Ñ¿FŽIýúõÙ·oŸÑ± „¶mÛâèèÈÞ½{ÍmlØ°jÖ¬IùòåY½z5¡¡¡ýôS:D¥J•èÔ©?üðÃïr/ÇÃÃ;v°eËbbbrtߣG¦J•* <˜€€€Ý÷í|ðÁ|÷Ýw„„„äú± šk×®±`Á*W®L³fÍð÷÷ç—_~aóæÍTªTÉèx’z‚DDDDDîÓNvÒ„&<ÅS|Ï÷¸£ i)xTd`%lmaÙ2ˆˆˆˆˆˆˆÈ#kG;ш‘Œ$ ΖüÃÎÎŽž={rôèQ¾øâ bbb %((ˆY³fqåÊ£#Z ///>ÿüs|}}slŸãÇ'33“ØØX†šcû½›%K–hÀ{;tèƒ ¢xñâŒ1‚5jpðàA¾ýö[ªW¯nt¼M=A"""""÷á;¾£9ÍiD#¾áœp2:’ˆˆˆˆˆ!Td`EþYh°}»Ñ‰DDDDDDD$Ÿz‡wø‰ŸXËZ££ˆ<0:wîÌÎ;9|ø0M›6å7Þ xñâôîÝ›üQ³ˆä ääd–/_Nݺu fëÖ­Lž<™³gϲxñb*W®ltDAE"""""÷´’•´§=Ïò,«X…FG1L:é*2°&· Ú¶U¡ˆˆˆˆˆˆˆ<´§xŠÎtf cH%Õè8"­R¥JÌ›7èèh>úè#:D:uà•W^a÷îÝ*8y)))¬[·Žž={R´hQz÷î——›7oæ?þ`Ø°axyySþF=A"""""w±Œeô ýéÏ–`‡Ñ‘DDDDD •A¶ØCr’­-,_mÚ¨Ð@DDDDDDDÚ;¼C$‘|Â'FGydŽŽŽôìÙ“ýû÷³ÿ~zôèÁwß}G½zõ(S¦ £FâçŸVÁÈ]$%%±jÕ*ÂÂÂ(\¸0:t **ŠiÓ¦qîÜ9Ö­[GÓ¦M1™LFG•ÛP‘ˆˆˆˆÈÌc½éÍ«¼Ê\æên­"""""Ü,20¡ü­Ž­-,] ͚ݜÕ`Ï£‰ˆˆˆˆˆˆH>SšÒ b“˜D<ñFÇÉ1U«Våí·ßæÏ?ÿdÿþýtíÚ•5kÖP³fMÊ”)Ã!CX·n FG1\dd$ .¤S§NøúúÒ¥K.]ºÄŒ38wî[·n套^Â×××è¨r%%""""rӘƆ0iLeªÑqDDDDD,F*ÀµVöö°r%4i-ZÀ/¿HDDDDDDDò™ L tÞå]££ˆäŠ[üöÛotïÞ={öо}{¼½½iØ°!o¿ý6ûöí###Ãè¸"¹.!!uëÖ1dÈ)]º4#FŒ 99Ù\X°e˨‚|ÆÎè"""""–$“LF3šYÌâ#>¢ýŒŽ$""""bQTd`åntì¡¡°u+T­jt*É'¼ðbã˜Ä$2â7:’H® !$$„)S¦pñâE6oÞLxx8sçÎåµ×^£páÂ4iÒ„&MšP§N*T¨€~[•ü-))‰_~ù…]»v±yóföìÙCZZ!!!tèÐfÍšñôÓOãàà`tTyD*2ùd0,þÿÿõ¢—Ñ‘DDDDD,ŽŠ €B…à믡U+xæر*T0:•ˆˆˆˆˆˆˆäCÂ<æñ:¯ó1G$O)R„nݺѭ[7>Lxx8ááá >œÄÄD<==©]»6µkצN:Ô¬YWWWƒ“‹ÜÝ™3gسg{öìáÇäÀ¤¦¦âïïOÓ¦Méׯ¡¡¡š¥À ©È@DDDDH'y‘¬àk¾¦ŒŽ$""""b‘ÒIÇ[£cHnsr‚õë¡yó›3ìÚ¥KJDDDDDDDòxƒ7xÁ* ›HÁS¹re*W®Ì«¯¾JZZ4Ò^´h'NÄÖÖ–Ê•+óôÓOS³fMBBB ÂÞÞÞèøR@]½z•C‡±ÿ~saATTvvvS§N†N:u0:®ä2ˆˆˆˆHwët¥+ᄳŽu„jt$‹¥™ ggX·š4† açNPÇ‘ˆˆˆˆˆˆˆÜ‡ô`³˜ÀV±Êè8"†²³³£zõêT¯^!C†pîܹ,w‡ÿä“O¸~ý:T¬X‘‚ƒƒÍOOOƒÏB¬Mdd$äàÁƒ8p€ƒò×_‘™™IáÂ…©U«/½ôuêÔ¡F¸¸¸Yò˜Š DDDD¤@K$‘t`û'œ:Ô1:’ˆˆˆˆˆES‘Aãáß³È 4ôf¡ŸŸÑ©DDDDDDDÄÂÙ`Û¼I[Úò?Q‹ZFG±(þþþtêÔ‰N:ššÊü‘eÐ÷š5k¸|ù2¥K—&88˜J•*Q¡BÊ—/Oùòåqww7ò4$8{ö,Ç'""‚?þøƒC‡qàÀâââ0™L”-[–úôéCpp0!!!<öØcFÇ  ")°âˆ£­ø/ÿeÛ!ÄèH"""""OEPá°e 4hÍšÁ¶màãct*±pmhC0–±lg»ÑqD,š½½=UªT¡J•*<ÿüóæö¨¨¨,…«W¯æĉܸq€bÅŠh.: $00Ò¥Kcg§!Âŵk׈ˆˆ ""‚ãÇ›‹ """HHHÀÓÓ“ÀÀ@‚ƒƒyî¹ç¦J•*¸¹¹œ^,•þ‘)†šÑŒXbÙÅ.ÊSÞèH"""""ù‚Š ¨¢Eaóf¨_ÿæŒ[·‚¦h‘{˜ÂêQpÂiF3£ãˆä;%J” D‰´jÕÊÜ–––Fdd$;vÌ<˜|ýúõœ;w¸Y´@É’%)Y²$æGÉ’%yì±Çppp0ê´äÅÅÅqúôiN:Edd$§OŸ6?"##¹pápó}/S¦ 4iÒ„—^z‰òåË„¯¯¯Ág!ùŠ DDDD¤À9Ç9šÒ”TRÙÉN0:’ˆˆˆˆH¾¡"ƒì±ÇþWhЪlÚ®®F§ V—º´¦5£MSšêw%‘`ggG¹rå(W®-[¶Ì²ìïw´?yò¤y`úÞ½{‰ŒŒ$99“É„ŸŸŸ¹ð xñâ+V ___|}}ñ÷÷§H‘"øúúbkkkÄiÉÉÉDGGsþüy.^¼È… ¸pá111œ:uÊüˆ7oS¤HsñH­ZµèÒ¥ eË–Õ ’ãôI‘å4§iBì°c;Û)Nq£#‰ˆˆˆˆä+é¤c‹:• ¬rå <6„gŸ…uë P!£S‰ˆˆˆˆˆˆˆ{›· !„¯ùš0ÂŒŽ#bÕÜÜܨ^½:Õ«W¿íò˜˜˜lwÁ?uê»wï6tOII1¯o2™ðõõ¥H‘"øùùáççg.>ðôôÄÓÓ//¯l*€¿&$$G\\±±±YþÇÅ‹9wî111ÄÄÄpîÜ9²ìÃÝÝÝ\ì@•*UÌ3Q”,Y’R¥JáììlÐJA£")0þâ/šÐ'œø(F1£#‰ˆˆˆˆä;šÉ@¨Téæ,CÏžðÅ`£Ï„ˆˆˆˆˆˆˆÜ^e*ÓîL`éˆ=öFG)°nÍTðä“OÞq«W¯f»³þÅ‹9þ<ÑÑÑü÷¿ÿåâÅ‹æAôÙöáì윥øÀÃÃœqppÀÝÝ;;;<==±··ÇÕÕ'''qssÃÎÎ//¯,û´±±ÁÃÃ#Û±<==1™LÙÚ¯_¿NRRR–¶ôôô,³ܸqƒÄÄD’’’¸~ý:ñññ¤¥¥Gjj* $''“’’µk×HKK#66–øøø,E©©©Ù28::š_‡[…U«VÅ×××\LP¤HüýýñõõÅÑÑñŽï‹H^S‘ˆˆˆˆÇ8FSšâ‡›Ø„>FGÉ—Td T¯kÖ@‹0x0|ø¡Ñ‰DDDDDDDÄ‚Mf2A±ˆE d ÑqDä.<<<ððð ((è¾ÖÏv×þ>'!!Ë—/“’’’e ÿ?ùíïEÿ,€¸UQ¦L<<<ÌEwšÑAE’Ÿ©È@DDDD¬ÞÐŒfÈ6àŽ»Ñ‘DDDDDò-ˆYÆ°b<û,+&HDDDDDDD,T)J1€Lf2Ïó<.¸IDrˆ»»;îîî”,Y2GöwõêUÒÒÒ¸zõj–ö[Å—‘‘‘m=€!C†P¥JFmÙ?gH°³³ÃÍÍ GGGœœœrà D¬ƒŠ DDDDĪýʯ<Ã3T¢ëX‡nFGÉ×Td Y´kóæÁÀàáC‡HDDDDDDD,ÔxÆóŸñ0ŽqFÇ åáá€ÏCï£téÒ¸¸¸P½zõœŠ%Rà¨'HDDDD¬ÖnvÓ˜Æ<ÅSld£ DDDDDr€Š $›à7`øpøúk£Óˆˆˆˆˆˆˆˆ…*B†3œéLç2—Ž#"VÌËË‹ØØX£cˆäkê «´ƒ´  hÀ·|‹šÒNDDDD$'¤“Ž-¶FÇK3qâÍY zô€ðp£Óˆˆˆˆˆˆˆˆ…ÉHp`:ÓŽ""VÌÛÛ›+W®C$_S‘ˆˆˆˆXl¤-hMkV³ŒŽ$""""b54“ÜÑ̙Сtêû÷FDDDDDDD,+®Œe,ðg8ct±RšÉ@äÑ©'HDDDD¬Ê:ÖÑ<˳,g9öØIDDDDĪ¨È@îÈÆ–-ƒ§Ÿ†æÍáøq£‰ˆˆˆˆˆˆˆÄ üñçMÞ4:ŠˆX)///Íd òˆÔ$""""Vc+èHGúЇ%,Á[£#‰ˆˆˆˆXÈ]ÙÛÃêÕP®´hçÏHDDDDDDD,L! 1‘‰,f1ð‡ÑqDÄ y{{k&‘G¤ž ± Ÿð ÝéÎp†3Ÿùô$""""’ 2ÈÐõ¶Ü³3¬Y… Ý,4ˆ7:‘ˆˆˆˆˆˆˆX˜çyžJTb"Ž""VÈËË‹¤¤$RRRŒŽ"’o©'HDDDDò½,`Å(¦3Ýè8"""""VKErߊM› &:w†´4£‰ˆˆˆˆˆˆˆ±Á†ÉLf5«ù‰ŸŒŽ#"VÆÛÛ@³ˆ<;£ˆˆˆˆˆÜK:éØb{ÛeÓ™ÎÆ0™ÉL`B'±n}éËù¯ùy&™ØcÏ{¼Ç VdY÷>À¿¼Ž(–, Ö¯‡ `à@ø䣉ˆˆˆˆˆ…ûöÛoùòË/³´íÛ·€Î;giïÚµ+:tȳl"’óÚÒ–:Ôa,cÙÎv£ãˆˆñòòàÊ•++VÌà4"ù“Š DDDDÄ¢md#Ÿò)_òe¶BƒiLcãx÷Æ0ƒŠˆˆˆˆX/o¼ÙÉN2ÉÌÒ¾›ÝYž—¥¬ äöªUƒ•+¡];([Æ3:‘ˆˆˆˆˆX°2eÊðõ×_ßvYddd–篽öZ$‘Ü6•©Ô£›ÙL(¡FÇ+¡™ Dæ´‹6‘‰|Í׼ȋdanŸÀÆ1Ž9ÌQˆˆˆˆH. #,[Á?ÙcOúäQ"É—Z¶„yóàµ×`ùr£Óˆˆˆˆˆˆ æñÇ¿çzeË–%888‰Hn«K]ZÑŠQŒÊÒ,"ò(n\¹rÅà$"ù—Š DDDDÄbmb¿ð ËXÆ@’AÃÆ;¼Ã§|Ê˼lpJëUêp×uÒH£Ýò(‘ä[ýûÃðáðâ‹°u«ÑiDDDDDÄ‚õìÙ{{û;.···§O»‹X“wx‡Ãf«ŒŽ""VÂÁÁ'''Íd òTd """"ëu^Ç;2È`‹hCæ3Ÿ•¬¤½ N(""""býzÐ{n?¸ÃjR“Ò”ÎãT’/½û.´n Ï=ÇFDDDDD,T×®]IKK»ãòÔÔT:w‰D$·U¦2ÝèÆxÆ“JªÑqDÄJx{{k&‘G "±Hßó=?ñiüïGä 2øžï #ŒgyÖÀt"""""GawìÜ5a¢7½ó6ä_66°|9A‹mt"±@eË–¥jÕªØØdÖd2™¨^½:?þ¸ÉD$7Mf2§8Åg|ft±ÞÞÞšÉ@ä¨È@DDDD,Ò¼ažÅàï2Èàs>g£ H%""""RðT¦2sûÁ&Lt¢S'’|ÍÉ Ö¬;;hÓ’’ŒN$"""""¨gÏž·-2°µµ¥gÏž$‘ÜVšÒ¼È‹La ×¹nt±^^^*2y*2‹³‰MÙf1ø» 2˜ÉL¦0%“‰ˆˆˆˆL=è=öYÚì°£%-ñÁÇ T’o. 7Bd$téééF' FFFF¶öôôt:uR±»ˆµšÀ.q‰…,4:ŠˆXooo®\¹bt ‘|KE""""bq&0[lïºN&™L`³˜•G©DDDDD ®0ÂH%5K[:éôDw”‡T¶,¬^ áá0fŒÑiDDDDDÄÂøùùQ¯^=lmÿ×_dkkKƒ ð÷÷70™ˆä¦bc y›·IB³ŠÈ£ÑL"FE""""bQ gûHçÎw²´ÅgœËXºÑ-Ó‰ˆˆˆˆLòO`ÂdnsÆ™–´40•ä{õêÁâÅ0k|ú©ÑiDDDDDÄÂ<ÿüó÷Õ&"ÖeãH$‘¹Ì5:ŠˆäsšÉ@äѨÈ@DDDD,Ê$&a‡]¶v&l°ÁoÆ3ž³œåÞÁ?RŠˆˆˆˆ<=èa¾V·Çž®tÅ 'ƒSI¾×­ŒÀŽF§ òÜsÏacó¿¡M666tìØÑÀD"’ S˜¡ e:Ó‰'Þè8"’i&‘G£"±á„ó?‘Fš¹Í[L˜xŒÇ˜Å,Îr–×yO< L*""""Rðt¡‹ùZ=•TzÐÃàDb5¦Löí!, Îœ1:ˆˆˆˆˆXwwwš7oŽvvv´hÑOOõ‰#IÌf¶ÑQD$óòòÒL"@E""""b1þ>‹Á­ÿ>Á|Ægœä$¯ð Ž8QDDDD¤À*Mi‚  ŨG=ƒ‰Õ0™`ñbðõ…¶m!1ÑèD"""""b!zôèAzz:ééétïÞÝè8"’G<ñd8ÙÅ,® Â"òp¼½½‰‹‹#33Óè("ù’ÑDDDDÄ2\»v´´´;.OLLäÆ*T—;®kgg‡››Û3›ù‰Ÿ0a 1yרOýÚˆˆˆˆˆ<¼»M––F ›ð9@‡«¸šqõŽëÞë;ƒH6®®°v-<õôì «VÝ,>‘­mÛ¶8:Þ¼U›6m N#"yi8ÙË\f0ƒ·yÛè8"’yyy‘––Ƶk×pww7:ŽH¾£" ‘@rr2×®]ËöwJJ ñññwüþW™™I\\ÉÉɤ¤¤ÿ,týúu’’’ 9Gggg0™Læélqrrâø¢ã˜*™(º£(å¿)GŒs™ËR÷¥899áââ‚»»;ŽŽŽ¸ººÞño777œ5 IDDDDò­ëׯ“˜˜H\\III$%%™¿$%%‘@||¼yYll¬ùÚ?##ƒ«Woþ¿õáV' @||<éééÿ½à1à|øô‡|øû‡¼¹§§'&“ '''±±±ÁÃà *”¥hÙÝÝÝü}ÀÓÓ'''œñôôÄÅÅ'''ÜÝÝqssÃÉÉ WWWó1$*U ¾ùš47ß„‰N$"""""y 11‘ÈÈHþúë/"##³=’““xì±Ç(UªT¶GéÒ¥)Uª”ú†D¬Œ+®Œb¯ó:¯ð E)jt$Ég¼½½¸r劊 D‚Š DDDDrP||<ÑÑÑ\ºt‰¸¸¸Û>bccoÛv·YlmmqwwÇÅÅGGG<<<Ìö½¼¼ðññÁÉÉ øßÀœ›ƒslmm±··ÇÕÕ5Û1þ¾îíü}»„„RSSï¸nRRׯ_ÏÖ~k»ôôtsqDRR'ýN’–’FÐëAxÄxÜ ÅÍâˆS§N™@]½z•””̓£î–×ÓÓ3ÛÃËËë¶ížžž)R„¢E‹>ð, """""·gþ~{_[…wrkþß‹k=<<̃ïmll(]ºôò»ººbooÇï·ú߉‡‡3/Îdè¡wÍy»ï /ˆ¾[Ä7HLLàôéÓ$''“œœl.¦¸õÝànþ~ý???? .lþN%ª[æχ¾}!(:w6:‘ˆˆˆˆˆ<¢äääl…/(¸xñ¢yÝ"EŠ˜‹7nL©R¥¸té&“ ó6;wîdéÒ¥wÜöVáÁßúÎ'’ÿ f0³™ÍT¦òïGDò™[ãibcc)Uª”±aDò!ˆˆˆˆÜEJJ gÏžåüùó\ºt‰èèh.^¼È¥K—¸té/^4ºtéR¶õvvv·ä^ªT©ÛzwrrÂÍÍ WW×,ÛÛÛô äL21a‚š¶Ý­ÁGñññ¤¤¤ðìÝy\Tåþð›l €²© #®˜™[¨©hWÓüyÓ2Ór»j.•auoZfjÖMËʥͫÆxóºeŠû˜ ʮȢ¢,Ïï/ç20à°ÏÛ×ygÎœó}†Ãx¾gžïó //÷ï߯´ C£ÑTXW¾ö,‡ IDATÀÃÔÔvvv°³³C›6m`oo/ýìààØÙÙÁÉÉ NNNÒ4½DDDDôä»yó&RSS‘‘‘Û·oãÖ­[ÈÈÈòƒŒŒ ܺuKg~`ff¦³ƒ»»»»””®+¹ß°¶¶n4¹Á?ñO˜;Èß1ãîÝ»¸ÿ>rss¥< ??_ÊJóÒï“’’pþüy­¢Ž’’­}¶lÙmÛ¶…ƒƒìííáàà åöööhÛ¶-Ú¶m WWWŽYŸ&Obb€I“@¥’;""""""ªÂÇ‘’’Fƒ´´4¤§§C£ÑHËõë×¥üK¡PÀÑÑNNNèÚµ+F¥R ¥R OOO©H¾¬ÒÏqŒ+vszðàRSSµŽ§ÑhpìØ1ìܹFÚV¡PHÇR*•RJ¥^^^:‹ñ‰H^f0ÃB,Ä›xs0.p‘;$"jBÊÎd@DÕÇ""""j–„¸yó&ÒÓÓ‘ššŠÔÔT¤¥¥!%%éééHNNFzz:þüóO­çµjÕJê`nootëÖMú¹l§ôV­Zñf¤ž `P£çµhÑ-Z´ªÏk*//wîÜ‘:‰•-$)-,ILL”~.?jªÔѨmÛ¶h×®œœœàìì ggg8::¢M›600¨Y;‰ˆˆˆ¨þ !‘‘””¤¦¦"99Yú>))IÊʘ››KÏKgÅêÒ¥K…NéöööP(Ö‰æhí°´´„¥¥%ìììj¼œœdeeI×ý·oßFzzºô½F£Á‰'¤ŸËΦfccƒvíÚÁÕÕNNNpqq‹‹‹Ö÷œ‚»>þˆFNž䎈ˆˆˆˆ¨Ù*,,Drr²ÎFƒ7nHù’™™™Ôq_©T"$$DúÞÃíZµªöñu”255•ö¯KAAÒÒÒ*ĬV«¡Ñh-m[¾¡l!‚O•3‚Qý‰@þŽ¿ã]¼‹õX/w8DÔ„ØØØÀÀÀ@ëÿ{"Ò‹ ˆˆˆè‰uûömiºÕÒ)WK¿¿qã†Vç ©C¸““áèè8::ÂÙÙvvvhÑ¢…Œ-¢úbee+++¸¸è7òÅÇ‘™™)ÍrQZ”’’’‚äädœ8q©©©ZŦ¦¦ZÓ󺻻k}_›ŽQDDDD¤Ÿ»wïVèT ªX6GhÓ¦”#téÒƒ ‚‹‹ Úµk'å ,*núlll`ccww÷Çn+„f«([x’””„7nàøñãHJJÂÝ»w¥ç”ï âáá!}ïââReG™fÏØرèÑ98x05•;*""""¢'Vvvv…œ¹ºE¥ò33³*‹îß¿¯³pB­VãÚµkZŸõè*B(]\]]™ãÕ˜`a:¦c.æÂr‡DDM„‘‘Z¶l‰ÜÜ\¹C!j’xuKDDDMZff&bcc‡øøx\½zU*&ÈÏÏð(ipvv–:t÷êÕ îîîpuu• 8òUG‹-àääôØå÷îÝCRR’Ô ©´ÈåÊ•+ø׿þ…ÔÔT馼•••TpСCx{{K ˆˆˆˆôW\\ FƒË—/#66±±±R!AFF†´£££Ôá»GP*•pss“ LÙ™™Ê100€ƒƒÐ¥K—J·ËÉÉ‘ ¯_¿.³8pFú@ËÄÄíÛ·‡R©„§§'üüüàãã???8pÔþGlmŸ~z÷fÏÖ­“;"""""¢&«ª"‚¤¤$x4h’³³s…"‚Òww÷'jæfssó*‹²³³uÎà V«ñÇhuZdQý™„Iøâm¼¯ð•ÜáQbcc£U4HDúãÕ+5z%%%Ðh4ˆ‹‹“Š J þüóO@Ë–-áåå…Ž;",,Lk„xWWW˜˜˜ÈÜ jŽ,,,¤B]>|ˆ¤¤$©0¦ôë¡C‡°aéP¦uëÖðññ··7¼¼¼¤ïÝÝÝahhØM""""j4ŠŠŠpíÚ5©˜ ôk\\}´f&pssãge( (ŠJs¼²¯{ÙbµZøøxé³ØÙÙi½Öe—öíÛÃÈȨ!›FÔ¤ÁK°1ó1>ð‘;$"j"¬­­9“Q ±È€ˆˆˆ•ÂÂB\½z§OŸ––óçÏK7à |}}áç燰°0é{Þ𤦨E‹ðôô„§§§Îdz³³qùòe\¹rE÷ßÿþ7!„ž¯R©¤%003sѧ¨¨ñññZyÂÙ³gqïÞ=f%ðóóC¿~ý0cÆ øúúÂßßVVV2GNT‘½½=ìííÑ£G •ÍJ¿îÞ½ééé€V­ZÁÏÏO+ðõõ}¢FÕiøpà7€éÓ kW¹#"""""jpºŠJ;µÇÅÅáîÝ»´‹¡R©´:³ó3µº¥P(¤üL—ò¿·Òß™Z­Öú½™˜˜ÀÅÅEš1¢|oDÀ xïã},Ã2lÃ6¹Ã!¢&‚ED5Ç""""’ͽ{÷pêÔ)œ}ú`æÌ™P©TðööfG j–ìììпôïß_Z—ŸŸsçÎáôéÓ8sæ öìÙƒ>øÅÅÅðññAß¾}Ñ·o_ôë×...2F_GŒmÛ• ˜0ˆŠx߀ˆˆˆˆšÒÎäå 4 þøã­Îo …Bê<^¾ˆÀÕÕÆÆìîó¤¨i‚Z­FRRŠŠŠ¦¦¦pvvÖšý l!‚»»;?{¥'ÂóxAÂ[x »±[îpˆ¨ °¶¶FNNŽÜa5IÌ:ˆˆˆ¨Þ¤¥¥aÿþý8tèŽ;F###tîÜÁÁÁxíµ× WWW¹C%jò Ñ¡CtèÐáááÒú¤¤$=zÇǯ¿þŠ?þÅÅÅððð@ï޽ѿ 4ŽŽŽ2FODDDÍÍ¥K—°wï^©¨ //­[·Fpp0.\ˆ¾}û¢k×®ì4AT…–-[V˜õ ''ÇŽÑ#Gpøða|õÕWxøð!Ú·o§žz ¡¡¡ÿüs´oßÿûßñÒK/ÁÂÂBîЈ¨–LLL0bÄŒ1‰‰‰X½z5-Z„>øK—.ÅäÉ“ׇqcÇÇ@·n€‡‡ÜQ âöíÛ: 4 nܸââbU(•JÎÀÝ;v Æ “1«¯çÖæ8 ±¿æÌÌ̬FEjµ ¸s玴­®"„ÒB¥RÙøŠâ©Á,ÇrtC7ìÃ> Æ`¹Ã!¢FÈÚÚBäçç³Ð”¨š 倈ˆˆŸ?þøƒ ÂóÏ?ÄÄÄ **ªA ¼½½a``ðØeñâÅõ‡Z­†.]ºT¯Ç©K›6m’^Ÿõë×WûùuÙæ H±ìß¿¿ÖûÓGmÛ_õÝ>9^?} 0»wïÆ… йsg<ÿüóxæ™gpåʹC#""¢z&„À–-[àç燨¨(¬]»±±±˜>}zƒ0Wh8Ì ô×X¯ÝkËÝÝkÖ¬Áõë×ñâ‹/bæÌ™èÖ­N:%whÚ>üðö^xxøPîhˆˆˆˆ¨ ÙÙÙ8}ú4vìØU«VaÚ´i …‡‡,,,àììŒnݺa„ ظq#4 ”J%"""ðÝwßáÈ‘#HHHÀ½{÷€èèhlØ°óçÏGxx8T*U¤¤¤TÈ7ŒŒ`kk‹àà`¼ÿþûÈÍÍ­×ã­^½ºÎößJJJ „€¢Nž[YŽZÓãT–ÇÕ&î¦îìÙ³7nœÑªU+ôîÝ?ÿüs½¯´!$$ˆŒŒÄöíÛƒììldeeáÒ¥KˆŽŽFdd$BBB<:æ΋ÐÐPtêÔ °µµE·nÝ0fÌ,X°7n„Z­†F£AQQQ½µäˆ@ ÅP,Á4¿¿["z¼Ò‚œœ™#!jz8“iÙ²e ^~ùexyyáèÑ£èÝ»·l±$&&ÂÍÍ ðá‡bîܹZëV¯^ÌÌLÙâk¬¦L™‚_|±QŒØ‰I“&ÁÇǧÁŽÙí¯ïöÉñúU‡ŸŸ¶lÙ‚Y³faÆŒ ĪU«0kÖ,¹C#""¢z››‹I“&aÏž=˜1cÞ}÷]´lÙR–X˜+4=Ì š6;;;DFFbÊ”)˜6mz÷îwÞyóçÏ—;´GLL€­[À@à­·€ÈH¹#""""j¶²³³uÎB––†ÄÄDÜ¿`jj ggg­™¥Ù ÜÝÝa`` sk€víÚAI“&açÎÈÏÏGqq1222pàÀ,_¾ë֭Þ={йsçz9^SÕ·o_deeÕûskzœÊò¸ÚÄÝÔ <ýû÷ÇéÓ§allŒ%K–`Ø°aØ»w/nøâ  üüüt>^ú~“––¦5ŠZ­ÆÕ«W‘——àÑŒ}vvvZ³Ÿ”]Ú·o߸fì£j+;›Á ‘;"jd¬­­ N C‰š  ¸¸S§NÅ–-[°xñb,Y²„7SˆH/*• ÇÇŠ+0gÎ\¼x6là{Ñ$-- Ï<ó òóóñŸÿüGÖbd"’§§'Ôj5"##±hÑ"$$$`ýúõ04l“&{z«WS§¡¡rGDDDDôDÒUDPÚÁ766÷îÝ´hÑíÚµƒR©„££#T*•V§^77·ÆqYFFFpvvÆäÉ“ñÜsÏ¡{÷î2d®\¹+++¹Ã#ª1 lÚ´ –––€O>ù[·nÅçŸ.K‘Áã( ¨T*¨T*—¿*}¯R«Õˆ—ŠxLLLàââ¢UèÄ"„¦%CÞÂ[ŒÁ0€üEjDÔx”ÎdÀ"¢êkšÕ¹ÿû¿ÿömÛ°{÷n,[¶Lö%qqqÒ(¤•yýõ×ñÎ;ï`íÚµÒt¦k×®ÅôéÓakk <÷ÜsZSÊöìÙpýúu­õº,X°¡ÿí”йsghÅtàÀôìÙæææhݺ5&L˜€ôôôÇÆ\zÌÒé\wîÜ)­ûæ›o¤c—®Ûºu+ÆŒ+++ØÚÚâµ×^ôöû駟¢}ûö°°°@ÿþýñÇT8vff&æÎ OOO˜™™! QQQÕjóÏ?ÿŒnݺÁÌÌ mÚ´ÁË/¿\ãDìqûÒ'Þê´ÿqǬê}ϲ۹¸¸`ôèÑøý÷ß+ÝïúõëµÎÝï¿ÿ¾Ú±Õ###,_¾»wïÆ·ß~‹)S¦ÈÕ‘;wî ÿþ022©S§d/0xRs]Ï-}¨ï5k]\k37¨}nPÕµ{eÇÏÏϯñyÙP °páBìÚµ [¶lÁœ9sdGËäÉÀ /&rGCDDDÔ$eggãôéÓرc‡4sí˜1cЭ[7ésŠnݺaüøñظq#._¾ sss„„„à£>Btt4pïÞ=$$$ ::[¶lAdd$"""¥RÙd ÊsppÀÊ•+‘’’‚õë×k=V›ÜP!!!ZyBQQàÝwß…¯¯¯´ÝâÅ‹¥mÞyç½ã+›­_¿¯½ölllàììŒ+VT[ÙçnÚ´©ZûÓõÜÊrT]ÛÕË#wFF†Öë\v=z´ôÜêæÉúäžÕi‡>Ÿ7=.FF#€±±1ÌÍÍñçŸ>öµkŒJ‹ÂÃÃ1þ|¬Y³Û·oGLL òòò••…˜˜lݺðóóÃýû÷¡V«1{öl„††ÂÃÃððð@hh(&Nœˆ `ãÆP«ÕÐh4())‘»©`–á Î`/öÊ 52¥3äääÈ Q$ˆˆH„ÿ÷QMmÛ¶MÔ÷«6l¨·}þùçÂÐÐP8p ÞŽQ[|ð u>ž——'777±}ûv‘ŸŸ/>ùä1vìXQ\\,Ú·o/Ưõœ›7o ???QRRRéq£££qñâE­õ{ö솆†âoû›¸}û¶ˆAAAÂÓÓSäææVÙ]û¼}û¶ ¾þúki]ll¬ ÜÝÝÅ®]»DNNŽøñÇ…………xå•W¤í¾ýö[@¼ýöÛ"++Kœ9sF 4HŸ}ö™´Ý¬Y³Ä¬Y³Dff¦ÈËËß|ó055—.]Ò«Í»víbÉ’%"++Kœ;wNx{{‹gžy¦Ê×°´ûöí«Ö¾ôWßöës̪Σê´Oßó£t»Å‹‹ÌÌL‘šš*Æ',--+ÝAA Ÿ~úi¥1Émß¾}ÂÐÐP|ñÅr‡ò?€Û¶ÉQ£SŸ×˜B@lk ÷ßúnKs6aÂáèè(222äE§')WÐçúðq׬uy­ÍÜ v¹Ae×î;~mΈ¶mÛ6a`` öìÙ#w(ÿsçŽîîB $D#z­ˆˆˆƒ˜˜£×¶Ì±ž\YYY"&&FìÞ½[lØ°AÌŸ?_„‡‡ •J%¬¬¬iQ(B¥R‰ððp1þ|±aÃ-DQQ‘ÜMiP/½ô’VnVVNNŽ000ýû÷—ÖÕ&7|ÜñÊ3fŒèܹ³ÖºîÝ» âÚµkÒº÷Þ{O¬_¿¾Úñ•æF]ºtQQQ"77W¬Y³F‡ª2¶Òç~þùçÕÞŸ®çV–£êÚVß?úè£*[Ù K///Ñ©S'­ugÏžDdddµ÷YU‘Áܹsµž?sæLabb"RRR„BtìØQhmóã?VèH£KXX˜˜6mš^mîرc…6ïÙ³G¬ôºnÐÖt_ºâÕ·ýúóq瑾íÓ÷üðòòªpó?//OØÚÚêÜÿ½{÷Ä Aƒ´n”7V³fÍíÚµ>”;”GXd@D¤‹ èq®^½* Ä?ü w(•z’r}®מº¼ÖfnPóÜ ªkw}Ž_ÓóRcÆŒ*•Jî0´<)„‰‰ðõ"""’‹ š‡¬¬,qéÒ%]¡ˆÀÆÆFï"‚ÂÂB¹›Ò¨<®ÓëÖ­…R©”~®Mn¨ÏñJmݺUFñ¨C¼R©ZùC¯^½Djjjµã+Í&Ož,­+**âí·ß®2¶ªŠ ·¿Úè¢+ԧȠ¼Ã‡ ###ññÇKëjšÛÖ$÷ÔÕ}î'T'Æììlñ駟 áääÔhž[mŠ¶oß.Ž9"änÆåŒ8# „Ø-vË 52 …‚95Ir<óÏQýöÛoHKKìY³ä¥NtéÒEçú)S¦ ¤¤ßÿ½´nÛ¶m˜4iRµ ×œŽ IDAT‘’’‚øøxôë×Ok}@@¬­­¡V««½ÏªtëÖMëç¾}û¢°°gÎœAVV®^½Šàà`­mzôè¡×¾mmmqõêÕÇn—’’‚«W¯â駟Öyœƒêu¼Úî«|¼ú¶¿ºÇ¬ì<Ò‡¾çGévO=õ”Öv-[¶Ô9íìÝ»w1tèPØØØ`Ê”)5Ž¯¡Ìš5 )))8qâ„Ü¡Q-DEE¡mÛ¶1b„Ü¡ÔZcϪ{}¨«=uy­]UœÌ ªVÕµ»¾Ç¯Ëó²¾MŸ>§OŸFRR’Ü¡üOPðÖ[À‚ÀÙ³rGCDDDT§rrrpþüyDEEaõêÕxýõ×1|øpøûûÃÆƶ¶¶èÔ©ˆ+Vàرc033Ã!CðüDBB>|ˆ¬¬,ÄÄÄ`ûö툌ŒDDDBBB T*all,wS›!„ô}C~Ž4dȘ˜˜ ** °gÏLœ8AAAسg ##B899Õ8¾Î;KßÁÎηnݪqÜu½?}è›÷VåÎ;xñÅñì³ÏbæÌ™ê&O®NîY¾úÜO¨nŒ¯¿þ:Þzë-Œ7gÏžE›6môŽ¯9Q(P©TÇüùó±aÃDGG#!!÷îÝCBB<ˆuëÖaüøñpttÄ•+WðÉ'Ÿà…^@ß¾}áááøûûcøðáxýõ×±zõjDEEáüùóÈÉÉ‘»™MJWtÅp Ç2,ƒ€xüˆ¨Ù°¶¶Fnn®Üa59Ì ‰ˆˆš¹¤¤$˜™™¡}ûör‡R'ÌÍÍu®···ÇèÑ£ñå—_bÚ´i8qâüýýѪU«j#33À£›xåµnÝZz¼®X[[W8¤¥¥!==]g,ºb»rå /^ŒãÇãÖ­[Ò ï€€€ÇÆPÚ¦µk×bíÚµ¯Ng}÷¥O¼ú¶¿ºñWvéCßó£ªít™9s&üýý±sçNüþûïz’ÈÅÝݦ¦¦¸~ý:úöí+w8DDDTCÉÉÉP*•04lúc•4ö\¡º×‡ºÚS—×Ú‹“¹A媺v×÷øuy^Ö·Ž;x»«««ÌÑ”ñ·¿¿þ Œœ9XYÉQµâôéÓøõ×_qèÐ!œ:u ÙÙÙ:·mݺ5žyæôïßO?ý4¼½½Ñ¢E‹Ž¸yÊÉÉAvv6üýý4ìçH­ZµÂSO=…¨¨(¼þúëسgV¬X,]º999ؽ{7¤çÔ$¾–-[jýlbb‚’’’Ç]×û+¯6yoU^~ùe૯¾’ÖÕEž\Yî©O;ô¹ŸP“×®]‹^xá±±“n&&&P*•P*•:øð!âââpèÐ!üúë¯8rä.\¸ s[…B   <ýôÓèß¿?T*LLLê3ü&mV ؃=†ar‡CD‹ ˆj†EDDDÍœ““ žžGGG¹Ã©W¯¼ò z÷î+W®àË/¿ÄŒ3j´;;;FÊ,ïÏ?ÿ„‡‡G•Ï/í UXX(­«*™)ÿAé¨#NNNÒï¬|,åGµ(,,DHHœqèÐ!tèÐFFF˜4iÎ;We¼ÀÿÚííí@ëÆQe#SÀÉ“'ñ—¿üEúùÈ‘#011J¥‚­­-:v숣Gj=çÌ™3Z?k4¤§§cÎœ9ðöö–Ö?xð Âñ*k³··7N:Uá1üíoÃرc+mCu÷ W¼ú¶¿.ã}ÏÒíþóŸÿhm—––¥R‰ÔÔTiÖ P*•011Á×_ •J…yóæém¦±X¿~=ìììЫW/¹C!""¢Z2d.\ˆèèh„††ÊN½i ¹Bu¯+ÛG]]kWÕFæU«êÚ½:ǯÉy)‡/¾ø>>>pww—;”Šœ<ÿ< ÔQÞGDDD$KKKøùùÁÏÏOçãHKKƒF£ÑZ¢££¡Ñh´5R(Ò(Û¥‹££#œœœàããÓ¨¹iLnÞ¼‰E‹ÁÅÅÓ¦MPûÏ‘ªkøðáxíµ×0{öl)wïÔ©”J%¾ûî;\¿~]ëœièøꊾ³,Vç31}%$$`æÌ™xíµ×0xð`i}§NpéÒ¥zÉ3õm‡>÷ª› —©*WXXˆäädiöùòï½7nÜ@qq1ÀÌÌ NNNP*•èÒ¥ FŒ!½÷zxx4Ê™ ›²Xøc7vc8†Ë5ÖÖÖ¤!¢ÇkúóœQ­XXX`þüùxï½÷pòäI¹Ã©w3fÌÀúõëñâ‹/êµ}éh˜qqqÈÌÌ„££#bbbðá‡"66‹-Ÿþ‰øøxL›6 žžž˜>}z•ûìر#°nÝ:dee!>>›7o®tûÿûßصkòòòðÓO?á‹/¾@DD„4Jä²eËpîÜ9¼óÎ;ÈÎÎÆ… ´F¤777ØÛÛã믿ƕ+WPPP€_~ùûöíÓ»ÍÿûßqøðaDFF"33™™™˜3gŠŠŠ0|xõnÎý‰å§1ä 5¹>,¯.¯µ™ÔNe×îÕ9~uψþóŸX´h éȀÇÓ§?ZÒÒ䎆ˆˆˆ¨^™™™A©T"$$ˆŒŒÄöíÛƒ¬¬,Ü»w ˆŽŽFdd¤Ô™\­VcÞ¼y6lºuëKKKØÚÚ¢[·n3f ,X€7B­Vãòå˸ÿ¾Ì-•Wqq1RSSñå—_"((†††Ø»w/¬¬¬¤mj“V—««+°cÇ­œbøðáؽ{7 Tá9 _]©,G-¯:y¤>ŠŠŠ0~üx´oß¾ÒYê#ϬN;ô¹Ÿ oŒÙÙÙpss«³Á¹š²ÂÂB¤¥¥áôéÓرcV­Z…iÓ¦!44077‡‡‡úö틉'bãÆÐh4P*•ˆˆˆÀwß}‡˜˜¤¦¦âþýûÒûï† 0þ|„‡‡C¥R±À tB'ŒÀ,Ã2¹Ã!¢F€3Õ ""þßD5µmÛ6Qßÿ­nØ°¡Þö]\\,† "lmmÅï¿ÿ^oÇ©)@Z^zé%­Ç¿ûî;­ÇˆììlûJMMvvv¢  @ïãϘ1CØØØkkk1cÆ iý¾}ûD÷îÝ…©©©P(büøñ"--M¯}FGG ___ann.úõë'bbb¤Ø(„"66V;vì&LVVVB¡PˆW_}µBüŸ~ú©puu¦¦¦¢gÏžâøñãÒþzôè!„âäÉ“¢oß¾¢eË–ÂÅÅEDDDˆÑ£GKÛ¥§§?¶Í={ö¦¦¦¢M›6bܸq"99¹ÒvΟ?_ë÷2~üx½÷UxõiÿãŽYóHŸöé{~”ÝÎÉÉIÌ™3GÜ¿_!Ä| µÿW^yEìÛ·Ok¿¿•16´ß~ûM( &Š‹‹åç!¶m“; "¢F§>¯1…€ØÖ@ï¿õÝ–æ,##C899‰Þ½{‹œœ¹ÃÑò$æ U]êÛžº¼Öfn _nPÝkw}_¿šœ— åøñãÂÚÚZëwÙhݽ+D‡B„† QR"w4DDD²‰‰‰111zmË«yÊÊÊ—.]ÑÑÑbÆ bþüù"<<\¨T*amm­u}«P(„J¥ááábþüùbÆ "::Z$$$ˆÂÂB¹›Rg’““+äB¡PˆÞ½{‹U«VUš+×$7Ôu¼>úè±q.]ºT´jÕJëµ?tè :T£øÊçFãǯŸ‡‡‡Î}òÉ'ZÛõë×Oïýézn©ò9jeÛê“GêÊSuíoÇŽ~'¥‹———[uód}rÏêäÃUÝO¨NŒYYYÂÅÅE„‡7þYYY"&&Flß¾]DFFŠˆˆ"”J¥066–^kSSS¡T*EHHˆˆˆˆ‘‘‘bûöíâÈ‘#"!!A”0×lt.Š‹ÂPŠŸÄOr‡BDÀ´iÓDHHˆÜaU[Cåæááá:¯ÿ „,×#¢fo ƶc»Ì‘PSµ}ûvŒ;õùßêÆQoû/((ÀèÑ£ñïÿ}ô‘4¥ë“æ“O>AJJJ¥#4&qqqðññÁ¾}ûtŽ2CÔØ|öÙg˜3gBBB°cǘ™™ÉÒÿÛ¶cÆÈ Q£RßטضmÆ4Àûo}·¥¹‹GÿþýѪU+üøãðöö–;¤:×”rj>ëyùÕW_aÆŒxöÙg±cǘ˜˜ÈÒãýöз/°q#0y²ÜÑÉâôéÓ•JõØm™c‘.ÙÙÙÐh4Ðh4HKKCzzºôóÕ«W‘——011œœœ T*+,íÛ·‡‘‘‘Ì­!¢æ ìûVÙ%-- ‰‰‰ÒÌ,-Z´@»ví¤÷)GGG­÷0ww÷Æ;ƒUj4F#ñ8ó0„¡Üá‘ŒæÍ›‡C‡áäÉ“r‡BT- •›—~–»}»vÿYãz?25 fffˆŠŠÂÛo¿W^yÛ¶mçŸ~ ¹C«µwÞy¦¦¦xá…ðÉ'Ÿ ::Z(×®]ÃÌ™3qàÀÌœ9~øaÓèdDDDDzóòò©S§ŽÀÀ@Ì›7‹-jòÿç3W Æ¨1Ÿ—˜7o¾ù曦wíß«0k0{6¸ºÊQ“£P( R©*-T)ß™·´A­V#..wïÞð¨ÁÅÅ¥B'ÞÒÅÍÍ ††ì JD§«ˆ ô½§ìûNÙ"GGG¨T*¾ï4˱]л° #1RîpˆHFÖÖÖÈÍÍ•; ¢&‡EDDD$122²eËŠW^y]»vÅÔ©S±páB899É^­Ì›7+W®ÄòåËѾ}{¹Ãy¬ H#U<ãÇÇ7ß|#sTDÚRSSñÞ{ïáóÏ?‡¯¯/Ž=ŠÞ½{ËÕggg:t«V­ÂÊ•+ñÃ?`åÊ• “;´Zij¹5í¼ÌÏÏÇêÕ«ñÁÀÎÎ@hh¨ÜaUßÊ•Àf2ˆŽ~4ãÕ™ê!”vV«ÕˆÅ½{÷<~Dqv&j>ªšA%>>ùùù*/…„„ ""‚3¨4s~ðÃHŒÄR,ÅŒàlD͘ ‹ ˆj€EDDDTAŸ>}ƒ/¾øï¾û.6mÚ„—^z ³fÍj’3,^¼‹/–;Œj‰ŒŒDdd¤ÜaétåʬY³[¶lƒƒÖ®]‹É“'óæ,Q3ТE ,Y²cÇŽÅÂ… 1|øpôìÙóæÍðaÚ\'¦˜+Г¯1—wîÜÁgŸ}†Õ«W£  o¾ù&Þxã XXXÈZ͘š[¶={7Ó¦ÉQ³R“"FµZ¤¤$LMMáìì¬5 yÙBwww° ”¨I(ý»/_@ ÑhpõêUäååIÛ* éožE¤¯eX†.è‚ŸðFa”Üá‘L¬­­‘““#wDM‹ ˆˆˆH'cccL›6 “&M—_~‰Õ«WcãÆ Á„ 0bÄXYYÉ&5¼¼<üôÓOøúë¯qðàAtìØ«W¯Æÿýßÿ¡E‹r‡GDDD ¬cÇŽøá‡pòäI¼ûî»5j<==1uêTŒ?ŽŽŽr‡HDµpòäIlÞ¼_ý5 1}út¼ñÆ°³³“;´Ú ÞxãÑ2`àé)wDDDDDô_õY„P¾ˆÆýû÷+”.×®]ÓêðYU««+ŒÙͪÏ~…QXŠ¥xÏs6¢fÊÚÚ÷îÝCaa!LLL䇨ÉàÕUÉÔÔÓ§OÇ´iÓ°oß>lܸS¦LÁË/¿ŒáÇcܸq8p /‰ž@………Ø¿?¾ýö[DEE¡¸¸ƒÆÏ?ÿŒÁƒs$("""B÷îÝ…¸¸8|üñÇX¹r%,X€⥗^°aÃ`ff&w˜D¤‡´´4|óÍ7ؼy3®\¹,]ºS§N…µµµÜáÕ­eË€½{I“€Ã‡&6 QsUUBaa!nß¾­³3³Z­Æ7P\\ 033“f=(¿xxx U«V Ý4¢&«  iii:‹pçÎiÛÊŠáîîÞtgÍ£Fo–¡3:ãGüˆÑ-w8D$W´µµ•9¢¦ƒEDDD¤CCC :C‡EVVvìØ­[·bØ°ahݺ5FŽ‰Aƒ!$$„35a¹¹¹8xð öíÛ‡üYYYèÛ·/V¯^Ñ£G3á&"""¼½½±nÝ:üãÿ@TT¶lÙ‚qãÆÁÚÚ£FBXXBBBøa1Q#“œœŒ½{÷b×®]P«Õ°±±Á /¼€þóŸèÞ½»ÜáÕSS`óf GàÓO™3厈ˆˆˆˆjÉÄÄNNNprrÒY„ððáC¤¤¤@£Ñ --M«A­Vãúõë())Pu‚§§§ÔI¨9¨ªˆ ôo©Tù"‚ &HKÞÞÞ°´´”±%Ôœù£1Ë° #1’³5C¥ƒ¨äææ²ÏQ5°È€ˆˆˆªÍÖÖÓ¦MôiÓpãÆ |ûí·Øµk6mÚ###ôéÓƒ  Aƒàïï/w¸DT!Ο?ýû÷cÿþý8~ü8Š‹‹„7ß|ãƃ«««ÜaQaff†±cÇbìرHOOÇ7ß|ƒ;wâË/¿D‹-п„……aÈ!hß¾½Üá5;%%%8yò$~þùgìÝ»ç΃¥¥%ž}öY|ÿý÷ ƒ©©©Üa6Œ®]…  €Aƒ€䎈ˆˆˆˆêQ‹-¤ÎϺ”-B(_ˆP¾¡´#µ££c…b„:}úhÿ^^^hÙ²¥Ì­!ªÜR,EgtÆøá—;"j`¥…nwïÞ•9¢¦…EDDDT+íÛ·ÇÂ… ±páBdffâ—_~Áþýûñü ,€““BBBŒÞ½{ÃÇdž†€H.%%%ˆÅ±cÇpìØ1DGG#==mÚ´ÁÀ1mÚ4<ûì³hݺµÜ¡Qçè船sçbîܹ¸yó&öîÝ‹½{÷bÞ¼y˜1cºté‚ _¿~èÓ§ìììä™è‰‡#GŽàðáÃ8pànß¾ 777 :‘‘‘èׯÌÌÌäSK–{÷/½9ÉÉäqEº:bk4\¾|jµºÒŽØå Ø›ZU4¦BMÙs6$$„4ôÄð…/Æ` –c9Fag3 jfJ‹ òóóeŽ„¨ia‘Õ;;;Œ7ãÆCII Ξ=‹ýû÷ã×_Åœ9sŸŸ…B^½z¡wïÞFPP,,,äè‰u÷î]œ:u GÅo¿ý†ãÇãÎ;hÙ²%zöì‰W_}ƒ B×®]a`` w¸DDDô„jÓ¦ &OžŒÉ“'ãÁƒ8|ø0öîÝ‹ƒbÍš5BÀ××O=õ‚ƒƒÑ¯_?8;;Ë6Q“S\\Œ .HEGÅÍ›7aii‰ž={â7ÞÀСCÑ©S'¹CmŒ/¿‚‚€u뀙3厈ˆˆˆˆ)SSÓ*‹ ––V¡áرcR§îRe‹Ê"x{{KàˆôQ¶ˆ |Aù"333­¢—²Ežžž°±±‘¹5Dõk – 3:ã'ü„Q%w8DÔ€J‹<9“Qõ°È€ˆˆˆê…¡¡!T*T*-Z„¢¢"\¸pÇŽÃñãDZ~ýz,^¼&&&èÒ¥ ºv튀€ K—.°²²’» DMN^^.\¸€³gÏâܹs8{ö,.\¸€¢¢"¸¸¸ 88+V¬@pp0ºté#ŽÒIDDD2055Ehh(BCCÙÙÙ8zô(Ž9‚#GŽ`Ó¦M(,,„»»;‚‚‚(å …Bæ艗„„œ9s§OŸÆéÓ§qêÔ)äää@¡P 88o¼ñúöí •J¹ÃmœºtÞ|X´9`Õ€™™YŠÔj5pçÎiÛòEe‹ÜÝÝ9xW3SXXˆÛ·oW((]nܸââbU(•JÞW¡fϾ‰‘xoc$F€¨¹àLD5Ã""""jÆÆÆ D`` fþwdÀääd=z§N¹sçðÃ? ;;†††°··G¿~ý´ 8’)Ñÿ¤¦¦âÂ… R1Á¹s瀒’( à駟ÆܹsѧO¸¸¸È2‘N …aaa ðh$¡'NàèÑ£8}ú4>ù䤦¦ÜÝÝ¡R©¤Âƒ€€888È>Qƒ(..†F£‘ J¿Þ¹sFFFðññJ¥ÂðáÃñÔSO¡S§N044”;ì¦cñb`Û6`öl`ûv¹£!"""¢'Ð㊲³³uŽB¯V«qíÚ5äääHÛVV„ T*áêê ccvjj²³³uh4$%%¡¨¨À£œ+” ƒˆª¶‹Ñ]ñ3~FÂ䇈ˆ©©)LLL8“Q51³ """Ù¸¸¸à/ù þò—¿222ðÞ{ïáŸÿü'²²²PPP€o¾ù‹-‚¦¦¦ððð€ŸŸ”J%|}}áççŽÚBO¤ÂÂB$''ãòå˸rå 4 ._¾Œ‹/"77àèè•J…‘#GÂ××*• ¾¾¾00àÈDDDÔ4YZZbÀ€0`€´.;;—/_–Fk/›'”v®(ÍJ¿º»»óšˆšœ¢¢"$%%UÈÎ;‡»wïÂØØ;v„J¥BXXT*ºví*ÄE5dn¬[ üü3ðÜsrGDDDDDÍŒB¡€B¡€ŸŸŸÎÇK;¡—/DP«Õ¸zõ*òòò´öUYBûöí9˱ jSDP¶€€÷:ˆjÏþC–ažÃsœÍ€¨±°°`‘Q5±È€ˆˆˆd÷Ç`íÚµøüóÏaee…9sæàÕW_… ++ /^D||<âââ‹íÛ·ãÆ())±±1ÜÜÜàããƒ:ÀÝÝnnnpww‡R©„¹¹¹Ì-$ªÜ½{÷˜˜(-ׯ_ÇÕ«W‡ÄÄDÃÐÐnnnðòòB=0qâDxyy¡sçΰµµ•» DDDDõN¡P 88ÁÁÁÒº[·náÂ… ˆÅ•+W‹}ûö!33`ccøúúÂÓÓJ¥P*•¼†"Y•”” %% Ðh4¸zõ*®\¹‚?þø………044„»»;|||ЧOL™2~~~èÔ©sÜúòì³À˜1À«¯ýû,Ü """¢FD¡P@¥RA¥Ré|¼|'öÒbµZøøxäççLLLàââ¢ÕqEµ§«ˆ ôw‹{÷îZ´hvíÚI³¨T*­×ßÍͳÒ5€¥XŠnè†8€A$w8DÔ@,--¥k""Ò‹ ˆˆˆH6gÏžÅ[o½…½{÷ÂÛÛkÖ¬Á„ `ff¦µ­­-úõë‡~ýúi­/((@||¼T|‡#GŽ`Ë–-RÇ"hÓ¦ ÜÝݵŠÜÝÝÑ®];¸¸¸pÄGªWùùùHNNFJJŠTDP¶ àæ͛Ҷöööpssƒ§§§THPº”ÿ» """jî‚­õ™™™¸téâââpùòeÄÅÅá×_Err²42`«V­¤‚ƒÒ¯J¥®®®pqqážäcd IDATµÕZVVRSS‘˜˜(”~½~ý:}‰‰‰¸ÿ>í"„Ò¥l®ìîî^íÏûƒòŸ÷”} ®^½Š¼¼<f‚°³³“Ú‚—ÿúWŒýðCX%&B<û, æÍžzJæÑã,ÅRôB/üÿÆ3xFîpˆ¨XZZ²È€¨šXd@DDD æÆX¹r%¾üòKtìØÛ¶mÃèÑ£ëåf£™™™tsS—ÌÌL¤¤¤ %%iiiHKKCjj*ÒÓÓqèÐ!dddàÖ­[Zϱ··‡ììì`oo6mÚHëtýÌ‘Hš–¢¢"dff"33·oßFFFF•?—-x4’®££#Úµk'Mqëìì,­k×®ìììdj™˜˜T™#ܾ}ÉÉÉHMMERRÒÒÒ¤©~ÿýw$''K*ÀÂÂmÚ´AÛ¶maoo{{{8::Jß·mÛÒÏM±“EsñàÁܾ}7oÞÄÍ›7+|ëÖ-dddàöíÛ¸}û6 ¥ç–v.quu…³³3ºwïWWW899¡]»vpssãŒMÍŒÀÖ­À«¯‡üÛ%"""¢f ¦EjµIIIÒ̦¦¦pvvÖ*B([ˆ WBùbñ²ËüÜÜ\iÛ²…âåg"puu…±±Ž®V/¿ <ƒ5k€~ý€À@`Ö,`üx€Ÿ5J=Ñ0oãm5–––ÈÏÏ—; ¢&…EDDDTï²²²°téRløöî<.ªzÿãøkX‡A›‚K®‰[YšÖÏ¥2—²ÜÒ¼åšfQ]÷ìªii¶\M+3ë*VXV×%[-Ó¬›•få’šŠ(.²Ã(àüþ˜;£¨ àay?ó`æÌ9ßó>€Èa¾Ÿóyýu¢¢¢ˆ‹‹£ÿþ†Þ­Ñ^,sÁmNŸ>ͱcÇHJJâøñãN“Ì“““Ù½{7›6mr¬+,,tÚ? €ÀÀÀ .ç¾îïïO`` ^^^xyy¨‰H%dµZIOO'77‹ÅBzz:™™™¤§§»ddd»®(WWWGÁHHHaaa´hÑ©¨¤V­ZDFFR«V-<== :{) öb€V­Z]p›ÔÔTŽ=Jbb"ÉÉÉNÑùñÇÑ‹^˜L&‚‚‚ $((Èi¹Ð:¼½½ñõõÅßß_EÌ‘——G^^éééäää8®ÒÒÒ˹ϋ®;÷%///BCC$ááá´nÝÚñ=ADD‘‘‘* ¨Š\\àõסukxë-6ÌèD"""""†»XB~~>'Ož,vÿúõë9tèã¹hçÀâ–Ë혗—Wìñ8ÀþýûIOOw:—âŠj×®Mttôå]ç™LÐ¥‹mÙ¶ æ̓¡CaÆ xôQ9tý(Rá<ÅSÜÂ-|Ë·ÜÌÍFÇ‘ræëë«N"¥¤")7gÏžeÉ’%<ù䓸¹¹ñÚk¯ñ÷¿ÿ½ø;|T@žžžÔ­[—ºuë^r[«ÕzÞ]îO:uÞDö””þüóO§Iî«”öôôÄÛÛ›€€¼¼¼Íf3>>>àâââ˜täî¯/`»@rwwÇÕÕÀV™íáááôzq\\\(Õç+##ƒ³gÏûZ~~¾ã<Ïœ9ã¸pËÌ̤°°Ðéõ¬¬, (,,t¼ž™™INN‹…ŒŒ rssÉËËs<>}úôsùúú[àé´®FNE!!!*ò'ÁÁÁÓ¼yóKnk/6°ß¿¸Éî8oÕj-v<ûµA`` ÞÞÞŽÂd¼¼¼ð÷÷ÇÍÍ ???Ç5‚}?ÀQÈìåå…ÙlÀÏÏï‚×hf³¹T“+.vMPô÷vûïù§OŸ&77€ôôt¬V+yyyX,Ξ=KFF†ãz ;;›ÜÜ\²³³ÉÌÌ$77—ÜÜ\Ç~úœWÌéTÔQ£F G!qXX˜ãšJª±ë®³u2˜0zö„ÐP£‰ˆˆˆˆTXîî‡; ´ÏuæÌ:DBB‚Ó²cÇÖ®]˱cÇ .à‰Ž÷æêÖ­KTTQQQŽ÷ê8xð t«hAíÚµûÜvÛmNãÕ©SÇñ>Y¹iÕ ââ`Ú4xùe˜<ž}F¶Ô¨Q¾Ç‘ëD':Ò‘YÌâs>7:Žˆ”3u2)½Ê1ÃODDD*Ÿþ™Gy„Ÿþ™1cÆðôÓO—zÒzeb2™wµlܸq©ö-((pºó¾ÅbqL–)ú¸èÝ9í:下¿Õjuš¤cŸàStES\!„··7žžž˜L&Ç$¨ÀÀ@BBB©¼½½1›ÍÅ>öòò" ÀÑ¢²µˆˆˆˆHÕb¿>(-{1²}"ý¹“êÓÒÒœŠní“ïSRRœ zÏ´o¿n¨HJRáââB`` aaax{{ãç營ŸÞÞÞøøø8ºÁÙ 0|||×öqD.ËÓOÃØ –.5:ˆˆˆˆH¥åááAƒ hРA±¯[,Ê{ˆ/=¿ä©¸§…ÿýï‰çĉNÛ‡……9ŠìEö¢‚:uê8Šê W¯ÌŸO> ¯¾ À‹/Ú:üãpÍ5F'`*SéF7¾ã;ÚÓÞè8"RŽ|||HMM5:†H¥¢W"""R¦’““?~<Ë—/§sçÎüöÛo¥žt_ݸ¹¹9îœ_Þì].5Á¨h±BIÙ‹.ÄÞâ¶èD"qfï´Užrrr8sæÌ%¯ ì×vGŽá®»îbùòåÅ^ç]ìš ¸®k"šŸŸmBн÷ƒB§NF'©’<̬7¯g#x衇Î{=//ƒUªŽ{Bh(LŸãÆÁ›oÚ ^{  °57kftB‘j­+]iO{f3›ùØè8"RŽ|}}ILL4:†H¥¢")3}ô=ôÞÞÞ|ðÁôéÓÇèHrŽ¢“ûk¨«ˆˆˆˆHµäããƒPºëûÝ [µjE“&MÊ%›H…Ò§Üv<þ8üü3¸¸HDDDD¤ÊÙÈF’H¢ýŠ}ÝËË«j\ƒúúBl,Œññ0w.4oíÛÃĉг§Ñ Eª­)LáNîäg~¦ mŒŽ#"åÄÇLJììl£cˆT*ú‹¸ˆˆˆ\±ŒŒ FÅ=÷ÜÃí·ßÎï¿ÿ®‘*&//ø«Ø@¤Zxé%øýwX¶Ìè$"""""UÒ VÐ’–4¡ ”„»; b»ÎøòK ‚^½ U+ˆ‹ƒÂB£ŠT;ÝéN[Úò,ÏEDÊ‘999FÇ©TTd """WäóÏ?§Y³f¬Y³†Õ«Wçt·|©ìE^^^'¹Š7†#`òdÈÊ2:ˆˆˆˆH•’O>ò!`t”«Ïd‚.]`íZغš5ƒ¡C¡aC˜?þw ."WǦ°ŠUüÆoFG‘r¢"‘ÒS‘ˆˆˆ\‹Å¨Q£¸ãŽ;èÔ©»ví¢W¯^FÇ‘rb±XH54cX,ð F'©R¾à NqŠ~ô3:Š±ì] öì=lEÎuëÂôépê”ÑéDª…Þô¦9Íy†gŒŽ""åÄl6;n¤#"%£")µ„„Ú·oÏÊ•+Y¹r%o¿ý6AAAFÇ‘rdÆl6œDä*«Yž|žN#""""ReÄÏMÜD]ê¥b¨WÏÖÅ !F†  NˆÕµˆH93aâIžä>`;ŒŽ#"åÀËËKE"¥¤")•O?ý”Ö­[SXXÈO?ýÄ=÷Üct$¹ òòò0™LxzzEäê‹…ˆ˜:Õè$"""""UB.¹¬f5ht”Š'4ÔÖÅàÐ!˜5 þó¨_úõƒŸ6:H•u/÷Ò„&<ÇsFG‘r "‘ÒS‘ˆˆˆ”ˆÕjå¹çž£GÜqÇ|ÿý÷Ô«WÏèX""""""r•X,Ìf3&“Éè("WŸ‡Ìž o¿ ?ýdt‘Jïc>&<îå^££T\¾¾¶‚çýûaÉؽÚ¶…`íZ£Ó‰T9.¸0‰IÄÏ^öGDʘ——………äççE¤ÒP‘ˆˆˆ\RVV={ödÚ´i,\¸·ß~ooo£c‰ˆˆˆˆˆÈU”——‡———Ñ1DŒsｶÉ<±±`µFDDDD¤R‹'žÎt&Œ0££T|îî0düþ;|û-A¯^ЪÄÅAAÑ EªŒ  õÔÍ@¤ ²ÿmÛb±œD¤òp3:€ˆˆˆTlÉÉÉtïÞ£G²qãFn¸á£#‰ˆ\ÜôépìØùëßx¾úÊyÝ´i~Ub‰ˆˆˆTvyyy˜Íf£cˆkþ|hÓ>üÐVt """""¥–I&Ÿñ¯ñšÑQ*Ÿl˶m0o  3fÀ£Âˆ ʼn\W\™Ä$F1Š©L%Š(£#‰H±äååáççgp‘ÊA DDD䂸ùæ›9uê” D¤òÈÊ‚Å‹áßÿþkqw‡ÿz¾d‰­•píÚF§©4,‹:ˆ´l ƒÁ„ pú´ÑiDDDDD*¥ù+Vîâ.££T^ö.{÷B0y2Ô­k»Ó©SF§©Ô3˜H"ÕÍ@¤Š)Zd "%£")ÖŽ;èС~~~üðÃ4hÐÀèH""%3p íc~þ…WW[ka“Éج""""•H^^žŠ Dfφädxùe£“ˆˆˆˆˆTJ+XAwºH ÑQ*¿èh[ǵ„3,€:u 6N'R)¹ãÎ&°”¥å¨ÑqD¤ŒØ»ôªÈ@¤äTd """çÙ¸q#:t aÆ|ýõׄ††ID¤äÚ´¨K´.ÍÏÿ«ADDDDJÄb±8Þˆ©Ö""à‰'àÙg!-Íè4"""""•ÊINò5_3€FG©ZBCm] ‚Y³`Õ*Û{%={ÂÏ?N¤Òy &˜ñ/££ˆHQ'‘ÒS‘ˆˆˆ8ùæ›o¸ãŽ;¸í¶ÛøôÓOñ÷÷7:’ˆHéÝ?¸»_øõèhhÑâêå©ÔÉ@¤ˆ ÀÞÞè$"""""•Êû¼'žÜÉFG©š|}m] þüV¬€ãÇ¡m[èÐÖ®5:H¥á‰'ó8¯ñ)¤GDÊ€Š DJOE"""âðã?Ò«W/î¸ãÞyç<==Ž$"ry´u+(Ž»;<øàÕÍ#"""R¨È@¤__?^~Nœ0:ˆˆˆˆH¥Oø¥jsw‡¾}᧟àÛo!(zõ‚V­ . ŒN(Ráf4^x±…FG‘2 "‘ÒS‘ˆˆˆ°cǺwïÎ7ÞȻﾋ›››Ñ‘DD._£Fм9˜Lç¿–Ÿýû_ýL""""•œÅbÁl6C¤â;Ö6Qgöl£“ˆˆˆˆˆT ‡9Ì÷|Ï@¥z±w1ض š5ƒ¡C¡aC˜?rsN'RaùàÃÃ<ÌE–ÑqDä Ù‹ ,‹ÁID*ˆˆˆû÷ï§[·n\{íµ|ôÑGê` "UÃ!àêê¼Îd‚–-¡Ac2‰ˆˆˆTbêd r³&O†×^ƒƒN#""""RáÅO t¥«ÑQª§–-m] öî…ž=m×3uëÂôéšjt:‘ éQ%Ÿ|±Èè("r…Ìf3&“I DJAE"""ÕÜ‘#GèÒ¥ ‘‘‘|öÙgøø¨5©ˆT÷Ý……Îë\]áï7&ˆˆˆH%§"‘bŒ‘‘ðì³F'©ð≧/}ñÀÃè(Õ[t´­‹Á¡C0f ,Xuê@l¬mˆ8Ô £Å‹¼Hš˜,R™¹¸¸àáá¡"‘RP‘ˆˆH5–——GïÞ½ñññáÓO?ÅÏÏÏèH""e'<n¼\Š\ö½÷—IDDD¤³X,˜Íf£cˆT,îîðÏÂÒ¥pà€ÑiDDDDD*¬?øƒílg Ž"v5kÚº:Ï<«VÙ zö„Ÿ~2:H…1Žqd’ÉR–ED®———Š DJAE"""ÕØÃ?ÌÁƒY½z5ÁÁÁFÇ){÷ß&“í±‹ tìÆf©¤ÔÉ@䆨(˜3Çè$"""""V<ñÔ¦67s³ÑQä\¾¾¶.þ +VÀñãpýõС¬]kt:Ã…Æ<ÀæO¾ÑqDä ˜Ífˆ”‚Š DDDª©—_~™eË–ñöÛoS¯^=£ãˆˆ”~ýþ*20™lE""""rYTd r®®0q¢­›AB‚ÑiDDDDD*¤¬`pÅÕè(r!îîз¯­‹Á·ßBPôê-[B\PÄ0“˜Ä1Ž±‚FG‘+àåå…Åb1:†H¥¡"‘jèûï¿güøñÌœ9“îÝ»GD¤üÔ¨]ºØ L&èÓÇèD""""•–ÅbÁl6C¤b2"#aî\£“ˆˆˆˆˆT8ÛØÆ^ö2€FG‘’²w1ض š7‡aàaC˜?rsN'rÕ]Ã5 `³™ÍYÎGD.“‡‡gÎœ1:†H¥áft¹ºŽ;Fß¾}¹ãŽ;˜¾ÝºúÙgävêÄñ´4HK+ñ¾^^^¥žHg2™ ,Õ>""""•:ˆ\„»»­›Al,L™b+8â‰'šhÚÒÖè(RZö.Ó§Û ¦Lgž1cà‘G 8Øè„"WÍd&ÓŒf¬f5ws·ÑqDä2xxxŸŸot ‘JCE"""ÕHaa!ýû÷' €åË—c2™ŒŽ$"aµZIOOlºìmûÒÓÓ±Z­œ9s†œœ233),,¤  €¬¬,à¯IüEÇÈÏÏ';;ÛéXiçL¼ÏÎÎvº¸.z¬s³ÙeeeQP¤U®Åb!//ï²Ï¿¬øÉÀЯ¾â½zõŒŽƒ››~~~Nëpqù«Ñœ··7žžžŽçîîîøúú:íäôÜ××wwwÇsOOO¼½½‹Ó¾¯ÙlvL´g(z,???ÜÜÜpuuÅßß¿Øl"""R}äåå©“ÈÅ<ø m²ÍܹðòËF§©¬XYÉJ3zo²ÒŠŽ¶L ¯¼  /À A0n4h`tB‘rטÆÜÅ]Ìd&wq—~¦‰TBêd R:*2©FæÎË?þÈ?þxÞW¹0ûû$úôôt ÈÌÌtL¾ÏÍÍåôéÓŽmÒÒÒ(,,,v›¢öíÛWp9\\\œïÂè(,*º}2¹]HHˆÓ]j‹N2·»œ‰ñÊZ—3É=gæL&ŽÇøRÜy÷rº&Ž¯ó…œ>}šÜ"­„‹;Î¥ 6ìß[E%%%qöì_íYÏÍa/)ºï¥²^JIŠì_gûsû÷¥½(" 77·‹nˆ««ëe}¿ˆˆˆHÙ±X,êd r10a‚myê)¨YÓèD"""""†ÛÌfqˆ 0:Š”…š5m] ƃ7ß„—^‚%K {wøç?¡­ºUHÕ6•©´¦5_ò%Ýèft)%ˆ”ŽŠ DDDª‰íÛ·3}útžyæ®»î:£ãˆ”¹ÜÜ\rssÉÌÌ$33Ó1©?==ììlrrrÈÉÉ!--ÜÜ\rrrÈÊÊ*QAIØ'YûøøàááA@@®®®:&NרQ///§;ÙÛ·7™LÅxxxàã〿¿?®®®ÅŽ#°t)!ÿûüJñìÝ,Š4dddpöìY§î—*V8w{Mvv6ÉÉÉŽmíÝ7ÒÒÒœŠl.Åþoáb…žžžøøøˆ¯¯/>>>ŽçöÇ~~~øûû;ž«€ADDäÒòòòTd r)C‡ÂÓOÛîê9c†ÑiDDDDD O<ÍiN3šEÊ’¯/ÄÆÂ#Àºu0s&\=´o'B`Ò]Þ¥êiIKºÒ•ÙÌV‘H%äîî®"‘RP‘ˆˆH5púôi† Âõ×_Ïã?nt‡´´4222HOOw|ÌÍÍ%++‹ŒŒ Ga@ff&YYYŽçéééŽÇÙÙÙ¤§§cµZ/x¼½½ñóó# Àñ< €ÀÀ@¼½½“ôíøƒ‚‚“ø===ÏÛÆÍÍÀÀ@§‰þR©Àà’ì] Œdïò`ïæ‘™™éTü“••å(^¸Ø6ééé$%%û³ãbüüüÿÖƒ‚‚¾¾¾ŽŸ>>>øûûãçççT¼`_ì?WÜÝݯÒgMDDäê°wß²ÊÈx{Ø1°`Œo›x#""""RMPÀ‡|ÈcpÓô4©Z&3™[¹•ïøŽö´7:Žˆ”‚:ˆ”Ž~‹©¦NÊ¡C‡øõ×_quu5:ŽT!yyy¤¥¥]t±O>wýÉ“')(((v\³ÙLPPãNåAAAŽç4mÚ´Ø׊{¢;ü‹Tžžžxzz–{ÁCÑŸIçþŒ*ú¼èããÇ»­½ÛùìÝMŠþ\*ÉbߧvíÚ˜t—'©@ìŽÔÉ@¤}^xÞzËöXDDDD¤šZÏz’I¦/}Ž"WC‡¶eûvxé%6 ¦O·u<1ÂV”-RÜÂ-´§=s˜ÃZÖGDJÁÃÃüü|£cˆT*2©â¶oßμyóxå•W¨[·®Ñq¤ÊËË#55Õ±œž7¦‡‡Çy?kÔ¨Appp±KHH!!!”É9‰ˆˆœËb±*2)‘5àÁm…£Gƒº\‰ˆˆˆH5O<íhG}êE®¦˜ˆ‹ƒ3`Þ<˜2fÍ‚‡†Gà`£Š\±ÉL¦=ØÆ6ZÑÊè8"RBêd R:*2©ÂΞ=˘1chÕªÇ7:Ž\YYY,°/)))¤¤¤8žçææ:áâârÞäÕÐÐP6lxÞ„×s uçm©vì].WVVÖE ìSSSÙ½{·ÓÏôÓ§O;åîîNpp°SA‚½!$$ä‚E úÙ-""—bïd`6› N"RI<ñ¼ö¬\ ÷Ýgt‘«Î‚…Õ¬æiž6:Š%* æχ©Sá•W`áB˜3úõƒ§ž‚ ŒN(rÙîäNZÑŠ¹Ìe+ŒŽ#"%¤"‘ÒQ‘ˆˆH¶xñb~úé'~úé'\\\ŒŽ#—Ájµ’œœLrr2ÇŽãĉ$''sôèQ’““9qâÇŽsL6=÷bÈ>Ù´èÅõ×_Á‰¦Áº{ˆˆÈUåç營ŸûÛßJ½ovvv±dE—ÄÄD¶nÝêx~n'“ÉäTTFíÚµ©Y³&áááŽuáááÔ¬YOOϲ:u©DìEêd RBQQpÏ=ð¯©È@DDDDª¥Oø„,²¸—{Ž"F«Y¦O‡ñãáwl]ß5‚îÝmÅ×_otB‘Ë2 b{ÙKCGDJÀÃÃÌÌL£cˆT*2©¢’““™2e ±±±ÄÄÄGΑššz^ÁÀñãÇ9~ü¸Óºääd ûyzzJíÚµ £N:´k×î¼"š5kŒ¿¿¿g)""åÍ××___êÔ©Sâ},K±nRRR8yò$ÇŽã—_~áäÉ“$%%‘í´PPµjÕ"44Ô©¡víÚNÿG…††âæ¦?;ˆˆT‹P‘H©<ú(tèß7Ýdt‘«*žxná 7:ŠT>>0r$ ëÖÁÌ™pà о=Lœ=z€ºîJ%r/÷òOþÉó<ϼat)wwwu2)½Û/""REÍœ9³ÙÌôéÓŽR­œ>}š#GŽ””Dbb¢Óã£G:ŠŠ^´¸¹¹9MÐ #&&Æ1AÓ[^n IDAT>³V­Zxv""R˜Íf"""ˆˆˆ(Ñöyyy?~œcÇŽqòäI§n:G姟~rtÛÉÍÍuÚ744ÔñYDD×\s ‘‘‘NÊã4ED¤ŒÙ;˜Ífƒ“ˆT"íÛCÛ¶°`Š DDDD¤ZÉ"‹u¬ãe^6:ŠTD..г§mÙ¼ž{zõ‚-à‰'lÝàt©\qe<ãËXf0CEU"•€‡‡‡Š DJA¿‘‰ˆˆTAdñâÅ,X°___£ãT§OŸ&))‰#GŽœW@`|âÄ Çö„‡‡É5×\C§Nœ ìw®Y³&&Ý•CDD*(///¢¢¢ˆŠŠºä¶ÙÙÙÅvèIJJ"))‰-[¶pøða²²²ûøúú[| B‘ŠÅ^d N"¥4v, Ï?‘‘F§¹*V±Š ¸›»Ž"]‡¶eûvxé%ÛõÓ´iðØc0bx{Pä¢îç~¦1yÌc.sŽ#"—àááA~~¾Ñ1D* ˆˆˆTA“'O&**Š¡C‡¥RÉÈÈ`ÿþý8p€päÈ:ä(,¸XAçÎ#""ˆŒŒ¤V­Z*‘jÅ××—† Ò°aËn—™™ÉáÇ9|ø0IIIN/UˆÉßþö7¢¢¢ˆŽŽ¦^½z„‡ëî@""åÍb±*2)µ`âDxýu˜9Óè4"""""WE<ñÜÎílt©,bb .fÌ€yó`ʘ5 ~ØV¼btB‘byâI,±ÌbS˜B FG‘‹P'‘ÒQ‘ˆˆH³uëVÞÿ}>øàÜÔFÒ‰ÕjåÈ‘#8pÀQLPôcjj*®®®Ž Œö‚sשּׁ‘ËçïïOÓ¦MiÚ´é·¹X!Â?üÀ^Íf3õêÕsý…§§çÕ:5‘*ËÞÉÀl6œD¤’ñð€‘#aÑ"xòIп!©âRHa=ëYÆ2££HeóçÃÔ©ðÊ+°p!Ì™ýúÙÖ]â7"Fxˆ‡˜Íl^ã5&3Ùè8"r*2)Í<©bž}öYÚ´iÃÝwWÏö£gÎœqœ»üñÇäääàééIDDÑÑÑÄÄÄЧO¢££‰ŽŽ¦qãÆx«õ¦ˆˆˆ¡JRˆ––vÞÿ÷;vì`Íš5‚ûî3:ˆˆˆˆH¹ú€pÇžô4:ŠTf5kÂôé0~<¼ó¼ð4n Ý»ÃSOÁõ×PÄÁâ!æ1Çx /Ô T¤¢R‘Héè!‘*äÀ¬^½š+VTù»ìŸ9s†}ûö±k×.vîÜéøøÇpöìYÀyBa—.]9r¤ãyݺuqqq1ø,DDDäJѺukZ·n}Þk‹…£GžW„°~ýzvîÜéè‚H½zõhÒ¤ M›6u|Tñˆˆ3‹Å‚——Þ ¹,ááУ¼ñ†Š DDDD¤Ê‹'ž^ôÂ_££HUàãcë7|8¬[3g 7@ûö0q¢íZ«Š¿/.•Ãc<Æ|æG£et¹wwwˆ”‚Š DDDª_|‘k®¹¦Ju1ÈÎÎæ?þ`ÇŽìÞ½ÛQP€ÕjÅÓÓ“ÆÓ¸qc D£F¨_¿>ÑÑÑøúê—"""Õ•Ùl¾`§‚‚‚:äètdÿýbݺuœ:u €àà`š6mJãÆŠjÕªuµOED¤BÈËËS‘È•1î¼vï¶Ý}SDDDD¤ :ÊQ6³™ÿð££HUãâ={Ú–Í›á¹ç wo¸î:xâ [A·º/ŠÂcC˜Ë\†1 7MË©\\\7.‘KÓí{EDDªˆÔÔT–.]Êã?Ž«««ÑqJ­  €ß~ûeË–1aÂî¼óN¢¢¢ð÷÷§mÛ¶Œ=š/¾ø†·~ÈÞ½{ÉÉÉá—_~áÝwßeÊ”)ôéӇ뮻Nb¨#GŽ`2™.¹Ì›7¯DãMš4ɱÏgŸ}vÁuFY¿~=&“‰;v8Ö-\¸Ð‘oÉ’%e~L#ÎÉ’%Žc.Z´èª ==Áƒ“œœ|ÕŽ)RÕ¹¹¹Q¯^=ºvíÊ#<¢E‹Ø´i©©©;vŒõë×3mÚ47nÌîÝ»™1c]ºt¡víÚÓ±cGzè!,XÀ¦M›ÈÌÌ4ú”DDÊ]^^f³Ùè"•×í·C:ðïDDDDD¤Ü¬`þøs·Eª²`íZضÍVd0l4hóçCNŽÑ餛ÀqˆøÈè("r*2)•Ì‰ˆˆTË—/ÇÍÍ¡C‡å’òóóùý÷ßÙ¶mÛ¶mcëÖ­üöÛoX,Ìf³£3ÁÈ‘#wŽŽŽ®”ÅR=EFFbµZyàøàƒÈÎÎ>o›I“&•x¼9sæðÀиÈÝ.‹[W‘Œ;–x??¿r߈ó>|8ƒ¾ªwðÍÌ̤cÇŽÄÆÆzÕŽ+RÕªU‹Zµjѹsg§õ'Nœpt<°üàƒHMMÅÅÅ…úõëÓºukZµjåX : ‘²g±XÔÉ@äJ¸¸ÀС°`Ìœ žžF')sñÄÓ‡>x¢ßwå*ˆ‰¸8˜1æ̓)S`Ö,[ÑAl,Ô®mtB©f¢‰¦}x–géK_L˜ŒŽ$"çP‘Hé¨È@DD¤Šˆ‹‹£oß¾øøøå<ÇŽãçŸfëÖ­|÷ÝwlÞ¼‹Å‚‡‡‡cBÞ}÷ÝGëÖ­iÛ¶-žz£]D€ &Á°aÃŒŽ"Rí………Æÿýßÿ9­?zô([·nu,/½ôÇŽ víÚtèÐöíÛÓ¡CZ¶l‰‹‹šJŠHå”——§"‘+5l˜­À`Õ*èßßè4"""""ej?ûÙÊVf3Ûè(RÝDEÙº<õ,\h[æ̓~ý`êThØÐè„RLa ­hÅzÖÓ•®FÇ‘s¨È@¤tôζˆˆH°sçN~ùåî¿ÿ~££°{÷n^}õUî»ï>êÔ©Cxx8wß}7«V­¢Aƒ¼þúëìܹ“ÜÜ\vîÜI\\±±±tèÐARmÌ™3‡Ç{ €Ï?ÿœvíÚáååEpp0÷ß¿c‚êå¸Ôx]ºtÁd29–‚‚žyæš4iâØnêÔ©ŽmfÍšUì±&MšD×®¶?5oÞ“ÉDݺu¶)((àÑG% €ˆˆž~úéóÆùøãiÓ¦ f³™°°0zè!233/ëü/5VJJ ãǧ~ýú˜ÍfbbbX½zu±c½òÊ+Ô©Sooon½õVöíÛWêc.\¸Ðñy\¸p!£G¦F˜L& pÁóÈÊÊbÙ²e<òÈ#NëKšÿþýôîÝ›üüü¸ë®»øá‡ÈÎÎvúú·k×€„„§õEs/Z´è’_âßwûÛ߸÷Þ{Ù²eËÏO¤ª§gÏžLŸ>µk×rôèQ’’’X»v-Æ #%%…©S§Ò¦M‚‚‚¸í¶Û˜1cß~û-gÎœ1:¾ˆH‰Ù;ωȈˆ€Ûo‡¥KN"""""RæÞå]jR“[¸Åè(R]…„Àô锋Ö-и1ôìi{,rÄCg:óÏEDŠáêêJaa¡Ñ1D* ˆˆˆTË–-£N:Ü|ó͆ÿÀ¼ùæ› <˜ððpš4i¤I“HOOgøðá¬_¿ž´´4¶oßΫ¯¾Ê!ChÒ¤ ®®®†ä1Rzz:µjÕr<ÿøãéÞ½;;wæðáÃ|÷ÝwìÙ³‡Ž;’••UêñK2ÞúõëéׯÍ›7Çjµâæfkp¶fÍvïÞÍþýû˜5k³gÏfÑ¢EL:µØãÍ™3‡/¿ü€ßÿ«ÕJBB‚Ó6¯½ö]ºtáÈ‘#L™2…iÓ¦±qãFÇë«W¯¦W¯^tïÞcÇŽñÅ_°qãFî¾ûn¬Vk©Î¿$cÍš5‹üü|¶lÙ☰߿vîÜé4V||}:mÚ´¡^½z<òÈ#$$$0dȾüòKRRRøä“Oxꩧèܹ3~~~W%›HE“““ãtwø   §×ÇG“&Mxæ™g ¡Q£F,^¼˜?ÿü“W_}µÔÇ+éx½{÷æ÷ßçàÁƒ?~œ””L&k×®ul·fÍzöìy™goÓ¦Mzõê…ŸŸcÆŒÁ××—o¿ýÖñú„ hÚ´)O?ý4AAA´hт矞¯¿þš 6”êX%kÞ¼yÌ›7àà`|}}4hݺu;oÒÿôéÓ‰‰‰aêÔ©ѲeKFŽyYÇ´ëÒ¥ }ûöÅÇLJ±cDzbÅŠ žË¶mÛðóó£FNëK’ßb±ð믿ҧOBCC ä…^ÀÇÇ°ýñdÔ¨Q|ôÑGddd8ö‹‹‹cøðá˜L&§cý>üðÃx{{;} ÇGÓ¦M™9s&ÁÁÁ„‡‡óú믫;Èÿ¸ººØ1cX¾|9 $%%±téR¢¢¢xíµ×h×®¡¡¡ôë׸¸8ÒÓÓŽ-"âDE"e¤gOðóƒøx£“ˆˆˆˆˆ”™_ù•ìd Ž"ò—¿º|û-AïÞв%ÄÅÁÿ:}‹”µ.t¡5­y‘Ž""çP‘Hé¨È@DD¤’;tèûöíã¶Ûn+×ãœ9s†µk×2xð`BCCiÓ¦ ñññÜrË-|ýõפ¥¥±yófæÌ™C—.]ððð(×<"•…V«Õ±¤¥¥9^;rä{öì¡S§NNûÄÄÄàïïÏúõëKu¬ÒŒ×½{wÜÝÝY½z5k×®eÈ!´mÛÖQdpüøq¬V+ááá¥Êq®æÍ›;›L&jÖ¬Irr²#óÞ½{¹å–[œö¹á†øꫯJ|œ+«FìÝ»×ñüÔ©SìÝ»—:;Öåóºë®+ѹ€íóX¢mÏÍo6›iÛ¶-“'OfåÊ•äååáææƉ'Û >œ³gÏ::¼÷Þ{<ðÀç_ôkèêêJHHˆÓ×ÐÞ-£(___RSSK”_¤: §oß¾¼þúë$&&²sçN&OžÌÉ“'6l¡¡¡tîÜ™W_}•””£ãŠˆ`±X0›ÍFÇ©üÌfèÛ–/7:‰ˆˆˆˆH™‰'žk¸†v´»ôÆ"F°w1øå¸î:6 4€ùó!'ÇètRM`ò!ûØgt)BE"¥£"‘Jî‹/¾ÀÛÛ››nº©ÌǶZ­|ûí·Œ=šððpîºë.>Ì“O>ÉÞ½{Ù³g/¼ð·Þz«îV-RB?~À1iôÜ;Õ—zRiiÆ ¤cÇŽNE½{÷¦wïÞlÚ´‰ŒŒŒ2éb¶ÉæE½p·gZ¸p¡SLJÐÐPK|œ’Žµk×.úôéC­ZµpqqÁd2±lÙ2§cÇŽç.Ï}^Úü¥¹ûoZZîîîç­/I~€Ï?ÿœ»îº‹Ç{ŒÀÀ@î¼óN¶lù«-kÍš5¹÷Þ{yë­·øá‡hÑ¢E±… ç~ ÝÝÝÏû÷}'"%פIÆdž HNN&..Ž°°0&NœHxx8=zôàÝwß%Gox‰ˆAÔÉ@¤ Ý?lß¿ýft‘+fÅÊ{¼Ç}܇ Ó¥w1R‹¶.ûöA¯^0e DD@l,üï½!‘²p÷Pz¼À FG‘"Td R:*2©äÖ¯_OÇŽËt’ÿéÓ§‰‹‹£yóætìØ‘M›61vìXöíÛÇÆyâ‰'hРA™O¤º lwÍ?Wjjªãõò¯wïÞlÞ¼™¤¤$þüóObbbèÕ«|ú駬^½šÞ½{—*CiÙ3M˜0Á©ãƒ}Y^Š»{–d¬üü|ºtéÂáÇùæ›oÈÏÏÇjµò÷¿ÿ«Õê«víÚÀùŸËŒŒŒrË®   òóóÖ•4¿}ÿ_|‘¤¤$6n܈Åb¡cÇŽìß¿ß±ÍÃ?Ì?þÈ®]»xë­·3fL©s^ìûND.OPP àÝwßåäÉ“|øá‡x{{óàƒNll, FÇ‘jFE"e¨CˆŽV7©þËI 4:ŠHÉÕ­këbpèLž +WBT {öNªW\ùÿ`Ë8† XD* WWW Ž!Ri¨È@DD¤’ûá‡èرc™Œ•‘‘ÁÔ©S‰ˆˆ`Ô¨Q´k׎íÛ·³sçN¦OŸNttt™GDl"##¹öÚkùæ›oœÖoß¾ÌÌLºtéR®ãõîÝ›‚‚üqºví @³f͈ŽŽ&>>ž„„š6mzÉ㺸\þeEdd$5⧟~:ïµ-ZðÞ{ï•éXàرcôïߟFáêê ØŠ«ŠªQ£ 6dóæÍNë·mÛVnùÏU«V-ÒÓÓÖ•4ÿñãÇiÞ¼¹ãy»víxã78sæ ?ÿü³cý7ÞHË–-Y¸p!‡¢U«V¥Îiÿ¾Û¸q£Óú£Gb6›IMM-õ˜"ò³ÙLÏž=yÿý÷9rä“'Oæ£>¢~ýúôíÛ—;vQDª ‹Å‚Ùl6:†HÕ`2Á AðÞ{pN±°ˆˆˆˆHe³‚4¢×qÑQDJ/$&N„ƒañbزš4ž=mE®À<@ jð2/EDþG DJGE"""•Xff&‡všHz9 X´h 4`Ñ¢EÄÆÆ’˜˜È’%KhÑ¢E¥‘â¼ð ìÞ½›'Ÿ|’ÔÔTöìÙèQ£¨_¿>£G.×ñ®¹æbbbX¹r¥SÇ‚Þ½{³fÍn¿ýöÓ~×ÿ?þøƒ””j×®í4‘ýR^|ñE6mÚÄœ9sHII!%%…'žx‚‚‚‚RwR¸ÔXuëÖ¥fÍš,_¾œ]»va±Xøâ‹/øôÓOÏkúôélß¾Y³f‘––Æo¿ýÆܹsË5Q­Zµ"++Ë©C@iòïرƒýë_dff’žžÎ믿ŽÙl¦mÛ¶NÛ3†E‹1xðàËÎú /°k×.žzê)RSS9tèC‡åïÿ;ÁÁÁ€­ ƒÉdbÎœ9—}‘ê®fÍšLš4‰°|ùröïßOLL £GæäÉ“FÇ‘*N DÊXß¾pø0üðƒÑIDDDDD.[!…¬d%÷qŸÑQD®Œ§§­‹ÁîÝ°j$'C»v¶Ntkת@\.‹'ž<Ê£¼Ê«¤“~éD¤Ü©È@¤tTd ""R‰íܹ«ÕJ³fÍ.{Œ„„ÚµkGll,÷ß?ûöí㩧ž¢fÍše˜T¤z9rä&“‰eË–‘““ƒÉdºà¿Ó=z°nÝ:Ö¯_ODD7Þx# 4`Ó¦Møûû0iÒ$7n ÀwÜÁàÁƒ‹]WÒñŠêÝ»7NQìã{õêU¢ómܸ1cÆŒaøðáÔ«W>}úðçŸâççÀˆ#ùä“r;Æ‚ 8räÏ=÷\¹`Ø°a8p€ 6”ëqDª£œœžxâ Þxã ¦NÊŒ3Jü»aq¿c–%“ÉÄ{ï½w^!Sy(ïs©îêׯϰaØ""àñÇaøpðñ1:Y…¡k׋ÏxÞå]r<ŒŽ#R­-_¾œ#F`±XŒŽ"R"WëÚÜþ^îûï;ÏŸU'‘J,11‘¨¨¨Ë*0øïÿK=èܹ3ß}÷ DD* çž{Ž#GŽðÖ[o•é¸³fÍâùçŸçðáÃ,X°€1cÆ”éøçÚ±c‰‰‰¬Ô]ZEÊ…¯¿þ:‹/fÎœ9L›6ÍèH"R©“H9¸÷^8rÄv‡Ì‹°Z­<öØc<ñÄ$%%ñè£Ëÿû_Ç6_}õ·Ýv­[·æÀlݺ•¼¼<Ú·oObbbyŸ‰ˆˆˆˆTCg8Ã*V1ܦgÏžÔ®]›7ß|Óiý¡C‡øꫯ1b¿þú+7Þx#>>>lݺ•¤¤$n¿ývºvíê(Ûä§ääd6oÞLrr2‹-bõêÕœ8q¢|NR E [ƒ}ûl] ž|ÒVl ÇŽ®Âеë…=Æc¤Â»¼kt‘jÏÕÕ•ÂÂB£cˆT*2©ÄN:E5J½_AA>ø ·Þz+ï¼óÞÞÞåNDD®T@@7n䫯¾"99¹LÇž0a×]wcÇŽ¥N:e:ö¹š5kÆ—_~IHHH¹G¤º>|8 .ä™gžqzóUD¤,äååa6›Ž!Rµ´hÁ%Šq“““4h7ß|3L˜0¨¨(–.]êØfêÔ©4mÚ”ùóçS«V-êׯϊ+°X,Ì;·œODDDDDª£Où”tÒéÇ…;Xº¹¹1tèPÞyç§;¿õÖ[x{{sß}÷¶¿W_sÍ5ÄÅÅM5øç?ÿI»ví˜9s&ùùùlÙ²…AƒQ¯^=¼¼¼hÕªï¾û.aaaå{²"uëÂüù“'ۮ墢`ÈسÇèt†Óµë…EÁ0—¹X±G¤ZsqqáìÙ³FÇ©4Td ""R‰]n‘Á† Ø»w//¿ü2®®®åLDDÊJPPï¼ó¡¡¡e6æÔ©S±Z­¤¥¥×¦VD*·‘#GÃâÅ‹Ž""UŒ:ˆ”“»ï†U«.º‰««+]»vuZ׸qc°X,üøãôèÑÃi›àà`Ú·oÏ7ß|S–‰EDDDDˆ'ž›¹™H"/ºÝˆ#ÈÌÌäÃ?àìÙ³,]º”àççÇ™3gØ°a=zôÀÍÍÍißN:±yófÜÝݹöÚk™={6ï½÷éééåsb"—'ÂÁƒ°x±­;]“&г'üðƒÑé £k׋Ç8þà>ã3££ˆTk*2)ˆˆˆTb§N"((¨ÔûíÝ»—š5kR¯^½rH%""Rö¾ûî;‚ƒƒÙµk—ÑQD*¼o¼‘?þøÃè"R…X­VNŸ>­"‘òг'8ù=788ø¼ÉV~~~dddžžÎÙ³g‹-L #55µl3‹ˆˆˆHµ—Có1xÉmëÔ©C·nÝxóÍ7øòË/ILLdĈ¤¥¥‘ŸŸÏóÏ?ÉdrZfΜɩS§c­ZµŠ† 2dÈ‚ƒƒ¹ñÆY±bEùœ¤È¥xzÚºìÞm+?yn¼:t€µkÁZ½îX¯k׋kNsºÑçyÞè("""%¦"‘J¬°°ð¼ õ’¨U«§NªòêR¶Ö¯_ÉdbÇŽ—=Fvv6f³™ï¿ÿ¾ “oÉ’%Ž?B/Z´¨Ôû—ÅùJ»ßtù IDATå±uëVÚ·o&“‰ÈÈ‹ßy©¼\é÷mY+ï¥ÿìÙ³X­V¬çüQ¾¢|íªºx€;wCJhïÞ½DDDCDªÓ§OcµZ1›ÍFG©zn¸ÂÂlP.Àd2]tˆÀÀ@\\\8yòäy¯%''|Å1EDDDDŠZÍjNsš»¹»DÛ5Šo¾ù†ýû÷³dÉZ´hÁõ×_@@@®®®Ì˜1Ãñ7à¢KÑ» _{íµ|üñǤ¥¥ñ駟ÁÀY·n]¹œ§H‰¸¸üÕÅàÛo!(z÷†˜ˆ‹ƒü|£^ºv½´ñŒgØ££ˆˆˆ”ˆŠ DDD*1///òòòJ½ßí·ßN@@3fÌ(‡T"¶nÝ:‚‚‚h×®]¹køðá—õïCª§AƒÁ‰'صkÞÞÞ†äÐ÷í…Ý|óÍœ:uŠ¦M›:­¯(_»ªî‘GáŽ;îà×_5:Š\† X¿~=ƒ 2:ŠˆT!ößOÔÉ@¤¸¸Àí·ÃLŠ2›Í\ýõçM¬:uêß}÷:uºÒ”"""""NV°‚nt#”óïH^œ=zP»vmæÎËš5k9r¤ã5³ÙÌ-·ÜÂêÕ«),,,ÑxÞÞÞtëÖ÷ßOOO¶lÑ„]© ì] ¶o‡-`Ø0hО{þwGÿêJ׮Йδ¢ó˜gt‘Q‘ˆˆH%v¹E>>>,X°€… 2wîÜrH&R¼ÿüç?Üu×]¸¸è×P©8, {öì¡K—.øúúÒ¸qcöîÝkt,)}í®žÖ­[3jÔ( Tâ7úäêûñÇéׯ÷ÜswÞy§ÑqD¤ Q‘H9ëѾÿRR.{ˆ§Ÿ~šßÿÇœ'NpàÀˆ»»;&L(ð""""RÝ¥‘Æ|Á”x777†ÊâÅ‹qss;ï/¾ø"{÷îeРAìÞ½›¼¼<öìÙüyóøÇ?þ@bb"½zõbýúõ¤¦¦’••ÅâÅ‹9sæ ·Þzk™ž£È»î:[ƒ}ûl] f΄k®ØX8zÔèt†ùöî;>Š:ÿãøkÓ{!BB ˆ†^E)¤Yz1 x èÅŸV,§ H,`;°ÐÎ ŠX@Ž£¨'HQ¥…–ÞËüþÈíšM#$ÉûÉcìÎÌμ§í$›ïg¾úÝàÖ²–xâÍŽ"""rVjÝ%""róðð ;;ûœÞ{óÍ7³`Á~øan¹å–sžÔ­7Ò«W/ÜÝÝ eôèѶ»±,Y²‹Å‚ÅbaÉ’%Üu×]4iÒ‹ÅÂM7ýùÅî† ¸üòËqss£yóæÜy礧§ÛÆ'%%ñàƒÒ¦MÜÜÜèÚµ+~ø¡]Ž3f0tèP:uê„Åb!""¢ÚËÈËËã“O>á†nà¾ûî³eß»w/ÿüç?mÃÞyçÛr­ÃÞ}÷]ÆŒƒ··7Mš4áÞ{ï%//Ïn/½ôáááxxx0pà@~ýõ×rÛ´¾Ö·´ÚÚWPõ1QÑ4Üzë­?~€ÌÌL[‹ÅbëUâ?þ°^LU­×ðáÃk´œÚ\DzÙ^}õUî½÷^|}} ᩧž²›ÎÚXnÊ”)X,®½öÚJ÷euŽŸê.Ûª:ÇmYµuLÅÇÇ3räHñööfÔ¨Qlß¾¨ú<¨év¨(ãÙγÊæõúë¯Û†UwßÕtŸÔÖ¹x>Û¿:ãÏw?Ôô<:u*à믿®p;‹¹^ýuú÷ïÏ•W^ÉòåËÍŽ#" Lnn.PrÇ9©×^ NNðÙgç<‹¡C‡ò駟²cÇ"""èÚµ+...|÷Ýw„‡‡×bXiìÞç},XÁˆ½oÒ¤IŒ3___»q]ºtaçÎ 0€&Mš0jÔ(Ž=j+2 cÊ”)Ì›7ÈÈHZ¶lÉòåËùç?ÿ©"¹pEDÀ¢Epì<õ¬] ­[Cl,©­s±6¶ÿÙÆ×Æ~¨éyÚ¡Cã¯ýëY··ÔŸýû÷Æ 3,‹1}út£¨¨¨Fï¯ègÌÚ«W¯®ÓeXÕõºˆ4f{÷î5€rשEC†Ƹqf§‘n×®]Æ®]»ª5­~Ç‘Ê 2£Ñ5~ßæÍ› Àø÷¿ÿ]©D.¹¹†±|¹aDF†ƒƒa n:'ùÆ|ÃÃð0’Œ$³£ˆ4:õѾK¤6Õ×ïæ111FLLùö³:[DD Èù3«Èàõ×_7¼¼¼Î»0 11ј8q¢áàà`ôîÝÛøôÓOUlpˆŒŒ4:uêd7,##ÃhÒ¤‰ÝkÀ¸ãŽ;*œÇe—]ftìØÑnØúõë ÀøòË/+]vtt´1eÊ»a•5®î2&NœhŒ?¾ÊùUUdðàƒÚ-ãoû›áììl=zÔ–£lÑÍ¿þõ¯j5Ö®‹õ-«6öUuŽ‰ÈÈÈróùᇠÀ˜={¶mØÌ™3 www#55Õ6lîܹÆSO=U£Lg[¯gŸ}¶Âå,X° Âéks­Ù&NœhVXXhxxxO?ýt¹éªÓP½"?Õ]öù·ç{Låää”kü[PP`4kÖÌöº&EUm‡Ê2šQdP}Rçâùnÿê쟊œË~¨Éy:|øp£GUfúñË/¿“'O6ž={[·n=§ù¨È@DªcçÎ`ÄÇÇ›E¤ázî9ÃhÞÜ0ô}ŒˆˆÔ!ˆÈù:n7 Gã}ãý½/))ÉèÕ«—Ñ¿ÿ:J&r‘)*2Œ>2Œ+¯4 0Œ¾}K^ëwÂF!ÓÈ4Œãã³£ˆ4:*2‹ÙEçÕ ‚ˆˆˆ˜ªM›6dffròäÉóšO‹-xã7صkÞÞÞ 6ŒöíÛóꫯ’••UKi¥&Ž=ÊÁƒéß¿¿Ýp///’““ËMß¹sç çñË/¿pÕUWÙ ¿òÊ+øòË/+]~“&Møå—_ª•³:Ë(**â£>âÆo<ë<+sùå—Û½îׯ|ÿý÷œ9s†_~ù…¨¨¨ sœMm¯oUÎu_U瘰N3`À»iºv튛6m² ›0a999¬ZµÊ6lùòåÜvÛmç´®­ÀwÜAqq±ÝrV¯^Íí·ß^áv¨Íu´êÔ©“í¹££#œ:uªÂ¼ç¢ªã§ªeŸïqku®Ç”››={ödæÌ™¬]»–œœœœœÎùšRÕv¨ìø0CUû¤6ÏÅóÝþçºÎe?Ôä<õóóãĉUfºc_|ñǧmÛ¶lÚ´‰7ß|“íÛ·Ó·o_³ã‰H–›› ”\ŸD¤Ž  'O¾}f'©ÔjVã‰'×q]µß3dÈZ´hÅbá­·ÞªÃt"ˆŽ†íÛaËð÷‡‘#¡KX± ÌN(uÈOþÊ_y‘É%×ì8"""•R‘ˆˆÈEìÒK/ààÁƒµ2¿nݺ±qãF8ÀÕW_Í<@pp0±±±¬_¿žÂÂÂZYŽœ]RRPÒX´:ÜÝÝ+Ç’%K°X,¶G³fÍ8|ø0ûöíãÆo$((, Ë—/'%%¥Ú9϶Œ-[¶Í5×\S­õ©ˆÝ뀀9~ü8P~{U´ýêc}«r®ûª:ÇDUÓØÆCÉçGTTo¾ù&Û·o§E‹„……ÓºV´^M›6eôèÑvËéÒ¥ ~~~5Ê.ëhåååe÷ÚÙÙ™âââJ—Q•š?U-»&ÇmUÎçü߸q#£Fâ¾ûîÃÏÏ믿žÿüç?g]fM·CeLJªÚ'µ}.žïö?ÛøÚÚ59O9sæL¥ë.uãСCÌ™3‡ÈÈH®¾újRSSYµj$66}½#"u+''¸°®é" N÷îФ TP8-""""r¡XÉJnàÜ©þ6m"??Ÿm۶Ѻuë:L'r‘ŠŠ‚õëáÇ¡kW¸ã‡Y³ -ÍìtRGîã>ÒIçÞ1;ŠˆˆH¥ôWh‘‹Xpp0!!!lÛ¶­Vç{Ùe—±hÑ":Ä“O>Éþýû1büßÿý[¶l¡¨¨¨V—)öΫ!§u=ô†a”{¼ýöÛ0dÈŽ9Â×_MAA†apÛm·aF­,àƒ>àšk®ÁÃÃÃö^kƒÈ‚RwâHOO¯tYeÍZïL‹-€òÛ+­Ìoõµ¾5UùV瘨jšäädÛx« &°cÇöíÛÇ›o¾Éĉëd]ï¾ûn»åL:µÆùÏukÓù?eU÷¸=ÕÝþþþ¼ð ;vŒo¾ù†ÜÜ\ú÷ïO|||¥ó®ííp!©ísñ|·Uãk{?T÷<-((¨q!Œœ›ß~ûùóçså•WÁ‹/¾ÈðáÃùïÿËÖ­[‰‰‰ÁÉÉÉì˜"ÒH¨È@¤8:€PòDDDDDÌ@;ØÁÍÜlv‘†©sç’^ ~ýbcaáB ƒ¸8HL4;Ô²æ4gã˜Ç<Š9·›£‰ˆˆÔ5ˆˆˆ\ä À7ß|S'ó ä¾ûîcçÎ8p€‰'²~ýzú÷ïOóæÍ?~<«W¯&55µN–ߘµlÙ’ÈÈÈrû611777[û³Í£mÛ¶ìܹ³Ü¸.]º°zõj8~ü8cÇŽ¥mÛ¶8::——Wî=Ý%¹:ËX·n7ÜpƒÝø¦M›pâÄ Û°Ÿ~ú©ÒõÙ±c‡Ýë-[¶àììL=hÒ¤ —]v[·nµ›æûï¿·{]_ë[SÕ™ouŽ ë4_ýµÝ4?þø#ééé 2Änø˜1cðôôdÉ’%¶;¦×źöîÝ›nݺ±dÉ:D÷îÝ+œ®.Ö±6Õäø©Žê·ç¢:ûïĉtêÔÉ6¼W¯^¼öÚkäçç³k×. âó 6·Ã…v÷õÚ<ÏwûŸm|mÕ=OSSS :§eHÕ ùæ›ox衇h×®—^z)Ï<ó íÛ·ç‹/¾àðáÃÌŸ?ŸÎ;›UD¡ÜÜ\, ®®®fGi؆¯¿†RÅø"""""ŠU¬"@3Øì(" [x8Ìž ‡ÃSOÁ?ÿ ­[—8`v:©Eò ¿ò+Ø`v‘ ]X-:DDD¤Æ Àwß}gw7øºÉSO=ÅÁƒ‰ç±ÇãäÉ“ÄÆÆÈå—_ÎŒ3X¿~}•w£—ê›7oûöíã±Ç#99™C‡1qâDn»í6ª5^xo¿ý–Ù³g“””DRRÓ¦M£°°‘#GAÓ¦Myûí·Ù·o¹¹¹|þùç|úé§åæe½ëúHJJ¢E‹ìÚµë¬ËصkÇ'::Ún~—]vÍš5ãå—_æÌ™3>%½$$À²e°ctèÑÑ°}»Ù餴¥-×qs™kv‘Š""bÄüïŸÈ¹Z½zµQ×—Õ¥K—V8ü—_~1ãÛo¿­ÓåWæÌ™3ƪU«Œ)S¦—^z©®®®Æ€Œ'žxÂøôÓO””S²5Ÿ~ú©qÅW®®®Fpp°1mÚ4#''Ç0 ÃX¹r¥Ø=*ÚÖ7n4zõêe¸ººÍ›77Æg9rÄ6~ÇŽF¿~ý ///#44Ô˜©­sñ|·ÿÙÆ×æ~0ŒŠÏÓÒRRR GGGcÓ¦MŽ—ªåçç;vì0^|ñE#&&ÆhÚ´©Mš41n¼ñFcñâÅƯ¿þZï¹*û³¶ÆêÕ«ëtVu½."Ù+¯¼bøûû›C¤á+.6Œ¦M ã…ÌN""" Ô®]»Œ]»vUkZýŽ%"¥í3ößæü=RD Ã(*2Œ>2Œ^½  £oß’×ÅÅf'“óðµñµ±ÍØfv‘F¡>Úw‰Ô¦úúÝ<&&ƈ‰)ß~Öb†q.Å "" ÉÆ°†5&'‘‹Õš5k;v,uyY]¶l“'O®p\çÎéׯ/½ôR-¿ºŽ=Êæ͛ټy3[·n%>>Ú¶mKïÞ½éÝ»7½zõ¢mÛ¶º#o#Ñ®];âââ¸óÎ;Ïéý ]»v|úé§\{íµµœNêËâÅ‹9zô(sæÌ1;ŠˆTâlçésÏ=Ç»ï¾Ëÿû_]ë!11‘;v°mÛ6¶oßή]»ÈÉÉÁßߟ>}ú0pà@ D—.]pp0¯£Éª~Ƭ ‹…Õ«W3f̘:[†U]¯‹Hc¶páBæÎ˱cÇÌŽ"Òð®®°v­ÙIDD¤Ú½{7=zô8ë´úKDJ{”GYÁ þà0ï»,ùŸ­[aÎøøcèØþïÿàæ›ÁÙÙìdr®à  g-ú.@¤®ÕGû.‘ÚT_¿›[ÿ–»f}ûYõa&""ÒŒ7Ž¹sç²`Á\\\LÍÒ²eKbcc‰ --;w²uëVvïÞÍ<@ZZ^^^DFFÒ¾}{zôèA=¸âŠ+LÏ/µoÿþýfG“<óÌ3¸ººrÓM7±xñb¾øâ ³#‰HÕ=OwïÞÍ+¯¼ÂG}¤ƒ $&&²{÷n»ÇñãÇhݺ5}ûöå/ù QQQtëÖÍÔ¢‘s‘““ƒ›››Ù1D‡>}`ñb³SˆˆˆˆˆØYÃÆ2V"Š¨¨’ÇO?Á¼ypÇ0}:L™÷ß¾¾f'”˜Æ4Æ3žhMk³ãˆˆˆØ觑à–[n!55•Ï>ûÌì(åøúú2dÈfÍšÅúõëIJJâûï¿gþüùôìÙ“0cÆ úõ뇿¿?½{÷æ¯ý+ ,àóÏ?çèÑ£f¯‚˜hÆŒ´k׀뮻ŽñãÇ›œHjꡇ¢sçÎÜsÏ=„‡‡›GD*PóôÅ_ä“O>¡k×®õœîÂ’žžÎöíÛyã7xà¸æškhÞ¼9!!!Œ9’÷Þ{///î¿ÿ~¾üòKÒÒÒˆgÅŠÄÅÅÑ£GˆÈE)77www³cˆ4}ú@b"üñ‡ÙIDDDDDØÁ~åWnæf³£ˆHY;Êðë¯pÛm°p!„…A\\Éï–rQÍhBa KÌŽ"""bG=ˆˆˆ4¡¡¡ôïߟ7ß|“#F˜§JNNNtëÖnݺن°oß>vïÞÍ?üÀ¾}ûøè£8uêPR¨Ð¾}{:tè`÷Ë–-ÍZ ©'³gÏföìÙfÇsôè£ò裚CDªPÝótùòåõæ‘––ÆþýûÙ»w¯ÝÿGŽÀÃÃvíÚÑ¡C¦OŸN÷îÝéÞ½;>>>&'©999*2©/={‚‹ lÛf§a%+iCºÓÝì("R™ðp˜=~Þz ž^}ÆŽ-Ö¶­Ù ¥ N8q7wó ÏðOà‹z¢‘ ƒŠ DDDˆ¿ýíoŒ=š}ûöѾ}{³ãÔˆ³³3]ºt¡K—.vÓ““Ë5î;[ñA»víhÙ²%‹ÅŒU‘‹Hjj*¨´˜ÀÓÓ“¶mÛÒ¡C dû™#""B=ˆH£¢"‘zäî]»Âöí0nœÙiDDDD¤‘+¦˜µ¬åî0;ŠˆT‡OI/wÞ «WÃsÏA‡0lXI±AïÞf'”JLf2Oó4oò&÷s¿ÙqDDDˆˆˆ4£F¢mÛ¶,X°€×^{Íì8µ" €0`À»á©©©ÄÇÇóóÏ?³oß>~þùg6oÞLBB...´lÙ’Ö­[—{DFFâååeÆꈈˆˆ RRRHHH¨ô%?;´iÓ†:0qâD[1A»víTL "äææâææfv ‘Æ£Gøþ{³Sˆˆˆˆˆð ßpŒcŒaŒÙQD¤&\]!6Ƈ?†gŸ…>} o_˜>†Ý´ï‚⇘À"ñ7þ†“šuŠˆÈ@W#‘ÂÁÁiÓ¦qÏ=÷ðä“Olv¤:ãççG=èÑ£‡ÝðäädöïßO||<ñññ$$$ðÓO?±nÝ:[ï‹….¹äZ·n]îÿÀÀ@3VIDDDÎQNN ¶kéÿÿøãòòò€’^ ¬×üÎ;sà 7Ø #""Ô ’ˆHÔ“H=ëÚÞ~Š‹A""""b¢U¬¢ÝhÏÅÕ‹ºˆüƒDG—<¶n…9s`äHèØî¹n»­¤ A.÷s?/ó2ëXÇhF›GDDDE""" É­·ÞÊã?Îœ9sX´h‘Ùqê]@@QQQDEE•—‘‘a×ðÐú|Ë–-:tˆ‚‚|}}m CCC £eË–„„„FPPŽŽŽõ½j"""Ö™3g8zô(‡æèÑ£;vŒÃ‡óûï¿Obb¢mÚ   [áàW\aWHdâZˆˆ\ÜTd RϺuƒÌLˆ‡K/5;ˆˆˆˆ4Rð>ïó™EDjCTTÉcϘ;·¤Èà‰'`ʸï>ðó3;a£×ŠV g8 X "¹ ¨È@DD¤quuåÉ'ŸdêÔ©Lž<™:˜é‚áííM—.]èÒ¥K¹q………9r¤Ü·mÛÆêÕ«9qâÅÅÅ899Ñ¢E‹rÅe t§A‘³JII©°€ ôóììlÛôþþþ„„„n×µÀÃÃÃĵi¸rssñöö6;†HãÑ©8;Ã?¨È@DDDDLó9Ÿs†3ŒaŒÙQD¤6uê+VÀóÏë¯ÂÂ…°`Ü~;<ô„„˜°Q»Ÿû¹Š«ØÎvzÑËì8""ÒÈ©È@DD¤™4iK—.eúôélØ°Áì8'''ZµjE«V­*_PPÀñãÇ9räGŽáèÑ£¶ç[·nåÈ‘#œ8qÃ0ló+]ˆÐ²eKBCC !((ˆ¦M›‚——W}®¦ˆˆH½ÉËËãôéÓ$&&rêÔ)9vì‡âرc¶ç¥ |}}iÙ²%ááá´nÝšþýûFHHˆm¸ŠDDÌ¡ž Dê™›DF–ŒQƒ.1ÇJVÒ‡>Dav© AA0kL›o½õgÑÁر0s&´kgvÂFièIO²U¬2;Žˆˆ4r*2i`˜7oä³Ï>ãÚk¯5;ÒEÏÙÙ™°°0˜æÈø IDATÂÂÂ*¦  €Ó§OsüøqHLL´=ß¾};|ð‡¢¨¨Èö777üýý ¦E‹•> ÁOÝSŠˆÈ %%ÅvKLL$%%¥Âç'Ož´õ%×¼àà`Z·nM‹-èÑ£‡íypp0—\r‰®u""0ˆ˜ kWøñG³SˆˆˆˆH#•M6ò!³™mv©k>>wÞ «WÃsÏAÇŽ0l<ü0ôîmvÂFç^îe8Ìa¨¼‚ˆˆH]S‘ˆˆHtÕUWqà 7pß}÷ñÃ?¨1H=pvv&88˜àà`zôèQá4………œ:uŠ“'OrüøqN:ʼn'8qâ§OŸæðáÃìÞ½›“'O’œœl÷^OOO‚ƒƒi֬͛7'88˜¦M›DPPØ‹¥>V[DD.r$''“””DRRR•×©S§NÙ½×ÃÃÃvjÖ¬¡¡¡ôèуæ͛ӢE š5kf¯DD.n¹¹¹¸¹¹™C¤qéЖ.5;…ˆˆˆˆ4RØ@9Œf´ÙQD¤¾¸ºBl,Œ\RlЧôí Ó§Ãðá ¿A׋±Œe&3YÌbæ2×ì8""Òˆ©È@DD¤Z²d ;vä‘GaþüùfÇÀÉÉÉVˆÐ­[·*§ÍÏÏ'))©Ò;D8p€Í›7“’’‰'0 ÃîýÖ^ÊöŽPöaˆ‹‹K]®¾ˆˆÔ±œœRRRÊ];*zìÙ³‡œœ ìæáêêJ“&Ml׈àà`.¿üòJ{Ü‘ÆA=ˆ˜ 2†ìlPÁ¦ˆˆˆˆÔ³•¬d0ƒiNs³£ˆH}sp€èè’ÇÖ­0gŒ mÚÀÝwÔ) ›QÔ)gœ™ÊTf3›Çx |ÌŽ$""”Š DDD¨àà`æϟϤI“9r$ 0;’Ô€‹‹‹­qg‡ªœ677—¤¤$’““íîFm}mvðàA’““9}ú4éééåæãããC```¹^¬???|}}ñóó³{îååUW›AD¤ÑÉËË#--ÔÔTRSSmÏSRRHMM-÷ù^úQTTd7/www»Ïñ¦M›F‡8sæ ¿üò nnnôë×Ñ£Gsã7hÒš‹ˆÈ…,''G=ˆÔ·ÈH(.†ß~ƒÎÍN#""""H:é|Æg¼Â+fG³EE•<öì%KJz4˜=»¤Ðà¾ûÀÏÏì„ ÖÜÉßù;ÿàÜ˽fÇ‘FJE""" Øí·ßÎG}Ä„ ø駟Ô¼rss£eË–´lÙ²Úï)((¨°jÙ…„„’““9sæ ©©©åîx àèèh+:ð÷÷/W„PQaBéÿýýýkssˆˆ˜*++Ë®8 ôóÒ•ÏÉÉ©p¾þþþøúúÒ´iS[Ñ@xx¸]AÙB1*îx;{ölRRRX¿~=k×®åî»ïfúôéDGGõ×^‹³³s]m&¹Èäææª'‘úÖ¦ 89Á*2‘zõ>ïc`0ŠQfG‘ E§N°t)<ù$¼ú*,Z Àí·ÃCAHˆÙ ü‰%–…,änîÆG³#‰ˆH#¤"‘^¢cÇŽÜwß}¼þúëfÇ‘ „³³3AAAÕè}YYYåË–þßz§íÔÔTÙ·oŸÝøììì ç[¶gÛsOOO<==ñóó³=÷ööÆ××<==ñõõÅÛÛOOÏ*ÕŠˆ”UPP@ff&©©©dgg“••Ezz:ééédee‘Mjj*™™™dee‘••EJJŠmÚŒŒ »‚ÂÂÂrËprr²ûœ³ ´lÙ’Ž;VX€U¶H«.øûûKll,GeÍš5¬\¹’#FмysÆŒÃM7ÝDïÞ½±X,u’ADD.999*2©o..šDDDDD™U¬bÃðCw(‘2‚‚`Ö,˜6 Þz æÎ-):;f΄víÌNØ ÜÏý,e)ëY¯Â/1…Š DDD¸-Z°|ùrFŽIïÞ½™4i’Ù‘ä"fmä|NïÏÏϯ´0Á:ÌÚˆ755•'Nؽ¶>ÏÈȨr9þþþ¶¬^^^øúúÚ^ûøøàãデ‡øøøàè舿¿?NNNx{{ãêêjïêêj7ˆÔ¿ÌÌL HKK£¨¨ÈÖ³Jff&¹¹¹äää••E~~>iiiäççÛÿ[ ÒÒÒÈÈÈ°½NII±½§2®®®¶"§ÒEOþþþ4mÚ”ˆˆ¼¼¼ÎZ àééY[ëÜ´lÙ’iÓ¦1mÚ4:ĺuëX±b‹/æÒK/eܸqLš4‰ÐÐP³£Šˆˆ rrrpss3;†HãÓ¶­Š DDDD¤^æ4›ÙÌ»¼kv¹ùø@\Üu¬ZÏ=;°a%Å}ú˜°A¸”KÆ0°@E""b ˆˆˆ4ÇgÆŒÜsÏ=tíÚ•=z˜I)š5kF³fÍÎ{^ÖÆÃÖ»Ž[[ ÊÞuÜú<))ÉöÞÜÜ\RSS)**"--­ZËuttÄÇǧ\!‚··7NNNøûûÛ¦qqq±õ®PzOOO\\\pppÀ××www[Ã-k1ƒu¾¾¾888œ÷v9WéééQXXh+ô±6þ/..¶CÙÙÙäååa©©©@ÉùZXXHZZš­8 ''‡ÜÜ\»‚²ÓX ªÃzyyyáêêj×ɧ§'­[·>kï(e‹‘g׳áááÄÅÅÇ?þÈo¼Á‹/¾È³Ï>ËðáÙ4i×^{m£Ý>""õú¯ž DLк5ìÚev iDÖ°W\¹žëÍŽ""ˆ…ñãáãKŠ úö-yLŸǃzJ>/÷s?ƒÌvpW˜GDDˆˆˆ4O?ý4»wïæ/ù »wï& ÀìH"çÅ×××Ö@¿¶äçç“••U®ñséB„ê4ÎÍÍ%))ÉÖ@ºô4ÅÅÅdddPXXxNñòò°-X‹[ƒÅbÁϯ¤+ãÒ ¥‡[Yçcåææf׈¬t1„UÙØ¥—Øz…(«²áU±k˜Íz|ÔDeäË/ÝHßÊÚ¨ßÊzÌY•nìo•ššŠa¶×¥ïÔ_TTDzz:Pq!@éLÖùœË:—f-Ž)ÝHÙBfÍšÕ¸PÇÏÏ'''»i¤ntíÚ•Å‹3oÞ<>úè#–-[Ftt4AAAÄÆÆ2yòdZ·nmvL©CÖŸTd b‚ÐPø׿ÌN!""""ÈJV2ŠQx¢ï\E¤ :ºä±u+Ì™#GB›6p÷Ý0e ¨—Ìs2ˆAt¥+/ò"ïðŽÙqDD¤‘Q‘ˆˆH#áààÀÛo¿M÷îݹùæ›Ù°a...fǹ ¸¸¸àââbëI ®•n$nm ^º±yéFå)))€}CpkÑBé†àÖ†ÒóNJJ²54¯iÃt8·Æõf+[(QVEÛáBW¶1}E#^^^8;;Û^—î þì%£I“&¶íc-(=kcþÒE!Öy—.:±6þ/=ï³m{¹8¹ººCLL ¿ýöo¼ñË—/gîܹ <˜I“&1jÔ(Ûñ "" ‡õgO7ý!X¤þ……ÁñãPP¥~Ω G8Â6¶1“™fG‘‹YTTÉcï^X¼fÌ€Ù³K î»ÊümKÎ.Ž8&3™çxŽPBÍŽ#""ˆŠ DDD‘¦M›òá‡2pà@n»í6Þ}÷]ÌŽ%ÒhYïÚÔ[aCm±=XY{k°ÊËË#;;»ÜûÎ¥‡²wô¯Ž²…)Û ¿:ÊöàP~~~XÊt[Qe§³öJ!r¡iÓ¦ Ï=÷Ï<ó _}õË–-ãÖ[oÅßߟ &ð·¿ý³cŠˆH-QO"& …¢¢’Bƒ°0³ÓˆˆˆˆH·’•øáÇP†šED‚ŽaéRxòIxõUX´æχ ࡇ@G¨¶qŒãaæe^æ9ž3;Žˆˆ4"*2idºwïÎ|À°aØ:u*/¿ü² D¤ÆÊE\lE"rþ2dC† áرc¼üòË,[¶Œ… rË-·GçÎÍŽ)""çÉÚ³–Š DL`-,8|XE""""RçV²’bpA7¿‘Z³fÁ´iðÖ[0wnIÑÁر0s&´kgv ž .ÜÅ],`ð^x™IDD µ(i„ ĪU«øÇ?þÁøñãÉÏÏ7;’ˆˆˆ\ÄBBBøûßÿÎÑ£GY¶l;vì K—.DEE±víÚ÷F"""kOnnn&'i„‚‚ÀÙŽ1;‰ˆˆˆˆ4p8ÀüÈÍÜlvi¨|| .àµ×`×®’Þ¢£á»ïÌNwÁ»›»É#¬0;Šˆˆ4"*2i¤FÅ'Ÿ|† 1biiifG‘‹œ««+±±±ìÙ³‡-[¶àïïÏرc‰ŒŒdÑ¢Edee™QDDjÈZd ž DLàèÍ›Ãñãf'‘n%+iA úÑÏì("Òй¸@l,ìÝ ëÖÁ™3UòX»tÓ¢ 5¡ ·p YH1ÅfÇ‘FBE"""Ø AƒøꫯسgW\q{÷î5;’ˆˆˆ4QQQ¬_¿ž½{÷2xð`fÎœIhh(<òÉÉÉfÇ‘jÊÍÍTd bš€ÐÏN""""RÇV±Š›¸ GÍŽ""…ƒÃŸ½lÙþþ0v,´m ‹Áÿ¾“’?=ÀÄÏÇ|lvi$Td ""ÒÈõèуݻwÓ¼yszõêÅ»ï¾kv$i@Ú·oÏÒ¥K9|ø0Ó¦McÙ²e´jÕŠ‡~˜¤¤$³ã‰ˆÈYX{2pss39‰H#¨"©Sßó=¿ð 7q“ÙQD¤±ŠŠ‚õëáàA6 fÌ€ˆ˜5 RRÌNwÁˆ$’«¹š,0;Šˆˆ4*2‚‚‚øòË/ùë_ÿÊ­·ÞÊرcÕèODDDjU`` >ú(üñO?ý4o½õÄÅÅqâÄ ³ã‰ˆH%¬EêÉ@Ä$ ïhDDDD¤­d%­iMOzšED»K/-éÅà÷ßáÎ;Kž‡‡C\=jvº ÂýÜÏW|Åü`viTd """8;;³`Á6nÜÈ¿ÿýo:uêĺuëÌŽ%""" Œ§§'qqqüöÛoüýïgÍš5´iÓFÅ""¨ÜÜ\œœœprr2;ŠH㤞 DDDD¤¬e-7s3,fÇ)TÒ‹ÁáÃðôÓðþûpÉ% ûö™ÎTWs5éÌBšEDDˆˆˆˆ¡C‡òÓO?qÍ5×pà 70|øpâããÍŽ%""" LÙbƒµkתØ@D䔓“£^ D̨ž DDDD¤VüÄOæ°Ý°­l凸‰›LJ%"Roï’^ àµ×`×.èØ¢£á»ïÌNgš{¹—•¬äÇlòÈb)KÙÄ&“‰ˆHC£")ÇÏÏüã|õÕWüñÇtìØ‘ÇœÌÌL³£‰ˆˆHc-6øõ×_yâ‰'Xµj—^z)O>ù$ÙÙÙfÇiTöîÝËàÁƒ¹æšk3f ·Ür o½õ………̘1ƒÇ{Œ9sæ0þ|>üðC³ãŠ4þþšjv iÖ³ž"¸’+y‰—8Å)V²’Nt¢#ÍŽ'"R9—’^ öî…?„3g * .¿V¬€¢"³Ö«ñŒ'€^åUþàäAZЂ;¹“=ì1;žˆˆ4 *2‘J]uÕUüðÃ<óÌ3,Z´ˆ6mÚðÒK/QPP`v4i`<==yðÁIHHà‘GaþüùDFFòöÛoc†ÙñDD…ÈÈHvìØÁçŸÎÚµkYµj;vì °°… 2wî\üqxà¾ÿþ{³ãŠ4îî ÂK©,8âÈNvG-hÁ{¼Ç¥\J**l‘‹€ƒÃŸ½lÙ­[Ä ж-,Z¹¹f'¬®¸r ×ð*¯r —°ˆEd .œáŒÙñDD¤Q‘ˆˆˆTÉÙÙ™x€øøxÆÇ<@»víxûí·),,4;žˆˆˆ40žžžÌ˜1ƒ„„n¼ñF&L˜@Ïž=Ù²e‹ÙÑDD O? ÿú\r Äƾ}5›WvvD<@>ù@I108ÍéúŒ%"" œŠ DDD䜴jÕŠW^y…„„nºé&ž}öYBCC¹ë®»8pà€ÙñDDD¤2d?üðÏ?ÿ<ï¼ó‘‘‘,_¾Cwõ©U^^^ <GÇŠ”x{{3räÈzN%Òˆ¹¹•ü6p‘†g+hK[³cˆˆœ;oï’^ âãáµ×`÷nèØ¢£aÓ¦³¿ûvèß’/¬®à 6°G«,S‘ˆˆÔ&ˆˆˆÈy fÞ¼y9r„gŸ}–M›6Ѿ}{ ÄÛo¿M¶þ."""µÄÙÙ™¸¸8~ýõWbbb˜4iƒæàÁƒfGiPn¼ñÆ ‡;;;3aÂ\]]ë9‘H#æî^òn®¹9DDDD¤AsÀÇxŒ¿ð³£ˆˆÔ—’^ öì?„”:.¿V¬€¢¢Šß÷Üs%… QQpêTýf>‹A b kª,28ÙzL$"" Š DDD¤Vx{{sï½÷rðàA>úè#|}}™4iÁÁÁL™2…ÿûßfG‘¢I“&,^¼˜]»v‘™™I§N˜1cyyyfGiFŽYaO1Ü~ûíõHDDDDDDÎ[eRqf yœÇë9‘ˆH=pp(éÅ`ëVزZ·†‰!2-‚œœ?§=xÖ¯/y}ûBb¢9¹+1ŠQ¼ÄK•ŽO%µÓˆˆHC§"©U >œ>ø€“'OòüóϳsçNúôéCÛ¶m™5k‡6;¦ˆˆˆ4]»veÛ¶mÌž=›%K–Э[7þóŸÿ˜KDä¢×¬Y3zöì‰Åòg‹ÅB÷îÝéÒ¥‹‰ÉD¡ââ’ÿ-•ß¥PDDDD¤:**2p‰f4c5«qÄÑ„T""õ(* Ö¬àúëaÆ hÕ fÍ*éé`Þ#€S‰ˆÔ³6mJz1øí7ˆ… <–//).°*(€“'Kz4øí7óòVàq'Ž8Ê4ÿ40HCm0DD¤vè[i©={öäå—_&11‘+VPXXÈĉiÖ¬Æ ãõ×_çôéÓfÇ‘‹Txx87näÍ7ßäå—_¦{÷îìܹÓìX""­n¸ÂÂBÛkgggÆŽkb"‘FÊZd ž DDDDä>>&§i„Š‹KþW‘ˆˆˆˆœ' [OŽ8ò 2“‹ˆàà_mß‹Aiš ýûÞ=õ­*,¼Îë aÎ8Û†«È@DDj‹Š DDDÄ4~~~ÄÄÄ°bÅ Nœ8ÁªU« å±Ç#,,ŒÞ½{3gÎ~üñG Ã8û EDDD(éÕàóÏ?ç…^`Îœ9 4ˆÃ‡›KD䢃££#Lš4Éì8"“z2‘ZbÁB1Å8ãLQ<˳fG¹0,[99UOSXééз/\@½(;ãÌû¼OWºâ„ "©=*2‘ ‚§§'111¬\¹’S§Nñá‡Ò¶m[æÏŸO·nÝæ¶Ûnã½÷ÞãôéÓfÇ‘ œÅbáÞ{ïe÷îݤ¦¦Ò¹sgÞ{ï=³c‰ˆ\TFEQQ­Zµ¢oß¾fÇiœ¬ÜÝÍÍ!"""" B!…ÈZÖ∣ÙqDDÌWPóæAuzD.,„ìl<vì¨ûlÕä‰'Ÿñ­i @ )&'‘†ÂÉì""""e¹¹¹Mtt4?ÿü36l`Ó¦MLœ8‘¼¼<Ú·oOtt4C† ¡_¿~¸ººšœZ¤ö;vŒO>ù„O>ù„Í›7Ó®];®¿þz† F÷îݱèN–""gÕ¾}{þóŸÿ0}útÆÏ7ß|ÃâÅ‹qqq1;šˆˆé HLLäðáÃ:tˆÃ‡säÈŽ9ÂüÁ‘#G°X,dggMhh(¡¡¡„……NXXÁÁÁ8;;Ÿ}a"rn22ÀÁ<<ÌN""""rÑ((( 33€´´4Š‹‹ÉÍÍ%''Ã0HMM ++‹üü| ÉÈÈ ==¢¢"òóóÉÊʲÍ3##ƒÂÂBÛëÌÌL l¯­ó²ÊÎÎ&//Ïö:''‡ÜÜÜrY‹ŠŠHOO¯ÑúË{q™áAÖµYDþ7²Zïñññ©pœŽŽ)x{{ãäôg$///»ß===í¾swwÇÍÍÍöÚßßßn¸ƒƒ¾¾¾vórvvÆËË ___pssÃÝÝ‹Å‚ŸŸ_u6…ˆˆ½µkáĉ?_[,àì\òaaù⃢"ÈÊ‚áóÏKz6(ÅúyŸššJ^^YYYdff’——GZZšm|ZZyyyå®W¥?ãóòòÈÎÎì¯;))öEÖqÅ-ŠqøÌ;ßÁݯßm7MÙ÷”Uú3¶2e?Ë]\\ðôô,7Îú]ú:RöóÚúÚÇÇWWW¼½½móð÷÷ÇÕÕrש_*2‘ ^‡èСÓ§O'##ƒ¯¾úŠ7òÏþ“9sæàííÍÀ¹êª«èׯ]»vµû"SäbQTTÄŽ;øøãùä“OøñÇqwwgàÀ<ñÄ8p€¥K—òøãÄ°aÃ6lC‡­ô‹~)ùòzÑ¢EDEE1iÒ$þûßÿ²víZBCCÍŽ&"R§Îœ9Ñ#GlEÖçÖ"‚'NPô¿?”º¸¸BXXaaatëÖÐÐPÖ¬YC§NHNNfÿþýlܸ‘cÇŽÙÏ8::DDD„­!44ÔV„J“&MÌÜ "·ÌLðò*iÜ """r‘+..&--ôôtòòòÈÈÈ°kl™ŸŸ_nXAAééé¶aéééäçç“žžn+°”mø_]¥¨[8:99áíím›ÆÃÃÃî†WeÉà^ª÷)kãH«Ò 1Ë*Ûh³:júžÌÌL>m÷)Û=èss¸¹zï©l{–m¬šššŠa¶×ÖƲVÇ·ýþ öEÖã¢ô2KŠÔ”uÛX÷‘¯¯/ÎÎÎøøø؆ùøøàââ‚]CWggg|}}mü½½qqqÁ××www¼¼¼ìŽ iƃ#àØ12âãɉ'ïÐ!ŒÄD,'Nàtò$îII¸fdàním°¸²³Éíߟ)!!|^P@vvvµ ÀJ7ž·~}q–µðÊËË‹   Àþ:dmÄoUúsê»S~~múÙ-÷lײÅr)ûÙo½ߟ륋ùrrrHJJþ,¾³Q”¾Ö—¾^TÄZ¬àíí¯¯¯íáããc{îïï_á8???ì®É Žè IDAT""R}j}'"""oooFŒÁˆ#ˆgãÆlÚ´‰gŸ}–¤¤$¼½½éÓ§ýúõ£ÿþ\qÅêé@.XgÎœáË/¿dÓ¦M¬_¿žãÇÁÕW_ÍÌ™3¹îºëÊÝ5¢tï7ß|3ÅÅÅôêÕËÖ»G=LZ‘ [LL ]ºtaôèÑtëÖwß}—k®¹ÆìX""礠 €Ó§OsüøqHHH 11Ñö:>>Þö=(ùãd‹-¦C‡DGGÛ^·nÝšððð ï 6f̘ ï™’’Rn™ |÷Ýw?~œßÿÝÖÀÄÍÍͶœÒË´¾nÕª•þÐ'R™Œ P#&1Qaa!)))ddd’’Bff¦í‘––FFFdee‘žžNZZšÝ4¥ßc½se¬ û«j‚³³3~~~¶†šÖ†™Ö˜¥ïœl½Ã~é˜~~~X,Û<‹«¹šB`€ÙIjÆZDRúŽÞÖ†®e­ÂŸ ]­ fSSSÉÏÏ·ƒyyy9r¤Â–‚‚[ÁCUüüüðòò²=üüüðööÆËË OOO[Wëxooo»÷x{{ÛÄŠHÝHOOçĉ$''ÛIII$''súôiÛóÒÒ=ÖXyxx”4\'ÀLJpwwÂœiéèHsà Öb¡×С¸øùáããc»ùùùáââb;ï]\\ê­Ç•bŠqèïpö /ÖÂ2ëçvJJŠí3?##ƒüü|ÛÏiii¶‚Å´´4Nž|8C‡µ»›’ˆHcwÙe—±mÛ6&OžÌõ×_ÏìÙ³ù¿ÿû?³c‰ˆ”“““c×x¿lƒþC‡ÙõB`k¼?dÈn½õVÛë6mÚœsÃ…ÊþøéïïO=*-pÍËËãرc"ìÞ½›C‡‘••e7¿ŠŠ¬ÃZ´hQåÏÈ" –µ'‘ó““Cjj*)))¤¤¤Ø=/ûºì¸Êî&ïààPiCê¦M›ÒªU«r¯½¼¼lwvuuÅËËËÖ³ì]™¥ö…bv„sbFOÎ¥·ššJnn.™™™UÓ¤¥¥qìØ1[XëøŒŒŒ —c±Xð÷÷·=üüüªýÚÏϯ›ˆ4t§OŸæäÉ“;vŒ“'OÚ¾w²>Nœ8Abbb¹Â6[ƒrkCò.]ºØ50ÿöî<.ª²ÿÿkØ7•AÙQQPpË…A(54T43ÈåËûÊJm*+·4Ë-+è‹Ê¸Knà* Š²¸°(ÛõûÃßœ30ø~ö˜‡Í9gÎy_g×\×9×bll ###ðù|î÷J<»@c¼Z+±Í¤†Žõ{ª©©ÉåmÊP^^Îu8xüøqƒ%>DQQ._¾Üh'cccôìÙæææèÙ³'wRüêÙ³',,,hðBH§F !„Òiðx<®ñõüùóùùùHNN†H$Âßÿ¯¿þjjjprrâĸ¹¹aÈ!Ô´šŠŠ $''#>>{÷îEnn.Œ1vìXlÞ¼'NlöÈÝ»w‡¿¿?üýýñóÏ?ãÒ¥K\†M›6AGG#FŒ€ŸŸÞ|óMX[[+9u„Òñà÷߇››ÂÃÑžžŽ7*<-=!„4Wee% eÎBpëÖ-‰‘ëÎB îD Ï,ª¤­­ÍuEÖliii‰D¸sç7]º¶¶6,,,dΆ`kkËJJH§RZJ3B!D‚¸>!‰ùáÇ(,,”xÕ]VRR‚çÏŸ7ض¶¶ÔFÌ–––R5‹G^w u¤3ÓÖÖæþF”åñãÇëð“Ý`½x¶Àººví >Ÿôèуk$mll SSSîÿŪ»wï®´ôÒ***pçÎäææ"77999ÈÍÍå–Ý¿•••Üö:::À SSSXXXÀÄÄæææ\ÇúÝzuèééAOOfff }îñãÇÜ,u;²ˆÿ½rå PPPÀݳ^ΚdeeX[[ÃÚÚ666ÜËÜܼÝÝ»%„yQ'B!„tjæææ\l¸ÿ>’““qöìYœ?{öìAii)ôôô0xð` :C† ‡‡ìììT=éÈnß¾¤¤$ˆD"$&&¢´´ÎÎÎxûí·áçç‡áÇ+}T&uuu®ãÌòåËQPP€ƒ"!!Ÿþ9/^ {{{øùùA(bôèÑr¾A!Õ’%KЯ_?LŸ>7oÞÄž={`bb¢ê°!@IIIƒ†óÍB`iiÉ5œæÞ;88¨dÔƶ ïlÒÎezz:nÞ¼)1ªª¬ÙÄïmmmidTÒñJlØD!„ö©´´÷îÝÃǹóóóñèÑ#q ðôéS‰Ïª©©q ‰Å¯>}úp ‹ŒŒ¤Ž†N- i[†††Íp xÙøUZ§„ââb‰ŽE™™™\~Quuõù…¸ƒ‚¸a¶©©),--abbBƒ²¥«­­Enn.222™™‰¬¬,®#ANN=zÄmÛ¥KX[[ÃÎÎýû÷Ç„ `nnsssn$yev"DœO;884º]MM $fиwïîܹƒ›7o")) ÷îÝã:ÄhhhÀ‚ët`kk‹Þ½{ÃÑÑԌҮQ'B!„¼RÌÌÌ0uêTL:ÀË `zz:Ο?sçÎáèÑ£X³f ª««Ñ£G :ƒ ‚««+\]]aoo§âTö¨¦¦§OŸFBBD"RSS¡¯¯±cÇbÕªU˜0a,,Úv*bSSS!((ÏŸ?Ç©S§ ‰°ÿ~¬Y³FFFðòò‚@ €P(Tx4Bé |}}qêÔ)Lœ8Æ Cbb"U!¤«¬¬Ä½{÷dv"ÈÌÌ”hôÓÔ,Ôð]¶Öš AV''''´Uò‘σ@ÏžªŽ‚B!ÍTTT„H4BËÏÏǃŸŸÏŽ[VVÆ}FCC&&&033ƒ‰‰ ŒÑ«W/‰ÆÀõ B:?qãW[[[¹?#žù¤°°PêÌ'=Âõë×qâÄ Ü¿ÅÅÅŸ711‰‰ ,,,$F‹7ðקuuu•œZÒÑåççs įŒŒ deeáÅ‹^ÎÖÞ»woØØØ`̘1°¶¶†­­-7Œ„„ìرQQQ011„B!Ænݺ©:dBiU=zôÀ‘#G€qãÆ!..o¼ñ†ªÃ"„(YS€kp.îLJ Î;---¹fCvMˆ;æ*2u,!JWP˜šª: B!ä•TZZŠÌÌLܾ}wïÞåF¢½{÷.rssQPPÀm«§§X[[ÃÊÊ C‡… WN455¥!ힶ¶6×)AVg~¨®®FAA73˽{÷¸ü1==@~~>×HVSS–––°²²‚ —_Z[[s#ÙSçªö­¦¦ééé¸xñ"®]»†Ë—/ãÚµkÈÏÏðòÚ€àææ†9sæÀÉÉ ŽŽŽrwp!äU£¦¦[[[ØÚÚÂÛÛ[b]UUnß¾ŒŒ ¤§§ãêÕ«HJJÂêÕ«QYY 8::Jt>pww§¿7BH« !„Bˆôôô0tèP‰QkkkqëÖ-\¾|/^ÄåË—±víZÜ»w ¯¯¾}û¢_¿~pvvæ^Ô©c)++Ñ#G€„„äççÃÔÔãÇGXX|}};|#"===…B…B@ZZ ‰ðððà¶qvvVe¸„Òjôôô°wï^Ì›7“'OÆ–-[0sæLU‡EQ@IIIƒ™ê¾Wd'''¨8E¤-ðù|ðù|ôë×Oêú¦fCÈÌÌÄÓ§O%ö'«‚½½=Õ ‰ü>zöTu$„BH§õâÅ äåå!;;iiiHOOçÊ{uëu; 4o½õ–D9ÏÎÎŽ"„¼2444¸º#¾OSÿ^Mbb"²³³QRRàe'+++®-~9;;ÃÉɉ: ¨Àýû÷‘’’‚ÔÔT¤¦¦"99%%%ÐÔÔ„ƒƒúõë‡÷Þ{nnnèׯý¢Dšššptt„££#üüü¸åÕÕÕÈÍÍEZZRSS‘žžŽßÿ7nÜ@mm-ÌÌ̸_ÝÜÜ0jÔ(ª0%„΀Ja„B!ͤ¦¦ÆUîüýý¹åOžŸkì*Íãǹü833¸|ù2þüóOxY‡¶³³ãòd¸¸¸ ÿþ4 ·’ÔÖÖâÂ… 8~ü8Îœ9ƒ³gÏâîÝ»PSSCŸ>}àáá•+WÂÃÃ...ÔéƒÑÐÐàî#Š^¶OIIIÁ™3gpîÜ9lܸÐÐЀ‹‹ <==1lØ0Œ7–––*L!¤#¢_}B!„%ëÖ­† †aÆI,òä ®_¿Ž´´4\¿~×®]É'›› ÆÔÔÔ`mmÍÝ O#éàਫ««(E_EE’““!‰°wï^ܼyÝ»wǸqã°iÓ&…Bðù|U‡©ÆÆÆð÷÷‡¿¿?jjjpéÒ%®ƦM› ««‹áÇÃÏÏS¦L•••ªC&„ãñxˆŠŠ‚‘‘Þ{ï=TTT`Ñ¢Eª‹N¯9³ÔíD̽·µµ…¾¾¾ŠSD^%ò̆PXX(õÚ‰D¸uëžNŸ>””®3ANN€—÷ãÅ O¹ÿ744TqÔ„Bäaddxxx4XwçÎ.ß¿rå °jÕ*TUUAKK }ûöEÿþýÑ¿xxxÀÝÝý•Œ¢¬¬ ÿüó8€Ã‡#77=z4>ûì3xyyÁÅÅ…î·ÒIØØØÀÆÆÓ¦M 9èâáDZaÃðx<¸¹¹ÁËË 'N„‡‡å„¨Å!„BˆŠéêêbÀ€0`@ƒu%%%\烛7o"33'OžÄÖ­[QTTàåhËâQ\Å/;;;îÿ{öìÙÖI’éÎ;ðõõŇ~ˆÙ³gCOO¯ÍŽ]þ .p£ðùå—xë­·hz@ÙØØ 88ÁÁÁxþü9N:ÅÍÅÍ!0iÒ$˜ššª:dBQØG}øàƒ ­­÷ß_ÅÒ>55 ÁíÛ·Áèèèp ¡ÍÌÌàææ&ÑHÚÎήMˉ„´ºººÜß‚4UUUxôè͆ð*ÉÊzö^ñÆ0„Bð²,tñâEœ9s§OŸÆ¿ÿþ‹ÜÜ\¨««ÃÙÙ...xÿý÷¹Î666ª™BH+±µµ…­­-üüü¸e•••ÜLòâÎkÖ¬A^^4440pà@ 6 žžž>|8ì^ÎÜåååHHHÀîÝ»ñÿþßÿË/0lØ0Ì;^^^ððð YyEèêêB @ x9[ÌÑ£GqøðaÄÅÅaåÊ•°¶¶ÆÔ©SáïïO!êd@!„ÒŽñù|™£t<þœÕ²îkß¾}¸qã7¬––,--4.70±µµm³Ñ;>|ˆ[·naáÂ…ˆˆˆ@hh(-ZÔjÏ qôèQÄÇÇ#>>?†½½=–-[†ñãÇw˜Y Ú;îÆDdd$²³³¹ÿùÏ°`Á¸ººÂÏÏB¡ƒ¦„ã£>Bee%,Xmmm¼óÎ;ª‰6U·Ü)­Á;wP^^Îm/ž…ÀÞÞnnn 8Ó,„4¦¦¦Â³!Ôý›‰DÈÍÍEuu5·?cccš ¡=ËÊzõRu„BˆJ<}úÇŽCrr27[AEEºwïOOO¼÷Þ{>|8† U‡K!DÅ´´´0pà@ 8PbùÝ»wqúôiîƒÊÊJôìÙ“ëp0jÔ( 2¤Stį¬¬D||ááápqqiñ1ÒÒÒ€øøxœ>}jjjððð@DD„B!œ[| Ò4{{{„……!,, eee8rä°iÓ&¬X±¦¦¦?~<„B!|||еkWU‡L!úôÓOñìÙ3¼÷Þ{èÚµ+wã•Î@< ¬N„††7rfUU 66;v쀇‡–.] ???¹ÕmÀþÏ?ÿ //&&&ðññAXX5`oôõõ! ! Hv™>}:´´´0räHLœ8}ûöUqÄ„"]dd$JJJ0sæLcôèѪ‰&56 Avv6î޽˕ÉÉYFŒ!µq1!¤cãóùpssSÊlâýÕŸ1OœwôêÕ †††m•´ÎçÖ-`ÄUGA!„´šªª*=z{öìÁ¾}ûpÿþ}X[[ÃÛÛ¡¡¡ðòòB÷îÝUf»—œœŒ‰'âäÉ“ ²Ó’ÏBHG¤££ƒ#F`DºÖõëב””„C‡aÉ’%xÿý÷áææ†)S¦àÍ7ßl×Ï­ÒÓÓ‰?þø&&& Á¼yó}FL:>úý&­ÉÕÕ®®®øúë¯qòäIlܸ .ÄÒ¥K±xñb,X°€Ú_ò QSu„B!D5LMM1dȼõÖ[øàƒðÓO?aÏž=8wîòòòðâÅ äååáüùóØ¿?V®\‰©S§ÂÆÆEEEHLLÄ×_·Þz #FŒ€ ´µµajjŠb„ x÷Ýw±lÙ2lØ°ñññ¸zõªÔ©FÅ ÛRSS1qâD¸¸¸ &&†›m¡¾ììlÄÄÄ@(ÂÈÈ“'OFjj*æÍ›‡””¿A>´|ùrÄÄÄ@$!;;[¢Ã©£²ÈÍz÷Vu$„BˆÒ]»v K–,••|||pîÜ9ÌŸ?W¯^ENN6oÞŒ€€€v×Á@Z}Aü²´´ÄìÙ³ñàÁƒ6«¶¶Œ±fݯ–öYyËê›7onPGkír´4­]÷iÏu«ö›*I»6;ŠöøªâﺭõíÛ¡¡¡HHH@qq1áææ†5kÖÀÙÙC† ÁúõëQRR¢êP9……… Àpúôi¬[··oßÆŠ+Ú¼ƒÁ™3g¸kDWW—/_n°»»»ÄïfkÞ×ËÚãß]SZòû-–ššŠ#F@__Ÿ+Ë´Uä#mù[ЙóÉQ£Façθ{÷.Þÿ}|ûí·°³³CTT=Ç'äA3B!„©ÔÕÕ¹Ù SYY‰ÂÂBÜ¿Ÿá²î¿—/_F~~>ï¾û.þý÷_ôìÙùùùX³f ÆŒƒóçÏÃÝÝ]éÇm/^ýuŒ;©©©ÐÐÐÀgŸ}†‰'}=$O IDATâŸþÁ믿®”côèÑþþþð÷÷GMM NŸ>„„ˆD"ÄÄÄ@__cÇŽ…P(Ä„ `aa¡”ãBHs©««cÛ¶mðòòÂo¼3gÎÐtÓ¤Õ46šxvvv££‰×Ÿ…ÀÞÞ|>_…©‘TXXˆÙ³gc×®](..†·îСC€ƒ"((ˆ[^ZZŠ±cÇbûöí­öÐËÒÒŒ1Ì™3þù'JKK[¼ÏÈÈHÌ™3§ÉQïäÝŽ¼´hÑ"Ì™3]ºti•ý·ö÷¡ÈþÛ¢\®¨¦fC¨;‹Š´Ù›E¥îL¯ô,*ׯUU@ÿþªŽ„BQ Æ8€ÈÈHœ8qvvv˜?>‚‚‚`gg§êðä"«¾P^^Ž¤¤$âÆ8}ú4ÔÔÚn¼ÇQ£F¡¸¸¸Í?;oÞ<Ìœ9ºººÜ2ª×ö@ÚµIšïUû»ÖÒÒ‚¯¯/|}}±aÃœ\¥7M !¯.ìÛ·Æ ƒŸŸŽ;===U‡E: ’’™²³³%FGÓÑÑ‘ht[¿µµ544:Ö­>___lÛ¶ G…P(ä–:tVVV‰D`ŒÇãxÙÉàñãÇ­:ª!íQ[—Ë•AGG‡ËŸd‘•&''sïëîOZÇñ{++«V¯¶¹+WmmÀÉIÕ‘B!-vòäI,^¼/^„D"ÆÇ•õ;:===Lš4 ³fÍBtt4.^¼Øè½rB!ƒššFÑ£GcíڵرcV­Z…M›6!((+W®„©©i›ÅóâÅ Ì™3qqqX²d V¬XÑîÑ1ÉÉÉ ÁÎ;U‘áùóç¸yó&>üðC oß¾ÈÈÈPuX¤àóùøþûï‚9sæ`È!Ø°aæÌ™£êÐ!­„ZäB!„6¡««‹¢¢"®,uóx{ðàAxzzBWWVVV˜:u*Ξ=«ð6 pww‡ŽŽLMMñþûïãéÓ§Üú¬¬,Lš4 ÆÆÆèÒ¥ &OžŒ3gÎȽ>;;›kÈÜ÷Øìì쌸¸8<|øIIIصkFSSS`Û¶míjŠZBÈ«ÁØØû÷ïÇ­[·¢êpH;TQQÁ؃åË—#$$ÞÞÞèÕ«455addwww̘1QQQ\'F@€eË–!..)))())AEE²²²””„èèh,_¾ÁÁÁ°··ïp ÀÛÛjjj8xð ·¬²²éééøàƒððáC\ºt‰[wôèQŒ;ÀË™þ÷¿ÿ¡wïÞÐÑÑ««+öíÛ×àM•wZJÞ8ÄJKKñÎ;ï [·n022Bhh(^¼xÑäqš*÷Õooo@ÿþýÁãñ`kkË­¯[ÖìÞ½;fÍš%Ñ »1ëׯ‡ ttt0lØ0œ>}š+÷zzzÊsSñ)’àåŒh¡¡¡èÖ­,,,ðÅ_4Ø¢ç°1òž?yÊôumܸ‘;—<üñÕ—Ë[‹x6„……!22’Ë÷òóóä{ÁÁÁ°··Gvv6vïÞ>ø€ËSµ´´¸<5 aaaˆŠŠÂîÝ»‘ššŠüü|U'WqW®}û­ó!„WJyy9ÞyçŒ=&&&¸xñ"áååÕi:Ô%.§æääÿ¾ñºuë¸e7nl²l[WÝÏnÞ¼Y¡ýIûlSåoq}@OOcÇŽEff¦ÌØš[ÿ©¯±z]cñÊS_«{Ö­[‡ùóçÃÈÈ<Ó§OW¸î"Þ—x$ï.]ºÈL»<õŠæ¦hºîÖTÚÅZúBE¿yþ.¹6ëk¬î¦¬ó$Oš›úN­×Ê{Kî´ÆõÑ^uéÒóçÏÇÍ›7±mÛ6>|}úôÁ¶mÛÚäøÏŸ?‡¯¯/8€C‡áÛo¿mw `É’%˜0a~ÿýwî·¥)ò\ÛòlÓœ| 5î¡ ‰û;â™h¿þúk8;;sÛ}úé§Ü6_}õÅóGy¿ë^|í„„„€ÇãÁ××Wî4Ë›·ÈKù§˜¼×@cÇTVúdÝçkìø¥¥¥Ÿßk½sçŽÄòöÀÁÁ'NœÀâÅ‹1wî\üø㪉ÒZ!„æÿÿÿGHsÅÆƲÖþYŽŽnÕýÒz÷îÍH¼ÔÔÔÇãÞ°Q£F±-[¶°ŠŠ •ÆðѣG³ãdzÚÚZ™Û&%%1ìêիܲG1lûöíܲøøx¦¦¦Æ>ýôSVXXÈòòòX`` Ó××Wh›½{÷2Ç>ûì3V\\Ì.]ºÄúôéÃÆÇÅ9pà@6mÚ4VPPÀJJJXXX˜Ä>šZ_ßÑ£G¶xñbÅNd+ÈÊÊb?ýôLSS“©««3777¶lÙ2–’’ÒèwE!ÊtðàA¦¦¦ÆÖ¬Y£êPš¥µË˜Xlll«C¬-ËË•••,//¥¤¤°¸¸8ÉBCC™¿¿?sssc†††åfooÏ fË–-cÑÑÑ,))‰eee±ªªª6‹½½qwwg½{÷æÞ>|˜MŸ>]¿~`‘‘‘ܺ… ²0Æ caaa¬°°={öŒíرƒikk³k×®Iì_ÑòŽØìÙ³åÚNÞ8Äééß¿?û믿ؓ'OØßÿÍôôôØÂ… l—˜˜È-“§Ü'´ò)cÿWÖüä“OØ£GØõë×Ù!CXïÞ½ÙÓ§OMïï¿ÿΰ/¿ü’‹eÔ¨Q Û°aƒB1ËŠO‘´<{öŒ` `ûöícOŸ>eëÖ­cرcÇZ|¥}òž?yÊôõ÷ÿüùs& Ùúõë=í©\®jÅÅÅ\^üÓO?±ððp./633“¨kÖÍ‹gÍšÅÂÃù¼øÚµk¬¬¬LÕÉ‘4~¶¶¶,..Ž•––²µkײiÓ¦5ºiÄi·³³c{÷î•™vyëÍM»¼u·¦Ò®ŒçÒ(úÝ4uË{mJ#OÝMçIÞ4ËúN›[¯m*vy¯ÅæÞ¯hî}™Ž ´´”}øᇌÇã±?þ¸Õ7wî\ÆçóåÊ‹TeÆŒlÏž=¬¨¨ˆÙØØ0]]]‰xOž<ɼ¼¼$>#Ïu$Ï6-ÉZãZ@@ëß¿¿Ä²¡C‡2ìÖ­[ܲ•+W²7rïÍåùý–FÚçIsSy‹4ÍÍGä='ò^òSé“uŸ¯©ã×ÔÔ06cÆ ‰Ï°~ýúµËgàßÿ=SSS“ønÛ³¶hßEˆ2µÕóOæïß°ý,ýµB£N¤å¨“!òáóùŒÇã1555®¡Çcìã?–ûW[©¬¬dS¦Láb533c ,`gÎœi°­¼‹œœœÜTzöì322RhGGGæââ"±M||<À>Ì***4쬪ªb&&&Œ1ÖäúºJJJØúõëY·nݘ¹¹9{ðàô¦"EEE,..Ž3333î¦Opp0‹‹‹kòF!„´Ô—_~É455Ù‰'TŠÂ¨“tååå,++‹%%%±èèh¶lÙ2̳··g\ù@SS“™™™1777æïïÏÂÃÃÙO?ýÄâââXJJ {üø±ÒâêŒ>ýôS€eee1Æ g¿þú+cŒ1kkk6nÜ8n[V^^.s_B¡…„„pï)ïÔ'o'yâ`ìÿ2………I,ÿÏþÃ455Ù½{÷$¶«û@¦©rŸ,²,:995ØßÅ‹têÆÑÑ‘¹ººJ,ûçŸ<¬“'fev2˜;w.·¬¶¶–°/¿üR¡x¤‘ö}È{þä)Ó×Ýyy9óõõmðp·®ö^.o***¸ü|ëÖ­,22’ËÏ™¾¾¾D§0>ŸÏåç¡¡¡,22’ËÏóòòÚöA®©)c«VµÝñ!„¼Úª“Amm-óóócvvv,33³Ùûi¯ê×ÊÊÊؾ}û˜žž›ð÷ßËìdÐTýGòÔë©[H«¯‰ÏÁ¼yó¤~¦9 þ÷¿ÿI,¯Ÿvyê-I»¼u·¦ÒÞÒçŠhì»iê:–÷Ú”Fžº[KÏ“,ÒÒÜØwÚœc4»¼uÜæܯPæõÑžmݺ•©©©±mÛ¶µÚ1RRR¶wïÞV;†2ˆ;0Æعs瘖–ëÛ·/+--eŒIïd ϵ-ï6ÍÍZãÚÎ;–ÍcìþýûÌÞÞžñx<öã?rÛ 6Œååå5º¯ÆòGew27ÍMå-Ò(󾧴s"ï5 Ï1[š¾ÆîóÉsüo¾ù†éêêJ<[øî»ï$®öæí·ßfŽŽŽ¬ººZÕ¡4‰:ŽFÕ Ô@!„BH+*//‡H$Bhh(ž>} Æôõõ1yòdÄÅÅ¡¤¤øæ›oàææ¦êp%hjj⯿þÂÑ£G1sæL”••á矆§§'|||ðøñc…öwïÞ=ܼy¯½öšÄr)´MFFÆŒ#±‡‡àðáÃÐÑÑÁ!CðñÇc÷îݨ¨¨€††  Éõu-^¼Ÿþ9qñâE˜šš*”îÖfddDGGãÞ½{HIIÁûï¿´´4L›6 ¦¦¦ðööFTTnÞ¼©êp !ÐÒ¥K! €üü|U‡CšPUU…üü|¤¦¦b÷î݈ŠŠBXXàîîCCCèéé¡W¯^ðööFDDvïÞììlØÛÛ#88¿üò ’’’••…ŠŠ äçç#%%qqqˆŒŒDXXüýýáææ†nݺ©:É횀—Óp@RRÆÏ­;uêÊËË‘ ssóF§722BFF÷^‘òŽ2Õ£.qyMlÔ¨Q¨ªªÂ… ¤n/O¹Oâ²æèÑ£%–»ºº¢k×®‰D2?[\\ŒŒŒ Œ9Rb¹»»{«Æ,þýûsÿÏãñУG<|øPéñÈ{þä)Ó×UVV† & [·n˜7ožÌã·÷ry{¤££{{{!<<ÑÑÑHJJBZZJKKQ\\Œ””ìß¿‘‘‘€ÔÔTÄÄÄ`Ú´ipww‡………ÄïCPP"""‘H„´´4”••)'ð‚‚—¯”³?B!¤%''#!!¿ÿþ;z÷î­êpZEYYx<x<ôõõñöÛocõêÕø믿Z´ßºe[uuuse[UîOV} ~§±uMÕ¤Qv½®±úÚ%–½êדê¦]ÞzEsÓ.oÝ­.iiWÆsE4öÝ4v7çÚS´îÖÜó$KciVÖ1ÄdÅÞÜ{m}}´gAAA ÅÇÜjÇøí·ßàêêŠI“&µÚ1”mÈ!øá‡pýúu,X°@ê6ò\GòlÓ’|@––ü}Ào¼MMMìÛ·   2ñññ€€1ssóF÷%o^ÑRÍIsK~;•™Ê{ (zÌ椯±û|òÞ¼y¨­­ÅüÁm‹9sæ(O[Y¾|9222ðï¿ÿª:Bˆ’Q'B!„Ò*NŸ> ðù|x{{cíÚµ Djj*ž_â}÷îÝ@f%yË}òj¬¬Ù½{wn½4÷ïß—úÙ®]»¶jÌò000x¯¦¦†ÚÚZ¥Ç#ïù“§L_×þóhkkãÏ?ÿlòúì(åòŽ„ÏçÃÍÍ B¡ÁÁÁˆŒŒD\\N:…¬¬,”••!==Ä×_ WWW\¼xÛ·oGTTBBBàíí téÒ>|8f̘5kÖàÂ… Š— NŸÔÔ€vÖŸB‘×ùóçaaaOOOU‡ÒjôõõÁCmm-233áää„eË–Im˜¬ˆúe[MMM®l«Êýɪ4VæU´þ#KsëuŠÖ×ëÔ®¨úõ¤ºiW¤^Öœ´Ë[w«KZÚ•õBE¿›Æ®ãæ\›õÓ(oÝ­%çIÑ47çͽ9÷Zóú舦NŠ¼¼<äååµÊþ333ÛÝ@mòX¸p!¦OŸŽmÛ¶á·ß~k°^žëHžmZ’ÈÒ’¿044Äk¯½&ÑÉ`Ò¤I˜4iNœ8'Ož`ÿþý …ŸkI^ÑRÍIsK~;•™Ê{ (šŸ6'}Ýç“÷ø=zôÀÔ©S±eËÀ™3g0pà@*O[qttD×®]Û¤C !¤mQ'B!„Ò*† †;wâ—_~ÁôéÓÁçó±}ûv`ñâÅ8tè^¼x¡ê0¢­­‰'âСC°²²Â¹sç¸ujj/‹ÖUUUܲ§OŸJ|ÞØØÀËÑdQd›>úŒ±¯íÛ·xùgÕªUÈËËÃñãÇñüùs¼öÚkÈÊÊ’k}GvåÊDFFbúôéøä“OPVV†1cÆàÛo¿Ell,æÏŸOB !JehhˆØØX?~«W¯Vu8/; <¡¡¡Øºu+bcc‹;v`åÊ•X°`üüüп¢¨¨ÿý7–,Y‚€€L›6 X¸p!"##±sçNœ:u wïÞEuuµª“×ihhh`ܸq8räáííÍ­óòò‚ºº::$ÑÉ ªª wïÞűcÇPUUƘÔN>­YÞQ$±gÏžI¼7@’5b™¼å>y5VÖ,**âÖKcff&õ³õß+;æ–Rf<òž?yÊôu}úé§Ø¿?ˆ   ”——ËQžêêjܽ{§NÂÎ;‰… Jü&,Y²ÿý7ŠŠŠ`hhˆþýûÃÏÏ ,ÀÊ•+±cÇî÷fëÖ­ ÅàÁƒÁãñ &9èר×0Bé(ìììððáÃ7¸ïx<z÷î;w¢  K—.•X/Ï}ãŽ@V}àÉ“'2?£hýG–æÔëšS_S¦úAë¦]‘zYsÒ.oÝ­)ÊzQŸ²¿›æ\›õӨ蹑¶ÆÎSKÓÜZõì–Ü#h­ë££JKKƒ¾¾~« `bbÒagÐÝ´iúô郅 "==]b<ב<Û´$¥%b“&M©S§——‡[·nÁÕÕ'NDuu5±oß>‰Ù)TýÛ¥Œ47çxÊÈ?å½Úâ¾ec÷ù9þÂ… qîÜ9¤§§cË–-2gi/žýôS|öÙg –khh@SS“}x9¢ðrjK±+W®H|ÎÒÒNNN8~ü¸Äòüü|èèè ¨¨HîmúôéƒóçÏ7ˆmàÀˆÅƒ$¦öôôĦM›PYY‰”””&××õÛo¿aúôé2ÏS{PQQ‘H„°°0ØØØ`àÀXµjzöì‰-[¶àÑ£G8zô(þ÷¿ÿ¡_¿~ª—ÒI¹¹¹á³Ï>CDDDƒß¢Z°²²ÂÈ‘#ˆˆˆ¬_¿ñññ¸rå JJJPQQ¬¬,$%%q³˜››###›6mÂ;＃Q£FÁÚÚššš022âfC CTTvïÞÔÔÔûÐOU|}}ñìÙ3|ýõ׳ÂÃɉ‰ÈÈÈà~ó³³qÿþ}L›6 }úôá:ÖïÀªHy§9䣮úÇ=yò$455eŽF'O¹Oqc¦úûsrr±cÇ$–_ºt OŸ>…@ ¹?###8::âÔ©SËëÇ&oÌÒâ“E‘mëkÉ9”¶/yΟàVqqq½½=víÚ…;wîH<³liþØ’ûg€rÒ¬èñ”•Ê{ ´V~ZWc÷ù9þ°aÃ0hÐ ¬[·999>ÿüsX[[£W¯^ ƒH$’ÙI•V¯^;wâÑ£G¨¬¬ÄíÛ·Š;wî`þüùÜvŽŽŽ011ÁÏ?ÿŒââbܼy[·nm°¿ï¿ÿéééøì³ÏPTT„œœÌ;³gÏæ:-ȳͪU«pâÄ DFF¢°°………øðÃQ]]ÍzqíÚ5üøãxúô)?~Œèèhèèè`È!r­^Ž¸dkk‹iÓ¦µÚ9n®Û·o#&&011··7D"Þ~ûmœ’Ò¹,]ºžžž ÄóçÏUQ€ŽŽìíí!„ððpDGG#)) YYY¨¬¬Dqq1RRR‡eË–qV’““±zõjL›6 îîî°°°€®®.zõêooo!""111‰DÈÎÎn7eö@ܱ 77£Fj°.##Æ ã–ÙÚÚ¢Gؾ};ÒÓÓñüùs:t‰‰‰ ö-Oy§¹‰C쯿þB||<ž={†={öà—_~Appp£#yÊSî“F<‚×7PXX333¤¤¤àûï¿Çõë×±téRáæÍ› AïÞ½%Ê·Ò,_¾—.]ÂW_}…’’\ºt ÑÑÑÍŠYV|Š¤E^Í=‡ÒÈ{þä)Ó×çì쌕+WâçŸFRR’ĺö\.o/JJJššŠÝ»w#** aaa€»»;ø|>ôôô¸|9,, Û·oGvv6ìíí1kÖ,¬_¿žË󫪪$òüÈÈH„……Áßßnnn­[¿xñ¸p:Béкté‚;vàðáØ0aB‹F ïH–-[uuu|øá‡Ü2yï·'²ÊßõëW®\Á·ß~+s?MÕªªªÀãñÙhÐ÷ÐÀÚÚ®®®Ø½{·Äßé¤I“°ÿ~øúúJlßÒü±¥÷Ï€–å Í¡ÌüSÞk µòÓúdÝçSäø ,ÀÆ1sæL¥ÅÕRSS±lÙ2,Y²|š•“·BaãÙx6ÍSu¤‹e­ý³ݪû'D•***XRR g}ûöe˜‘‘ó÷÷gÑÑÑìþýû*‰ëÙ³gì×_e¾¾¾ÌÆƆijj2>ŸÏ¼½½Ù¡C‡lŸ””Äœ™®®.=z4KIIaæããÃm—˜˜È†Ê´µµ™¹¹9ûðÃYEE…ľäÙæàÁƒÌÓÓ“ikk3SSSÈî޽˭OHH`ÞÞÞ¬{÷î¬k×®läÈ‘ìðáÃr¯gŒ±ââbfeeÅüýý[t.•¡ººšŸ¹¹¹1Ê"##Y\\KIIayyy¬¶¶VÕIj3NNNÌ××·Áò3gÎ0ì?þX~îÜ96jÔ(f``À¬¬¬Xpp0›:u*wnÅåGyÊ;uݽ{Wâ;À~üñG™ÛËGxx8÷~ÇŽÌßߟ0>ŸÏ-ZÄž?ÎcÛ`3fÌàŽÓT¹O– °nݺ±®]»² pËë–5ù|>›1cËÏÏorŒ1¶~ýzfmmÍtttبQ£Xjj*À6oÞ,±<1ËŠOž´ìÚµ«ÁùªÿýõêÕK¡xêjìû÷ü5V¦ÿî»ï$ö¿páB–˜˜(±làÀܾÚS¹\ÊË˹ü5::š-[¶ŒË_íí홆†wÞ455™™™—¿†‡‡³Ÿ~ú‰Ë_KJJTœÆ:ÅÀXv¶ª#!„Ò ¥¤¤°””¹¶UF+55•YZZ2ccc¶iÓ&VSSÓâ}ª’´ú»ï¾+±Í¢E‹Ô%šºo¬hÙ¶®µk×Jl7zôh¹÷'í³b²Êêâú€¶¶6óôôdÿþû/÷y¹ë?Œ1–––Æ°ãÇ7zÞå©×I‹WžúZýs@jyQÞºËõë׶{÷n6kÖ,Ö¥K©igL¾zEsÓ^÷»’Uw“7í-}!Ms¾›¦þ.šº6ÓXÝMYçIÞû}§ŠÖk彩k±%÷+šs}t………,88˜©©©±É“'³gÏžµú1322ŸÏg£GnwõÚÓ§OK\#R·{ï½÷˜——Wƒåò\ÛòlÓ’| 5î¡1Æزe˘¡¡!«ªªâ–;vŒ`ÇŽk°½Ÿ¼ùy^^366nP†hO’““™‘‘óññaÕÕÕªG.mѾ‹ej«çŸþþþRŸ?ðc „òŠ‡q苾Xõª…tPqqq˜6mZóg5&&ÁÁÁ­¶BÚ“ììlˆD"ÄÇÇãСC¨®®Æ Aƒàçç¡PˆÁƒƒÇã©:LÒ=z„cÇŽ!>>ñññxüø17Ê´ŸŸ||| ¥E³BÚŸ5kÖ`É’%8þ<\]]UŽL­]ÆäñxˆE@@@«C¬£”—KJJüü|Ü¿ÙÙÙÜ+??<àê:::077‡½½=ÌÌ̸ÿ¿·³³ƒžžžŠSD^u—.] Aƒpüøq¼öÚkª‡t@UUUxôè‘DžX7ÌÊÊÂãǹíëæuóDñ2¨«««0E-ôÝwÀ?÷ï«:B!Pjj*ÀÍÍ­Ém•UÇ*++Ãwß}‡•+WÂÒÒ¡¡¡†®®n‹÷M:¶wß}ÙÙÙ8zô¨ªCQš7n oß¾HLLl0BµªQÝ–{ðà6n܈իWCWW‘‘‘˜5kV›=³LOOÇ믿555lݺ•þ– éäÖ®]‹{÷î!**JÕ¡4ÀÚ5k___ìÚµ«Ã”ïÛ¢}!ÊÔVÏ?ÅÏrãââ$–k´ú‘ !¤¨D%4¡©ê0!„üÿìííŒàà`”••áÈ‘#HHHÀæÍ›±bÅ ˜˜˜ÀÇÇB¡>>>èÚµ«ªC&JR[[‹‹/rLNŸ> ---Œ9˜8q"úöí«ê0 !¤I‹-Bll,BBBðï¿ÿvìÆŽD©ø|>ÜÜÜd6êyñâòòò¤vDHMMENNÊÊÊ$ö'­‚x™™™uÎ$J³yófìÛ·?ÿü3LMM‘••…°°0¸ººbØ°aª´SR;U‰—åää ¦¦ ¥¥…îÝ»sù™@ À¬Y³¸÷½{÷F·nÝTœ¢Vvø00z´ª£ „B”F__Ë—/G`` ¾ýö[|ôÑGøæ›o0cÆ Ì;...ª‘¨Àµk×››‹Ý»w«:”N‰ên„(OUUñ믿âŸþ‰‰ >ýôS„„„ÀÀÀ Mcqvvƹsç0oÞ<Œ7óæÍÃ_|“6ƒÒz¾úê+hkkcúôéX»v-’’’TRÉÉÉøàƒpùòe|ñÅøïÿKÏÀéĨ“!„x‚'è†Nþ€’B:(}}}…B…B@ZZéÓ§CMM Ü6ÎÎÎ*Ž˜(ªnG’„„äççÃÔÔãÇGXX|}}Ñ¥KU‡I! QSSÃÆáææ†7báÂ…ª‰tÚÚÚ\'Yd͆––‘H„Û·os£ðhkkÃÂÂBæl¶¶¶Ð××o«ä‘.00EEE˜0a²²²`ll @€¯¾ú šš4x먲²………2g!¸uëžþøcìÚµ ‹-BXXu6 ¤“wÊ]±blllTçܹsøæ›o°ÿ~Œ7©©©Ôa˜WѼ„KXb –à| êPHÕÓiµÕôG„t$………8zô(âãã‘€’’®¡ŠŸŸÆmmmU‡I¤ÈÎÎæ¾·'N ¦¦®®®ðóóƒP(ÄàÁƒiÔeBH§ðñÇcÆ ÈÌÌD=TN­]Æäñxˆå¦ØlMT^þ?âÙêwB7þÍÍÍEii)·½¬ÙÄïíììèw™WTIIIƒ¼DÞYêç%4 ]S||€Ü\ÀÊJÕÑBé„RSS@æÌjuµv‹1†'N`×®]Ø·o  êÐ!Duu5öïßõë×ãÈ‘#2d>ÿüsøùù©:´fk‹ö]„(S[=ÿ?Ë‹‹“XN3B€”€¾ªÃ „¢ cccøûûÃßß555¸té×p}Ó¦MÐÕÕÅðáÃáç燷Þz –––ªù•UQQäädˆD"ìÝ»Ÿß¼ _uu ²´„ÁäÉpœ<@#£6BHG´téRlÛ¶ +V¬À:•˜´‘ÿ½;«²Îß?þb‘]A17pÇRSpÁ̲ԙ6lšÊ¬)ÍiÊ¥o©Õ˜þšJÛ§&3m) l1÷qÇ%EQP0qCQÙ~œ9Gp7×ÓÇý8pß÷¹Ïu#‡³}ÞŸwetC8xð ¥¥¥–ã5nÜø²"óàáàààjo/"·îJ]Êÿ]HKKãÌ™3–ý¯×… $$[[[ÏÈ ,_mÚ¨À@DDêz÷îMïÞ½yçwX»v-ß|ó Ë–-ã­·ÞÂÉɉ¨¨(zõêE=èÖ­› ED¤Ò©~7Û… 00ÐrŸŽŽ&!!Áò}‹-pÓsòª·|9 ht ‘jgkkKdd$‘‘‘̘1ƒ#GŽœœLrr2óæÍãå—_ÆÖÖ–°°0zôèAÏž=éÞ½;-[¶T×5¹®üü|6mÚĺuëX³f ëÖ­ãÈ‘#888бcGÈ?þñn¿ýv\\\ŒŽ{ÓêÕ«Ç“O>ÉO<Á?üÀûï¿Ï_ÿúWž|òIzõêE||¼ D亶lÙÂ_|Á_|Á¾}ûfÔ¨Q$$$дiS£ã‰ˆA4ªJD꼓œP‘ˆˆ• !!!„„„ËfÑŸ>}:>>>ôíÛ—ØØXâââðòÒãÀ­º´›ÄæÍ›-Ý$¦Nzín¹¹°cìÚ;w¦MðñÇ`¼èïo*6 ƒˆÓ×ááàì\}'("ò;ýéOâwÞaüøñ$%%Gä†Üh7„+ `NOO¿ánÀ,òûqøðá«\«ÈÜ…@…@5LV–éõÐÌ™F'1œ¿¿?£GfôèÑüöÛo¬[·ŽµkײvíZ>ùäòóóñöö¦{÷îtéÒ…¶mÛÒ®];š5k¦ç5""uØùóçÙ¹s'Û·ogûöí¬[·Ž-[¶P\\L£FèÑ£ãÆ£GDDDÔÊ¢‚«±±±¡_¿~ôë×óçÏóí·ßòÅ_ðÜsÏY † Btt´&à XµjÉÉÉ|ù嗖‚{øøxºvíª‚^Q‘ˆÈaИÆ'‘ªâââBtt4ÑÑÑL›6ôôtË@øG}”’’:tè`)8èÔ©“^0ß ãÇóã?Z~ž§N² Zzùå—éß¿?NNN×?§'DE™–ò²³+¬^ sçBAØÛC‹‹Ì—­[ƒ>H‘ÄÖÖ–3fЯ_?~úé'úôéct$‘Jáåå…——áááWÜ~­AÐ)))¤¥¥qæÌ™ Ç»ZBhh¨AKrêÔ©Ë:”ÿþZE<—v!hÙ²%õë×7øŒäº¡^½Ë_‰ˆˆ5bèС :€ââb¶nÝÊÚµkY·nŸ}ö¯¼ò %%%Ô«W°°0ÚµkGÛ¶m-Å 64ø,DD¤2•””žžÎöíÛIMMeÇŽlÛ¶ôôtJKKquu¥M›6tëÖ±cÇÒ³gOBBBŒŽ]mêÕ«Ç=÷ÜÃ=÷Üc)8X´hS¦Laܸq4jÔÈRM“&MŒŽ,"U¬¤¤„-[¶’’BJJ «W¯¦  €-Z0tèPâããéÒ¥‹ÆIˆH*2‘:/‹,l°Q‘ˆHÊرc;v,çÎã‡~`éÒ¥¼ÿþûLž<???ú÷ïO\\Ô¬º—عs'K—.%%%…Ÿ~ú €nݺ1~üxâââ «¼ 0-ÑÑ×ÁîݦΩ©¦åã!3Ó´ÝÃÃTpжíÅ¥}{ÓzƒôíÛ—¾}û2yòdHáèèxËÝ222())±/00ðª…Í›7ÇÝݽºNOäw+,,$++ëª{öìáìÙ³–ýËw!'..N]¬Ñ¢E0x0XÑ,š"""UÅÁÁ.]ºÐ¥Kžzê)À4sõ®]»,ƒMSSSILLäرcøùùѶm[Ú´iCóæÍiÞ¼9·ÝvAAAØÙÙy:""r ùùù¤¥¥±oß>ÒÒÒØ»w/Û·og×®]œ?[[[š5kF»ví9r¤¥¸,44T¯•ÿ§|ÁAII ›7ofÅŠ¤¤¤ðøã[÷ë×îݻӭ[7Z´h¡Æ"µ\AA›7ofÆ ¬ZµŠü‘“'OZŠŒÞyç‰Èu©È@Dê¼,²ðÁgœŽ"""puu%..Ž¸¸8JKK-Õû‰‰‰Üwß}8::Ett4wÞy'­Zµ2:rµ;þ<+V¬`éÒ¥$%%‘••…¯¯/ à?ÿù ¨Þ}ŽŽ¦¢öí+®?sÒÒ.v=ص ¾ú rrLÛýý!"´”ï~ "RM¦L™BTT?ÿü3½{÷6:ŽHp#ÝŽ?~ÅØ¿§Bpp°I•û=]Ê$$$X¾ ÁÕÕÕà3’*wü8üü3|ú©ÑIDDDj­zõêѹsg:wî\aýÑ£GÙ¾};Û·ogÇŽ¬^½šýë_œ:u 0=kÚ´)-Z´°˜/›4i¢ª""Õ   €}ûöY Ì—iiideeQVV† Mš4¡yóæDEEñØcѾ}{¨W¯žÑ§PkØÙÙY õ&L˜@~~>«W¯fÅŠüôÓO|ôÑGâééI·nÝèÚµ«åR]Dj®²²2öìÙÆ Ø°aëׯgÛ¶mÓ AzôèÁK/½D¿~ýhÓ¦ÑqE¤Q‘ˆÔyYd©‹ˆˆ`kkKDDŒ?žcÇŽñÓO?‘˜˜È´iÓ˜0a¡¡¡DGGË€ptt4:v•0Ïœ˜˜Hrr2ÅÅÅtìØ‘‘#GKdddÍ›ÁÄÝýbÁèÑ×:[·š–-[`þ|8pÀ´­aCèØÑ´tè`ºlÞôᡈTÈÈHúôéÔ)SX±b…ÑqDjGGG ""âŠûÜl7Ÿ«!¨‚\¹ ÁÕŠ2228wîœeÿëu!hÚ´iÍ{^-Õ﫯ÀÁþ𣓈ˆˆX???bbbˆ‰‰©°þøñ㱦¥¥ñóÏ?óÁpúôiœiÖ¬¡¡¡„„„ФI‚‚‚ "88˜F©ADä’™™YaÉÈÈ ##ƒ}ûöqøðaJKK±±±!00ÐRð5hР Å_ÎΚ8²²¹¸¸Môÿ:Š±eËË å 0uêTš5kFDDmÛ¶µ,z_C¤ú°k×.RSSÙ±cÛ¶mã¿ÿý/¹¹¹899ѱcGzöìɸqãèÖ­Íš53:²ˆÔb*2‘:/ƒ ‚2:†ˆˆÔ@ 6$>>žøøxJJJX»v-K—.%%%…¹sçâêêÊwÜA\\ƒ¦qãÚ[´VPPÀªU«HIIaÉ’%ìÞ½oooúõëÇìÙ³2d52:æïÓ¤‰i‰‹»¸.7·báÁwßÁ?þÅÅàê íÚ],>èÜÙÔñÀÁÁ¸s«ñòË/sÇw°zõj"##Ž#b®× ¡¸¸˜cÇŽ]µ¾}û,ƒˆÌÇS7„ºëz]8@YY`tfþ½ð÷÷'""¢ÂïLÓ¦M5›¢Ü˜Å‹aÐ ¨_ßè$"""uFƒ hРÝ»w¿lÛ±cÇØ·o{÷îeß¾}8p€-[¶ðõ×_“]¡ˆ900°Báùë&Mš‚‹‹KuŸšˆHµ;~üx…âK ~ûí7˾...„„„D³fÍ0`@…BýÝ4–££#ݺu£[·n<ùä“œ8q‚õë׳aölÙÂG}ÄÁƒ)++ÃÍÍððpÚµkGÛ¶miÓ¦ íÚµÃÛÛÛà3©ýÊÊÊ8pà€¥˜`ûöí¤¦¦’––Æ… prr",,Œ¶mÛ2eʺuëF‡¬v’D1†Š D¤ÎÛË^nçv£cˆˆH gggGTTQQQ8p€äädRRRxöÙg3f aaaÄÅÅKÏž=kü,VGeùòå,]º”åË—sæÌBCC‰eöìÙôîÝkXïé }ú˜³ÂBرÃTt`.@ø׿àÜ9pv†öíM]:w6]†…½^R‰ÈÍéӧݻwç­·ÞR‘H5qpp¸n7„üüü+(7wCÈÌÌäÂ… –ã5hÐàªE·ÝvÕyŠrƒ ,].n¦ Ahh(—ýŸûûûk¶>¹u¹¹ðãðñÇF'‘ÿiØ°! 6¤GWÜn.L½ôõ÷ß~ËþýûÉÍ͵ìk.L5¿fð÷÷ÇËËë²uzn)"5Qùî‘ÙÙÙœ:uÊòµùòСCäååY®S¾£_‡6l˜:úÕr>>>üáàåºïååå±wï^vîÜɦM›Øµk_ý5999@Å÷TBCC #<<œ-ZàææfÔ©ˆÔHåŸ[îܹ“]»v‘žžÎž={8{ö,€¥KjLL &L <<œ6mÚàäädpz±v#"uÞ>öñCDDj™¦M›’@BBùùù¬^½šÄÄD>ûì3¦OŸNƒ ¸ãŽ;ˆeÈ!xzz™’’¶nÝJbb"K—.eóæÍ8;;É”)S¸ë®» ªÃÝ}œœLÅå–”À¯¿Â¦M—yóàüySgƒæÍ/^'"ºvÍ!"×ñÔSO1jÔ(yò$‡"33Óò¼ô·ß~#;;›µk×’MNNÅÅŖ븸¸X 5jD@@¾¾¾4jÔÈÒuÁ¼øøøTשŠˆ•)**âĉ?~œcÇŽ‘““cùÚüºùÈ‘#9r„œœË„ ®®®XþF…‡‡myÏÃÜÉÅÙÙÙÀ3”êâææfy,=z´eýáÇٱc{öì!--´´4.\Hff¦¥P`` Í›7·t¯0w ¦Q£F*B«SXXÈ¡C‡,^8`¹¤¥¥qæÌÀô|Ð|߈‰‰áñǧuëÖ„‡‡ãîînðYˆH]¥O²D¤N;Æ1NqŠæ47:ŠˆˆÔb...DGGͬY³HOO· ää‘G(--¥C‡ÄÆÆG§Nªí ²'NðÃ?’’Â7ß|ÃÑ£GiÚ´)111¼üòËÄÄÄè ßk±³ƒðpÓb~“ôÂصËTp°q£éò‹/  \]¡cÇ‹ºu3"ˆˆ”sÏ=÷ðÜsÏ1gÎ^{í5£ãˆÈ ¨înÍš5«Eª5Éõºì}j/v]€§+ΘyÅŸµH0oÜy'¨ŠˆˆˆÕðööÆÛÛ›öíÛ_uŸ²²2rrrÈÉÉáðáÃ=zÔ2À7;;›M›6qôèQŽ=Za†p0uÝ-_tаaC|}}/[×°aC4h€——— hE¬Tnn.'Ožäرc?~ܲäää\qÝéÓ§+\ßÖÖÖò7Ã\<кukðóó£qãÆøúúHýúõ :K©M dàÀÖ’žžÎÞ½{+ ®^¶lYYY–'''š4iR¡ð 88ØRÄÒ¸qc\\\Œ85‘«:yò$‡&##ÃRH™™iù¾üÄ'...4mÚ”æÍ›sÇwðè£Z Ud#"5ŽŠ D¤NK# @E""R©BCC;v,cÇŽåäÉ“¬X±‚””æÎËäÉ“ ¡ÿþDGG3hРJcvçÎ,]º”””~þùgJKKéÞ½;ãÆ#::úªãäÙÛC»v¦åÁM늋aç΋E«WûïBQøø˜:tëvñÒÛÛØsC988À[o½Åßÿþw}Ð/b%®× ®>Ó~JJŠå{³ºÖ áz]®õ³éÕ“•·¯dý#ë±™iÃHFò¤í“´¡g$rÒÒL¯¾ýÖè$"""RÍlllðóóÃÏ϶mÛ^wÿS§N‘Í©S§,Ë‘#G,ë233Y¿~=ÙÙÙ?~¼B—3ggg¼¼¼nzñõõµª×"5M~~~…ûö.Wº¯_z? K—.—­ó÷÷×ý[ª•““­[·¦uëÖWÜ^þ}¡òï%&&²wïÞ wæ÷…Ìï•™Ÿ/]çïï¯ÛrKÌÏ¿Ê?ç2m¾<|ø°¥˜&>1ÿ¶k׎¡C‡Vxo·iÓ¦ú½‘ZEÏE¤NÛÎvÜq§ MŒŽ"""VÊÛÛ›øøxâããy÷ÝwÙ²e )))$&&òþûïãììLdd$ÑÑÑ :”–-[Þômœ?ž5kÖ˜˜ÈW_ͬà IDAT}Å¡C‡hØ°!}úôáƒ>àÎ;ïÄC³bV-èÐÁ´<òˆiÝ… °giÐЪU°p!Lž eeàïQQy±ë:JˆÔ)cÆŒá•W^áË/¿däÈ‘FÇ‘jâååei%%ågë¿^7óñ®V„Š——WuÚ5]«ËCzzú5ÏëÒ.×:¯3œa ˜Ílæ2—"xŠ§Áp¨®Ó¹qÿú@LŒÑIDDD¤†3¾Qåg6ÏÍ͵ J6m¾ÌÌÌdûöí–uåÉ™ÙØØàåå…§§'nnnÔ¯_Ÿúõëãî§§åûúõëãé鉻»»å{77· ×ÓD b-rss9{ö,yyyœ={–Ó§OsæÌ™ ëN:ÅÙ³g-ËéÓ§9}ú´å{óöKÙÙÙáééi¹ß•/¿l½¹X AƒºI­u½÷ÌrrrÈÌÌ$;;ÛÒùçÈ‘#9r„µk×’MNNN…ÂüüüðõõÅÇÇ4h`¹¼Òz½d­ÎŸ?oéìb~Žtâĉ —å×=zÔÒaÀÕÕ•€€5j„¿¿?áááôë×ϲ.00ý«£"©Ó¶±ö´ÇU‰ŠˆHÕ³µµµ¼A6~üxrrrX¶lK—.åµ×^c„ „††K\\½zõÂÑÑñŠÇ24KLL$99™ââb:vìÈý÷ßOll,={öÄÖÖ¶šÏP*°·‡ðpÓ’`Zwì¬_6˜.'M‚Ü\SAÇŽ;tëטYDj????È'Ÿ|¢"±pvvþÝÝV¯^mX7„[éBpiÁ­drÇ„ÿý[Å*f3›‡y˜ñŒçü‘'x‚@où|E*Ei)üûß0z4ØÙFDDD¬Œ··7ÞÞÞ7=©MIIÉ‹Ì—æÁÓgÏžåÌ™3dddTT››K^^^…"âòlmmñððÀÝÝ'''ÜÜÜpqqÁÙÙwwwðððÀÉɉzõêQ¿~}ðòòÂÁÁÁR¨àä䄇‡¸»»ãì쌋‹‹e©»ÊÊÊÈÍÍàôéÓsæÌ ÈÏÏ'//ââbrss)**âܹsœ;wŽ¢¢"rss)..&//üü| 8sæ ÅÅÅœ>}šüü|ËïûÕ˜oëׯ———¥Ø¦~ýú„††âááQ¡§|Á€ùÒÝݽº~\"µ†¯¯/¾¾¾×ܧ¬¬Œœœrrr8|ø0999üöÛo–å'Nœ`ÿþý–å'Ož¼ìøøøàééiy¼òðð°,æï½¼¼.ÛîââRc&ü°6æ¿×æb-saWù¯sssÉÍͽl[nn.'Nœ ??¿Â1,Å%æB“V­ZY¾oÔ¨øùùѸqc=¿‘:KE"R§me+]èbt ©£|}}=z4£G¦°°•+Wòí·ß’””ÄìÙ³ñôô¤ÿþ <˜¾}û²{÷nËö´´4<==0`sçÎeРA4lØÐèS’ëiØbcM‹YvöÅn7Â{ïAa!øùA—.;téNNÆe‘J7jÔ(î»ï>>L` ½ŠÈ¹ÞÌnyyydff’‘‘Á¡C‡ÈÌÌäСCìÞ½›ï¿ÿž¬¬,ˬnvvvؤ MBBhÒ¤ Mš4!88˜   ‚‚‚ÈÌ̬pÌC‡‘‘‘a™AÎ<«—ƒƒ7¦I“&„„„0`À‚‚‚*ÓÍÍ­Z~NQÿû—M6s™Ë;¼ÃLfr'w’@ÑDWK‘«JIÌLS‘ˆˆˆH aggGƒ hРÁ-§  €¼¼<òòò,3¾›sÇ„¢¢¢ ƒ¿Íƒ¹ÓÓÓ),,äüùóœ={Ö2øÛ<ÀðFyxx`kkk)@°±±ÁÓÓ0ÍFìè舽½½å5ʵö.+`(]óÏ®üàpsAEy×|êéé‰ÍOŒçæævSÚçÏŸ§°°ð†÷7ÿ\É¥ÿ.\¨0𾤤¤BWŒÒÒRNŸ>mù¾|!@qq±eFÿÓ§OSZZjù½(¿ïÙ³g)..¾æþ7êJ…+žžž888àæ憷··¥øÅÑÑwww\\\,æóbd¬ÉŸDŒcccƒŸŸ~~~´mÛöºû—””XŠÌ…æâƒòƒÕOž<É* \?uêÔUëè舫«+nnn8::Z œ-…tõë×ÇÕÕÕò·°ìàîîŽ]…Ç"óqáâãXy×ût½Çë=N\éï¬ùï2`ù™”L0lœ9s†’’ËßðsçÎQXXHnnn…ç………äååYò\ïg]¾Ä\âëëKóæÍ-ë/-&ðõõUA—ˆÈ R‘ˆÔY¥”²ƒ<Â#FGÁÉɉ˜˜bbb˜9s&iii$%%ñí·ß’@áí… m¶á®»îâø‘‘‘•2ó¬, âãM @~>lÚd*¾äææRVVViçjsg"¸Xaþn.ð2w…ð÷÷ÇÍÍ ''§Ë¶›ãééY¡{„¹(CDDªŽF$‰Hµ—½œå,íiot‘Ë4oÞœ§Ÿ~š§Ÿ~š³gÏâVß·½Í_þÅèhRÕ\\L…QQ×¥§›:¬^ _~ ÿ÷PV¡¡¦.ænWL("5“³³3Ççßÿþ·Š D¤ÚØÚÚÒ¸qc7nLÏž=M³©·oϤ$Ë>çγ!–"ó€ÚÈ 'âÿ÷o›˜Ë\žæižçyîå^Æ1ŽVh°·T“¬,Óóú?6:‰ˆˆˆH­ãêꊫ«ë5»áÒÙýË϶—Ïî_ÞÍ,\Ú9Àlíڵ̜9“… ^¶íJ®çjÝ.=Ö¥]àÚ]DD¬»»;îîî4iÒ¤Ênãj]ÌÅrWÚïJ.í s%—vì¹Ô•GÌ…'Nä»ï¾ãû￧[·n×>1©5Td "uÖZÖâ‚ íhgt‘k2¿¡S³>I-jZF6}Ÿ“kÖ˜ Ö¬Ï?‡¢"SWsBŸ>Ц ÜD‹m©~ñññÌ™3‡}ûöqÛm·GDêš²2X¶ {¬ÂjWWW 3(XÕŠ ‚9Ìa*Sù˜y—wù€èK_H`ðÃÎè˜bÍfφ† /v3‘ZÏÑÑGGÇ ë|}}«5ƒyÖëx=ϱ ööö–¢­š^¼5þ|ú÷ïψ#X¿~½>×±5«wµˆH5ZËZ:ÓG¯¿³ˆˆˆHMâë C‡Âo˜Š rsaåJxâ 8w^z Úµ3íw÷ݦAL©©¦„"R£ôêÕ ///–.]jt©‹6o6ͨ>x°ÑI á‹/ãO:é,g9Î8s/÷Ò’–Lg:'8atD±FçÏÇÂ_þŽz_RDDDDDDDj?GGG-Z„­­-Æ £°°ÐèH""R Td "uÖÖЃFǹu..pûíð °t)œ8;vÀÿýØÙÁäɦ¢ˆ‰éÓaÓ&(-5:¹HçààÀ AƒX²d‰ÑQD¤.JJèÐÁè$†²Å–h¢I$‘_ù•{¸‡éL'@F3šml3:¢X“?6$$DDDDDDDD¤Ò4hЀ%K–ššÊc—tN‘ÚIE"R'æ4»ÙMwºEDDD¤òÙÚBx¸iàÒÂ…p옩èà7ÀË fÌ€ÎÁÓSE"5@\\+W®äÄ Í˜-"Õ,) bcÁÆÆè$5F Z0idÁ,f±…-t éÌ|æSL±Ñ¥6++ƒ·ß†Ñ£¡A£ÓˆˆˆˆˆˆˆˆTª°°0>ûì3þýïóüÃè8""r‹Td "uÒ*VQF‘DEDDD¤ê]©è`ófS‡ƒzõLE;ƒ¯/ ï¾ ûöZ¤Î4h¶¶¶$''EDê’cÇ`ãF<Øè$5’n$@*©üÂ/„ÊÃàæft’Z#€&1‰Ãæ3>£€bˆ!Œ0f1‹sœ3:¢Ôd[·Â?˜žs‹ˆˆˆˆˆˆˆX¹·ß~›Î;3dÈrrrŒŽ#""¿ƒŠ D¤Î9Á v°ƒ¾h¦>‘+²³ƒˆSGƒäd8y–-ƒÈHX¸bbÀÛÛt9}:lÚdtb‘Z¯wïÞìÞ½[o´‹Hõ(.6 lt’ZÉGâ‰'™d6²‘^ôb" €1Œa»ŒŽ(5ÑË/C§Nп¿ÑIDDDDDDDDªœƒƒ_|ñööö 6ŒÂÂB£#‰ˆÈMR‘ˆÔ9)¤`‹-½èet‘Ú¡~}ˆŽ†Y³ =öî…3ÀÕ^}ÕÔá iSHH€¯¿†sšÅWäfEEEakkË/¿übt© ~ùrsaÐ £“ÔzD0‡9d‘Ŧð=ßÓ–¶ÄÃ|A %FG”š`Ó&HL„)SÀÆÆè4""""""""ÕÂÇLJ%K–°sçNŒŽ#""7IE"Rç$‘DOzâ‡ÑQDDDDj§æÍáñÇM'N˜*Ž Û·ÃÝwCƒ¦A‹ï¼F§©<<¾ÿbb Q#>Ü´ýôi£ÓŠÔ(íÛ·gË–-”••ED¬Yz:ìÙ£"ƒjÐœæLcd0›Ùìa=éIg:3—¹P`tD©j¥¥ðÜs¦û[t´ÑiDDDDDDDD 3nÜ8yäFŽIjjªÑqDDä¨È@Dê”$’"ˆpÂŽ""""býììL]¦MƒÝ»aÇÓ «#GࡇÀ××Tx0k8`tZõoßžS§NqèÐ!££ˆˆ5KLˆŒ4:IQŸú$À6¶±‘„ÆóÉ ƒqŒãS>¥)M‰#ŽR(CÝl¬Æñãð·¿Á㛞犈ˆˆˆˆˆˆÔq,^¼˜²²2î½÷^JJJŒŽ$""× "©3¶³ìd#ŒŽ"""""ÞÞóç› –/7ͨüÏB«V¯½ûöT¤Ú¸»»ÄÎ;Ž""ÖêÜ9X¹6:Iç?ãÏ~ö³€P@ 1´¦5³˜ÅYÎQnÕSO£#Lžlt‘ÃÇLJ/¿ü’5kÖð·¿ýÍè8""r *2‘:ã3>£ MèA££ˆˆˆˆHyп?Ì ¿ýfêt³gCóæ;üú«ÑIEª\³fÍ8pà€Ñ1DÄZ}ÿ=›:Hàˆ#ñÄ“L2›ÙLoz3‘‰4¦1cÃNTxV+%%ÁgŸÁœ9àéit‘¥C‡Ì™3‡3fðùçŸGDD®BE"R'”QÆç|ÎF`ƒÑqDDDDäjììL³fAV–©à :ÚT€ÐºõÅ‚ƒÝ»N*R%BCCIOO7:†ˆX«¤$èÚüüŒN"WБŽÌaÙd3…)¤BÚE_ð¸`tD¹99ðÈ#pÿýkt‘iäÈ‘<ñÄ<øàƒlÞ¼Ùè8""r*2‘:aë8ÀF0Âè(""""r£lm/:+V@ïÞðÞ{íÛ믚¶‰X ˆH•)+ƒeË`ð`£“ÈuxàÁXÆ’FÉ$@#A0ÁLbÇ9ntD¹š²2xøapt„ÿ÷ÿŒN#"""""""R£ýãÿ [·nÜ}÷Ý?®÷¼DDjˆHðŸÑŠVt ƒÑQDDDDä÷°³ƒ¾}áwL~üÑT€0s&„„@L |ò œ?otR‘[Jff&EEEFGk³y³é1TEµ†-¶DÍB²‡=ŒbÿÿG g8kXctD¹ÔŒ°|9|þ9x{FDDDDDDD¤F³··gÑ¢EØØØ0bÄJJJŒŽ$""å¨È@D¬^)¥,f±ºˆˆˆˆX ;;èÓþùOøí7Ó@.//Ó¬±þþ0z4¤¤˜f’©eBBB())!;;Ûè("bm’’L“4CmÔŒfLcYd1—¹ìe/‘DÒ™ÎÌe.ùäQ–,‰aútèÞÝè4""""""""µ‚‹/fõêÕüío3:Žˆˆ”£"±z?ðÙds÷EDDDD*›DGÃÂ…¦‚ƒ×_‡ôtSgƒà`˜0öï7:¥È óõõàرc'«“”±±`cct¹Î83šÑle+ÙHa<Á„Â&A†Ñë¦mÛàࡇ`Ü8£ÓˆˆˆˆˆˆˆˆÔ*;vdÎœ9̘1ƒÏ?ÿÜè8""ò?*2«÷ŸA-hat©J^^«VÁÎpÿý0o´hQQ0w.äåRäš4hÀñãÇ N""VåØ1ظ6:‰T¢"˜Ï|2Éäžá?ü‡PB‰!†D)C]ªÅ¾}0htí ï¼ct‘ZiÔ¨Qüå/áÁdóæÍFÇTd "Vî,gYÄ"îç~££ˆˆˆˆHu ƒiÓàÐ!øê+ðõ…'Ÿ„€xøaÓ@K‘ÈÍÍ '''ˆHåJJèÛ×è$RшñŒg?ûYÀ†0„V´b:Ó9Å)ƒZ±Ã‡M´5‚Å‹M÷3ù]fÎœI×®]¹ûî»õ9‰ˆH  "±jŸð ÅóGþht1‚ƒ _~ ÙÙðê«°atéݺ™:äçR¤‚ èÍs©\IIЧ¸¹DªÄO2Éìf7ÈT¦L0cÃvѺìß½{ƒ‡¤¤€§§Ñ‰DDDDDDDDj5{{{>ÿüsJJJ1b%%%FG©ÓTd "Vm.s¹—{ñÁÇè(""""b4S7ƒÔTøùg …1c 0ž}öí3:¡>>>œ ÷Þ 'š æÍ'§ª¿]‘:læÌ™têÔ‰aÆ‘››kt‘:GE"b•>äCœqæ>î3:ŠˆˆH¥±³·ã¡‡âÓO?¥ Üìê}ôõêÕãþûïàùçŸ'((ˆùó犷·7/½ôÝ»wgêÔ©³~ýzxàš5k†‹‹ :uâ?ÿù~~~†œŸHѵ+,Xû÷Ã}÷ÁÔ©“'ÃÉ“•r999<ðÀÜ~ûíxxxðüóÏÓ´iSæÍ›gÙçÅ_$<<œY³fѨQ#n»í6,X@AA3f̨”R3©“ˆTšôtسÇRd Ç)Ï 'F3š-la#‰ ‚±Œ¥1ËXrð–Žoooo¯_Ö­3u¹Z¹–-ƒgž©ÚÛX¸p!çÏŸçOúeeeFG©STd "V§Œ2>àþÈqÅÕè8"""•êÑGåÌ™3,^¼€ÒÒRæÍ›Ç}÷݇››EEEüøãÄÆÆboo_ẽ{÷fÕªU€é ™–-[òÚk¯ñùçŸkæ‘+ ‚7ހÇáùçaöl†±c!;û–mggGLLL…u­[·æàÁƒ°aÃbcc+ìãããCdd$?ýôÓ-ݾÔlêd "•&1ÜÝ!2Ðã\]Ìa9È&ð5_ÓŒfÄC"‰”ñû>À­Õ¯_Š‹ME¦½z™º\mÝ ÑÑU»""""""""bѨQ#-ZÄwß}Ç믿nt‘:EE"bu–±Œýìg cŒŽ"""R邃ƒéß¿?~ø!ÉÉÉdffòè£pêÔ)Š‹‹yýõ×±±±©°L:•“åfaÿúë¯iÑ¢£GÆÇLJ=z°`ÁCÎK¤Fsw‡ñã!#^y-‚¦MaÌÈÉù]‡ôññ¹l ››§OŸ 77—ÒÒR|}}/»®ŸŸ'Nœø]·+µƒ½½=%%%FÇk”‚ƒ Ç¹>?üÏxÒIçk¾àNî¤%-™ÎtNrs]jíë—mÛ [7˜1^¾ýÔñMDDDDDDDÄ=zôàÕW_eâĉ$''GD¤ÎP‘ˆX7xƒh¢iMk££ˆˆˆT‰1cÆðÓO?±ÿ~>øàÚ·oO×®]ðððÀÎÎŽÉ“'SVVvÙRZZj9NË–-Yºt)§Nâ»ï¾£qãÆŒ1‚¤¤$£NM¤f«_ßÔÅ`ß>˜6 ¾ú Z´€©SáìÙ›:”Í5·{zzbkk˱cÇ.Û–““ƒÏMÝžˆˆÔAçÎÁÊ•0x°e•äFÙaGq$“Ìnv3ˆA¼Â+ÌÆ°í7|¬ZõúåÜ9xáèÚêÕ3u/;®sß(mMç IDAT‘ªõÌ3Ïp×]w1jÔ(²²²ŒŽ#"R'¨È@D¬Ê6ð?0 FG©2±±±øûû3cÆ –,YBBB‚e›³³3}úôá›o¾¹áY°ëÕ«GÿþýY¸p!NNN¬_¿¾ª¢‹X7öï‡gž7Þ€ÐPxë-(*ª”›pvv¦k×®— š;yò$«W¯¦wïÞ•r;""bžÿŠ‹aÀ€¾ŠäJZÒ’YÌ"‹,ÞäMV³šö´§3™Ï|Š)¾æõkÍë—Å‹¡ukxï=Óó»•+¡yóÊ9¶ˆˆˆˆˆˆˆˆÜ>úè#¼½½‰§¸øÚïI‰ˆÈ­S‘ˆX•Wy•®t¥/}Ž"""Reìííy衇˜;w.ööö<ðÀ¶¿ùæ›ìÝ»—x€Ý»w“ŸŸÏž={xë­·xöÙgÈÌÌdÈ!¤¤¤pâÄ òòò˜;w.EEEÜqÇFœ–Híãæ/½ééð§?ÁĉЦ $&VÊá§L™Bjj*ãÆãèÑ£¤§§3bÄxþùç+å6DDÄŠ%%™fc÷ó»©«éñG®ÆwH •T’I&”Pæa‚ f8Ìá+^¯Æ¿~Ù·þðˆ‡>}à×_áÉ'ÁVŸˆˆˆˆˆˆˆˆÔ$nnn,\¸mÛ¶1qâD£ãˆˆX=½K."Vc7»I$‘‰èI¤ˆˆX¿‡~€áÇãááQa[ûöíùïÿ @ïÞ½ñööfèС>|Ø2H'((ˆ1cÆðÆoвeKù׿þÅ¢E‹Td r³||`Æ Ø»ºw‡;ï„~ý 5õ–Ãwß}dž  ¡C‡8::²zõj‚ƒƒ+)¼ˆˆX¥²2X¶ ¾é«êñG®Ç¢‰f! 9ÈAHàC>¤ÍÎpRH¹ì:5òõËùó0i’©H4;ÛÔ¹`þü›.Ì‘êÓ¦MÞÿ}Þ|óM/^lt«fot‘Êò¯Ñ’–Ägt‘*wàÀÆŒsÅí­ZµbÁ‚×<ÆàÁƒü;ž‰ÈUš¦=ú(Œ :ÁãÃ+¯˜ºüÏo¼Áo¼qÙÕ¯tŸ‰‰!&&¦Jc‹ˆˆÚ²²²*èñGªBc3‰I¼À ,a oñ1ÄЉNŒa #I=êÕ¬×/¥¥ðÉ'ðâ‹—¯¿þ3Øëã‘ÚàþûïççŸæÁ$<<œV­ZIDÄ*©“ˆX…L2YÀ^àlõ§MDD¬Ü‰'˜8q"½zõ¢{÷îFÇ‘KÝ~;lÜï¾ Ÿ~jš÷ÛoN%""uÉÒ¥àï:Dê'œˆ'žÕ¬f#éLgžæi`̉1<;ñÙšñú几sgxè!4~ýž|R""""""""µÌ¬Y³hÑ¢#FŒ   Àè8""VI#qEÄ*Lg:þøs÷EDD¤JEGGãïï ü±ÑqDäjlmá‘G`ï^8Ð4“ôðápì˜ÑÉDD¤.HJ‚ØX°±1:‰ÔAD0‡9ä £ò¾ÿûl±ÙBÉÇ%|Á”PRý¡öì1=ë×||`óf˜3üüª?‹ˆˆˆˆˆˆˆˆÜ2ggg/^Lff&ãÆ3:ŽˆˆUR‘ˆÔz9äð13žñ8à`t‘*•’’BQQkÖ¬!44Ôè8"r=ÞÞ¦lK—ºu_}et*±fÇŽ™:ê lt©ã|ñ%-% EX²f .¡.Ü˽´¢Ó™ÎINV}ˆ'`ìXSg©;MÏÉ’“¡]»ª¿m©RÁÁÁÌ›79sæð駟GDÄê¨È@Dj½7ywÜyŽ"""""reƒ›¶Ýu þ3äçJDD¬QR88@ß¾F'À[âˆ#™d~åWîæn¦1Æ4f4£ÙƶʿÑÓ§aòdhÚ¾üÞRSU|#"""""""beâââxâ‰'xì±Çøõ×_Ž#"bUTd "µZ6Ù¼ÍÛL`.¸GDDDDäêÜÜ`î\Ó@·… !"¶UÁ :©Û’’ OÓãŽH Ó‚Lc™d2‹Yla è@g:3ŸùS|k7pú4L™b*.˜5 Ƈ={àO[}""""""""bÞ|óMÚ´iÃðáÃÉ×$_""•F懲H­6™É4¤!æÏFG¹1wÝ[¶€tïŸ|bt"±ÅÅ’¢ÙÚ¥ÆsÃH%•_ø…PBy˜‡ "ˆ Là‡nÁßþ!!ðÖ[ðôÓpà€i]½zUr""""""""R3888°`Á²²²xöÙgŽ#"b5Td "µVi|ÌÇLbN8GDDDDäÆÁO?Á“OÂèÑ0q"”–JDDj»U« 7 2:‰È ‹"Š…,$ƒ Æ0†øˆÛ¸á '…”«_±´ÔTT3l˜©sÁÇóϚŠ ^z <<ªï$DDDDDDDDÄPÁÁÁÌ›7÷Þ{O?ýÔè8""VAE"Rk½È‹„ÊHFEDDDDäæÙÙÁŒðé§0s&ÄÅÁ™3F§‘Ú,) Z·†Ûn3:‰ÈM €ILâ0‡ù„OÈ&›bhMkf1‹sœ3í¸iŒk*ÚŒ‰ÌLxçHO‡_TqˆˆˆˆˆˆˆHÇO<Ác=Ư¿þjt‘ZOE"R+mc‹XÄ4¦a½ÑqDDDDD~¿#L³ðnÜýúÁÉ“F'‘ÚjéR<Øè"·ÄGâ‰g«ØXº^¹í˜xáyŸ÷bÌ¿ë±{TgHN†‡†;MÏ¡ÀÙÙèè""""""""b°7ß|“6mÚ0|øpòóóŽ#"R«©È@Dj¥çyžÎtæNî4:ŠˆˆˆˆÈ­‹Œ„µkáØ1S¡ÁñãF'‘Ú&=öìQ‘üöî<®ªjãøç0#ƒ8”³¦b9d˜×Ä)ƒÔmPÌr¨‹B¥¦•¥ÕíJ¥e9–Š¦ýÈ-ÁÑÔ4s6ÍyH1áüþ8WAöAŸ·¯óB÷^gíg¯³#tïïZEGj*\¾ ¿ÿ{ö@l,ÌšŸn)hÒ/÷§™Zb§ªØñ鞬êèDÝ}6øî«@اuI­SÓè³+booÏìÙ³9q⃠2:ŽˆH‘¦é¿E¤ÈYÏzV±Šh¢1a2:ŽˆˆˆˆHþ¨V Ö¯‡Ö­¡E ˆŽ†råŒN%""EEx8¸»[ ×D RZœ>m)øýwK¡@|<\¹ò÷×[/³ÙòžË—ÿ~ÿÕ«YÍ$çà*À#@ãÆ–Õ 4À£aCØÛÓŸ4Ö°†ñŒ§+]©F5úЇÞô¦¥ áäEDDDDDDDÄÚU«ViӦѵkW|}}yþùçŽ$"R$©È@DŠ3f>àžåYZÓÚè8"""""ù«reX»Ö²šAÛ¶–}ÝÜŒN%""EAD„åÿööF'‘ûÉ_Á/¿ÀŽ°kìÜ Br²e¿ƒ”(Å‹ƒ‡Çß_«Vµ|µùßbÊîî`kkù½«ë߯âÅ-¯[¿/[öï÷dÁ|þ÷ë‡øŽïÍh†2üy—wyœÇ xPDDDDDDDDÄÚuéÒ…¨¨(þýïÓ°aCªT©bt$‘"GE"R¤Ìa¿ò+[Ùjt‘‚Q©¬^ ÿútëK–üýPžˆˆHV,«á„„DŠº„ËjJ?ÿl¹¦ââ 5Õ²º@ýúСÔ©c)"xä(_>Û¢€‚TƒŒbÿá?Ìa“˜DCâ…Ò“ž8ádH61Þ„ øå—_èÑ£k×®ÅV÷ÛDDrŘýɃk\c0ƒéMoÒÐè8"""""§reX¶Ì²ªÁFDD¬ÝªU’mÚDŠ¢sç`útðóƒÒ¥á…,…M›Â¼ypö,œ< ‘‘ðùçУ4k+V`p;W\ $]ìb+[©CúÑGx„! á'ŒŽ(""""""""prrbÎœ9lݺ•‘#GGD¤ÈùG+œ:uŠŸ~ú ‡üÊ#"rW?6þ‘˵/óèüG ¹®™ù$çŽ """’{^^ðÝwðÊ+а!tïnt"±Vи1”)ct)*þü-‚9s,+8:‚Lšd)6xøa£æ‰^„ÊhFJ(™ÈÆÐŽv `Ïð &LFÇ‘BR·n]FÍÀiÞ¼9O?ý´Ñ‘DDŠŒTdpöìYžzê)¼¼¼ò+ˆH–s˜·y›1Œ¡_Ï~FÇ‘"$$D)"""R„½ü2lÞ }ûZf ~䣉ˆˆµ1›aÅ xã £“ˆµKL„¥K-…+W‚­-tèaaж-+ftÂ|SŽr f0ïðKXB!øâKMjò&o@®¸SDDDDDDDD A¿~ýˆŽŽ¦gÏžÄÅÅQªT)£#‰ˆ Ưe,"’€'ž¼n˜‹ˆˆˆÈfÔ(KqA÷îšjt±6;vÀ©Sо½ÑIÄݼiYé¢{wËJ=z@J L›gÏ‚ðâ‹÷UÁíp  ]ˆ"Šml£%-ùˆ¨@‚bûŒŽ(""""""""Ìd21sæLlll 4:ŽˆH‘¡"±zËXF$‘Lbvÿl‘¢ÇÑ~ø¶n…/¿4:ˆˆX›eË \9xüq£“ˆ5Ù·† J•,«ìß#FÀÀòåг'¸»²P=ÁLe*§9Í0†Eu©‹/¾„ÆMnQDDDDDDDD H‰%øá‡X²d S§N5:ŽˆH‘ "±jÉ$óïñ2/ÓŠVFÇ1Fýú0l˜åuø°ÑiDDÄšDDX"7™ŒN"F;Ƈ† á±Ç , ÞxŽµ+eËÒpÅ)Îp˜Ã¬b%(A7ºñL0¹htD)-Z´`È! 8]»vGDÄê©È@D¬ÚW|Å)Nñ_EDDDDÄXï¾ 5kBß¾F'kqá‚åáñöíN"FIIŸ~‚N bEøïá‰'`ýzKaâСPµªÑ)­’ 6øàðŸýt§;™HE*â?ÙhtDÉgÁÁÁ<ñÄtïÞ7nGDĪ©È@D¬ÖQŽ2‚|ÄGT¢’ÑqDDDDDŒegS§ÂêÕ0ožÑiDDÄDD€½=´nmt)l'OÂèÑP½:tî —.Á¤Ipê|÷4o®Õ-rÁOF1ŠSœ"„ro¼iD#B!‰$£#ŠˆˆˆˆˆˆˆH>°³³cÖ¬Y?~œ>úÈè8""VMEˆÃ‡c2™ðññ1:J‘³uëVL&ÁÁÁÊQÈÞäMªQA 2:ŠˆˆˆˆˆuhÜ`Ð HÒÃn""¼ˆhÕ ÜÜŒN"…!5ÕRlèï<ãÇÃ+¯XV,ˆ…À@pu5:e‘æ„=éIqle+u¨C?úñ0„!ç¸ÑEDDDDDDDäªZµ*&L`ìر,_¾Üè8""VKE"@ll,&“‰#FEþçÿø?V³š©Lţ㈈ˆˆˆX‘#᯿`Ü8£“ˆˆˆ‘RR,œ·oot)h·V-¨ZÚ´Ë—aî\8~F²l—|ç…¡„r‚¼Ë»Ìf6Õ¨†~¬f5fÌFG‘<êÕ«þþþôéÓ‡K—.GDÄ*©È@ä5j„Ùl6|kÉQ.r‘÷yŸ~ô£)MŽ#""""b]zÞ}>ÿΟ7:ˆˆ%6®\víŒN"!»U ¢¢ K°·7:å¡,eÌ`Žp„yÌã:×ñÅ—ZÔb4£¹Â£#ŠˆˆˆˆˆˆˆHL™2[[[Ž""b•Td "VçmÞÆgF •%DDDDD²ôÞ{àâb™ÕXDDLP«xzDò“V-°Z8Ð….DÅ>öÑ–¶ g8•©LAìaÑEDDDDDDD$<<<øî»ïX¼x1¡¡¡FDZ:*2L’’’øôÓO©]»6NNN/^œgžy†•+Wfh·bÅ L&ãÆã—_~¡U«V¸¸¸PªT)zõêÅŸþ™©ïË—/óÖ[oQ¶lYœiԨ˖-ãûï¿Çd2±páÂe\¿~=¯¾ú*žžž8::òÐCáççdž 2µ5›Í|ÿý÷´hÑÜÜÜxòÉ'™6m7oÞdĈ4oÞ€O>ù“É”þغu+&“)Ó 9NYÉ*G~õ}{?ÑÑÑ4mÚ”bÅŠñðÃÓ§O.\¸p×öëÖ­£eË–¸¹¹Ñ¨Q£\ d¾&j¼]ƒ¹Ìåùèçq7¹g¸&îu\³ÙÌŒ3hÚ´)nnn8;;Ó A¾ùæÌæ¿—/OMMeâĉxyyQ¢D <<}:±±±üú미ººfjÿþûïsó¦å¦YZZZ®Æ&Ó5á‡ß9 sáÜ’sÙæ¼ó¸f³™=z0{öì mwíÚE¿~ýعs'!!!|øá‡|ùå—ÚmÛ¶mÛ¶áàà@¿~ýrÕNDDDDÄðÙg0q"|ú©ÑiDD¤0= À”)F'‘âÜ9KÁà”)–ß·m ‹Aûö`§º/ ŠSœ@éMoÖ°†BèF7ÊP†>ô¡/}yˆ‡ŒŽ)""""""""Ùøì³ÏˆŽŽæµ×^cݺuØÚÚIDÄ*h%É`Ò¤IlØ°Ê•+N||<'Nœ 88“ÉÄ;ï¼ÃÙ³g3¼gÎœ9¼þúë:tˆÄÄDbcc©\¹2ëׯgçÎéí&NœH\\5kÖ$::š¿þú‹cǎѯ_¿LEÙ1™LøúúÎüArr2çÎcÁ‚8::2jÔ¨ô¶3fÌ`þüù”*UŠ)S¦pâÄ ®]»Æ¯¿þJïÞ½±··ç?ÿù111 >³Ùœþ2bœò"¿ú £G:tˆk×®±~ýzêÕ«Çþýû=zt–í{öìɸyó&Û·oÏÕØÜyM¼yùMJT)Aï=½³½&²:î¬Y³˜={6õêÕ#22’K—.qíÚ5Ö­[Gƒ ˜6m›6m`ñ⟸¸°hÑ"®\¹BBBqqq 4(C!ENÛ‰ˆˆˆˆ¢X1èÛ&M‚¿þ2:ˆˆ¦ðppwoo£“H^lß½zA•*–bÁ×_‡cÇ ":uRAdƒ >ø°€à=èÁ$&Q‰JøãÏ2¯À+""""""""ÖÁÑÑ‘Ù³g³}ûö ÏŠˆ<èTd „……0þ|:t耻»;•*UbèСôîÝ›ëׯ³téÒ ïyöÙg™š^Á™›±¹ýš°imÃTû©Œ³Ç´‘Ó²½&²:îÌ™3±µµeåÊ•´k׎’%KâââB‹-˜3gK–, bÅŠ”/_žŽ;R¼xqŠ+Fƒ øòË/yíµ×Ò“Óv"""""†éßnÜ€ÿýÝGDD–YïííN"9•–f)ñõ//ؼF†ãÇaäH¨\Ùè„’OªSQŒâ$' !„C¢ÍhD#B!‰$£#ŠˆˆˆˆˆˆˆÈ{ì1FŽIpp0›7o6:ŽˆˆUP‘dpøðaJ•*E“&M2íëСCz›ÛµjÕ*SÛjÕªð×m³i9r„ *ðØcejߦM›gܸq#M›6%,,ŒÓ§OsóæÍ û“’þ¾I³ÿ~J”(OŽûω‚§¼È¯¾Ÿ}öYL&S¦~}ôQŽ9’©½O¦ö¹›[×D¥Ç*ñ¯á‡=é dMduܽ{÷’ššJ¥J•°³³ÃÖÖlllÒ¯¹'N0vìXÒÒÒðôô$((ˆo¿ý–;vd:NNÛ‰ˆˆˆˆ¦dIèÖ ¾ý²YMDDî# °~=´ootɉøx?ªUƒçŸ·l[º~û  °¬L$÷%'œèIOv°ƒ­lÅ /0€ò”gøߎ(""""""""·8p O?ý4½zõ"11Ñè8""†S‘drçÃÛ÷âìì|×>Ìw<är·¾ïl—Q£F‘œœÌСC9|ø0III¤¥¥a6›©Y³f.’ÿ39N¹U}g§T©RYnÏÍؘL&0€D™ÊÔôíÙåÎê¸iii¤¦¦’ššš~MÜÞOrr2 4`ÿþý„††RµjUbbbhÛ¶-=öX†•0rÚNDDDDÄP}ûZTŒ5:‰ˆˆ†¨(HI\LÚ!8xÐRDP¡ü÷¿–Ïkï^Ëççç¹ü·E)Ú¼ðb*Sùß³˜êTÇ_ njŠEEDDDDDDDŒf2™˜9s&çÎã£>2:ŽˆˆáTd xzzrñâE¶lÙ’i_dddz›¼¨^½:'Oždß¾}™öEEE帟£GR¦L‚ƒƒ©^½:NNN˜L&Ž9¡C‡2´­U«—/_&:::Û>ml,ÿ)ܹ*ÂÝä8iÕªU™î?zô(¤zõê9ê#7cS½zuNzä{¾g2“)C™ô¶¹¹&ÀòY+VŒ+W®¤ÜùZ¸paz{;;;Z´hÁ!C˜;w.ÇŽãêÕ«dè7§íDDDDD óøãðä“0eŠÑIDD¤0DD@ãÆP¦Ì½ÛJáJKƒÕ«-EµjAd$|ò ?S§Z¶É­ eÌ`Žr”Å, ¨IMF3šË\68¡ˆˆˆˆˆˆˆÈƒ­B… L˜0‰'²víZ£ãˆˆJE’A—.]èÚµ+‘‘‘\½z•“'O2|øpBBBptt¤cÇŽyêû¥—^Âl6Ó¹sg~þùg8~ü8ï¼ó«V­Êq?•+WæüùóLš4‰øøxâã㉌Œä¹çžKŸÍþ–^½zЭ[7¦M›ÆÉ“'IHH`Û¶m²nÝ:J–, @LL —.]ºg†‚'#mÙ²…€€>LBB±±±¼ð ¤¤¤Ð¹sçõ‘›±iÛ£-æ)f<~ô ÔÏ¥ò|M˜˜ˆË–-ãÂ… $''süøq"""x饗ҋMš6mÊ”)SØ·oIIIÄÇdzbÅ .]ºÄÑ£GÓûÌi;Ãýûß°x1\»ft)Hf3,_íÛDn÷×_u낯/\¾ óçÃþý0x0xxP¬Œ-¶øáGQüÆo´£ÃNªD»Ñ*ª""""""""FéÑ£/¾ø"ÿþ÷¿¹zõªÑqDD cgt)\ÑÑјî²÷ðáÃùàƒX´h7n¤}7+ÇGÙ²eótìþýû3{ölvíÚÅÓO?¾Ýd2Ñ¥K°··¿g?AAA,_¾œþýûÓ¿ÿôí 6¤nݺœ9s&}Û믿Ί+X¸p!™úò÷÷ FT¨P5kÖPºtéôýwÎêK¿~ý lœŒÔ¹sgBCC™9sf†íµjÕbðàÁ9ê#7c³ëÍ]8œsàÊëWxújÞ¯ °”¬[·Žï¿ÿ??¿,ÛôéÓ€íÛ·³iÓ¦lÛ䦈ˆˆˆˆáüýaÀXº^yÅè4""RPvì€S§Td`-΃ɓaÂHJ‚.]`ÁK±HÕ¤&ãÏp†3yŒg>>>”(QªU«ÆóÏ?ÏO?ý„›7o¦oß¾Ô©S'ý<½½½™>}:cÇŽMï3§íDDDDD W²$<ó Ìkt)HË–A¹rðøãF'y°íß½{C•*0u*|ðœ> ¡¡*0äC–°O<ñÅ—pÂ1£¥ÞEDDDDDDDò“¯¯/¼õÖ[œ;wÎè8""…F+H¡ªQ£3gÎ4:†Ü¡mÛ¶˜Í{óéçx×ñÇŸ^ôJß®kBDDDD$=÷|ü1ìÚ*Ú¹¤¤@T ft’û×ùóðí·0q"$'ÿÿ ｕ+L$Kó0ƒÌû¼OL`èDuªÓ›Þô¡%)itL‘ûÂW_}ETTo¾ù&?þø£ÑqDD …V2‘—FÝéŽ+®„bt‘ûWýúP²$ÄÄDDDòSl,\¹bY±Fò×ñãз/T©S¦À»ï‰–Õ T` E€ 6øáGQìg?/ñ£E*Гžìb—ÑEDDDDDDDŠú(={öÌ°=!!O?ý”úõëS¬X1ÜÝÝ©W¯ÄÄÄøìòÿÄ¢E‹0™LüüóÏ@Ñ;—Ûó>|“É”áåèèH•*UèÕ«4:n¡ËÏñÉHbˆa>óqÇ=þ­[·b2™ÎsÖÜöñO™Õûóãg8CWºÒô§–ÇjÔ¨f³ù­Û>ò㘅ÑçÝ 4ˆÝ»wVàÇ‘"ÈÛΞ…ãÇN"""ùáèQ8pÚ·7:IѶgôì 5kBl,|÷<€££ÑéD L3š±€ç8A1ƒT§:þø³šÕFÇ)RÜÝÝ™2e óæÍ㧟~2:ŽˆH²ž§¾ïc?þø#çÏŸ§gÏž¶oÚ´ €©S§Ò¹sg~øaœ©]»6¯¿þ:?ÿü3uêÔ1"ò=mÚ´‰ .àçç—þg(:çrgþ¬¸¸¸àííþðüîÝ» +žáòc|nr“—yW\ùžï1a*ÐÌ’gžy†råÊ1eÊ££ˆˆˆˆˆ5züq°±¸8£“ˆˆH~wwK™äÞÎàïõë[þß8cìÚe)8¸mr ‘û]yÊL0ð³˜ÅiNã‹/Oð!„@‚ÑEDDDDDDDŠ„víÚÑ£GúöíËåË—Ž#"R`TdPBCCqvv¦mÛ¶¶{xxP¼xñ{ö1bÄš7oÀ'Ÿ|‚ÉdJݲ~ýz^}õU<==qtt䡇ÂÏÏ 6dèkÅŠ˜L&ÆǺuëhÙ²%nnn4jÔ(GÇÇÎÎŽvíÚȹ$%%ñ駟R»vmœœœ(^¼8Ï<ó +W®¼ë¹DGGÓ´iSŠ+ÆÃ?LŸ>}¸páB–îÌŸ³Ù €««k¦í3fÌ iÓ¦¸¹¹áììLƒ øæ›oÒßššÊĉñòò¢D‰xxxШQ#¾þúks<&F~¾yŸ÷xÍ77S¬{1*ºU¼ëølݺ“É”a€ÛÏá—_~¡U«V¸¸¸PªT)zõêÅŸþ™áXwöq¯óÌ꘹ã¬dÕ§««k†cßùÚ¿†qÌÉõ`kkK§NøùçŸ9qâÄ=³‰ˆˆˆÈÆÕªUS‘ˆÈý""Ú¶{{£“-7‚Ÿ4l‡Áüù–‚ƒž=ÁÖÖèt"†qÄ‘.t!–X¶²•'y’ ¤"ˆßøÍèˆ""""""""Vo„ ØØØ0hÐ ££ˆˆ°›7oÃO<ý7oÍÿÆo°lÙ2®]»–çãœ={––-[2gÎŽ9Brr2/^dÙ²e´jÕŠõë×gzÏÆñññaýúõ\»v´´´/<<œæÍ›§äç¹$''ãëëKpp0û÷ïçÆ\½z•5kÖЮ]»,go߸q#mÚ´aÓ¦M$%%qá¦OŸN‹-²Ìrgþ¬$&&²aÃÞÿ}€ Ü›ÍfzôèA@@›6mâÚµk\¿~]»vѯ_?‚‚‚ÒÛ~øᇼýöÛlß¾+W®϶mÛxï½÷˜1cFŽÆÄèÏ7+ÙÏ\ó\&07Ø9{g¶ã“_ý•V­Z±nÝ:ùóÏ? å…^Èñ¹äT^ÆøŸ(_¾<?ü0»ëé–&Mš°fÍš|Í%""""÷‰Ç·e«¨K]|ñ%Œ0RI5:¢ˆˆˆˆˆˆˆˆUòððàÛo¿eÆŒ™&N¹_¨È €íÛ·k׮ѠAƒLûºuëÆ AƒØ·o~~~xxxP§N^{í5~úé§ …ÿç?ÿ!&&€áÇc6›Ó_&“ ___ÂÃÃùã?HNNæܹs,X°GGGF•éøaaaôìÙ“póæM¶oß~Ïã=z4=sAœË¤I“Ø°a•+W&<<œøøxNœ8App0&“‰wÞy‡³gÏf:—=zpèÐ!®]»Æúõë©W¯û÷ïgôèÑÚf•ÿ–èèèôYæ]\\hÖ¬{öì¡W¯^ôïß?½Ý¬Y³˜={6õêÕ#22’K—.qíÚ5Ö­[Gƒ ˜6m›6m`ñ⟸¸°hÑ"®\¹BBBqqq 4(}ökÿ|s3>»ÙÍ멯ÃX¨·ãÞã“9sæðúë¯sèÐ!‰¥råʬ_¿žÙ<0•“ó¼S^Æø^®]»–áØf³™+VP¹reÖ¬YCÉ’%Ü]O·<þøã9GyÕ­ ûöBDDþ©¨(HI6mŒNbýV­‚fÍ ysHL„µkÿ.8PqH¶<ð`8ÂV²'œèJWjR“ь旌Ž("""""""bu:vìÈK/½DPPýõ—ÑqDDòŠ Ø™3g(]ºt–û¿üòKŽ=ʸqãèܹ37oÞäÿþïÿxñÅyꩧ¸råJŽŽS¦L>ÿüsBCCù׿þE±bÅ(S¦ þþþ$$$°{÷îLïiÒ¤ Ó§OçÑGÅ6K„/]º ÓCèùu.aaaÌŸ?Ÿ:àîîN¥J•:t(½{÷æúõëéniܸ13fÌÀÓÓš7oÎâÅ‹±··gáÂ…9Ê75kÖdØ°aV¢˜9s&¶¶¶¬\¹’víÚQ²dI\\\hÑ¢sæÌ`É’%T¬X‘òåËÓ±cGŠ/N±bÅhР_~ù%¯½öZŽ2XÃç{7·Ï_ü…?þ8pÂöãœOvž}öY&OžŒ§§'ÎÎÎx{{óÁd[dyãÜZ»v-}úô!""‚š5k¦oÏÍõtËC=üý=FDDDD$OO8v R5û¬ˆH‘O> eÊÄz­]k),hÓÜÝaÈŽ†V­ŒN&RäØ`ƒ>„ÎЙΌf4©HOzGœÑEDDDDDDD¬ÊäÉ“ILLä£>2:ŠˆH¾S‘A»õ`½››Û]ÛT®\™0oÞ<<ȱcÇèر#[¶lI ú^6nÜHÓ¦M ãôéÓܼy3Ãþ¤¤¤LïñññÁ”‡Y¼ÂÃé]»6žžžr.‡¦T©R4iÒ$Ó¾:¤·¹Ý³Ï>›é\ªU«Æ£>Ê‘#Grœÿ™gžIŸqþÌ™3|ôÑGìß¿???’““ÓÛíÝ»—ÔÔT*Uª„¶¶¶ØØØ`ccÃc=À‰';v,iiixzzÄ·ß~ËŽ;r4·XËç›Ýø\O¾NwºO<öÝìIMÊÙød§U7ƒ«U«ïÕŸyãÜØ°a={ödÉ’%Ô­[7þÜ\O·¸»»ä¸xGDDDD05j@r2äàçn±Rf3,_íÛÄ:mÚ¾¾Ðºµe¬~þ"#¡iS£“‰ÜjPƒQŒâ8ÇÏx∣! iD#B %…£#Šˆˆˆˆˆˆˆ⫯¾âÛo¿%&&Æè8""ùJEÌÃÀ«W¯æø=<ò³fÍÂÆƆÈÈȽgÔ¨Q$''3tèP>LRRiii˜Íæ 3¦ß®T©R9ÎtK||<1119žå>/çäéáøœÈMþ²eË2räH^{í5víÚÅ”)SÒ÷¥¥¥ššJjjjúX›Íæô6·Š4hÀþýû ¥jÕªÄÄÄжm[{ì±ÏŽoŸïãóüÎçYÁ ƒÿM®Ÿ“ñÉŽ³³s¦m·®ÛûÊyãœÚ²e /¿ü2‹-¢aÆ™öçæzº%>>€%Jü£l""""rŸªQÃòõÐ!csˆˆHÞíاNÁÿ&ÝÿÙ¼üü,ÅII–U bc¡eK£“‰Ü—Üp#@v±‹b¨F5 •Âþà£#ŠˆˆˆˆˆˆˆªG´mÛ–Þ½{ÿã‰lED¬‰Š X¹rå¸xñb®ÞOZZþùgú6ËÇuç ëG¥L™2S½zuœœœ0™L9r„C¹|¨$»ã¬X±‚”””äå\<==¹xñ"[¶lÉ´ïV¡Â³ì¯Zµ*ÓCçGåàÁƒT¯^ýåÿì³ÏprrbÔ¨Q\¿~€ZµjQ¬X1®\¹’þ0ø¯… ¦÷aggG‹-2dsçÎåرc\½z•€€€‰5¾Ÿ}öö/Û³²ÑJ¾Nùo¼s=>!»óÌJ~ŽñívìØÁK/½Ä¼yóhܸq–mò2^·¾§Üú#""""’A‰à DDŠ²eË \9xüq£“X‡Ý»Áßžz .^„¥K-Å­[LäÑŒf,`'8Á;¼Ã,fQ•ªøáÇjVc&'†)*¦L™Â™3g9r¤ÑQDDòŠ X:upuu%...Ó¾>}úÄòåË9räׯ_çÒ¥K¬\¹’Ž;dx(¹dÉ’ÄÄÄpéÒ¥ }U®\™óçÏ3iÒ$âãã‰'22’çž{.}–ôœÊî8K—.¥T©R<õÔSv.]ºt k×®DFFrõêUNž<Éðáà ÁÑÑ1½Ï[¶lÙB@@‡&!!ØØX^xáRRRèܹó=óg§\¹rôîÝ›3gÎ0mÚ4HLLÄÇLJeË–qáÂ’““9~ü8¼ôÒKDGGдiS¦L™Â¾}ûHJJ">>ž+VpéÒ%Ž=š£11úóÍÎÙrg1ÿŸÆCÚ”´\OAÉî<³’Ÿc|Ëž={èÔ©¡¡¡x{{ßµ]^ÆëÖ÷”&Mšä)›ˆˆˆˆ<Ê—‡Ó§N!""yaYÅ €Vü,2öîµ4hÀüù°i“e51D9Ê1˜Áå(s™Ëu®ã‹/u¨ÃxÆ“@‚ÑEDDDDDDD U¥J•1b_~ù%{÷î5:ŽˆH¾°3:ÀýÎÎÎŽfÍšñóÏ?sãÆ Ó÷]¸p%K–’å{ÝÜÜ3fLúŸkÔ¨A… X³f ¥K—Nßn6›Óðïß¿?ýû÷Oß×°aCêÖ­Ë™3grœùnǹyó&Ë—/ÇÏÏ[[Û ïÉÏséׯ‹-bãÆ´oß>S_ãÆ£lÙ²¶uîÜ™ÐÐPfÎœ™a{­Zµ}ؾ};›6mʶ X÷ç{7g9K':ñTÚSlþx3_”Ìýø”ìÆ3+ù9ÆçÏŸÇÇLJsçÎÑú.3êýöÛoÔªU+OãõË/¿ðôÓOç*—ˆˆˆˆ<@Ê•ƒ\þ+""VâÂغ>úÈè$Æùí7øüs˜3j׶t +â€]þ÷k;Û™ÊT>â#þËy™—ÀêPÇè˜""""""""…¢_¿~Ì;—€€6n܈æ‘¢MßÅ A=¸~ý:‘‘‘¶OŸ>iӦѡC<==qttÄÙÙ™ZµjÑ·o_vîÜI£FÒÛÛÚÚ²páBš5k†‹‹K†¾:uêÄìÙ³©_¿>ÎÎΔ+WŽ   ¢££36äÄÝŽËåË—³|:?ÏÅÁÁÕ«W3tèPjÖ¬‰ƒƒnnn<ýôÓ,_¾œ7Þx#Óñ½½½Y¾|97ÆÙÙ™Ò¥KÀúõëquu½gþ{©X±"½zõâäÉ“|÷Ýw˜L&fÎœÉüùóñññ¡D‰888P­Z5žþy~úé'|||ؼy3}ûö¥N:éÙ¼½½™>}:cÇŽÍјýùfå:×y(F1–:-¥×«yŸ‚’Ýxf%?ÇàôéÓœ;w.Gms;^©©©,Y²„-ZP¥J•\g‘„V2)º"#ÁÞî2qÁ}í÷ß!(êÕƒmÛ`Æ Ø¹ºtQˆ{‚'˜ÊTNqŠa #Š(êR_| #ŒTRŽ("""""""R lll˜:u*Û·oç»ï¾3:ŽˆÈ?f2ßmZïضm^^^ùè~”œœLÅŠiÖ¬?þø£ÑqòlРALœ8‘‹/âææftV¬XA»ví;v,̶­5æ·&¹ŸXÈB6±I³Q²U«VѦMæÎËË/¿lt«B```gÁ‚tíÚõ®«eä‡Â:±N&LÌg>þøEDŠŠ,gÆÆDrÉßßò½~Á‚~¬‚þÓd21þüôs*HúyYî+þþ+W¤ðœ:ÆYŠ <=áÃáÕW!—+’ŠˆuH#5¬a<㉠‚ªT%@ 4¥ï݈ˆˆÜÜß×ß±DŠžÂ¸_)"""÷öÞ{ïñÝwß±oß>Ê—/ot¹~^’¢¦°þn~·ûÓZÉ 888ðñdztéR:dtœ< §eË–Eöý¢ž¿ åf|>ã3B %Œ0`̘1Ô­[—.]ºEDDDD¬™‡‡åU)ZRR * Ú·7:Iá¸t ÞjÔ°ULŸ{ö@Ïž*0)Âl°Á ç éB¾à *RüÙÄ&£#ŠˆˆˆˆˆˆˆˆáÇS²dI dt‘DE…ä­·Þ¢Zµj 6Ìè(yvàÀV­ZetŒ<+êù ZNÇg.sùÿa<ãy–g !™ÜnÆ DEE1fÌlu£]DDDD²S¼8\¹bt É­ØXË÷ïçž3:IÁJH€Ñ£-«Ìœ C‡ÂþýЫ—Š Dî3žx2ŠQœâ!„p€4¥)hD!$‘dtD‘|S¬X1¾ùææÎKDD„ÑqDDòLE…ÄÞÞžƒòÃ?E$Ïbˆáu^ç=Þã-Þ2:ÎÉÛÛ³ÙL›6mŒŽ"""""ÖNE""ESDÔªeyøþ~”’!!–• FŒ€  8r''£Ó‰Hr‰žôd';ÙÊVêP‡~ô£*UÂNpÂèˆ""""""""ù¢]»vtîÜ™¾}û’`t‘öÑ–¶ g8U¨BAìfw®úÛÌæJ*"""òÏ999ñâ‹/boŸq¢ÔÔT^}õUƒR‰ˆˆHvFŒƒƒÁÁÁFGÉ–Š D0fÌŒbT–ûÒH#…ÚÓ>}¦'1N;ÚáŽ{†m)¤ð ¯”HDŠ­d "býî,2°·‡Ž¡wïÂÏ2mŒ ii™÷ÙØ@íÚ°s'´o_øÙDDò¨µÏxNsš1Œ!–XêSŸf4#Œ°ôÕ~ïæ ghNs‚ .œÀ""""yðÊ+¯’’qÂ"WWWÚ´icP"ÉŽ»»;Ÿ}ö“'O&..Îè8""w¥"‘Ìr–óÿìÝy\”ÕþðË0€lÈ΀㖠©×@3Ëðj¢¥¹\S²L0ï½Qi©eiÝJK[Ôì^_V¦\7 ýxÓ4í¤¥hš"ë°ÈŽ²Ãùýá}žfŸg€™aù¾{ñšfxæyÎyx|ÎyÎ9ßsnàZ¡¥Ã÷ ZЂ٘D$š9u„BQ&‚s0"ü1‘Ü0 “,˜*BH—A+BHççìüÇÿ[[ °{·ùÓñÍ7ÀòåºßÜ|?ÀàÛoÍ—&Bé@.pA,bq—‘ŠTøÁOâIH!Åz¬×Xù—³ »ÐŠV¼Ž×‹X´ ÅÌ)'„B1lòäÉH$ü{‘H„'žxvvvL!„BôY´hÂÃñbÅ 0Æ,BÑŠ‚ éa6al`£wkXÃ.P@a¦TB!D—y˜‡&ÜŸHæažJÐ!„èTY ¸ºZ:„BôQ2` Ø·ðð0o22€Ù³ ogcÄÅj³cBHWb kÈ!ÇÀ ÜÀB,ÄGøÀÌAÒøm›ÑŒðZІÏñ9þŒ?£uÌ!„Bˆ&[[[• ‚¦¦&Ì›G+"B!™••¶nÝŠÿþ÷¿Ø»w¯¥“C!ZQ!=H:Òqguζd [¸Àk±ÙÈF<âÍœBB!„¨{à ^€&4á òñ¾@>ò1“1#°;ñ¾RYá -ø?üÆc<ÊPfÁ”B!„hzòÉ'ÑØØðòòÂøñã-œ"B!„2|øp,_¾/¾ø"ªªª,BÑ@A„ô oãm­3ÛÂÎpƬA.r±ëá ¤B!ê¬a§ðÀ>ˆ@„…SDé2h%Béü¸ûôàÁÀúõæ=vY0y2PSs?€@[[@¤Ô–äé œ?ožôBˆ™ˆ!F bpçð~À ÀJ¬Ä¬µZ7Z3šñ ~ÁCxyȳPŠ !„B4EDDÀÏÏ°`ÁX[Óp B!¤+xã7ÐÒÒ‚7ß|ÓÒI!„ ¶–N!=c EEEÈÍÍUù)//G@@¤R) ©T —ö ú¿Û8„Chų҉ ‚-l‡8¼„—àšå”B±´ÊÊJ( ñ¯…Ö…ÀjÀ7Õoœ~~~ðññ¿¿?|||àååK'ÒÙäçÿëX$„ž¦¹¹555¨««C}}=ª««ÑÒÒ‚ÆÆFÜ»wP__ºº:À½{÷økjjÐÜÜ ¨ªªB«Ò ÿÜþôihh@mm­Á4º¹¹a᯿b”­-^4wž^c•zž½½=ŽŽŽ‹Å'''ˆþ àêêÊ&‘üo¥ØÛÛÃÅÅ"‘®b1y%°²ºPÐÜ 0ØÙýû£G>   xxÌ!„teýï¿Ó8 ˜ u›&4á6nc Æà$Nâ<`ÞD©¥¥%%%(**BAAßÞP\\ŒÊÊJxyyñí \{ƒŸŸÜhU4B!¤KÉÍÍÅ<…BOOOܽ{NNN–N!„B H$X¿~=V¯^eË–¡ÿþ–N!„ð(È€©¯¯GNNrss‘——‡ÜÜ\dggóÿŸ——‡††€ |}}wwwdff"77ee,¹ìææ†ÀÀ@!((J¥J¥†ÞÁ…[°V°på;Øáoø^À \@!„˜Á;wP\\¬Ò¡Ï½ò?Ü@7°³³ƒ——àô„¬¿¶ÆW?|…üü|~pp¿.áåå???øúúª ($xyyñƒÏ!Ý\]PZ Y:%„bPSSª««QUU…ÊÊJTWWó?555¨®®FEEª««QWW‡{÷îñAhjjÂÝ»wQ[[‹††À!”ï÷êÕ vvvgggØÚþÑ„jcccp"+++ƒ3›ššpûöm”ÔÖbËÀøoiéýû¶’––TWW«|ÆåîÞ½‹¦¦&šÁúXø@  ÄÆ™öö¸íâ‚ÛÎÎPxz¢ªwoØØÙAbk ‡Û·áRVçŒ ¸ººÂÍÍ ...?4•Ò|‰/!‚MhÒúû&4¡%Ñ8†cG¸™Sx¿,)))ÑhgàÞB¡P ¤¤-J+ÖôêÕ ðöö†››²²²ŸŸ’’>ภæëëËÿ( pí ÞÞÞèÝ»·ÙóN!„û233qøða>|?ÿü3œœœ ‹±víZlØ°“'OÆÌ™3  '„B:­åË—ãÓO?ÅË/¿Œÿûß–N!„ð(È€6ª¨¨àfeeñ?ÜgÙÙÙ|箽½=üüü “Éàïï‘#GB&“ñ?Zü544   @e¿YYY¸~ý:¾ûî;dgg«ÌŒ'‘HTöË5ü{ðÄΡ;Ñ‚Øë±Å_áw³/B!¤»R®h{­¨¨@nn.îÞ½ËG,ÃÝÝ  …\.çßs¯ÞÞÞ|á×øOì~‚ßG}}=ÊËËu÷âÅ‹üÆÿ=‰D___H$ãq¯R©ÎÎÎæ;‰„Ž—}&êà`K§„Òƒ”••¡¼¼\Ыr r¥2±X¬2€ÝÕÕptt„··7D"$ D"œœœø™ý]]]!‰àââÂpÁÊAvvvèÕ«—9O‘¦K—€aÃл¬¨¨ ¤ €á™ ñåË8áãƒBOO”55¡©© •••hll„èÞ=8ü/ˆãöíÛ¨««ãÿV\ ˆrýR™««+œáââWWWxxxÀÝÝ]ãÕÓÓSå3‹ÿ!DI%*±{upšÑŒ{¸‡I˜„ƒ8ˆé˜Þ!Çohh@YY™Þv†ÂÂBäää¨pýÜó}xx¸Öç~n•mêêêt÷êÕ«HKKC~~¾Jœ<<<ô¶3øùùA*•ªîB!¤m233‘˜˜ˆàÚµkðððÀ´iÓ°~ýz<úè£8|ø0&OžŒo¾ù)))X¹r%–-[†1cÆ ** ?þ8úõëgélB!D‰ ÞyçL›6 ßÿ=Æoé$B 2 D«ÆÆF”––ê ¸qãjjjøí¹ÁýÜ Á¨¨(•þ¾¾¾°²²2:b±˜ß.éËÊÊBZZÿkôl?µ…÷WÞ¸ ¹€—e/«"Èd2é] BéI "//Ÿ=¸ß¡¯Ü©ªµs½-uƒ'ð„Ê{nð€ŸŸÂÂÂt~ÏÐà„~øAë̆ꃴ½úûûÓŒµ„tVÙÙ÷_i%BH;Ô××£¸¸˜¯+¢¸¸˜ÿìÎ;(--åÔž‹ÅbÁåR©Æ ãgÃç¤sƒÒ•gÉ‹Åʹ Öá»T<êé驹Áĉí>·Â„òŠÜjÜOee%müþûï*A&ú®OOOøùùÁËË >>>ðññ——¿2———¬­­ÛBÑå_øêQ/hÛ´ ­ˆF4>ÇçX„E:·54Y÷Z\\¬²:úó¹L&Óx> 4¸ÒŽû$ýÁ………ÈÈÈ€B¡@ee¥Ê÷¸‰ôµ7èš”‰Bé©ZZZpþüy$&&âСCÈÏÏGPPf̘?ü&LP ä{â‰û} .ÄÂ… Q[[‹“'O"11›6mÂK/½„ÄÄÄ`Îœ9 ±TÖ!„¢dêÔ©˜2e V­Z…ôôtj%„t d@z$m«(¿WžH,Ãßß_%ˆ 66–lÑÙÖ$ ÂÂÂt.¬­¯Å뵯cDú”;•C©ˆ››‹ææfÌ8ÄuT¨¯ˆÐ¿ÿé¨ „B,E9PÛ,€Üg†f ãß«ÏÔ×ÙˆÅbAÁܹQ?ÊRRRTê€ö`m³¶5è’ÒF99€›àêjé”B:¡––!77yyyÈÏÏGnn®F@AUU•Ê÷$‰Ê€ï>}ú¨Ðìô=‡³³3œáïïߦïë[墴´ …ÿýïQXXˆ¢¢"Ô×ÿ1Ø×Æƽ{÷V <ð÷÷G@@!•J¨w¦nBÑç{|/x¡5¨ƒöÕvla kXà V``hD#c1R/§bäFj}¶ÖµÒ ®‰ $ ‚‚‚àääd®¬ Ö–`mçäêÕ«‚ζv†Îzn!„ŽP__ÔÔT¤¤¤àèÑ£(..FHHæÏŸÈÈH„‡‡ nowttDTT¢¢¢ÐÐЀ3gÎ 99;vìÀ†  “ɉ˜˜£öK!„Ž÷þûïãÁÄÞ½{±h‘î‰ !Ä\(È€t;MMM¸sçŽÎ‚›7oªt’+¯B “É —ËUÖ÷éÓ§K?H;Ú;b³ýf`<îÿh¡o5„[·n©Ì6¤Ü¸¯„ “ÉL‘”„BÌ®£fTà^àÚéÚÙÙñÁ¡¡¡z·Õ·ÊCFF† Ut½úøøP]‚Ž“[:„ ©®®FVVòòò““£H››‹ÂÂB¾œ¶¶¶†‚‚‚àííÁƒó3ÈûúúÂËË‹ŸUÞÞÞÞÂ9#Ý…‡‡<<<пAÛWUU¡°°%%%|½¾¸¸˜ÿìâÅ‹ÈËËCEEÿ^½z!((AAA¦™² !ZÂ!÷ÈçåãFñ Ü®¸ìÊlÜ+@a]!î4ÝAYsª¬ªÐ nܸ_:‰#¿AÐwAzW9ì)³õ F0Ô®Ã#j×ÑöÚSÚu!„t}÷îÝÃþó$&&âÈ‘#¨©©AHH–/_Ž'žx<ð@»!‹!—Ë!—˱eËœ;w)))8tè>üðCbêÔ©ˆŒŒÄ”)SzD}…BéL „eË–á•W^ÁìÙ³i"BˆÅQér¸™o´ ŠW_…ÀÎÎ|ƒ²\.Gll,ÿ~àÀ4Ó ¯† ïœ§¥¥é=çêtÎ !„£®®NçÌúÆÌ(“ÉhÆ»$‘H ‘HŒFPþ[feeáìÙ³ÈËËCMM ÿ±X wwwƒ¸@…   ØØؘ:«„t]ÙÙ@P¥SA1!å Ônß¾Í×”ëE!!!˜4i’ÊózOäHº6WWW¸ººTS__…B¡uÓK—.iL> þU^ÑS&“Qû!Ý÷¬ª¯½AýYUy`___ ôøÇ3ªïÏ­AAA°y˜žUÅ ]…Q× •Ü*Œ†V¨Ô×Þ@!„˜Sii)Ž;†ÄÄD|ûí·hiiÁ˜1cðÆo`öìÙm^=NkkkDDD ""›7oFff&‘’’‚;wÂÝÝÓ§OGTT¦M›Fƒ !„3Ù°aöíÛ‡wÞy6l°tr!=NG׬úYYYgÕ—Ëå4«¾ šmHßêiiizWЈÐÕW „bÀ¦/p@¡P¨”û€fð@Ož °+Œ ïzà‚ TêÜþõ ðóóƒT*…­-=ö('9ÒÒ© „´SSSnÞ¼‰ÌÌLdffâÚµk¸yó¦Jûˆ‚ƒƒÑ¯_?„„„ ::}ûöEß¾}D«ÅÞÞÞàŒÙeeeÈÎÎÆ­[·øO¿ýöRRR P(VVVð÷÷Gß¾}Ñ¿„„„`ðàÁ 1é BHûè[u{5´êžL&Cxx8­º×I)¯Â¨/0¼ cJJJ›Waôõõ¥> B!m–““ƒ#GŽ %%§N‚H$¤I“°mÛ6üùφ———EÒŠÐÐP¬_¿·oßFRR1wî\888`âĉˆ‰‰ÁŒ3h• B!Ä„z÷î5kÖàµ×^ÃÓO? ©Tjé$Bz0mCÌŠ›MLWAnn.š››¨Î Ä(¯Bп¸¸¸X8GD"‘Á†}å}å¿¿¶ÕÄb1üýýu"ÓL „ÒI ÈÏÏGuu5ÿ®ÌWîÄ £Áâ=ˆ¡€FŽ¾ë«¢¢W¯^å?S&$! vvv¦Ì&!æÃpí°`¥SB¨¹¹·nÝ•+WpõêUdffâêÕ«¸~ý:accƒ>}ú 44&LÀÒ¥KÑ·o_ôë×R©”V÷!ÄðððÐÚ†U[[«|póæMüþûïHJJBII ÀÍÍ|Âÿ¿¯¯¯¹³BH!$x@¹oP¹ž ~—Ëå4X¼ië*ŒÊ¯™™™8{ö,rssq÷î]þ;†Vaä^½½½©žF!™™‰””$''ãܹspssƒ\.ÇîÝ»1sæL8;;[:‰*úôéƒøøxÄÇÇãÎ;8~ü8±téR<ýôÓ=z4bbb0gÎz"„BL ..;wîÄš5k°wï^K'‡ÒƒÑ(-Ò¡ô­B >à‹kä甩¯BD¯Ýˆ¡}åeŽ¹k†»n®^½Š7n¨,O­k5î3ê"„Že¨C¿¢¢999¸wïÿõWš ´—Ð`„úúz”——ë¼^¹`„ââb´¶¶òßS„¢ë500]Iç——TVC†X:%„-š››qõêU¤§§ó?¿þú+`mmÍLŸ>  Á AƒhEBÌÀÑÑC† Á-ehii)®\¹‚k×®áòå˸ví:„ÒÒRàFŽ‰#F`Ĉ0wé2PVVfp¢‚’’~r@ó¹-,,Lã¹ÍßßnnnÌéJ:rF!kÐ*Œ„Ò3´¶¶ââÅ‹HNNÆþýûñÛo¿ÁÓÓS§NEBB{ì±.3éMïÞ½±páB,\¸åååHIIAJJ Ö®]‹Õ«WcìرˆŠŠÂ¬Y³Ð¿K'—Béììì°yófÌž=+W®Ä¨Q£,$BHE-VD°††è ÈÎÎFmm-¿=7\[A¿~ýh =¢BÈ2ÇÚVCÈÊÊBff&ÒÒÒÍTbш‘HdÎ,BH§$d6@CKÇs³vÒl€¤³àêúê€áA-?üðƒ A-Ú^iP ±¨_¬¬€Áƒ-Bz<ÆnܸôôtüôÓOHOOÇÅ‹Q[[ GGG 6 cÇŽÅóÏ?ÐÐP 4ŽŽŽ–N6!D OOOL˜0&LPù¼¤¤W®\ÁåË—‘‘‘ƒbÓ¦Mhmm…¯¯/pÀôîÝÛ2 ÄL }s¯†‚¾µMR@AßÄ’:bÆÂÂBddd °°*ß² #õkBHçÖÒÒ‚óçÏ#11DAA‚ƒƒ]»vᡇêò.¹»»óuuuHKKCbb"6oÞŒ—^z !!!ˆ‰‰ATT”Þ¶yB!„6kÖ,Œ?«W¯ÆÙ³g-BHEA„Ç­B m·B¡@QQc4W!àü¹÷4ë 1C3 q0ڮጌ %•a´"H$seB:”òê0ê« (–““côl€\p!ÝX,ŒèÈÉÈÈ@JJ rssÑÜÜÌGùß”¾Y ) ‡t¸Ë—À@€]±ˆ¬¬,¤¥¥áìÙ³øî»ïŸŸ[[[ 0aaaˆ‰‰AXXFÕef-$„èæåå…‰'bâĉügwïÞÅ¥K—‘‘ŒŒ ìß¿ëׯcŒŸ%<<'N¤ÕH—¡oà´z{ƒ2õÓÚ&* 2‘tÆ#¨ÿÛѶ £rß ùoJ[{ƒT*…³³³©³J!àÚ§¤¤àÈ‘#())AHHžzê)DFF"""ÂÒI4DEE!**J%Àb×®]Ø°aúô部¨(ÄÄÄ <<œÚÀ !„6x÷Ýw1jÔ($%%!::ÚÒÉ!„ô@4 ¼‡Ð7øZ¡P ''÷îÝã·W|­@@ƒ¯Ig%‹ 6Þë ¦áÞ¾}[g0z ÓBÌ­£fÔ<@« ¢‡¡`GŽ¾`nõ%C«ƒèzõññéò3]3¹|xðAK§‚ã·ß~Ãwß}‡S§NáôéÓ(..†‹‹ ÆU«Vaܸq6l ž$¤qrrBDD„Ê€¢òòrüøã8}ú4N:…Ý»w£µµ¡¡¡xä‘G0aÂ<üðÃððð°`ÊIOdhÖu…B‚‚TUU©|OH𵟢›ƒƒÚ½ #Œ`¨=PÛ+µBHÛTTT -- ÉÉÉ8|ø0jkk1|øp<û쳘7o `é$š ÿ ´eË\¼xÉÉÉøúë¯ñᇢwïÞ˜2e bbbðØcQ !„"Ј#0{öl$$$`Ú´iÔÎB1;ºëtʃ™Ô ‘Í7.*œöõõEXX˜J5ü“îL"‘ ,,Lg£=£-áìÙ³ÈÎÎFmm­Êþ”ÿý("ôë×è !‚(wèë›ÁÌÐÌe\‡¾ò`å   899Y0w„ô,Æ#hû7ÏÕ;òòòPSSÃÇÎÎ:Wá> ‚©³J:³Ë—¨(K§‚n-33‰‰‰Ø¿?~ûí7ôêÕ cÇŽÅ /¼€ððpŒ="‘ÈÒÉ$„t"îîî˜6m¦M›¸wïΟ?ϯ|ò駟¢¥¥cÇŽETTfÍš…þýû[8Õ¤+S€Ööì¡þÌ!‹áîîÎ?_(OB¤üìAÏ„˜ÐUÕW6Uÿ7/dFCí „ғݹsÇGbb"¾ýö[´´´`̘1ظq#bbbè>©ÄÚÚšï_¿~=233‘’’‚äädDGGC"‘@.—#22³fÍ¢>,B!Ä€7ß|¡¡¡øâ‹/°xñbK'‡ÒÃÐHò. ±±ùùùž¹÷7nÜPéPôÌÍT¬<Ú××—–¢#DcVCPDHKKÓX\}5õ@Xâ IDATê#¤{2 B¡@ee¥Ê÷„ÌH×é¸`CôÝG¸`!÷m¯té¦7€!C,Bº•¦¦&œ>}‡ÂÑ£G¡P(пÌš5 Ÿþ9FŽI6BŒÒ«W/ÈårÈår@UURSSqøðalÚ´ /½ôÂÂÂ0kÖ,üùÏFHHˆ…SL: }«§q¯†VOS ÕÓéúììì#úï#\0­ÂH!ÀíÛ·‘””„ÄÄDœ?b±“&M®]»0cÆ šlN ÐÐP„††"!!ÙÙÙ8zô(ñ—¿üË—/ǤI“ƒèèh¸¹¹Y:¹„BH§Ó¿,Y²k×®Åܹsáàà`é$Bzêýì´­B ü>''---î€ö÷÷ç*‡††"66–ߧO8::Z8G„to†VC¨¯¯çþi DPž)H$ÁÓÓS#ˆ{ß¿¸¸¸˜3{„„äç磺ºšÿŽú äÜjBêq´¢!D™ƒƒƒÁH@û}IyÖ«W¯ ZEÛ¬…ÁÁÁèÕ«—©³J:Êõë÷ ¶tJé®]»†={öàŸÿü'JJJ‚eË–!**Êàà-B1†««+fÏžÙ³g£¥¥çÏŸGbb">úè#¬Y³ƒ ¢E‹°dÉôîÝÛÒÉ%Œ›\ßꆅ……g  Ó:˜&"„(3vFmí ÜĹ¹¹¸{÷.ÿõQt½z{{ÓL„N‰[¹0%%pwwÇôéÓ©S§Ò¬ûíŒøøxÄÇÇ£´´ÇŽCbb"–-[†%K–`̘1ˆ‰‰¡Õ!!„5ëÖ­ÃÞ½{ñ駟â…^°tr!=`3±¦¦&ܹsGgÁï¿ÿ®2‘[…€›MH.—« :îÓ§uÒÉÙÛÛ·k5„[·n©ÌF¬<øO[ Bpp0Í DH2`~~>ùï¨wœÑl€Ä~øáDGGãÌ™34£©šžzn„#Ô××£¼¼\ç}Ž F(..Fkk+ÿ=õÁKÚ^)8²3øñG W/`Ð K§„.«µµ‡Æ{ｇóçÏ£oß¾ˆ‹‹ÃüùóléäBzDDD ""[¶lÁÙ³g±gÏlܸëׯǼyó°zõjƒƒC‰å544 ¬¬ÌàD%%%üDC€öàõú·¿¿?ÍôJ:TO}ž¢§ž¡ÁBVa,((@UU•Æþ ­ÂH²BL­µµ/^Drr2öíÛ‡7n 00S§Nźuë0eÊZ ÖD<==±páB,\¸HKKCrr2^}õU¬^½ÇGdd$ž|òI 8ÐÒÉ%„B,Ê××qqqxë­·°dÉZQ‰b6Ô*ÓN\Ù¶ÁÂê«ØÙÙ! €o“Ëåˆåß8Ф‘ïùùù TùÌÚÚ®®® Att4–/_Þaƒ´oË–-Xµj•Æ¶Û·oÇÊ•+»víÂÒ¥K;$ ¦’––†É“'ãòåËü¿BM‡—^z o¿ý6àøñã˜2eJ‡î_›Ï>û Ë–-|òÉ'X¾|¹ÉŽeêü³ÿ‹/âÿøNŸ>{÷î!$$¯¼ò "##;,=†VCÐwoQ_ AýÞ¢ˆ`ê{ !]àm³*Ïþ ¹\n±Ù)WÛ²/à~ÀD¿~ý°dÉÄÅÅuhž¶²“´]kk+c*3Ò“ûºò¹1G„¬Ä TÒÅÐ`¨ŒŒ ¤¤¤¨<ï(ï_ß  ebçÏ£G4ƒ£µ¶¶â«¯¾Â† ••…èèhüç?ÿÁ„ ,6ჹëP¦:^FFâââpéÒ%ÔÖÖÂßßùùù¿§íYÞí#]™9Η9Ûó䩲²Ï?ÿ<Þÿ}xyyañâÅøûßÿn‘AýÖÖÖ?~<Æ>øûöíÃÖ­[1dÈLŸ>o¾ù&|ðA³§«§3¼Ë½êZIŒkoÐ6IT*…³³³YòAm DYW~ž6µ®|nÌÑÖОU•_322ø]” F ¤AÀ„Á”WKLLDaa!d2"##±{÷n„‡‡wH;@FF6n܈Ÿ~ú wîÜ¿¿?är9æÏŸñãÇÓä’J$ ¿‚A]]ÒÒÒ’’‚O>ù6l@HH¢¢¢ÙaB!¤«IHHÀÎ;ñî»ïâ7Þ°tr!=:0@y¶qõ¾†fW_…ÀÒ³€1†Å‹ãßÿþ7îÞ½‹––áĉØ°a>þøc$''cÈ!&9ž.Ï?ÿ</^l¶ÎS0u6oތŋcg$]ºt)žzê)888˜üX¦ÎŸ1ûŸ:u*yädddÀÖÖ¯¾ú*¢££ñÍ7ß`êÔ©&IŸ:CòúVIIKKÃÍ›7UfR^%E[ ‚¥ïO„´…¡°\‡”¡°ÚfôóóƒD"±`î4S®¶u_eeeعs'þú׿â·ß~ÃÎ;;*ù¤ƒ7ååå–NF§Ô•ÏMg¨ƒpÄb± `@ Œ -‹»ç*tY*«[9w˜=ÛÒ© ¤Ë9þ<ž}öY\¹r‹-±cÇпK'«Û˜?>|ðAœ8qyyy˜1c† ïi{–·DûHWfŽóeÎö#Àôyª®®Æøñã///ÀÊ•+1uêT$''cèС&9®NNNX¶l–.]ŠcÇŽaýúõ>|8.\ˆwß}K[w¡o,×ÎÀ}¦L}lhhh—Km DYW~ž6µ®|n:S[CG¯Âh(K[;ƒ9¹! 7p=11III¨ªªBHHbcce° ÔX/^ÄC=„§Ÿ~ç΃ >üðCL˜0?ýôFŒÑ¡Çì.…¨¨(|üñÇ|@È_|·ß~ÁÁÁˆŽŽFTT&L˜@+ÞBé1ÜÜÜ€×_+V¬€¥“Déztm»¾¾ž_ªSÛ*ê3…{xx¨(¯B0`À€.Ù(ecc,Y²‘‘‘5j¦M›†«W¯vÉüÒñÙgŸ¡W¯^€mÛ¶áË/¿Ä®]»ÌÞ讋H$28ðOyÀŸò}Ž[ AyàµX,†¿¿¿FˆÌŸBLMèl€ÅÅÅhmm忧< “É4:‘;lÅžîÈÃÃ/¿ü2’““±{÷nlÞ¼îîî–N!=FW¨ƒh#‘H ‘H Îì«/!33iiiÈÏÏGcc#ÿõUet½úøøPÀ$”•7ncÇZ:%„tÍÍÍX³f Þ{ï=Èår|õÕW ±t²º•úúz\¿~«W¯†““ „7nX:Y„èôâ‹/ÂßßO?ý4ÿYXXžyæÌŸ?¿üò lll,˜BÀÊÊ Ó§OÇ´iÓpàÀüõ¯Ehh(vïÞéÓ§[4m•¡Ù³  T&í„H¥Rؤµ5bY]±­¡£VaüᇠP(PRRB«0ÒC•——#%%)))8vìêêê0vìX¼üò˘5k–I'øç?ÿ ±XŒíÛ·óí–ÁÁÁxÿý÷‘––f²ãv7666ˆˆˆ@DD>øàdff"11û÷ïLJ~OOOL:111xì±Ç`gggé$B!&‡íÛ·cãÆؾ}»¥“Cé X;¤§§³ôôôöì¤ÊËËYzz:;pàÛºu+KHH`111,,,Œùúú2ü½½=“ÉdL.—³ØØX¶yóf¶gÏ–ššÊnݺŚ››-³hÑ"Ö«W/­¿ûòË/öÎ;ï¨|þÿ÷lôèÑÌÞÞž¹»»³§žzŠ)ŠvOYMM À>ùä¶råJæââÂüüü؆ T¶»sçûÛßþÆúöíËÄb1:t(;räˆÆ~¸ŸÑ£G3Æ»}û¶ÊçœäädÆÄb1óòòbÏ<ó «ªªÒ™Î„„•ý`AAAFå¡-ÇeŒ±k×®1ìøñãFíËÐ9S¶}ûv&•J™ƒƒ›0aûõ×_ù< =æ¶mÛøs³mÛ6¶|ùr&‘H6wî\£ó'ôúSÞ. €=þøãìÇÔ¹ÿO>ùDåï¸oß>ióòòbãÇ×ùû®¨¡¡ð÷ÉÍ›7³ØØXɘ³³³Êù‘H$,,,ŒEFFò÷É°ôôtVPPÀZ[[-%­vìØa–ãìß¿Ÿµ³X5È\y1•ÚÚZvëÖ-væ̾l^·nÊuçëëˬ¬¬4®½&—ËÙ‚ XBBÛºu+;pà;sæ »rå «©©±töÌF[¹zóæMÍ<<<˜““›1c;þ|›öÅc .dØ/¿üÂvÖ—}e'c”=b±˜3†;wN¥Z®*=O†ŽÏÿþòåËŒ1ÆùÏöîÝ«±}u„¶l·k×.£¾+ôœë"´Naì¹V¾F¾øâ Ãœœœ˜D"a+W®dõõõ‚þ&Bϳ³3 `Ÿ}öklld+V¬`...L*•²Ï>ûÌè<ëK×ôéÓª›ªëŽuC´• lÁ‚L.—³æââ¢rþìì옯¯¯J]eݺu*åEw{®Ó*)‰1++ÆÊÊ,ÒN111,&&Æ,Ç2uÛ¿¿IÁ16/UUUìÑGe½zõbŸ}öY§{¦ÑU‡2U¹c¨ÎÖ–¶!å´r?=ö˜QûÔÖVÐÖöƒI“&©¤¥©©‰1ÆØÆÙ AƒøíÖ¬YÃoóÆoèÌŸ:¡4µ·®Â˜áz0w¾ÙâÅ‹™‹‹‹F=˘¿ cúÛŒm›ÓFhž”¯!Cû­®®föööì›o¾ÑHSyy9³±±aiiizÓn láÂ…ÌÚÚš½÷Þ{–NŽY•——³+W®°ÔÔT¶gÏ­uG'''•kO,k­;îرƒ%%%ñm]ݾÚ¨­Ú¨­¡;ãúB”Ë‹Í›7³¸¸8ÃÂÃÙL&c¶¶¶*çë;g111,..NkyÑÙž# 1¦¿«÷Iž)77—íرƒEFF2‘HÄÄb1“ËålëÖ­¬°°ÐléXºt)³··7Ø%´ìçêºM{ë8m­+v¤+W®°Í›7³ððpfeeÅÜÜÜXLL Û³g«®®6kZ!„sÚ¹s'‰Dì÷ß·tRº$sŒï"¤#™ëÙ\Wÿt— 2¨««c·nÝRi Šer¹œÉd2&‰ø/‘HÄwÄÄĨ NLOOg•••Ƀ¥èô_UUŬ¬¬Ø#<–œœÌ¬­­Ù+¯¼Âîܹî]»ÆFŽÉúõë'èáÌØ ƒ|=z”UWW³íÛ·3ìÔ©Süvñññ,>>ž•––²ššöÅ_0±XÌ®\¹ÂoÓÜÜÌüýýÙüùóUŽQ\\ÌÈ7ø9r„YYY±W_}••——³K—.±x€Mœ8Qo£`jjªÊƒ¾±yhëqµu  Ù—sÆc_}õß^^^Î~þùg6eʾ¡Ü˜crç"88˜8p€Ý½{—mÛ¶Íè ¡×·ÝÚµkYii)+((`óæÍS¹öÔ÷___Ï¢¢¢ØG}¤3MŒ1öÝwß1lÕªUz·ëŽ”;jwìØÁkqîÖÖÖ îÊÁwìØÁk566Z$d`zBƒº¹¹itü °äuÓ™i+W‡ÊæÎËŠ‹‹YEE‹Töê*£ÇŽˬ­­YYY™àû°¡4è*;ÕËžK—.±qãÆi”=†Ê!eS[Γ¡ãjË×;w4ÿµÕ>øà:‚±ÛqÛÆ|Wè9×Fh¢-çš+§ûôéÃŽ9ªªªØ¡C‡˜££#[±b…࿉¾s3tèPvìØ1V]]Íþö·¿1+++¶xñb–””Ī««Ù‹/¾ÈlmmYvv¶ÑyÖ—®––¤µnª³þדë BPù£ÅË/3¦4H–t]dÐ6Æ䥹¹™=öØcÌ××—edd˜0Uíc¨ý¡#Ë}ÇkOÛ¶²Ù˜} 2º¿9sæ°!C†¨¤eÔ¨Q »yó&ÿÙ¦M›Ø§Ÿ~ª7o†ê®mJHð@KK‹¥³×éP[µ5(oGm ÔÖГõ„ò‡‚ HwtëÖ-¶uëV~°¹££#‹ŒŒd{öì18ÉŸ©ìܹ“`?ü0;}ú´ÑÏâÚÊ~!ýáB¶éˆ:N[늦’ͶnÝÊär9³µµeööö,22’íرƒ[,]„Bˆ)477³Aƒ±yóæY:)]®†‚ tàV!HJJRتm–cõU¸†n`+7+¹ÏР&“Éø÷dƒVÙæâÅ‹ Û¼ys»ÇáK—,YÂÖÚÚÊœœœôÎÇcQQQì™gžQùìå—_f*A$ÿøÇ?Ø믿ο0`€FÞ’““vòäIÇ3ÔÉo(m=®¶Ô¶îKÛ90`6l˜Êg‡ÒèrLî\,]ºTg„äOèõ7pà@A555ÌÝÝ]ëþkkkÙ”)SôvºVTT°>úˆ¹ºº2???VTT$8/=ECCƒÊà>õÕÔgŠãVCàfýQ_ Á(È í¸}ƒ7ÕW¼ ™¤MO½\­««Ó¼×ÔÔļ¼¼ŒÞWii){ë­·Ëv’]e§¶²ç›o¾ÑÙñ¯«\1T6µõ<:®±ÿÊu„ææfæèè¨RG0v;mÛ†¾+ôœ ¥^§hë¹æÊé¿ÿýï*Ÿ¯\¹’‰D"–ŸŸ¯’O]}çæ駟æ?ËÏÏgT:ã À¾úê+£ò,$]o½õ–Öºé–-[4¶¥:HÇRFàž/»íJ:?̘ÒuNº. 2hcòòÞ{ï1GGÇN`À˜áö‡Ž(w„¯=mCº‚ „îShÐýq«xfee1Æ+,,d2™ŒYYY©œŸ±cÇê}NR皦öÔU„´Ñpç+>>^e;õz–Ðô m?jëuhLž¸k@ÈßCÈ~W¯^Íœu¦«;tfëÖ­cvvvü5Þ™täLÒÜJÄʃ7IÛQ[µ5(oGm š¨­¨²’ŽzÛ¹¡•tÌÕvNA¤»¸rå [·n c˜‡‡[°`KJJÒX±ÍÙ¬Y³ø{€¯¯/{î¹ç4V`LxÙ/¤?\È6í­ã´§®h¥¥¥lÏž=,22’‰ÅbfccÃÂÃÃÙÖ­[ù²ŸBéꙵµ5»xñ¢¥“ÒåPéj,d` hhh@AA  ‘••Åÿ( äääàÞ½{üö‰2™ 2™ áááðóóãß÷íÛnnn–ÈF·Åãÿ???ׯ_ÇŠ+T¶6l\\\––†„„„=þ!Cøÿ·²²BïÞ½QRR¢÷;îîî¸qã†ÊgùË_°iÓ&|ýõ×xæ™g{öìÁ7ß|à~ÞnܸçŸ^å{£Gœ((/Ú½þtmçä䄲²2ýÞ»wÓ§O‡——–.]ªóø«V­BJJ æÍ›‡õë×ÃËË«Íyé®ìììøû±.üý]ùžŸ‘‘””ܾ}›¿ïØÛÛ«Üã}}}UÞK¥RØÚZ¤èêV***ø¿‡®×¼¼<455ñß±··‡D"ŸŸ|}}UÊdîïäëë X[[[0w=½½=FŽ‰—_~VVVˆŒŒ„ƒƒŠ‹‹}ÿÞ½{°²²pÿßt¿~ýðî»ï">>^ð}¸­iÐUöŒ1Bçw´•+B˦öœ§ö”gÊ”ë666ðôôÔZϺ±ÇhË97D½NÑÞkR=-ãÆömÛðóÏ?Ãßߟÿ¼-“Áƒóÿß»woϼ½½wîÜÑ»muOCéZºt)6lØ R7Ý¿?RSS5¶¥:HÇrpp0X_€úúz”——ë,¯^½ …Bââb´¶¶òßãê/Êå¡úk`` \\\L›Ñ¦&য়€§ž2íqéZ[[ñÎ;ï >>úÓŸ,œvéˆrÇS´ uô>Ùß´iÓ ‰pôèQ¬Zµ ÉÉÉX¸p!Ž;†ääd¬Zµ EEE`ŒÁÏÏOç1 ÕyÚ’Gcë*ƶѨ·ñ(׳c‚Ò+´ýhÛuØÖ¶.!!û-**ÒÛÖìæ憋/êü}gðꫯbß¾}xÿý÷±mÛ6³³¡¡eeezÛ JJJÐÒÒÂO½¦Q €«««YòAþ@m ÔÖ`Ìw©­Úz"‰D‰D‚ÐÐP½ÛÕÕÕé,³²²pöìY ªªJcÿúÚüüü¨¿„ô8---8þþøc|üñÇxôÑG±ÿ~£Æ¹éºM{ë8í-—MÍÃà .ÄÂ… Q[[‹“'O"11¯½öV­Z…ÄÄÄ`îܹ4h¥“K!„´Éã?ŽaÆaÆ 8|ø°¥“CéÆLÒòPTT„¼¼<äææ"77999üÿçææª4¤9;;C*•"((ýúõÃĉˆàà`H¥RøùùQ‰UUU¡¢¢C‡”––¸ß¸©ÎÃÃÿ}GrrrRyomm­2ˆçêÕ«X»v-Î;‡’’~pò°aÃT¾×¿DDDàóÏ?Ç3Ï<ƒü¾¾¾J¥þÈÛöíÛ±}ûvtäææš$y\¡ûrÎ hþ­Õß›~ÁùQ'ôúÓ·6+W®ÄСCñïÿÿýïµv„s¶oߎ'žxÂؤ%‰aaa ÓúûÚÚZdgg«”¹¹¹¸~ý:RSSQPP€ÆÆF€­­-üýýUÊ ©TŠÀÀ@!((HãßæÍ›‡ëׯ£°°P£CßÙÙþþþðööF@@F…€€x{{óŸûûû›~`$i—'N`ãÆXµjžzê)Èår¼öÚkzïoœ^½záîÝ»ZgL= -iÐUöè»Þ´•+B˦öœ§ö”gÊÔïQ"‘H¥žcìvÆ£-ç\™ÐzX{εzZ<<< …Båó¶üM”Ï ¥í³¶Ô= ¥«wïÞ˜={¶JÝtèС:;’¨b~Ü 7n€›. (**BAAŠ‹‹ù×üü|”””àÔ©S(..V¹^€ûƒýüüàããƒ?üÐà@£?ÔÖ<Ò±û%¤ÊÍÍEqq1fÎœi餴[G•;ú˜¢m¨£÷iÌþÜÜÜ0~üx• ƒ×_"‘ëÖ­CUU’’’eð¸úꨮ®ÖÙΛ›‹Ÿ~ú ¨©©á¿ccc///øúúbàÀøꫯLBÌ®¾¾iii8|ø0’’’PZZŠÐÐPÌ;³fÍêL˜0&L@CCNœ8­[·âÛo¿ÅÆñî»ï Þþpc¶io§=å²99::"** QQQ¨¯¯Ç·ß~‹#GŽ`ûöíØ°aŒ™3gbæÌ™>|¸¥“K!„fee…×^{ 3gÎDzzz»ý !Dšj˜¨HJJc Ó¦Mxzz€Öγ²²2þ÷æÒÔÔ¹\Ž¼¼<œ:u MMM`ŒaÑ¢E*x8ùË_páÂ\½zŸþ9–,YÂÿŽKû‹/¾ƘÆÏÞ½{M’‡Ž<®} =g¾¾¾4ÿÖ곧˜ó¼ ½þôm§ÍÚµk‘””„¡C‡ò3BHW&‘HðÞ{ï¡  §OŸF}}=Æ[·nµk¿ÆÔÚ’]e±ƒv„–M¦8O\G±òÊÕÕÕmÞŸ©µçœSkϹ®¨¨PyÏÍ°¤o&aS1¶îiÈŠ+Tê¦Ï=÷œ RMz¬ÔT 8èÛ×Ò)!„tUm¨£÷iìþf̘ÁÏ{óæM 6 ÑÑÑhnnÆñãÇqôèQ̘1ÃàqõÕyÌѦflò 4.Àýz–Ðô m?â{¶§ÝIÈßCÈ3ƒrÝ^]SS“àI.,›Ež¶¢¶jkŠÚŒCm „Ò3‰ÅbDGGãÛo¿E`` .\¸ÀÿNHÙ/¤?ܘmÚ[Ç1U]‘B!Â͘1#FŒÀ[o½eé¤Bº1“,àããŒ9RëïPPPÀ/ ™••…¬¬,ܼyßÂÌè… IDATÿ=rrrpïÞ=~{‰DÂÏòÀ-ɽïÛ·o›fa#šŠ‹‹±fÍò˪`àÀ8uê”ʶ—.]Buu5är¹YÓ˜••…ÂÂB¬^½<ðÿyCCƒÖíçÌ™ƒ¸¸8lß¾'NœÀ‡~Èÿ. <ð~úé'ï :¯¼ò æΫu¿Üƒ~[´ç¸mÙ×°aÃ3www 0gÏžUùüçŸ6Yú zýqÛ©ÏЦP( “ÉPPPÀÏL2™ "‘{÷îEXX^|ñE­35üë_ÿê|ôtÈÊÊÒ¸çsïoß¾Íwžp3Ëd2 8&LP¹çÓò¿m£>ƒQEE…Öå™ qáÂ=zyyy*™öööH$z—göõõ…O»î‘ÄxEEE˜jÚcÒMH¥RøøøàСC:Û®:‹öÔ­…–;†ŽgŠ¶¡ŽÞ§±û›1câââð /`òäÉ€ÁƒC&“aß¾}ÈÎÎ6¸âŒ¡:ÏܹsMÞ¦flMzz:bbbø÷Êõ,???AéÚ~Ä1ö:lk»“¿‡ýúøø¨Ìʯ®²²>>>zó`iÍÍÍ8rä¦Nj²c„††âäÉ“üû††”••é¬?]¾| …BcuEõz”¶×€€¸ººš,/D;jk ¶cP[ƒq¨­¡g©««3ØÎPPP °)‘HøòP*•b̘1å$õ—žÀÞÞ‘‘‘ˆŒŒDkk+Î;‡””ìß¿o¾ù&¤R)¦L™‚ÈÈHL™2Eïªdæ¶víZXYYá7ÞPùÜÖÖ"‘H¥ïZHÙ/¤?\è6í­ãôêÕËduÅŽV[[‹“'O"11GEuu5BBBðüóÏcîܹ4h¥“H!„´Ëš5k0sæLdddXä—ÒýY¤åA,\R× Ô~ø …EEEZ¥j D FÝZZZPTT„'N`ýúõ°¶¶Frr2œùmÞ}÷]̘1kÖ¬ÁêÕ«QZZŠgžyýúõóÏ>kÖô£wïÞØ»w/¦M›™L†ï¿ÿÇGpp°ÆöNNNˆ‰‰Á§Ÿ~ŠgŸ}ööö*¿ï½÷Í›7céÒ¥€·Þz ÍÍÍzg¬ãfæùí·ßàããƒ!C† 99Y¥AXŸ¶·-û²²²|ÎÖ¯_yóæaãÆX±bòòòðÎ;ï˜4ý†½þ¸í^}õU¬Zµ wïÞÅ3Ï<ƒE‹©4Ò( Á¦M›°zõj̘1ƒXÜ¿ >£GÆþýû;4OÝIcc#òóóu\¿~]eitå 1nƒú½›˜žD"D"18h† F¨¨¨ÐèøÈÊÊÂÙ³g‘——§2¦<<<4¨(ÁÆÆÆÔYí1®\¹‚-[¶à駟Fkk+vìØ{{û4'ô>l( ºÊNõ²''';vì0:†Ê¦ÊÊJ“œ§ÀËË ü1F;wî`Ïž=mÞŸ9´õœSkϹþÏþƒ#GŽ`Ò¤IHKKÃîÝ»k‘2Âغ§Ï=÷bccu^'TéxÚ:ôµ•mÊϘ€j‡¾¯¯/BCC5ʶ   899Y0wÿSY ¤§û›¥SBH—`mm„„¼òÊ+˜5kV§4hoûƒ¡rGèñLÑ6ÔÑû4fR©Æ Cbb¢Ê@é3f`Ë–-X½zµ cªó˜£M͘6šƒ"""&LÐZÏš^¡íGc¯Ã¶¶;ú{ÙïŸþô'ÔÔÔ ¼¼\늷oßÆC=$(–²aÃäææâ…^0Û1Åb1üüüàç秷3³±±¥¥¥Zëb………ÈÈÈ@JJ rssÑÜÜÌO[0‚®‰HÇ¡¶jk0µ5Gm ݃úD>ÚÊ6õ¶s±X www¾Ü’Éd×(Û¨íœí¬­­ˆˆlÞ¼™™™HLLDJJ vîÜ L›6 QQQ˜6mzõêeé$ãƒ>À<€G}®®®(((À–-[O>ù„ßNhÙ/¤?\È6QÇ1e]±½ÊÊÊðÍ7ß 11©©©hnnƘ1cðúë¯cöìÙð÷÷·t !„°°0¼õÖ[8x𠥓CéŽX;¤§§³ôôôöì¢ÍêêêØ­[·Xjj*Û³gÛ¼y3‹er¹œÉd2&‰€‰D"æëëËÂÂÂXLL KHH`[·ne`éé鬲²Ò"y0·¼¼<þœp?VVVL"‘°‡zˆ½ýöÛ¬ªªJëw?ÎFÅÄb1“H$lþüùL¡P}¼-[¶hÝvß¾}*ÛÍŸ?_ãû}ûöeŒ1váÂ6nÜ8æääÄYll,›={6¿]aa¡Ê¾¿ÿþ{€ýôÓOZ}âÄ 6fÌ&‹™··7›7oËËË3t:ÙsÏ=Ç\]]™‹‹ {î¹çŒÊC[Ž› ±¡û2æœ}ôÑGL*•2±XÌÆŒÃÎ;Ço7zôhAÇT?XEE…Þó©/B¯?åíüüüØêÕ«Y]]cŒ±üã*û_±b;~ü¸ÊgC‡å÷U^^ÎYLLŒÞtwwåååìÊ•+,55•íرƒ%$$°˜˜Îd2³¶¶æÏŸ½½=“ÉdL.—³ °„„¶cÇ–ššÊnݺÅ-’‡;v˜å8û÷ïgí,V 2W^ŒU[[ËnݺÅÎœ9Ã8À¶nÝÊØ‚ ˜\.g!!!ÌÍÍMã¾ ‘HXHHˆÊ5ÕÏgÎœ±èuÓé+WSRRØäÉ“™‡‡sqqaìäÉ“Fí+<<\çöBîÃBÒ ^vr¸²ÇÞÞž7Žedd0ì³Ï>cŒ /W •‡Æž'¡ÇMMMe!!!ÌÁÁ=üðÃ,==ßþ±Ç\GºÝ¶mÛT>{øᇮ‡:纭S{®cìÚµk KLLd ,`ÎÎÎL"‘°çŸžÕ×× ú›=7©©©*Ÿ=þøãì»ï¾SùlÒ¤I‚ólLݧ  €yzzòyRGuá¨üQrð cÖ֌ݹcé”c¶û€©ë˜ØþýûMz Ž1yinnfS§NeÞÞÞìÂ… &LUûjhO¹#äxœ¶´ ©—ÍØ™3gïS[[AG´0Æغu똛›kjjâ?;uêÀN:%è\ ©óJS{ë*Œé¯+Ÿ¯/¾ø‚ÅÄÄ0'''z–±çPhûcm»…æIùò÷0ôÌP]]ÍìííÙ±cÇ4ÒTQQÁlllXZZšà|˜Û;ï¼Ã¬¬¬ØîÝ»-”vSn“âÚÿãââTÚ¥”û¸¶)® 22’ÅÆƲuëÖ±;v°¤¤$–žžÎ XKK‹¥³×iP[µ5P[µ5U=¡ü1¦¿³öI¢îÖ­[lëÖ­,<<œYYY1GGGÉöìÙ£süƒ©ÕÔÔ°þóŸlÊ”),((ˆ‰D"&‘HØäÉ“Ù·ß~«±½¡²Ÿ£¯?ܘmÚ[ÇiK¹lJÙÙÙlëÖ­L.—3[[[fooÏ"##ÙŽ;Xqq±ÅÒE!„˜ÃáÇ™••»té’¥“Ò%˜c|!É\Ïæºú§­SšªÑHÐi—ZáVCж"÷ž£¾‚úŠ4k!¤»áfS¾7*ßoܸ¡2Ë· új1Üg¾¾¾°²²²`Ž´Û¹s'bccM~œ`îܹhG±j¹òb*5“´¶Ù ;ÍLÒ¤C\ºt ÇÇéÓ§1~üxK'§Gè çü·ß~àAƒpüøqL™2Å"i0‡mÛ¶!??o¿ý¶¥“Òii+/ ÍØãWÒyöY #¸pÁÒ)!dÎœ9î×1MÍÔuL+++ìß¿ŸÏ“)›—ššÌ™3ßÿ=¶lÙ‚eË–uÊgšö r‡t]í:\¾|9rssqìØ1•Ï7mÚ„/¿ü¿üòK§«OTTT ..ûöíÃû￸¸8K'Él:b&i]¯ÞÞÞîoMÚ®3<÷ö4áœS[ ¯¤Ã½¶u%ÎÚ7¢‹1ýû]½O‚ôLyyy8~ü8’““qâÄ X[[cܸqˆŒŒÄܹsáããcé$’’™™‰””$''ãܹspuuÅäÉ“‰™3gÂÙÙÙÒI$„BÌ‚1†Q£FA*•Òj˜c|!É\Ïæºú§mM~d ’H$ ÓÙHR__…B¡5!--M¥1IyЊ¶@„ÐC !¤SQîdÕvŸËÉÉAKK €û«þþþü=M.—#66–¿ÏwŠeEI×æààÀ—ŸúÔ××£¼¼\g‡ÏÕ«W¡P(P\\ŒÖÖVþ{Ú:}Ô_áââbê¬#|öÙg8zô(>þøcx{{ãÖ­[ˆÇ°aÃ0vìXK'¯[¢sn~7n„X,ÆO1’åz=z4Ó¦McÊ”)šŽSªJR}úÊÔ©SYµj•é8"R8ä^ÃÜ+§XQY Úä{i§püýý/+D°~ݼysªW¯nø‰ˆ=(î¹åÒI+zn)\Y *‹qZM|õ®vlA㯵VD*ƒâ ¹RSS9|øðUr©`)›3¦N…£GAº*ŒÂÆQ^×û5¦ƒƒ+V¬°Ý§ëéZïKnn.+V¬ **Š;vpï½÷2räHºwﮢl± ééé,[¶Œ7ß|“]»vÑ»wo^yå•b»øKé+­ °½^ÖX©¨Š*äºôxC^*ä*]W²¾¯5 ©¨òn\ŽŽæðáÃ4jÔˆððp"""èÒ¥‹Ö} ²„ÄÆƲfÍRRR&<<œ^½zéßGDD¤«W¯¦ÿþ$$$hšAÊb—Hi*«÷æ…­O«¤÷ש¨nãqqqìܹ“ôôtÛåóv/h³p£Fô†I¤È;%åÒ‚⦤„……åûZSDŠçååeë”]”¢Š‰‹‹ãàÁƒdeeÙ®ãêꊷ·w±›jýüüô»*"Æ•t3ÔÑ£GÉÉɱ]ïÒâ   Ëžç4h€‡‡‡Á{WI}öôꥑRàààÀ<Àý÷ßOll,sæÌ¡G4jÔˆ¡C‡2xð`5jd:¦ˆT2/^dýúõ|øá‡DGG“““ÃÃ?ÌgŸ}V®'®”w¥5…qÆ $''“’’¢â])·JRû,}ûö¥E‹¦#ÊuVSããã9|ø0 ;;Ûv77·B›h £HåSÔóˆõ´¸ç‘Šô<""å]HH!!!DEE±wï^>ÿüs¢££yàpuuåŽ;î ""‚Þ½{«ø÷*$%%ñÙgŸÍÆmé{ｧÇTDD¤ôíÛ—V­Z1}útþùÏšŽ#"€V?Ë—b724 aÏž=¶GIIIäääù7=_Z„¤ÍFR©effrèСB ’’’8{ö¬íòy‹z.- hÒ¤ 5kÖ4xoDÄ´«)F(¨óøöíÛINNæÈ‘#äææÚ®WX1BÞ¿ÀÀ@uÖ±CÖ×ï…MHNNæÀùŠ‰ ëxéf!1V"Ÿ}aaP£†é$"ž““wÜqwÜq ,°u6\¹r%³fÍ¢jÕªtîÜ™.]ºÐµkWn½õVM‘"9s†M›6G\\ 888бcG^~ùeú÷ï¿¿¿é˜b‡®¶!ï{ëÆâÞsv¼Aï9DìOVVÇ/vºaqQBCC Œ""RÙ4jÔˆ‘#G2räHŽ;Æþó¢££yüñÇ:t(;v$""B]÷‹‘˜˜Hll,111lØ°///ÂÂÂX²d ÷ÝwŸÖ°DDDJ‘ƒƒ£Gæ‘GaêÔ©4nÜØt$)ç´“¼‚(naÁºqº°B„}ûö‘‘‘‘ïö ›†„——WYÝ5‘ReBPØ‚½{÷Ú6ð^:…ÀÚ•G9"RÚJZŒpþüyNž>>DFFIjj*qqqÄÄÄ0qâDž{î9Ú¶mK¯^½4hÍ›77רœœˆ‰‰aùòåìرzöìɘ1cèÙ³§&u‹ˆˆ\Gd„ Ì;—yó晎#"圎îV®®®%š†PÐæë 6\Ö=ùÒÍ×—"hñ@L(ª˜&99™ýû÷sæÌÛåóÓX°«˜FDì™õï¯ua°0—v.»´ƒ™uÒѾ}û¸xñâe·_Ô&ëFh‘ʦ$ÅEu´nî »ì÷ªnݺ888¼wR.}õœ?áᦓˆTzM›6¥iÓ¦ 6 €={öÇúõë™?>£GÆÉɉfÍšjû¸é¦›Tx RA>}š-[¶oûøã?ÈÍ͵MÃ3f ·ß~;µk×6W*¹«™ÂXÐñkãƒÃ‡ç»^IŠüýý5H*sçÎ9ÝÐzZÜdÓ   Ë~¯¨¡‰w""×———m‚Á¹s爋‹#66–… 2eÊ‚ƒƒ §W¯^tíÚÕtÜ2qñâE6nÜHtt4«V­"99™FÎûï¿O—.]t \DD¤Œ8991jÔ(ÆÏK/½¤ã"rM´ \l¼¼¼l ݱnà.¨aýúõ$%%qöìÙ|·—wÃvÞB„&Mš¨;Š\±¼›û.- 8|ø0III¶N=y aêÖ­Khhh¾ŸÇ ¨C‚ˆTX...%*F€¢7N[‹Šëº^Ø©6N‹½+i7À”””" r êX¿~}<== Þ;©ðþýoèÔ üüL'‘K1lØ0[ÑÁÎ;Ù¼y3¿üò ›7ofõêÕœ9s777Ú´iCûöíi×®-[¶¤E‹T¯^Ýð=‘+qøðaÙºu+ñññlÞ¼™;w’››Kýúõiß¾=ƒ ⦛n¢}ûöÔªUËtd‘«bS4h€‡‡Çõ¾«"פ¨‚œ¼§©©©ù®wiñ@HHÈe¿Z×±?îî‡ÎÛo¿mÛhÿÉ'Ÿ0cÆ 6lHïÞ½‰ˆˆ sçÎjR­µÀ"::šÏ?ÿœS§NÌã?Nxxx±ëS"""rý<þøã¼òÊ+¼õÖ[Lž<Ùt)ÇTd %v%Ó.-Dˆ‹‹»¬‹Ñ¥Ó.-D ¤J•*eq×Ääíº]PÁŽ;8}ú´íòÖ"ëÁöððð|?GÚØ*"R2^^^¶ëE)ª!11‘õë×_61ÆÕÕooïb7 Ô©SGó¥T·©ÅzZܦ–.]ºhS‹Ø§³ga͘>Ýt)뤃Aƒ–î~þù'›7o¶¼ÿþûœ;w6lHpp0-[¶$88˜‚ƒƒqww7|OD*·cÇŽ±mÛ6¶oßNbb¢íãäÉ“øúúÊÀmuëÖ5œZ¤ì•t cqEßÖ ËÅ}tª¢o¹JR|8aaa„‡‡Ó·o_»(`Þ³g111DGGóã?âîîN÷îÝY¼x1}úô¡fÍšËÈ‘ ƒÃÌ™:^)""bGjÕªÅ#<Âœ9sxâ‰'´f."WEÏbwŠ›†`Ý„VX!BÞŽø...øûûZˆÐ¼ysªW¯^–w¯\*ê1¿t Á¥yXXÆ Óc.""ù\I1Â¥êóžnß¾äädŽ9Bnn®íz—n>/hÃ@@@€]wª.]Ð/èßòÀœ>}ÚvâºZÿ-Õ P*§ôtøê+X¸Ðt)NNN4oÞœæ͛ӿÿ|ÿ/o“ëÇ_ýÅþó’’’ÈÉÉþîä\PsM ”Š&ïռǯ¬ŸïرÃöÚÓÙÙ™ Dpp0½zõ²ýžÜpà ú½)‡®´¡ ÷¨Ö)Œ—¾WÍ;y¹¨ã z¯z}eeeqüøñb§æ]³€ü“-¬Ó3 û·¹Þ9r$#GŽäøñã¬]»–èèhžyæžzê):vìHDD  ~ýúe–+11‘èèhbcc‰ÇÛÛ›{ï½—‘#GrÏ=÷\þ)4~øV­‚_„&MàùçaÜ8pu-³Ü"""R¸^xwß}—U«VñÀ˜Ž#"åŠ ¤Ü)nSbQ]õãââŠìª_Т{eèª_Øôˆ={ö;=",,LÓ#DDäºqwwÇÝÝݶ\˜ÌÌLNœ8Ql1ÂÑ£GmîàòEæ‚Nýýý-]yĦ4º^Z< î"%´j•å´_?³9Dĸ¢š4dddØš8p€ƒràÀþüóOÖ­[Ç¡C‡ÈÊÊ,Ûýüü N:¶¿Ç¾¾¾Ô«W___üüüðóóÓT1æĉ=z”£G’œœl;öwäÈRRRl?ãyùyxxРAñ÷÷§C‡ØÎÓÄ+‘ÊËZŒPœ¢¦îY‹’““ó?·Þ~Þ÷¹õëÖ¥‘‡^Íši c!Ο?ÏÉ“'‹rXÜq‚Št\GDDìYíÚµ‰ŒŒ$22’ŒŒ ¾ù梣£™4iÏ>û,ÁÁÁDDDðÀpà 7”ê÷ÎÉÉáÇ$66–ÿûßìܹ“ p÷Ýw3yòdzöìYü눈€{ï…×_·|üóŸðê«–óEDDĨFÑ¿¦OŸÎÀ+üH)}ZE‘ ÇÙÙ™zõê¹1ïƸ¼›ë š†àêêJýúõ -D°÷Žy;¸TD°ÿ~.\¸äïÆd- È;… iÓ¦xxx¾G"""—suu-öï?\ÞñîÒNwñññÄÆÆæûûE#\Úµ°¼²>6EMŽ8|øp±Mhhh“#êÖ­«ƒ"¥aùr¸çÐ&)BµjÕhÕª­Zµ*ðÿçææräÈöïßoÛœ½oß>RRRø믿øá‡8zô('NœÈw½êÕ«ç+<¨[·.ÞÞÞÔªUËöáííÞÞÞ:† ÊÉÉáĉ¶“'OæûÜZLpôèQ>LJJŠ­(,>òþ úùùѾ}{üýým 4ÐÏŸˆ\³+™ÂXXgý ZµŠƒ99Üž•UäÆ‚Ž7–ëÉÀy›¢Ž77¡2$$ä²c0åý±¹TµjÕ'<<œóçϳnÝ:bccY´hS–M!Ø%˜ððpzõêE—.]®êx{ff&?üð111¬\¹’#GŽD¯^½øàƒ®úv©Z¢¢à±Ç`üx8Þ}ÞxZ¶¼òÛ‘R3nÜ8Ú¶mK\\=zô0GDÊH¥TÜØä¼­›ñ­ô·oßžo´ºõö *B°žw=7Õ5…Àúµ•u 5Û¥S4ÊYDD*:—#@ÑÝú­ÅÅuë/ì´,»õ7åÁzš’’b+²´Þ—âºÖ¯_OOÏ2¹"9ÿûŸ¥Ð@Dä888ØŽWtèСÐËeee‘’’ÂáÇ ì"ÿÛo¿åÛž÷uXAä->ÈûyÍš5ñððÀÃÃ5jàáá——W¾ó49Á~¥¥¥qúôiÒÓÓIOOçôéÓœ:uŠ´´4Ûyééé—X‹Z/UµjUÛχ~~~4oÞü²i¾¾¾øúú¸Ç""…+°aÓ&;~ùÂÂh}§[·nENi;{ö,ÿýŽæ³Ï>#==àà`†Îý÷ßOpppé÷÷‡¥KáÑGaäHh׆ ±L6¨]»ô¾ˆˆˆ”XëÖ­éÞ½;3gÎT‘ˆ\1ˆ $ š†°gÏvnÛÆà?æÅÌL~þÿ y7÷TˆPØAóÌÌL:ThARRgƒÎBgàÝ¿‹ * hÒ¤‰F‹ˆˆ\⊭ +FHMMewÒnVݲŠÌ×2ÉØža»Ž««+ÞÞÞÅn¨S§N¡€çÏŸ/rsBqÝ­Å]ºt¹ìûP£FÒy E¤ô,] Õ«C¯^¦“ˆH%áââ‚¿¿?þþþ%º¼uSùñãÇ íRðàA¶lÙ’o#úÙ³g ¼='''<<<ðôô¤fÍšÔ¨Q777jÖ¬‰««+U«V¥zõê8;;ãåå…‹‹ ÕªU£Zµj¸¸¸àé鉳³35jÔÀÝÝ777jÖ¬i+øôòò*ËNdee‘‘ayÝyþüyÎ;Xþm²³³9uê™™™œ={–3gÎMjj*ÙÙÙœ9s†ŒŒ ²²²HKK#;;›Ó§OsæÌ™|§N*ôû{zzÚ E<<<¨U«¾¾¾Üpà ¶"‚Úµk_Vxbý·)÷¶o·tñŽ†.]à‡ kWÜ DŠk`m|w"3”¬¡^½zEþí+IñÀÁƒIOO·]'ï„dkchhèeß7  È """r¹‹\d’ã$0€&“!11‘èèhV®\ɼyó¨U«÷ÜsÜy縺ºrâÄ ¾øâ ¢££Y·n.\ cÇŽŒ?žþýûÓ¤I“ëü¶Ûà×_á“O`ôhXµ ^z ž~ÔôPDD¤Ì=š»îº‹-[¶ÐºukÓqD¤ÑÑ<‘«TèÆ×^‚­[ù~ûv’€ýû÷sàÀöïßORR`ãÆ8p€ÌÌLªT©bïëííÍÁƒÙ¿?'Nœ°Ý¬§§§mä{óæÍ # €„›x£ñl^¸™ÖŽz ""RÖŠ*F˜ÈD6° m Á±=z”C‡qäÈ[@rr2‰‰‰ÄÅÅqøðaÛF0°,Ôûúúâïﯯ/iii=z”ƒÚ6嵄¯¯/õêÕã‘ìlžJKcõ!øùùQ¯^=üüü¨_¿>¾¾¾ê(Rž}ø!<ô¸»›N""R ëÆòF]Ñõ.^¼Hzz:©©©¶®øy;ä[ÏOOO'33ÓÖ-ÿÈ‘#ù6Ê[7Ø[7Ê_ GGG[sgggªW¯`+f˦}WWW\]]©V­Z‘·W£Fb;IZ ¦ Žÿiiiäææ’››këm}ü®”µÃËËËvŸóh¸»»ãããCõêÕóM˜¨Y³¦­˜À:…Âú!"RiíßoéÐûþûТ¬\ WuS®®®%*FÈÎÎ&%%å²ã Ö¯7lØ`9oürßÍ… –ëU«V êÔ©ƒ§§')))ÁÁÁ´k׎ìììB¿oNNN‘ÝþÛý¢X7üç•·xÁÓÓðôô,¥õ’u²@õêÕmÅyÏ‘ktü8Ìšsç‚Ÿ¼ý6<ö˜eSÝuæììLýúõ©_¿~¡—ÙÉNšÑŒ•®¤É¾&ù Ž=JZZAAA¶ã Öã õêÕ³ým3ÎqŽ—y™a £)M ¼Œµà`âĉìÛ·Õ«Wóûï¿óâ‹/r÷ÝwÛ èòô´  Ï?={Z&µÎ AA¦Ó‰ˆˆTO<ñ/¾ø"3fÌÀÛÛÛt)'Td RZrsáÉ'¡Y3˘¿b888Ø:uèÐᪿ­#Ž¼É›t¥+ÿæßÜÇ}W}["""R:p€x€"Æ°+º®§§'žžž_Ý7ïÚºuƒ‰á‡®î6DÄþ,Y­ZALED$?kÑ‚¯¯ï5ßÖsÏ=‡““Ÿ}öY©Üžˆˆ”sgÎÀ[oÁ´iàê QQ0j\RfZ4ÑÔ¦6ý¼ûáäíDÛ¶mMG‘z“7I%•‰L,ÑåyöÙg¯sªkЬÄÄ@\<û¬eúÓOÀ+¯@1Eø"""ríyä&L˜À’%KxþùçMÇ‘râú·R©,–.…o¿µŒøs*ÛúÎtf0ƒÅ(Îr¶L¿·ˆˆˆä—M6ƒD-jñ.ïš ñÊ+–‰ß~kæû‹HéÊÈ€•+áÑGM'©”¶nÝÊüùóyýõ×U` "RÙegÃüùШ̘ãÇCRŒcw«XÅ}܇“zŽ‰ˆˆ”+©¤2‹Y<Ïóøág:Né ƒ„xýuË‹n€wß…œÓÉDDD*´êÕ«É‚ ¸xñ¢é8"RN¨È@¤4œ< £GÃSOAÇŽF"Ìd&i¤1“™F¾¿ˆˆˆXŒf4¿ñÿæßÔÀP÷.] {wË4)ÿ¢£áüyxðAÓIDD*œœ†N§N:t¨é8""bJn.|ú)[Ö† Ý»-ÅU«šNW ½ì%0Àt¹B¯ò*U¨Âs>¦“ˆˆT:óçÏç×_eÑ¢E888˜Ž#""&üü3tëж-$&Z:ïzy™NV¤•¬¤µ¸ÛMG‘+pƒ¼ÍÛ¼ÄKxàa:Îõåí sç¶mP«tí ÷ßû÷›N&""R!5mÚ”°°0Þzë-ÓQD¤œP‘ȵZ¿Þ²égþ|ðô4åYž%€^äE£9DDD*£Ýì&’H†1ŒH"MÇÎ-c‡'M2DD®Eb"|ÿ=<ö˜é$""•Nrr2“'Of̘1. ‘²·oDFZ¦gg[ÖV®„  ÓÉJd«èK_œp2EDDD®À&à‡Ãf:JÙiѾü>û 6o¶|e™î*"""¥jĈ|ùå—ìرÃt)Td r-²³á‰'àî»-cü sÁ…ùÌg«øŠ¯LÇ©42É$‚Ó˜7yÓtœ¿½ü²e¼p\œé$"rµ.„&M,EC""R¦FŒãÇ7EDDÊÒÉ“0v,4o¿ü+VXÞ[wîl:Y‰%‘D<ñ Àüº…ˆˆˆ”Ü6¶ñ Ÿ0i¸àb:NÙ ‡?ÿ„iÓ`ÎhÖ –.5JDD¤B §aÆ,Z´Èt)Td r-,€Ý»-S ìDzN8ÏñÙd›Ž#""R)Œa »ÙÍJV↛é8ëÔ î¼^zÉt¹gÎÀÇÃSO£Þ¾‹ˆ”¥µkײfÍ.\ˆ››½¾‘ë'+ æÎ…Æáý÷aÆ Ø¶ ""L'»b«XEMjÒˆˆˆÈËXnäFîç~ÓQÌqq‘#-Åwß C†Àí·Ã–-¦“‰ˆˆTŽŽŽ >œ%K–‘‘a:ŽˆØ9íR¹Z))–îÀ/¾hwã‘ßà ö°‡·xËt‘ ïK¾dóx‹·hLcÓq.7e lÜh5,"åËÒ¥–éiü÷{ k!ÐîÝ–nªÁÁ–ŽªÙÙæò‰ˆTóæÍ#!!E‹á`çÇ|DDä åæBt´¥¸`öl=Ú29ÌŽ ]ØÎv1Èt)¡÷xÝìf SLG)?4°3ýæHI¶m-N2LDD¤\ùÇ?þÁ§Ÿ~JZZšé("b§Td r%ÒÓaÒ$xê)KWÑr` Ø–²Ôt‘ !ƒ 3˜Ntâ^0§ä6„G…É“áÜ9ÓiD$¯”pº¤Ë¨µØ௿àÿ€€xã ˆˆ\'ÉÉÉDEE1fÌ‚ƒƒMÇ‘ÒôË/е+<ðÜvìÚeé€ëêj:Y©[Îr ,? DDD*¹ 2˜ÊTžâ)ÓØtœòçöÛ-Ó©/†ý 7†¹s-cEDD¤Xƒ ÂÁÁèèhÓQDÄN©È@äJLž ™™–Ór"„†3œ1ŒáªÜ¹V#Á NðOþIª˜Žse&M‚´4X¸ÐtÉëÈp,äíyNŽ¥ëê‘#pè8;—m6‘JbĈøøø0~üxÓQDD¤´:‘‘Сƒ¥ à×_-oëÔ1ìºÈ%—•¬d0ƒq@yDDDʃ9Ìá gÇ8ÓQÊ/GGËk¾?ÿ„dzL¬ºé&øþ{ÓÉDDDì^Íš5éÓ§K–,1ED씊 DJjûvxë-˜>jÕ2æŠLe*9äð2/›Ž"""R®-g9KYÊ|@=ꙎsåêÖ…gž×^³LhûpäHÑݵaȘ9³ì2‰ˆT"k×®eÍš5,\¸777ÓqDDäZ= 3fÀ 7ÀwßÁ‡Â7ß@ëÖ¦“]W?ðûØÇ ™Ž""""%pœãÌb£/¾¦ã”^^–½Û¶A½z– Váá°w¯éd"""vmÈ!lܸ‘?þøÃt±C*2)©çžƒV­àÑGM'¹b^x1•©ÌcÛØf:ŽˆˆH¹tƒ<ÅSŒ`á„›ŽsõÆŽµtFŸ3Çt±:t.\(øÿU©ƒ[F~;¨©ˆHiËÈÈ`Ĉ<üðÄ……™Ž#""×êÓO-ÅÓ¦Y¦ùíØaél[ ,g9­hEKZšŽ""""%0•©¸áÆHFšŽR±4k±±°nìÙ!!–u‘Ó§M'±KwÜq|ôÑG¦£ˆˆR‘HI|ó |õ¼þº¥‹h94Œa´¥-£e:ŠˆˆH¹“K.ó8u¨Ãë¼n:εñô„矇ٳ!%Åt8x°àóœ,ݶ>ø Ü¾±w'N$==Y³f™Ž"""×bçNèÙ"" {wKqÁèÑàêj:Y™È&›U¬Ò‘r"‰$Þá^æejPÃtœŠ), ~ûÍ2ÙyÑ"hÑ–.…Ü\ÓÉDDD슣£#‘‘‘|ôÑG\(¬)šˆTZÚ¥ RœœxáèÝî¸Ãtš«æˆ#oò&ßð «Ym:ŽˆˆH¹²…ÄÇG|„;î¦ã\»‘#¡zuËÁu1¯ ‚''Ë&©•+-Ÿ‹ˆH©Û²e ,àõ×_Ç×××t¹çÎATÜx#$%Á×_ÇB:†ƒ•­/ù’œà0EDDDJ`$¡ 5¥bsv¶¬‡ìÞ ýûÃССlÜh:™ˆˆˆ]2dG嫯¾2ED쌊 DŠóᇰu«e¼r9×™Î<ȃ<˳œå¬é8"""åÂö0†1Œe,è`:Né¨V &N„… aï^ÓiD*·Ü\8y2ÿyNNpË-mY‘R—““ÃðáÃéÔ©C‡jS‡ˆH¹!!0kŒc9Žf:•ñ·qhd:Šˆˆˆc [øÿbÓpFÇþÊD­Z0w.üò ¸»C—. GŽ˜N&""b5jÄ­·ÞÊ’%KLG;£"‘¢X» nY¬¨f1‹4Ò˜Å,ÓQDDDì^9 aiÌ$&™ŽSº†‡ÆaÜ8ÓID*·'àâÅ¿¿vv†ÐPˆ)77s¹DD*¸yóæ‘À¢E‹ppp0GDD®ÄÁƒpÿý–éÃ!!ðÇ–ãø..¦“q’“ÄË#>|;r$ß}GÕªUm—ñôôÄÉÉ <<<¨R¥Jé>>Ô­[—:uêàããC½zõðõõÅ××—ºuëâççG`` :{‹H…5qâDÒÓÓ™9s¦é(""R”;aøpøáxê)˜6 ªU3Êî\ä"ËYÎ†àˆ½Ç‘Ë,b9Ȧ˜Ž"Eqp€ÈHèßß2:* –,9sàž{L§¹î0`+V¬àÕW_5GD쀊 D.õí·ðÅðõ×`O›®ƒÆ4f£x‰—È@üð3IDDĨL2y˜‡¹Ûx’'MÇ);þþðÿ/¿l)²¬UËt")§²³³INNfÿþý$%%±ÿ~8ÀþýûÙ·oûöí###Ãvy777|}}©W¯>>>4lØN:áãミŸ~~~¶Ï+l'þ#,‹WÿŸ››nnnxyy]ÓÍ^¸pãÇsìØ1[AGÞÏ“““‰·ñâEÛuëÔ©C@@ 4 €† `;Ï××÷š²‰ˆ˜°eË,XÀ;ï¼£ç1{•mÙÀ5y2´h7Bûö¦SÙ­¯ùšƒd0ƒMG‘"œá ¯ò*Oó4˜Ž#%Q­š¥Àࡇ`üx¸÷^ ƒyó,¯SEDD*°ˆˆfÏžÍo¿ýF›6šÀ$RÙ©È@äR'ZÞ öèa:I™˜ÈD–±Œ‰Ld1‹MÇ1ê5^#‰$bˆÁ‡â¯P‘ŒoéÈóÚk0k–é4bÇ.\¸ÀÞ½{ù믿ø믿عs';vì`×®]$''Û6«»¸¸Ø6©7hЀ›nºÉöyƒ ¨_¿>†ïp¸>Ï5NNN¶BV­ZyÙœœ[BÞ‚°qãFV¬XÁáÇm—www'00fÍšåûhÞ¼9~~*\û“““ÃðáÃéÔ©C† 1GDD ²~½ezARL™b™6\¥ŠéTví#>ân¡9ÍMG‘"Ìd&™d2–±¦£È•jÒV®„o¾±LnÝž|ÒÒ°©fMÓéDDD®‹›o¾™† ­"Q‘H>_~ 6À?šNRfªR•éLç!bø™›MG1â/þbúÿÿ/ˆ ÓqÊ^0i<÷<ñ„åà¹Tj©©©lݺ•;v°cÇ[QÁÞ½{ÉÎΠ^½z¶ æ÷Üs­Û}@@uëÖÅá:m —ÒåèèH:u¨S§mÛ¶-ð2™™™¶©`Ïž=ìرƒÿýï¼óÎ;œ>}|EÍš5ã†n 88—²¼[""6óæÍ#!!„„ým±7§NÁK/Á‚г'¬] ¦SÙ½Sœâs>g>óMG‘"¤Â¼Á&P M.·ºw‡_…>°4­\¶Ì²žòôÓ*Œ‘ ÇÁÁþýûó¯ý‹W_}Õt1LE"yEEYFÝuêd:I™Ä ÞáF0‚Ÿø GMG)S9äðq#7ò ϘŽcÎðá°p!L˜+V˜N#e(99™øøxâããÙ¾};‰‰‰üñÇäææâêêJãÆ ¡_¿~Ì7Þ¨I•ˆ««+Mš4¡I!H©©©ìÙ³‡ÄÄD¶oßΞ={øüóÏILLäüùó899ѬY3BBB&44”›nºI“DäºKNN&**Š±cÇl:ŽˆˆäO=ÙÙ–Éz‘‘¦•ËX†#ŽDa:Šˆˆˆa S¨AʽîPQ89Á°aaÙWò °t)¼ù&Ür‹ét"""¥*""‚Ù³g“Phƒ2©Td bõùçðóÏðÓO¦“±€´¥-KYÊ?ø‡é8"""ej! ÙÄ&~ægªP‰»Î89Á´iз/<ó tíj:‘”²‹/òû￳iÓ&~ûí7¶lÙ¶mÛ8sæ UªT¡Y³fÜxã<üðônÝšo¼‘úõ뛎-倗—¡¡¡„††æ;?;;›;w²e˶lÙÂo¿ýÆ»ï¾ËáÇË4ŒÖ­[ÓºukÚµkG§Nð÷÷7qD¤‚1b¾¾¾Œ7Ît±:|Øòžóßÿ†‡‚9s vmÓ©Ê•Å,f ñ@…ß"""öj{XÌbÞæmªRÕt)-^^0w®¥XvÔ(¸õVèÕ æχ† M§)7ß|3 6$::ZE"•œŠ Drsaòdè×nºÉt#ZÒ’a c,céG?jRÓt$‘2‘L2™ÈhFÓ½A¦wo¸ë.xòIHH°H¹uòäI6mÚÄÆùñÇùùçŸ9sæ ¶MÝC† ¡M›6´lÙwwwÓ‘¥‚qvv&88˜àà` d;?%%ÅVt°e˾øâ f͚Ņ ð÷÷§sçÎtîÜ™Ž;Ò®];œ Þ )¯Ö®]Ëš5kX·nnnn¦ãˆˆHn.¼÷¼ø"øøÀ×_CX˜éTåÎOüD ¼ÍÛ¦£ˆˆˆHÆ2–F4â1E®‡æÍaíZËt®Q£ $ÄRH;q"T¯n:ˆˆÈ5qpp`À€¬X±‚iÓ¦™Ž#"iÇÀªU°u+|ø¡é$F½Â+¬d%S™Ê,f™Ž#""R&žâ)|ða"MG±óæA«V–ÍO>i:\]»vñí·ßòã?²iÓ&þüóOrssiÖ¬:uâþûï§sç΄„„àèèh:®Tb¾¾¾ôèу=zØÎËÈÈà—_~aÆ lÚ´‰©S§râÄ ÜÝÝ ¥S§NtéÒ…Ûn» OOOƒéE¤<ÈÈÈ`ĈDFF¦ ¬""æýþ;<þ8lÞléú:mT«f:U¹ôïÑŠVt¤£é("""Rˆ_ø…U¬b5«qÒ¶œŠ-<zö„·ß†—^‚O>±¼Ö}øapp0NDDäªEDD0kÖ,4Í@¤Ó»‘œ˜:î¿Z·6Æ(/¼˜ÊTžáþÁ?hIKÓ‘DDD®«•¬äs>gëpGÜmš6…§Ÿ†  "j×6H qæÌ6mÚDLL 111ìÝ»—ªU«Ò¶m[zôèÁ”)SèÖ­>>>¦£Š«ZµjtëÖnݺÙÎKNNfÆ ¬_¿žo¾ù†Ù³gãàà@›6m #,,Œ[o½sÁEÄ.Mœ8‘ôôtfÎœi:ŠˆHåvþ>ž3fУG¼½½éÙ³'6làþûïgݺuœsæÀìÙðË/*0¸FŸð ¸À`›Ž""""…øÿá¿ü—YÌÂu²¯Tj׶üü3¸¸@—. GŽ˜N&""rUî»ï>V¬Xa:†ˆ¤"©Ü.^„W^‡‚-L§± Ž82—¹|Ã7¬fµé8"""×Ís’n¦2•ßø™Ì4’=Ô¬ ¯½ƒC\œé›Ãûï¿O… xî¹çˆŽŽf„ DGG³uëVú÷ïO©R¥LgŠd+‹…ºuëÈñãÇùá‡h×®K–,¡B… 4mÚ”•+W’˜˜h:UDÒ‘¿¿?NNN >ÜtŠˆHÎ UªÀܹÿ?{ŽaÒÕüÁR–Ò›Þ¦SDDDä‚â7~ãüÏtŠdE¥K[gûÚµ þüÓ:û—ŸÄÆš.¹«æÍ›³oß>¢££M§ˆˆd 9SJ Lž ;C™2¦k²¬7y“2”a(CM§ˆˆˆ¤‹h¢y÷ÎpçqÓ9ÙÇ„ pó&ŒoºÄ¦„‡‡ãç燫«+&L I“&8p€­[·âïïOÉ’%M'ŠØ &NœÈ‰' ¥H‘"tìØ‘2eÊÀ‰'L'ŠÈCúæ›oøòË/ùðñ··7#"’3ÄÅY/Œzþy¨VM³d ÏøŒ›Ü¤ ]L§ˆˆˆÈmÄÇ{¼Ç@ê܃Ü]ݺ°c|ü1„„@ÅŠÖ¥oÞ4]&""r[Ï>û, ä›o¾1""häL«W[§£<ØtI––—¼Ì`+XA(¡¦sDDDÚ`ó(òo™NÉ^ŠƒwÞ±~Ñ}äˆéšlíæÍ›,^¼˜jÕªáááÁøðÃ9{ö,sæÌ¡zõê¦Xtt4‹%ÍO®\¹(V¬ 4`Ò¤I\¹rÅtæmܸ‹ÅÂÁƒÓu»—/_¦K—.ÄÄĤëvåÁØÙÙáííÍòåËùõ×_yíµ×X°`O>ù$mÛ¶eß¾}¦EäüùçŸøûûÓµkW¼½½M爈ä ëÖY|ù%,__ =fºÊf}ÄG´¥-Å(f:EDDDnc“H&™! 1"Ùtí ¿þ ýúÁ°aÖÏÖß~kºLDDäùòå£I“&¬]»ÖtŠˆ A’3 /€››é’,¯)MiA 1ˆDM爈ˆ<°ílg9Ë™Æ4ìÑÝmï[ïÞP¹2 hº$[JJJbþüùT¬X‘W_}wwwöíÛGXXÝ»w'þü¦ZéÒ¥III¡[·n,X””nܸADD=zô`Ö¬YT¯^ˆˆÓ©™æÊ•+4lØgžy'''Ó9ò/...Œ7ŽÓ§O³dÉŽ?N­ZµhÞ¼9{÷î5'"÷aĈ\¹r…ÀÀ@Ó)""¶ïòeëì¾¾àé ‡AÛ¶¦«lZa„Ž~¦SDDDä6Îr–éLg#4 PîO¡B0f DDX4k-[±c¦ËDDDÒhÞ¼96làêÕ«¦SD$“iä<›6AX Ñ]îÕ4¦qœãÌf¶é‘’D}é‹>´ …éœì)W. ‚ÐPX¶ÌtM¶òÍ7ßàîîNïÞ½iÒ¤ GeÑ¢E¸»»›NËp¹rå¢T©Rôèу~ø___âãã —eŽ¡C‡RªT)^{í5Ó)ryòä¡}ûö„‡‡³fÍ.^¼H:uxùå—9qâ„é<ùááá¨]""míZ¨Z¾ú V­²Î`P¼¸é*›7ƒÔ¤&õ©o:EDDDnc4£)BúÐÇtŠdWO>iýl½q#œú(½{÷æÊ_'P‚‚‚°X,X,‚ƒƒéß¿?<ò?þ8ü1‰‰‰ôíÛ—Â… ãââÂÇ|Ëk¯_¿OOOòçÏ££#¯¼ò gÏžM}< ªU«†ÅbÁÕÕõžúî$>>ž Я_¿4Ëccc2dåË—ÇÞÞwwwBBBnyþ±cÇhݺ5Å‹ÇÁÁ^x°°0þøãÔýa±XðôôàäÉ“i–ßn¿.\˜R¥J1vìØ»î£Çœ¶mÛ²{÷G[c±Xðõõe×®]„††AÅŠ àúõë¦óDä6’’’ðóó£^½z¼úꫦsDDl×¥KÖÙ Z´€zõààAxñEÓU9ÂYβ’• `€é¹£å>a<ãÉOöŸ­V kÒöí³ÞðiɨT æÎ…ädÓe""’Ã=öØcT­Z• 6˜N‘L¦A’³ìÛgÉ ÀtI¶3’‘¡£e:EDDä¾\àcË›¼IE*šÎÉþ¦L;;>ÜtI–•’’ÂôéÓ©X±"‡fóæÍlØ°ªU«šN3®E‹X,Ö­[—º,$$„V­ZáëëËÙ³g eË–-¼ø⋤¤¤Ð·oßÔ™‚ƒƒiÖ¬gÎœ¡cÇŽôêÕ‹×_çž{Žèèh:vìHïÞ½9uêTêö׬Yƒ¯¯/Mš4!**Š;vpôèQ6l˜ºÝ‰'¦~)AJJ 'Ož¼§¾;ùꫯHHHHð·qãÆ‘˜˜ÈîÝ»StèÐC‡¥Y綾^"þüDFF…««+ÞÞÞ*Tˆ¤¤$\\\èܹ3aaa¸ººrþüyÜÜÜHNNN³ßfÏž··7ÑÑÑ 6ŒÑ£G§¹ÓÈßûÈÇLJèèhvïÞM¾|ùhÒ¤É}ý~m‰··7ûöíc„ Q³fÍÔÙ8D$ë˜1cc±XL爈ئ+ bEøúkøòKëVMWåÁS˜Ât ƒé¹á §2•éBÓ)b+rç†×_‡£G¡];ð÷‡:u`ûvÓe""’Ã5mÚTƒ Dr 2œeüx¨Q¼½M—d;(À&0Ÿùìaé‘{6ŒaØcÏF˜N± … ÃäÉÖ»çèKí[ÄÄÄТE Ì!CØ¿?52•e<òÈ#+V,Í €¡C‡âææÆرc)Z´(5jÔ 00ï¾ûŽï¿ÿ>Íó=<æù… ýÿ`!;;»;.ûûNóoÿß³ 8::¦>~'÷Û÷O—.]"Ož<·,ŒŒ¤M›68;;cgg‡ÅbaÁ‚\ºt)Ízëׯç…^`àÀ)R„æÍ›³{÷îÔÇK”(AÛ¶m™?>aaaÔ¨Qã¶þ¹òäÉsOûHþŸÅbaÀ€„††²e˼½½ÓÌ"!"™kõêÕ„„„ðá‡bo¯;;‹ˆ¤›¸8ðóƒ¶m¡qcØ·O³D/ñ¥)m:EDDDþåk¾æ{¾g"M§HNѲ%> ï½|*ÀÂ…ð›úˆˆˆd¤üùóS¯^=6lØ`:ED2‘HÎ ‹ÃðáKS>ŒªTåu^gÈCÕˆˆHÖõ)Ÿ²—½„Íb”î\\àÿƒqã¬_lç`ááá´lÙBCCu±ö]|õÕW¤¤¤àëk½»WñâÅ:t()))·ü,Zôp[ÿÞþÅ‹oyìÂ… ©ÿ×ó¤¯hÑ¢$&&¦Y–˜˜ˆ··7QQQlÞ¼™ÄÄDRRRèÖ­[šþ~þ”)S8sæ [¶l!!!† rìرÔuüýýÙ³g‘‘‘ÌŸ?Ÿ>}úÜ}‡Üå=ÞnÉ­5jÄÎ;9}ú4-[¶ÔŒ"ÄÇǾюí IDATÓ¿ºv튷.|I?ë×[g/ ±þ,_E‹š®Ê±Âc{èG?Ó)"""ò/I$1œá´¥-^x™Î‘œ$o^0Ž__xõUxæØ¿ßt™ˆˆäM›6eãÆ·œ×Û¥A’3L %J@—.¦KlÂ8Æ‘Dïò®é‘ÛŠ'ž·y›Þô¦5LçØ®Aƒ¬Só¾ñFŽ½[ιsçhÞ¼9õêÕã³Ï>#_¾|¦“²¬óçÏ3bÄüqüüü(]º4•*Uâ‡~¸eý5j°lÙ²‡zÍÒ¥KS±bE6oÞœfùþýû¹råJš‹SÿžáßÏÐ>ggg._¾œfÙñãÇ9{ö,:t R¥Jäúkôõë×Ó¬wîÜ9ªU«–úgOOO>úè#nܸÁÞ½{S—{yyQ³fM‚‚‚8uêO=õÔ{îäï}´eË–4Ëûí7ìíí¹páÂ}oÓÖU¬X‘ÐÐP>L=Lçˆä8£FâêÕ«j6%‘ôqåŠuö‚fÍÀË ‚V­LWåx3™IMjRÍ’'""’Õ,`G9ÊXÆšN‘œªdI˜3öì›7¡V-èÚΟ7]&""6îÙgŸåüùóÎá7!ÉI4È@lß•+ðñÇÖݺè+]¥(cËt¦sƒ¦sDDDn1‰I\ç:ïðŽéÛ–;·õ‹ìíÛaÁÓ5™îï;Ð;88°|ùròæÍk:)ËIJJâÌ™3ÌŸ?ŸÚµkcggÇÚµkqppH]gÊ”)lݺ•‰'Kll,o¾ù&7oÞ¤uëÖÝ0yòd>̈#¸páGÅÏÏòåËóÆo¤®W²dIŽ9Bll,%K–dïÞ½Ü÷ÔSOŸf†WWWJ”(Á¢E‹ˆŒŒ$!!ÐÐPÖ­[wËó<È|À•+W¸|ù2sæÌÁÞÞžÚµk§Y¯OŸ>Óå!TOž<™ÈÈHFÅ… 8uê=zô [·n8::ÖY, 'j x€*Uª°téRV¬XÁ'Ÿ|b:G$Ç'((ˆÀÀ@œœœL爈d;vX/HZ½V¬°Î^ð×ç?1ç,gù‚/ÀÓ)"""ò/ $ðïЋ^T¦²éÉéjÕ‚mÛ`éRغ*U‚÷߇ÝÔFDD$½Ô¬Y“Â… ³uëVÓ)"’I4È@lßüùÖÑÛ¯½fºÄ¦øá‡;î d`ºnwõêÕX,öíÛwËcM›6ÅÃÃ#õÏ´nÝš¢E‹’?~êÕ«wˇ˜¨¨(^~ùeJ–,‰ƒƒuêÔyè;⊈HÖö¿ñð6oãˆ.ŽÈpµkƒ¿?¼õÄÄdÚË<gggbccyñÅ)T¨¥K—fÆŒ·¬»iÓ&4h@(\¸0­ZµâÈ‘#݆ X¼xqš‹æsªèèh, ,àÏ?ÿÄb±'OªU«ÆÇLß¾}ù駟ÒÜ¡À××—o¾ù†J—.MÕªU9þ<ëׯÇÞÞž¥K—¦îß^½zÑ¥K6nÜHžÉJråÊ…———ˆä d ¶-9‚‚àÕW¡X1Ó56Å;¦3ïøŽ/ù2ݶ۲eKJ–,ÉÇœfù©S§Ø´i½zõàÀxyyQ°`AÂÃÃ9sæ Ï?ÿ<>>>„‡‡§>¯}ûöÄÄÄ°}ûvbbb&$$„óš*PDÄfd$%(A?ú™NÉ9ƃ‚aðàL}Ù””È›o¾É™3gèß¿? `×®]©ëlÚ´‰çž{ŽZµjqüøqÂÃùvíõë×çôéÓõúãÆ㥗^¢nݺûVlBéÒ¥IIII󓜜ÌÅ‹Ù±cC‡MsQÿ?5mÚ”]»v‘À¹sçøì³Ï(]º4;vL³ÍÅ‹ãíífÙ_|AãÆÓ,Û¸qcêöŸþyvïÞMBB/^dñâÅ©3üÓ¬Y³¸|ù2qqqÌš5ëžúîÄÁÁnݺ1sæÌ4Ëk×®ÍÖ­[‰çôéÓÌ™3‡+V¤v;;;мysBCC‰%..ŽmÛ¶ñì³ÏÞò:+W®¤Gäû׬m·Ûoÿþýú믷ÝGgÎœaÊ”)iQL™2…ÆÓ°aû¾ïœfäȑܸqƒÙ³g›N±y3fÌ ""‚àààÔAe""ò fM†Ù³aíZxì1ÓU÷Ìôq`F»Á æ0‡×y{î<¨YDDD2ße.ó>ïó&oR’[¿[ü/:.ª@ëƒ_~OOèÐ||àÐ!ÓeÊÖDD²¢† ²yófÓ"’I4È@lÛ×_Ãñãз¯é›Tzt¢ÀU®¦Ë6sçÎM=øì³ÏHHHH]>þ| (ÀË/¿ ÀСC)S¦ .¤lÙ²+VŒÿýïxzzòî»ï˜˜ÈîÝ»éܹ3åÊ•#þü<õÔS,Y²„G}4]zED$k‰ ‚…,d<ãu2>398Xv.Z6dÚËÆÄÄйsgž~úi .ÌСCyâ‰'øôÓOS×9rdêÙ)_¾éºý;xð §OŸfE¸ûÕýrtt¤cÇŽÚ7",**ŠÑ£G@•*UL爈dO‰‰Ö‹Ž4ˆˆ€×_7]ußLf†¥,åèMoÓ)"""ò/ïñvØ1˜»ÙŽÎƒK¦(].„ﾃß·0öóƒØXÓeÂÖDD²¢FqöìÙ473Û¥AbÛ¦O‡fÍ R%Ó%6k S¸Ìe¦0%ݶ٫W/®\¹ÂÊ•+HNNæÓO?¥cÇŽ888pãÆ ¾ÿþ{Z´hAîܹÓ<·Q£Flß¾€ûì³Ûþwø FŽIJJ —.]¢¿ŽwŒ+[¶,)))œ9sÆtŠˆMZ½z5!!!|øá‡ØÛk¦*‘ûró&¼ÿ>xx@°? ‹é²fò80£}Ã7àoò¦éù—QŒÂ §‡žmHçÁ%SÙÙA×®pô(ôêÆAµj°~½é²tcËÇ""YUž>žþýûÓµkW¼½½M爈d/‘‘àåcÆÀر°m<ù¤éª‡fò80£Èó<;î¦SDDDäŽp„E,b<ãÉG¾‡ÞžÎƒK¦+R&N„Ÿ~‚råàùç¡eK8~ÜtÙC³å㑬ÌÓÓ“°°0Ó"’ tUŒØ¦øxøüsèÝ;[ß•)»ÈK^f0ƒ¬ ”ÐtÙf‹-(Y²$“&M⫯¾âõ×_O}ÌÞÞžÆrÏò(P€¦M›²|ùròåËÇîݻӥSDD²†¯øŠ0Âx‡wL§H«VÖ;ãôî 11FSìíí©S§Î-'U.^¼ÈŽ;hÔ¨Ño»T©RDGG§9©#w¶qãF, 4¾ýäääÔ“rÙݼyóRïdœi¯úºß~ûí}=·{÷î:t(ƒÊÌ8uê`ý{ADÒרQ£¸zõ*¦SDD²äd˜:jÕ‚\¹`ß>ëKsÈ éŒ<ÌH?ð›Ù̆˜N‘ÊPÜp£#Óe{:.ÆT¨_ 6À‰P¹²u¦³øxÓe&»ˆˆduuëÖåðáÃ\¾|ÙtŠˆd°œñ­²ä<‹Ab"¼òŠé’£)MiNs1ˆDz{¹sç¦GÌ;—ܹsÓ¹sç4O™2…Ÿþ™Î;søða®]»ÆÑ£G™6mo½õ§OŸ¦U«Vlܸ‘ .Ïܹs¹qãÏ<£»\‹ˆØŠRÍhÚІ:Ô1#3f@þüÖ†;–ˆˆ Äùóç9~ü8:u"Ož< úà³05hЀ˗/ë„M6ôôÓOsñâEÜÜÜL§<´ž={ríÚµL݉'røðázn¿~ýhÖ¬Hç*sÖ¯_OõêÕ)R¤ˆé›NPP·Ò^DDn#* || FŒ€íÛ­³ýæ0u˜‘&1 ‘@ƒ ÄöìØauíçgº$Ç)G91ˆQŒâçz{'NœÀï¿ËJ•*±téRbbb¸ví‡fòäÉ<öØc©ë4oÞœo¿ý–ØØXâââصk—.±!I$ñïЙÎT§ºéù'ooëL½{Ù3é¾ùÉ“'sîÜ­Ÿ7–.]JXXXše>>>ìرƒk×®qåʾþúkªT©òÐ o¿ý6‹-"22ò¡·eJPP‹‹ÅBPPo¼ñÅŠÃb±Ð±ãÿOÿ½fÍ<<<°··çÑG¥wïÞ\¹r%õñcǎѺukŠ/Žƒƒ/¼ðBêï! ªU«†ÅbÁÕÕ€ØØX† Bùòå±··ÇÝÝûj¼Ûöïö~çÍ›w˲àà`ú÷ïOáÂ…)UªcÇŽÍ”ýw/ûáo³fÍÂÅÅ… ðÌ3ÏðË/¿ÜµñŸÖ¯_§§'ùóçÇÑÑ‘W^y…³gÏÞöýÜϾøã?RŸg±XðôôàäÉ“i–ÿ­OŸ>9r„Í›7ßs{V5eÊnܸA¿~ýL§ˆØ”3fApppš¿?DDä6ââ¬3úvè`½8hï^ðð0]•!²Âq`F˜ÌdÊP†6è{k‘¬d«ØÅ.&39Ý·­óà’e+Ó§[# „  }{8}ÚtÙ²Õã‘ìÂÓÓSƒ Dr 2Ûl½cS:¦Kr¤QŒ¢EŨ‡ÚÎ… xûí·iØ°aêEJ"""ÿ¶€ã£m:En'0J”°4°A:u¢V­ZtîÜ™ëׯ›Îy }ûö%>>°^,ýì³ÏÅÌ™3S× ¡U«VøúúröìYBCCÙ²e /¾ø")))¼ôÒKäÏŸŸÈÈH¢¢¢puuÅÛÛ€‰'²aÃ"""HIIáäÉ“Œ7ŽÄÄDvïÞz¡}‡8tèÐ=7Þmûw{¿·[6{öl¼½½‰ŽŽfØ°aŒ=š-[¶døþ»—ýðùçŸÓ·o_zõêÅ™3g˜:uê=Oç¼fÍ|}}iÒ¤ QQQìرƒ£GÒ°aÃÔ÷ð û¢P¡B$%%áââBçÎSOฺºrþüyÜÜÜHþÇTÛE‹¥R¥J,3<ÓÉÃÚ»w/ï½÷£G¦˜fÑI7QQQŒ=šáÇS¹reÓ9""YÛ¦MPµ*lØ_}sæX/ ’lãw~çS>å-Þ"79sæ ‘¬(‰$F1ŠŽt¤µÒuÛ:.Y’»;lÞ !!ÖUªÀ˜1`ºLDD²¨ºuë²gÏÓ"’Á4È@lËÅ‹ðÅÖ©ÜĈ`˜Ï|öð`$¼½½)Y²$‹…O>ù$ EDÄVÜä&ïñÝéN9ʙΑÛ)X>ýÖ­ƒùóMפ»\¹r±dÉŽ?Îk¯½–æ"êìÈÛÛ›víÚQ°`AúöíËÒ¥K:t(nnnŒ;–¢E‹R£F ùî»ïøþûïIHHàÀ´iÓ'''Š)ÂäÉ“)x7M›6iÓ¦áèèH¡B…èܹ3M›6Ms‘þ½4¦Zµj…ƒƒþþþ(P€mÛ¶ÝÓstÿÁ½ï‡1cÆàîîÎÈ‘#)Z´(5kÖäõ×_¿§¾ÁƒS¥JÆOñâÅ©T©sçÎå×_åÃ?|è}agg‡ŸŸ«V­"...uùÂ… éÙ³ç-w"â‰'øñÇï©=+ŠŠŠ¢mÛ¶4nܘšÎ±)ýúõÃÙÙ™€€Ó)""YWB@Ó¦P·.:-Z˜®’DùÉOwº›N‘ø˜ù…_ËÝg:½_:.Y^Ë–päŒS§B… °p¡é*É‚ÜÝ݉‰‰á·ß~3""Hƒ Ķ|öäÊ;š.ÉÑ^æežæiüñ'™û¿ØnãÆܸqƒ;wR¶lÙ ([°˜Åœæ4Ãn:Eî¦^=8 ‚S§Lפ»²e˲råJ¾øâ üýý³õ@ƒêÕ«ß²,::šŸþ™Æ§Y^·n]6mÚ„½½=µk×føðá¬X±‚k×®‘;wnΟ?ÿ@ÅŠã矾çÆôT­ZµÔÏ•+Å‹'&&æžžû ûïNþ½.^¼ÈÏ?ÿLƒ n»­»‰ŽŽæèÑ£4jÔ(Írwwwyä6nÜxËsd_ôìÙ“äää4ƒ?–-[F÷îÝoY·H‘"·Ê:;8sæ >>><òÈ#|þùç·  ‘·zõjBBB˜={6ööö¦sDD²¦ˆëÀ‚Ù³­?_|ŽŽ¦«ä\å*ò!ýèGA4…ˆˆHVqk¼Ë»ô¦7å)Ÿ®ÛÖypÉòæ…¬ƒ š5ƒW_…gŸ…L—‰ˆHR³fM, û÷ï7""Hƒ Ķ|ò ´o¦Kr¼ ‚ØÏ~±ÈtŠˆˆØ $’˜ÈD^ážà Ó9ò_Ƈ2e GHI1]“î¼½½Y¶lŸ~ú)íÚµãÚµk¦“HþüùoY @PP‹%õÇÉÉ €Ó§O°~ýz^xáH‘"EhÞ¼9»wïþÏ׌ŒŒ¤M›68;;cgg‡ÅbaÁ‚\ºtéžÓS¡B…Òü9Ož<÷þO²/J”(AÛ¶m™ÿ×Ì%aaaÔ¨Qƒ"EŠÜ²nž‘OÿþýéÖ­Mš41#"’õ$'Ãôéàá… Á?Â=Îj%YÓ<æq•«øão:EDDDþa*S¹ÌeF0ÂtŠˆY=sæ@X\¿O=]»Â=Þ˜GDDl[‘"E(S¦ŒˆØ8 2ÛûöYGQ‹qU©J/z1ŒaÄg:GDDlÌç|ίüJ¦Sä^äËg ºu+™®É­[·fÆ lÙ²…zõêqôèQÓIé¢xñâ :””””[~-²(-Z´(S¦LáÌ™3lÙ²…„„6lȱcÇî¸íÄÄD¼½½‰ŠŠbóæÍ$&&’’’B·nÝH±‘Á(÷²ÿîu?”,Yà– óãâÒ~ÖnÑ¢Eš×X³fMjÇí.ê¿páBêãéÁßߟ={öÉüùóéÓ§Ïm×KLLÌvè/^¼˜úõëS®\9¶oߎ³³³é$›2jÔ(®^½Ê¤I“L§ˆˆd='OÂ3ÏÀ!0|¸õت\9ÓUò’Hb3x•W)A Ó9"""ò—K\b SÂåQÓ9"YCíÚ°};,] [¶@ÅŠðþûpã†é21ÌÝÝšéFĦiØŽyóàÉ'¡AÓ%ò—ñŒ'‰$Æ1ÎtŠˆˆØ$’ÏxºÐ…'yÒtŽÜ+9†…Ÿ~2]“!4hÀîݻɓ'µjÕbîܹÙþbùÒ¥KS©R%~øá‡[«Q£Ë–-ãܹsT«V-u¹§§'}ô7nÜ`ïÞ½ØÙÝzèyüøqΞ=K‡¨T©¹råàúõë÷Ýy»íg÷²ÿîu?+VŒ *°}ûö4ËüñÇ{ê¨X±"›7oN³|ÿþý\¹rooïû|gwæååEÍš5 âÔ©S<õÔS·]ïòåËÙæ"ý‹/òÊ+¯ÐµkWzöìÉúõë)Z´¨é,›NPP“'ONíEDDþ²p!T¯.ÀîÝ0f üõ™Q²¯ÏùœSœbƒL§ˆˆˆÈ?¼Ë»ä!þ-òo ´k‘‘0`€õ¸¤Z5X»Öt™ˆˆäîîξ}ûLgˆHÊšWbˆÜ¯7`Éë,‹éùKQŠ2–±Lc9h:GDDlÄ Vð ¿ð6o›N‘û5jxyA§Npõªéš Q®\9vìØ¿¿?þþþ4jÔˆÈÈHÓYeÊ”)lݺ•‰'Kll,o¾ù&7oÞ¤uëÖø€+W®pùòeæÌ™ƒ½½=µk×þÿ.üGŽ!66–’%KK‰%X´h‘‘‘$$$ʺuëî»ñvÛÿ{€ƒiÿµÿ\]]ïy?Œ3†ýû÷3nÜ8.]ºÄO?ýtÏwüžÃö_¾|ùîi?têÔ‰Y³fñÑGQ²dIüüüxï½÷xã7ðôô¼ãïºE‹¬]»–ãÂC IDAT7RªT)¼¼¼xòÉ'Ùºu+<òÈmßÏöE@@•+W Y³f· &ðõõÅÑÑ‘öíÛ߶åòåË9r„:ܱ״ݻwÓ¸qcºtéÂsÏ=GDD¾¾¾¦³DlÒŒ3ˆˆˆ 88‹n!"bµv­õ® ûöÁwßÁôé/Ÿé*I'KYª›'ˆˆˆdA#I)JÑ“ž¦SD²¾Ç·žïùî;ˆ‰wwë qq¦ËDD$U©R…äädŽ=j:ED2ˆ%%%%åAŸ@­ZµÒ-Hä¼ð\¿p×SÉx;ØÁÓ<Í*Vñ/˜Î‘fîܹ¼þúëþ:Ë—/§C‡<ÄÿVÿSf½‘¬ì[¾¥Íø‘©‰îà˜m­^ mÚÀçŸCÇŽ¦k2TRRŸ|ò cÆŒáÊ•+ 8àèèh:M$ÃÌœ9“èèhÞÿýÛ>>aÂ>ûì38@®\¹2¹îîÂÃÃy÷Ýw ¡qãÆLš4)uFŽÌô÷åË—gøkeôgL‹Å²eËî8è$=éórö…››o¾ù&cÆŒ1#"bÞµk0x0Ìž ]ºÀÌ™P¸°é*IGI$Q•ªÔ¦6 Yh:GDr û9¿¯c,ÉI~â'jR“Ïùœödüw"6%9/†!C )É:«u߾žû‘ôwóæM *ļyóî:ÃyV’×w‰¤§Ì:6¿ÓùiÍd ÙߥKðí·Ð©“鹃úÔ§x‹·H ÁtŽˆˆdcÒ”¦`ݽø"øùÁoÀÉ“¦k2T®\¹èÙ³'?ÿü3ÇgöìÙ¸ºº2dÈÎœ9c:O$ÝŒ7ŽÀÀ@¢¢¢˜9s&}úô¹ízáááÌž=›Å‹g©[·nÅ××¢££Y»v-ßÿ½‘"9I¿~ýpvv& ÀtŠˆˆy‡§§õ✅ ­?``s–±Œ_ø…Œ0""""ÿ0Œa<ÅS´£é‘ìÇκv…#G gO:j׆­[M—‰ˆHË;7åË—çÈ‘#¦SD$ƒhd+V€Å­[›.‘»˜Âb‰%@Ó)""’M…Îw|dž˜N‘ôðÁPªtî ‰‰¦k2\>|8'Nœ`̘1|öÙg¸ººòâ‹/òí·ß’œœl:Qä¡ :”êÕ«Ó·o_\\\n»ÎŒ3øæ›opwwÏäº[ÅÅÅDµjÕhÔ¨ñññ¬[·Ž½{÷âëëk:OÄæ­^½šfÏž½½½ésRR`út¨U „}û¬³ˆÍI&™ L ¨HEÓ9"""ò—-lá[¾e"±`1#’}- 'ÂO?AɒШ´l 'N˜.‘ T¹re>l:CD2ˆHö·d ´j¥»:eqÎ8@˜ÀINšÎ‘l(@jPƒ&41"é!~X¶ €áÃM×dšB… ñÖ[oqòäI–,YÂü¯¯/eÊ”! €èèhÓ‰"däÈ‘¤¤¤péÒ%ú÷ïÇõ,X@ÕªU3±ìVáááøùùQºti† BåÊ•Ù¾};Û¶mãùçŸ7Ú&’SÄÇÇÓ¿ºuëF“&úl'"9Øùóм9  °m”-kºJ2È2–q˜ÃšÅ@DD$ I!…hF3{I/+ÂÚµ°a?nnÖãøxÓe""’*Uª¤™ Dl˜HööÛoÖ/:™.‘{ðoñ83Œa¦SDD$›9Á V²’t'![âæ}S§ÂÊ•¦k2UÞ¼yi×®6l 22’víÚ1wî\Ê•+GûöíYµj×®]3)b3¢££™>}:xxx°sçN&NœÈ¹sçX¾|9õë×7(’£Œ5Š«W¯2iÒ$Ó)""æ¬^m=&:r¶l1c W.ÓU’A’Iæ=Þ£¨D%Ó9"""ò—¬`{ÏxÓ)"¶ÇÛöï‡  8*W†… ­³¹‰ˆˆÍ¨\¹2¿üò ‰‰‰¦SD$hdoŸ}fÁ Y3Ó%rò’—™Ìd9ËùžïM爈H62i”¦4mik:EÒ[§NЫ¼ú*=jºÆˆJ•*ñÁpæÌæÎËï¿ÿNûöí)Q¢:t`ÅŠüù矦3E²Ó§O3uêTêÕ«G™2e=z4nnnlÛ¶ˆˆüýý)¬ñD2]xx8AAALž<'''Ó9""™ïÚ50Ú´__øé'¨WÏt•d°¬ ’H†“sfòÉêId$#éLgjRÓtŽˆmÊ“Çzüsì¼ôôèuë®]¦ËDD$”/_žÄÄD¢¢¢L§ˆHÐ ÉÞ–/·žŒÉ—Ït‰Ü£¦4¥9ÍéK_ÑFùoñij€ô£¹Ém:G2ÂŒP¡´oo½à&‡ÊŸ??ݺuãûï¿çÌ™3K§Nprr¢mÛ¶,]º”¸¸8Ó©"YÖñãÇ™tí çΙ.‘‡äêê ÀÉ“'vˆHÆÐ ɾNž„ðph×Ît‰Ü§éLçÇ&ØtŠˆˆd X@"‰t§»éÉ(ùòÁ²epú4 hº&KxôÑGyã7Ø´i111Ìž=›ëׯӵkWñðð €7rãÆ Ó¹"ÆüùçŸlܸ‘€€<<<(W®ï½÷*T $$„sçÎñ駟Ҽysòipºˆq3fÌ ""‚àà`,‹é‘Ì“’b½ ¦~}(]""¬³ºIŽð_pˆC¼ÍÛ¦SDDDä/ðã‡?þ”¥¬é‘œ£fMغBB`Û6(_ÆŒ„Óe""ò€œœœ(X° ˆØ( 2ìkÕ*(\žyÆt‰Ü§r”cƒÍh~çwÓ9""’…¥Â,fñ ¯PŒb¦s$#•+Ÿ~ }lº&K)V¬]»vå믿æìÙ³,^¼˜5j°dÉ|||prr¢M›6Ìž=›cÇŽ™ÎÉP7oÞdÇŽŒ3†zõêQ¤Hžþy¾ûî;žþy6oÞLLL .¤eË–äÍ›×t²ˆü%**ŠÑ£G3|øp*W®l:GD$óDEÁ³ÏÂ!0|8lØ`h 9B )Œg<íi¯Y DDD²)LáOþ$€Ó)"9SË– £FÁ”)P­¬XaºJDD‹‹‹ˆØ¨Ü¦DتUЪ袑li#XÄ"F0‚¹Ì5#""YÔ6p„#,c™éÉ ­[ÈðÆP¡<ý´é¢,ÇÑÑ‘Ž;Ò±cGŽ9Bhh(6l`èСôéÓêׯ——^^^Ô¨QƒÜ¹uè'ÙS\\aaa„……±k×.víÚÅ•+WpqqÁÇLJAƒñì³Ïâèèh:UDþC¿~ýpvv& @pˆH²r%¼þ:89AX<õ”é"Éd_ð9Èb›N‘¿üÎïLe*à„“é‘œ+~6 ºt±ÈîЂƒáƒ zuÓu""r\]]5È@ÄFéJɞΟ‡]»¬w’l©…xŸ÷éJWzÒ“:Ô1$""YÐLfò ÏP}™˜c¼óìßíÛÃÞ½Pª”é¢,­R¥JTªT‰þýû“˜˜ÈÎ;Ù¼y3aaaŒ9’¸¸8 ,ˆ‡‡õêÕÃËË OOOJ”(a:]ä)))=z”°°0vîÜÉ®]»ˆŒŒ$99™²eËâååÅ„ hÒ¤ +V4+"÷aÕªU„„„°qãFìííM爈d¼øx<æÎ…W^Ù³¡`AÓU’É’Hb,ciK[ªQÍtŽˆˆˆüe,c)HA0ÀtŠˆ€õ<Ð…з/ µjA0nè\†ˆH¶àêêJDD„é Éd ÙÓªU`o>>¦Kä!¼ÌË|ÄG d ;Ø‹é$ÉBŽsœoø†å,7"™ÉΖ,OOhÛ6o†|ùLWe yòä¡Q£F4jÔ€ääd>œzÁö—_~ÉĉIIIáÉ'ŸÄÝÝwwwjÔ¨AõêÕyüñÇ ¿ÉI9rä?ýô`ÿþý„‡‡sñâEòçÏO­ZµhÖ¬ï¾û.žžž8;;›N‘Ï€èÖ­Mš41#"’ñv_¶4øê+hÙÒt‘²…æ°¾×ÉBNp‚¹Ìe&3)ˆŠd)uêÀŽ°h‘u†ƒ+¬ÿ4òæ5]'""wQ¦LÖ®]k:CD2€Hö´z5øúB¦Kä!X°0ixàÁ"Ñ•®¦“DD$ ™ÇÍñãÇ9~ü8‡"22’ãÇsâÄ RRR(R¤nnn4hÐ???ªT©‚‡‡ööö¦óE$ƒìÝ»—Y³fñÑGáääd:GD$ãœ?ÿÇÞ}ÇGYæëÿ¤÷^H$–aEYDÑe­»PVÙƒ‹ØV`EÁ³¸GʪèZÖÕsÖ²û³+bA–ª E:B€ô„@’IHI~ ó˜H‘òL’ëík^3y¦ä!äžû¹¿÷wÂX¾{Ì^hàêjv*1Ñ ¼Àa3“™fG‘㶲•÷xø@çDœŸÌž ·ßú\s \y%<ÿ<¤¨ˆWDÄÙDEEQWWGEEÁÁÁfÇ‘V¤ONÒþ,_V+\u•ÙI¤•<ÅS|ÀÌaó™ovq_ð…2 fG3 þ3<ð$&B³…ñÒzÉÈÈ ##Ã8fµZÙ»w/Û¶mcÏž=ìÙ³‡M›6ñÎ;ï`±X W¯^$%%Ñ»wozöìI·nÝHHH &&ww}Üìhª««9xð 999:tˆ½{÷²gÏöîÝË°Z­ÄÅÅÑ«W/zõêÅÕW_MŸ>}èׯ111&¿iK6›I“&‘žžÎ„ ÌŽ#"rá|õŒÞÞ°z5\z©Ù‰Ädå”3—¹<ÄCÄgv9îa0ƒù¿2;Šˆœ©ž=á½÷`Å û¹¢‹.‚ÿú/û¹£À@³Ó‰ˆÈqÑÑÑ©È@¤ƒÑªi¾ü†í€×a„Ê“<É<ÀD&’L²Ù‘DDÄd¯ó:Wp=èav1ÛŸþ‡Á-·Àš5ö d¹àÜÝÝIMM%55õ„ûŠŠŠŒ…åŽË;ï¼Ã¨¯¯7žCBB‚QxOBB]»v¥k×®øûû·õÛ’ŸQTTDnn.999F!Á¡C‡Œ¯KKKÇE&wÜq‡QT””¤?[`áÂ…lß¾-[¶¨óˆtLuuðè£ð öÝ5_z 4àiž¦‘Fæa³£ˆˆˆÈq_ñËYÎ Và‚>£Š´;W\›6Áÿþ/Ìœ ÿþ·ýú77³Ó‰ˆtzQQQ€ý\cr²Öý‰t$*2ögéR˜8ÑìÒÊ&1‰ð¦0…e,3;Žˆˆ˜¨ˆ"¾äKÞà ³£ˆ³øÛß`ÿ~{;Üõë!N;Aš)**Š¨¨(†~Â}eeedgg“MAA………dgg³téRãkoooBBBˆ‰‰!::úgo˹©©©¡°°‚‚ÊÊÊ(,,$??ŸÂÂB cyyyF‘@HH‰‰‰$&&’‘‘ALLŒñutt´þLDä´rss™5k3fÌ E-ìE¤#Ú½~óûç”7ß´ˆð/0‡9„bvšhb3ËX~Á/ÌŽ#"çÊÝî½nº fφG±{î96Ììt""Zxx8îîî™EDZ™Š ¤}Ùºrr`̳“H+sÿñ7†1ŒE,âz®7;’ˆˆ˜äÿø?`ãÌŽ"ÎÂÃÃÞ÷²Ëàúëaõjðó3;•œDHHiii¤¥¥ô~‹ÅBNN¤¤¤„ÂÂBJJJ())!''‡¬¬,Š‹‹9räH‹çùúúJpp°q iñõOáííþþþxxx´Åÿ‚VuìØ1êë멨¨ ¾¾žòòrÊËË)++ûÙÛååå=z«Õj¼ž‡‡4440räHRSS‰ŠŠ"""‚ØØX£ë„§§§Yo[D:€)S¦ÍôéÓÍŽ""ÒúÞ|&O†ÔTûNš={šHœÈ¿9WWד[ÄÇÇÓ¿ÿ÷ÅÄÄAdd¤Q`••Å7ÞȺuëxà¸øâ‹ö=‰ˆœ©>úˆE‹±|ùr¼½5ž‘¤¢Â¾ˆåw`ʘ?T˜)ÍìaoðÿËÿâ…—ÙqDDD¨§žYÌâNî¤/§Ÿ“‘v&9Ù¾†hñbxðAèÓÇþYmæLð÷7;ˆH§Ôâ\¨ˆt *2öeéR=\]ÍN"È|æ“L2 XÀLfšGDDÚØw|Ç^öò/þevqF={Â'ŸÀUWÙ÷üýïp ¾¥ýñôô$66–ØØسzžÍf3 ***ŒÅúUUU444œt!¿£[@yy9MMMTUUqøðá^Ûb±œð=ëêêðòj¹€ÈÍÍÀÀÀ„ëñÏ2ÁÁÁøúú]š@xyyáëë‹ŸŸžžžáééiœìµÏFZZ7näöÛogøðáÌ;—©S§ž×kŠˆTVV2uêTÆÏÈ‘#ÍŽ#"Òz6l€ßþŽ³/b=ÚìDΩ1R¸ÛÌŽ""""ǽÌËä“Ïlf›ED.”±cíŸÑ^z žxÞ~þòû†U:‡$"ÒfTd Ò1©È@ÚšX¿î¾Ûì$rÅËŸøæÏÜÁt¥«Ù‘DD¤ ý?þ=éÉ`›EœUF¼÷ÜpøúÂsÏ™Hœˆ››aaa„……™Åé…‡‡óå—_2oÞ<zè!233yå•Wðóó3;šˆ´c3gΤººšyóæ™ED¤uØl°`<þ8üâðÆev*qBØÀ§|Êç|Ž+Ú$IDDÄTQÅÿð?ÜÏý$`v¹<<`êT¸ùf˜=Û¾®èÅaáB{‡l¹à‚‚‚(//7;†ˆ´2ÍtJû±nÔÕÁˆf'‘ ìa&žxåQ³£ˆˆHj¤‘÷y_;þÉÏ;þïÿà…`î\³Óˆ´[...L›6Å‹óÅ_0dÈvíÚev,i§233yñÅY°`‘‘‘fÇ9¹¹pÅ0kÌŸK–¨À@Néaføš«ÍŽ""""ÇÍe.uÔ1ifG‘¶mï‚ýÝwàé éépçP\lv2‘/88X D: HûñŸÿ@R$h—ŽÎOþÊ_y÷øÿ1;Žˆˆ´‘¯ùš|òù ¿1;Š´·Ýf/2˜1Ã>a,"çl̘1deeáççÇ¥—^ÊÇlv$igl6“&M"== &˜GDäü}ò ‡Æ ö1]\ÌN%Nj‹ø†oxš§ÍŽ""""Ç•PÂB2ƒ„jvikƒÁš5öÏv«WCöuuf'é°‚‚‚Td Ò©È@ÚÿüÇÞ’Z:…k¹–k¸†)L¡³ãˆˆHøü?†0„Þô6;Š´“'Û'…'O†wÞ1;H»Ö­[7Ö®]Ë­·ÞÊ 7ÜÀ“O>ISS“Ù±D¤X¸p!Û·oç•W^ÁE‹pE¤=««ƒûï‡_ÿÚ~ÉÌ„‹.2;•81+Vã1~ͯ¹”KÍŽ#"""ÇÍbòþ`v1Óر°s'<þ8üõ¯Ð¯¼ÿ¾Ù©DD:¤€€,‹Ù1D¤•©È@Ú‡ª*û t* YÈ>öñ ¯˜EDD.°Zjù˜ÕÅ@ÎÞOØK–˜F¤]óòòâ•W^áïÿ;O=õ7ß|3ÕÕÕfÇ'—››Ë¬Y³˜1c)))fÇ9wÂðáðÏÂ[oÁ?þ¾¾f§'÷*¯ò?ðþbv9n/{y×ù3ÆçD:=__˜6 ví‚K.[n+¯„íÛÍN&"Ò¡x{{S__ov ie*2öaÍ°Zaij“HêAàf1‹Ã6;Žˆˆ\@ËX† ·p‹ÙQ¤=zæøíoí;.]jv‘vïž{îaÅŠ¬ZµŠôôtrrrÌŽ$"NlÊ”)DGG3}út³£ˆˆœ»?„¡¡²²à¶ÛÌN$í@eÌb÷s?½èev9î1£=¸ƒ;ÌŽ""Î$.Þ|¾ýÖ¾ÑéÀ0i”–šLD¤CðôôT‘H¤"iÖ®…Þ½!*Êì$ÒÆf2_|yŒÇÌŽ"""Ð'|ÂP†CŒÙQ¤=rq×^ƒ[o…뮃O>1;‘H»—‘‘Á·ß~K}}=—\r ß}÷Ù‘DÄ }ôÑG|ú駼üòËx{{›GDäìÕÖÂÔ©pã0v¬}:)ÉìTÒNÌb®¸2“™fG‘ã¾ã;>äCæ1wÜÍŽ#"Îèâ‹á›oàõ×aÑ"ûZ¤… ퟊ˆÈ9óððP‘H¤"iÖ¯‡K/5;…˜Àæ2—×yïÐÂ&‘Ž¨‘F>çs®çz³£H{ææfŸ¾ç{«Û>2;‘H»×£GÖ¯_ÏàÁƒ¹üòËyë­·ÌŽ$"N¤²²’©S§2~üxFŽiv‘³·{·}qÉoÀ»ïÚw´ôñ1;•´»ØÅ+¼ÂS<ú¨½ÐàÙgÍN#Ò!Œ1‚µkײwï^ÒÓÓÉÏÏ7;’ˆ´±… ²}ûv^yå\\\ÌŽ#"rfÞ|†…ˆغÆŒ1;‘´Só™O>ù<Íý¾{ IDATÓfG‘ãê©ç¿ùo&2‘TRÍŽ#"í»;Ü{/ìÙ7Ý“'Û?O®[gv2§fµZq×úN‘GEâü¾ûÎ>`—N/”PžäI²Ýì6;Žˆˆ´‚¥,%„.á³£HGõ—¿Ø yÄÞÚV»Íˆœ·¾}û²fÍ6l{÷î5;’ˆ´‘ÜÜ\f͚Ō3HII1;ŽˆÈϳXàÖ[á®»`ÊX¾bbÌN%íT9<ÍÓ<Îãt£›ÙqDDDä¸xB yœÇÍŽ""Ih¨½Þ¶mö‚õaÃàæ›áСÓ?ï£à«¯Ú&£ˆˆQ‘HǤ"qnuu°k—½“0‰I¤Â¦˜EDDZÁr–3’‘¸¡¶yrM o¼/¿ ãÇCCƒÙ‰DÚ½nݺñÍ7ßÅe—]ÆúõëÍŽ$"m`Ê”)DGG3}út³£ˆˆü¼ÌL4V®„/¿„§Ÿµl—ó0•©Äö¦Æ^à>nlßÞæQED̤"‘ŽIEâÜví²/ë×Ïì$â$ÜpãE^äk¾æS>5;Žˆˆœ‡zêYÍj®äJ³£Hgpûí?N••f'i÷BCCY¶lC‡åÊ+¯dÉ’%fG‘ è£>âÓO?åå—_ÆÛÛÛì8""§ÖÔdßm2=ºuƒ-[`Ô(³SI;÷_ñ Ÿð<Ïã…—ÙqDDD丹̥‘Fá³£ˆHG7v¬} “£ƒv¯^ðæ›öÏ  @q±}CÕQ£ °Ð¼¼""mLE"“Š ĹmÛ^^”dvq"é¤s+·ò RK­ÙqDDä­g=UT©È@ÚÎÈ‘°b|ÿ½ýöáÃf'i÷üüüX´h¿úÕ¯¸þúëyçwÌŽ$"@ee%S§NeüøñŒ9Òì8""§VZj_øñÈ#0c|õDE™JÚ¹:ê˜ÂnæfF3Úì8"""r\<Ïó<Îã„bvé <=íݳwï¶ohu×]ð‹_Ø‹Ûóóí6›ýRZ ¿ü%;fvj‘6¡"‘ŽIEâܶm³·Ó/ ù‰ù̧„° ÅñRJ™Ä$Ê)7)™ˆˆœ©å,§ÝèA³£Hg2x0¬YcŸÜ6 0;‘H»çááÁ[o½Å}÷ÝÇí·ßÎÛo¿mv$ie3gΤ¦¦†yóæ™EDäÔV¯†ìEÅ+WÂìÙàªS rþæ2—<ò˜‡~Šˆˆ˜¡‚ nâ&ö²·ÅñYÌ"‚&3Ù¤d"ÒiEGÃßÿß|µµösOW]e/.phh°w>¸ñÆ–ÇED:¨ªª*üýýÍŽ!"­L3ìâܾÿúõ3;…8¡Xb™Á þÂ_8Ä!¬Xù#‘D^åU6±Éìˆ""ò3–±ŒQŒ2;†tF½zÁºuàã—\ß~kv"‘vÏÅÅ…gžy†éÓ§3aÂÞzë-³#‰H+ÉÌÌäÅ_dÁ‚DDD˜GDäDMM°p!\y¥}aÇ–-žnv*é ö±§yšÙ̦+]ÍŽ#""Ò)íf7ð©¤ò{~OEìaÿäŸü™?ã…—ÙE¤³ºøbû9¦?ýÉ^PÐÐÐò~«–-³wÛéà, fÇ‘V¦"qnÛ¶©È@Néa&–XîáúÓŸ©L¥’J<ðP‘ˆˆ“«¢ŠL2¹‚+ÌŽ"Ut´½£Á¥—Ú[ÙžnAôP®.I"gbÎœ9LŸ>»îºK…"€ÍfcÒ¤I¤§§3~üx³ãˆHgTUuúûKJàê«aÚ4˜?>þBCÛ&›t ÿÅÑ“ž<ÀfGé´v± W\±aãù_ºÑ;¹“Þôæ6n3;žˆtvMMðÙgàî~òûm6{aüßþÖ¶¹DDÚXee¥Š D: SŒpDœ@y9@Ÿ>f''u„#ô¤'KX‚;î4Ò@#d‘er:9ïø+VÒÑî’b"û"¤'Ÿ„ñãaÇøË_ÀµY-vM \w½ãÁ?þa^V‘vdÎœ9Üu×]ÜqÇfÆ‘ó°páB¶oßΖ-[pqq1;Žˆt6eeöqø¢Eœ|âý+VÀí·ƒ—¬ZeßAR¤½Á¬`kY‹fÇé´v± wÜ©§ž컄of3Þx3ŸùLe*Þx›œRD:­7Þ°wÔkj:õcšš`êTHH°Ÿsé€, ‘‘‘fÇ‘V¦Nâ¼öí³_'%™›CœN ,d!I$±‚X±÷Û°ñ-ßšODDÎÀzÖOzfmj‚[n±$ˆˆt@‹E D: ˆóÊζï"›`vq"«XEOzò0SM5õÔŸôq9ä`ÁÒÆéDDäLm`—p‰Ù1D~4q¢}'Ôÿü~ñ (*‚_„ý íãÒ Z ˆÈi=õÔSL›6 &¨Ð@¤š2e ÑÑÑLŸ>Ýì("Ò-^ Žñþ}0y²ýv^ž}¼>w®½Høƒ 8ؼœÒa=ȃ¸ãΟù³ÙQDDD:½ml£‰“ïÞH#E1‰Iü‰?µq2éôÞ|Žµ¸»ƒÇi: 55Ù‹æùKÈÏo»Œ""mDE"“»ÙDN);âãíí®EŽK%•h¢) à´k¢‰Ílf8ÃÛ(™ˆˆœ làQÎpg‘¶’‘k×ÂرöEL‹ÿØÞÖf³/hš3žzÊÜœ"íÈSO=E}}=&LÀÓÓ“›o¾ÙìH"r>úè#>ýôS–-[†···ÙqD¤³©¨€{î±ú66Úa¼ñtébï0™™Ð§ÙI¥ƒZÉJÞæm>â#‚2;ŽˆˆH§VGyäö1.¸p ·0‡9m”JDä¸ûï†mÛì 6o¶^ݾjkíŸk==¡®îÇ"ƒ#Gì…ß~ gü­ÊËËijj¢©©‰òòrl6‹}ó͆†ªªª¨««£úxGÀššjkk8vìõõ'ßȳùsN¥¾¾žcÇŽqf???<==Oû___¼N±6ÌÓÓ???¼½½ñññ9á9þþþx/ð ÄÍÍ €àà`\\\pqq!Xˆ\pEEEDFFšCDZ™Š Äy8‰‰f§'A«XÅ=ÜÃÛ¼}Ê]+<ñd›Td "Ⅎɦ˜bu2çÔ»7|ú) þcƒÕjß1õÆaà@sò‰´CóçÏÇjµrÇwÎW\av$9ÊÊJ¦NÊøñã9r¤ÙqD¤3úÃì;A66¶<þì³pà öB__s²I‡WM5¿ãwŒc¿âWfÇéôö²—FOy¿ .Lf2 Yˆ+®m˜LDÄÎÒЀ%&†Ê€ªÓÒ¨¼ålõõXwîÄß>ü÷í#4;›È¼<¼jjì›ZíØÁ÷ÉÉüå²Ë°;FCCeeeF¡@uu5uuuX,l6ÛyåóòòÂ÷øgèæ‹ôêL⇄„œÕ÷/++;¯Çœi‘Ä™rss#00Ðøÿâ(P ÁÃã€!00‚‚‚~öñhwé´¬V+¥¥¥DGG›EDZ™Š Äyeg«È@NÊ /ÞäMÒHãA8¡ØÀ†,²Ìˆ'""?cðÄ“h‘¶8!«~÷;(+³Oôþ”‹‹}WšÌL8¾Šˆü¼gžy†#GŽ0nÜ8V¯^ÍE]dv$9…Ç{ŒššæÍ›gvéŒ>ÿÞ~ûä÷55Á† '§‹´’Çyœ#á^0;Šˆˆˆ»Ø… .§Üxî ž`6³Û6”ˆtÇŽãèÑ£TVVb±X¨¬¬¤¼¼œŠŠŠÇ, åååX,ãÒüøé8vá÷õõÅ+:š$__Rê긨±‘¤cÇóì4‚ƒƒñðð À(8›ú‹çcñ{GVYY‰Õj ¢¢‚Æã8 ©¨¨ì‹Ÿ+++»18Š*++ihh ¼¼Ü¸¯´´”ºº:***°Z­TTTΤÐ!$$Ä(8p\ÜâX@@AAA·8jtoi/Š‹‹ill$**Êì("ÒÊTd Î+;FŒ0;…8±©L%–XnçvlØ°b5î³aã[¾51ˆˆœÊf6Ó‡>øpò2DL5m¬_ê…KV«½ííÂ…ðÐCm›M¤sqqáµ×^£  €1cÆðí·ß’`v,ù‰ÌÌL^zé%^{í5"""ÌŽ#"MELœ®®'v1ûX<7×^üî»mŸO:¼õ¬g! y׈!Æì8"""‚½ÈÀêùqA§Ëñÿ^æeîå^Ó‰ˆ3¨­­åèÑ£9r„£G·—“ÝwôèQêêêNúzAAA',&>>þ„…ãÁÁÁ_ûøøˆ»»ûuHîlåÿE@@€qûl»+´†òòr¨¬¬¤ººÚ(‘ 00/¾ø‚K/½”qãƱdɼ¼¼ÌŽ%"ÀsÏ=ÇöíÛÙ²e ...fÇ‘ÎæóÏáÍ7þqžžP_³fÁ¯ ¡¡>›t 3™Iy,a .è÷ ˆˆˆ³øžï"7ÜðÁ‡ÏùœË¹Üäd"r¶………Ÿ²ˆàðáÃ45Û ÊÇLJ.]ºMdd$III\rÉ%-Rÿ´x@ §¥= >§ ™¬VëI‹·KKK)..&;;Ûø¹«©©1žïââBddä)‹"##‰‰‰1n‹œNaa!øûû›EDZ™FUâœJJì×*23K,ßò-·p KXB#xàÁ&6©È@DĉX°K.}ècv‘ÝwŸýrð ,Zo¿ ™™'/8°Zá?ÿ±/„ºSÍlEÎFll,_|ñÆ c„ üûßÿÖ‚f“åää0{ölf̘AJJŠÙqD¤³©¨€‰ÁÕõä] ãñ  ¸î:¸é&øå/í"­`x–gy…Wˆ'Þì8"""r\#ìg?xH ËYΘœLD~ª¦¦†ÂÂB²³³)((0n;¾ÎÍÍ¥²²Òx¼——¡¡¡„„„Ctt4©©©ÆmÇuHHÑÑÑš?9 wwwºtéB—³X[WSSCYY………ÆÏjAAqlÆ ”••ôg666ÖøùLLL<á¶~V;·ììlºwïnv ¹Td ÎÉQd JH9 þøó)Ÿòñ<ÏÓ@Yd™KDDšÙÁšh¢/}ÍŽ"rjݺÁÔ©öËÎðÞ{ð¯Á¾}?îœêpÿý0z´Æ­"g©oß¾¼ûî»\{íµ<ýôÓ̘1ÃìH"Ú”)SˆŽŽfúôéfG‘ÎèþûáèÑ–Žqwt´½¨`ìX1´¥´²:ê˜ÈD†3œ‰L4;Žˆˆˆ4sƒÔS .ÄË VÐ-^ikV«•C‡±ÿ~rssÉÍÍ%''‡üü|ãö±cÇŒÇG×®]éÑ£—_~9]»v%..ÎXˆ|.»¶‹ÈùóññÁÇLJ˜˜ÒÒÒNûز²2£ûˆãg=//¼¼<>ÿüsòòò(//7ïççgü¬ÇÅÅOBBñññ$&&ÒµkWuéÀöïßOÏž=ÍŽ!"€þåçT\ ..nviÊÊÊhll¤¢¢›ÍÆ–;ñ õä™nÏðuõ×¼ÿÅûÔÔÔP[[ÛâyuuuTWWŸñ÷qww'  Å1777û<==ñóóÃÛÛ|}}ñòòÂßßVy¿""íÙv¶ã?]éjv‘3“š ³gÛ/[·Â»ïÚ rrÀÍ;ëê”)ðî»TWWSWW@UU Ç»X,l6{Kïòòršššhjj2&Þl6‹å”N6Ž9ŽqË©8Æ/€1†ZŒc‚‚‚puu $$WWW‚‚‚Î;Ÿt>£Fbîܹ<òÈ#¤¥¥1jÔ(³#‰tJ~ø!‹/fÙ²ex{{›GD΀Õj¥²²ÒG:æÅcÌæcͲ²²ž.ãËæcÅæcBÇupp0...Ƹ100777N~"ù‹/ìÝÁÀ^@`µBÏžð›ßÀ¸q0pàYå9[Oð9äð9Ÿã‚v\‘Χ¶¶–ššàǹËæçP+**hll¤¾¾ÞXDÜ|¾³¹æãÐsåOlKØ£!æh S—L%Ó#“L2ñóóÃÓӳż¤c>ÓÅÅÅX¼ìãã£Ï¹"g¨®®Žüü|£û@óËŽ;ŒÏÞÞÞ-º 8°ÅŽæ=zôPHBHHÈi;ßÖÖÖRPPpÒ&[·neÿþýÆØÀÝÝ„„O¸ôêÕë„õXÒ¾ìÛ·«®ºÊì"r¨È@œSI „…Ù[aK‡gµZ)--åðáÃ>|˜ââbJKK©¨¨Àb±PQQAYYY‹¯—æñ'ùÿÎçæ 7ãeóÂ××·ÅÝ?·ÐO²5ÏÞ¼EØéxzzD`` ÁÁÁ_·ÃÃɈˆ ""‚¨¨(ÂÃÃOzWD¤=ÚÉNRIÕ‰{1MCCƒ1ž(//Çb±PYYi\WTT'Ñ…eee444PUUe/$ˆˆ ¾±‘«-®¯ª¢Ë{ïqí{ïñù9fj~âëdÎvÌr*?·˜ìرcÔ7ïÒp–9 Ðððð 88Ø(Ätœ ÆÃÀ€|}}ñ÷÷7ÆDÁÁÁ@`` ~~~çœIœßƒ>Èúõë¹ãŽ;ÈÊÊ"..ÎìH"Jee%<ðãÇgäÈ‘fÇé4êêê(--åÈ‘#”––RRR‘#G¨¬¬¤¼¼œŠŠŠcT‹ÅÒbìê(l=Í‹D<<<ð÷÷?«Ì•••X­ÖÇÅ gÊËËËã…„„íãÙ™„yaalïÛ—œ´4 #²¬Œðï¿'<<œ°°0£V¤µ¬e-å¯¼Ê«Ú ADDÚÚÚZÊËË)//§¬¬Ì¸ÝüÜiMM ‹åŒnŸ‹Óm®vÊâÒ3ðÓó®µ÷Õâîãα;ŽñxÙãTUUÓëâãミŸ_‹ÛAAAx{{Ÿô¶cž2$$Ä8¯¬¢i×, »wïfß¾}ìß¿¿Å¥°°°Ÿ¯p ôèу_ÿú×üñ4¾v˜‹ˆ€½ðÈQ(p*G=áߜݻwóù矓ŸŸo<.::šž={ÿÞ8.ÉÉÉ­ržT.¬ììlzôèav ¹Td Ω´""ÌN!究¶–¼¼<£M^^^žQÁê((p4çîîNxxx‹…÷!!!$$$œ°(ßßߟàà`c§ GGÇÉÒlïl"ŽE@Â}¯Ž…‡ŽE‰Ž‚Çb¾êêj㤰cQ£c¯¤¤„~øÁ˜ü+--=¡˜ÁßߟÈÈH"##„øøxââ∉‰!!!ØØX}¨§wƒjg,çÍb±pôèQŽ9b\_=z”²²2c–ã÷­ÅbÁb±œr‘}óOÍÉ»»»‚¯¯/‘‘‘ÆŽXAAAT¸»³(0¸¼<žÊÉaÒ¯ÏññÍwÉr,¬Œ\¡å®\Φù"2ÇneðãN¸Í”9Nþ9®ã‹ÅBCCÆNg%%%Ô××S^^NCC•••ÆcOu’ÐQ¼bü9.!!!„††JXXX‹kÇíŸ.ªçóúë¯sñÅsÓM7±jÕ*ãçED.¼Ç{ŒššæÍ›gv‘v¯®®ŽÂÂBòóóù¯‚‚ŠŠŠ(--5æÀJKKOØ°ÂÍÍ°°0Œ‚KÇüW\\œ±a…£Ó××—   Ū?í$ЖcÍŸvR€ÇÍ;+8²9Æé}¿ûŽ/úöeeHÙV«½˜bÑ"*++)--5Æ „‡‡IXXáááDEECll,ÑÑÑƵ äçTPÁíÜε\ËÝÜmvé„šššŒñ¡c¬è؈íðáÃ9rÄ8ŸØ¼ àdó›îîîÆxÑÏÏã¶cñŸãvPP¾¾¾øøødl?Î]6/JuÌ‘ž¬ãû…ö9Ÿs%Wâ•Órl瘋l^”p²cŽMdÿߪ««©¨¨ ¦¦Æ¸]UUEqq1ÔÖÖrìØ1£È÷§E¶`_LùÓƒàà`ÂÃÃK—.]ˆˆˆ06– wÚy`é˜, ?üð;vì`çÎÆõÁƒillÄÃÃøøxIMMåÚk¯5 ÷îÝû¬‹ÒEDNÇqÎnÈ!'ÜW__O^^Þ T>ùävîÜi¬ !55•>}ú×]tZ[è «È@¤cR‘8§ª*P$§W[[Ëþýû[Tºçää…Í‹<==‰‰‰!..ŽèèhúõëgLªDEE ç“-­%´V{­ÓñððhÕþÇŽ£¸¸˜’’c‚±¨¨È(ÎÈÍÍeýúõäææ¶81íëëkÄÅÅ•½Žjß°°°VË("r.rÉå ®0;†8™²²2ŠŠŠ())¡  €’’ŠŠŠ(..6 š444´x¾cQ–cQyHHDEEµ(Züé¥ù®ÖX„tÑy¿‚óh~°­Š‹Óš…4¿8vïu\*++Ù¹sg‹¿?-Ôtä kQ|àèEdd$111FA§£DÚŽ¿¿?}ôC‡eÚ´i<ûì³fGé233y饗xíµ×t2Fä 8Z¾8pÀ¸’——g8¸ººÒ¥K¢££‰‰‰!""‚””cþ˱@Þ±Cxx¸‰ïìü¹ººcÆÐÐг~þí§8îXpçèøpäÈJJJŒã¥¥¥lÛ¶ÂÂBŠ‹‹ijj2žëï9æ"»wïn\‰ŠŠ:—·*ÈøÕTówþnvé`ªªªÈÍÍ¥  €üü|òóó),,¤°°°Å&l'+ªl¾(=,,Œ¨¨(’““…ì?]Üîøº£.¾†kNz¼ùÜå…ü<[UUÕ¢[ÄO»G4?¶eË–…Å͹ºº¶ø³ŒŒ$**Ê(’uŒYããã;쟥\åååFÁ®]»Œb‚¼¼<À¾Rrr2©©©Ü{､¦¦’ššJ×®]Ϲ㈈Hkòôô>>FÁ£ø wïÞ$''Ó­[7uÃêà>æcþÅ¿ø”OéB³ãˆˆH;RSScŒ£   Åüx@@qqqÆßGWûnݺãV­£è”>Lff&YYYdff²iÓ&rssûßÕäädúôéÃÈ‘#ŶݺuS i·ÜÝÝUÇŽkolläàÁƒFñÁÎ;Y³f ¯¾ú*ÇŽìÅiii¤¥¥1xð`ÒÒÒ´ÑδeË‚ƒƒIHH0;Šˆ\:Ó!Ω¦¼½ÍNÑé>^ÏÀ‘#GZdee‘““@·nÝ]ˆsª©Q'ƒ ¬°°Ðø@š••ÅúõëÖ!!!¤¦¦’‘‘Á¤I“HMMeÀ€jÏØN¹¹¹KW]uU‹ûl6‡2 JvìØÁÒ¥Kyæ™g¨­­ÅÝÝ^½z쌌 ¨ÝÜDäœä’KM*2hGÊÊÊÈÎÎ>åÅÁÛÛ›˜˜‰eÈ!$&&MLL IIIšøNDN-00ÀÀ@’““ö±ŽŸ‰‚‚ Ÿ…õë×óñÇsèÐ!l6`ßå8..Θèl~IMMÕŽc§0þ|V®\ÉÝwßÍ×_­ I‘ ''‡Ù³g3cÆ RRRÌŽ#rAUUU±eË–-Õ333©««ÃÝÝ„„>|8“'O&11‘þýûivt¹ÜÜÜèÚµ+]»veĈ'Ü_QQÁ¾}ûÈÎÎ6æÊÞ}÷]öìÙƒÍf;až¬OŸ> 4ˆÐÐж3rNšhâ.î"”P°Àì8""b¢ºº:öíÛgŒwîÜIvv6;wî46Nqt!HLL¤_¿~\ýõÆÜN=6ù]HGáØ|碋.:åcjkk)((8aŽ~É’%üðÃX,À^|oÌA:v´ïß¿¿æèTCC7ndÍš5dff’™™ÉÁƒHHH --I“& cµ VDää\\\ŒµQcÆŒ1Ž—––¶(Üzùå—N0Ý»w'--!C†‘‘Á!Cððð0ë-´[[·nåâ‹/6;†ˆ\ *2ç¤"ƒVeµZÙ¸q#«W¯fÆ lܸ‘¼¼<\\\èÝ»7C† á‰'ž`РAôéÓG“bˆ›››1!Ú¼½X}}=»wïæûï¿gãÆlܸ‘÷ߟÚÚZHKKcèСddd0lØ0ý‘3’ƒ}—•Ô&Ï™TTT'ÓöîÝkt¼Ù¿¿ÑRÒÛÛ›ÄÄDzöìI¿~ý7nÝ»w'!!øøxœN#$$ÄXTv2õõõFg?K[¶láƒ>0vsww§k×®F›×¤¤$úôéCJJÊY·_ïh¼¼¼xã7¸øâ‹y饗¸ï¾ûÌŽ$ÒáL™2…èèh¦OŸnv‘VU__OVVß|ó YYYlÚ´‰~øÆÆFÂÂÂ4hÜÿý¤¦¦Ò«W/uo”‚‚‚Œ±ÞM7Ýd¯««cïÞ½ìܹ“M›6±iÓ&þû¿ÿ›£GâêêJRRƒ "--Ë.»ŒÁƒë„´“ú+å?ü‡µ¬Å?³ãˆˆH¨®®æûï¿gË–-lß¾Ýèh•——Øç!zõêErr2W_}5?ü°ÑÁJ…§âLsô‰‰‰'½¿¤¤„°ÿ~£[ÛŠ+øûßÿN]]qqq$''“œœLŸ>}0`ýû÷Ç××·-ßJ§W__ÏÆY¹r%«V­â›o¾áرcDGG3tèPî¾ûnc—mý;$"rþÂÃÃ=z4£G6Ž›òfffòì³Ï2mÚ4üüüHOOgøðá >œ¡C‡jŽçgØl6¶mÛƽ÷Þkv¹@Td Ω¦´Ô9³Z­dff²råJV®\ɺu먪ª"&&†‹/¾˜ûî»!C†0xð`‚‚‚ÌŽ+NÈÓÓ“þýûÓ¿n¿ývÀ¾‹Â¶mÛŒ¢ƒ/¿ü’ àââ€1b#FŒ`Ø°aú{%"'UB î¸N¸ÙQ:¥ŠŠ víÚÅöíÛë;w'ÔüüüèÕ«={öd̘1ôìÙÓX§ÝÄE΀§§§q²ïòË/?áþ²²²…<û÷ïgûöí|øᇔ””öÅm)))ôíÛ׸NMM%..®­ßŽiÈôéÓyôÑG5jIIIfGé0>üðC/^̲eËðöö6;ŽÈy±X,¬[·ŽuëÖ±fÍ6nÜHMM ‘‘‘ 2„›nº‰Aƒ1hÐ ºvíjv\iǼ¼¼èׯýúõã–[n1Žúùù‘’’Bjj*#GŽ4Z&wëÖMØ"XHHƒfðàÁ'ÜwäÈvìØatÙ¹s'‹/¦¸¸À8>hÐ ã5’““;ì‰ð'žx‚/¾ø‚ &°zõêû>EÚRee%<ðãÇgäÈ‘fÇ9k ¬]»–/¿ü’eË–±mÛ6l6½{÷æ²Ë.c„ ¤§§kþKÚL·nÝèÖ­ãÆ3ŽíÙ³‡o¾ù†5kÖðÞ{ï1gÎÜÜÜèׯ£FbôèÑdddh¼6VE·q#Á#>ûì3¾øâ Ö®]KMM ñññŒ1‚gžy†áÇӳgO³c¶º¼¼¼SvêeäÈ‘Ì;—¨¨¨6Nf®ÆÆFšššhjjj•ç._¾œ«®ºŠm۶ѷoßóþ>Ó§Ogîܹ|ùå—Ænïç“»5¾¿²²²¸ÿþûÙ²e ÕÕÕÄÆÆ›µ9““ý¬¹ººDjj*×]w¿ÿýïO¹áÁŽ;˜7o+V¬ ¤¤„ˆˆzôèÁèÑ£¹á†ZÌ­eee1gÎ6nÜÈáljåÊ+¯ä¶ÛnãòË/ïPã+Ç&bwÞy'`_<¿jÕ*V­ZÅ¿þõ/æ΋¯¯/\sÍ5\sÍ5ôèÑÃäÔæËÊÊÂÓÓSÅH"˜Š Ä9¹»ƒÕjv §·mÛ6>ýôS-ZDff&>>>üò—¿dþüùŒ1‚ääd³#JƸq㌓©¥¥¥¬ZµŠ¯¿þš×^{Ù³gÏu×]Çu×]Lj#4%Ò‰9Š ¤u>|¸EAAff&¸¸¸””ÄàÁƒùÃþ b'Æå—_~B„#GŽH]¥þñPWW‡¿¿?4ŠLRRR‡øwwwçõ×_gèС<ÿüó<øàƒfGi÷{ì1jjj˜7ožÙQDÎXnn.K–,áË/¿dùòåTVVÒ»wo®¾újžxâ ÒÓÓ‰ŒŒ4;¦ˆ¡wïÞôîÝ›»îº €ââbÖ­[ÇêÕ«Y´hóæÍ#00+¯¼’«¯¾šÑ£GwªŽUfù=¿§œrþÉ?qÅÕì8""rŽ=Ê7ß|úuëX»v-™™™ÔÖÖÅe—]Æý÷ßoDDD˜WÄ鹸¸Œo¼ñFãxII‰ÑdÆ ,X°€ââb¼½½2déééF·®ÐÐPßsilldíÚµ,Z´ˆÏ>ûŒ½{÷¨Q£xñÅ>|8‰‰‰fǼàâââhjjb„ |ðÁTUUP]]ͲeËøíoËîÝ»ùöÛoquí<ãòaÆqôèÑ þÜsý>O?ý4&L %%¥U^¯µ¾¿n»í6ú÷ïÏÒ¥KÉÍÍåúë¯7;ÒIìgÍf³QTTÄÒ¥KyòÉ'y饗X¼x1ýúõkñÜwÞy‡ &0uêTV¯^Mll,eee¬\¹’?þñ<ùä“ÔÖÖ°yóf.»ì2&NœÈ7ß|CTT<ÿüóŒ1‚7žtS­Ž"))‰¤¤$~÷»ß°ÿ~V­ZÅÒ¥K™5kS§N¥wïÞ\{íµ\ýõ¤§§wªÛÖ®]KZZš:'‹t`*2çäáaïf 'ÈÉÉá7ÞàÍ7ßdß¾}DEE1vìXžxâ FŽ‰Ù¥ ÿÿìÝ{\Œéÿ?ð×4¥QéHD*ÇD„Q*ç‘Chm$²Èyb³Ë’ó"çÃòqX±vUl$+k -Ö!*’tRm•Þ¿?üš¯ÑT353÷T×óñèñÐ5÷}]¯ûš[sÍ}ß×}c̘13f víÚ…û÷ï#((!!!صkttt0vìXL:¶¶¶\ÇeFÁrÃ&È@bb""""ëׯãùóç€V­Zá‹/¾À¼yóðÅ_ÀÚÚ:::§e¦¦ àààaYqq1¢££‰»wï"""(..†ŽŽúöí 8::¢[·nµö)VVVX¹r%V®\‰1cÆÀÄÄ„ëH SkEFFb÷îÝ8xð »à†Qz/_¾Ä©S§ðóÏ?ãÁƒÐÐÐ@¿~ýàïï¡C‡¢U«V\Gd‰5mÚ®®®puu<þ¡¡¡øý÷ß1oÞ<äççÃÚÚ'N„›››p q?ãg\ÄEÁˆë8 Ã0Œ„ÊžbuñâE\¼x=XXXÀÖÖ^^^°µµ­“wg.bðàÁ-Z´€§§'œÑ³gO 6 ±±±hÔ¨ ::S¦LÁÂ… ±nÝ:áºM›6…››:uê„ž={ Ëúé'¨««# @xñ¼™™¶nÝŠððpÅn (›œçé鉒’ܸq.\Àùóç±eË4oÞcǎŸqãЧOŸz3áàÆ:t(×1†‘£úñ׌©}4`“ >QPP€'N`РAhÕªvîÜ gggüõ×_HNNÆþýûáììÌ&0J…ÇãÁÚÚßÿ=²‘ °‹Þ¥•˜˜ˆ£GbêÔ©hݺ5ÌÌÌ0sæL$$$૯¾ÂÅ‹‘žžŽçÏŸãôéÓX²d ú÷ïÏ&0L¦¦¦†nݺaúôéØ¿?îß¿œœܾ}kÖ¬@ ÀÆÑ£G`Ĉزe "##ñáîãKeéÒ¥hÙ²%/^Ìu†©µ>|øoooØÚÚÂÃÃë8 #VFFöíÛ˜™™aýúõ°¶¶ÆÅ‹‘‘‘óçÏcÖ¬Yl‚SëµnݳgÏÆ… ‘‘‹/¢k×®øá‡`jjŠ~ýúáÀ ¹[d}‹XÌÃ<,ÅR  ®ã0 Ã0UHJJÂþýû1zôh ÿþ BÿþýŒôôtÄÄÄ`ÿþýððð` FAÚµk8p111HKKCPPñÛo¿¡_¿~hܸ1\]]±ÿ~¼|ù’ëÈrU«VÁÌÌ ööö¸rå f̘˜˜ÄÆÆbãÆppp` Ä033ðñ¼×§ÎŸ?/¾øM›6ÅÌ™3‘““ÇÇÃÞ½{1wî\hkk£eË–8t芋‹1gÎèèèÀÔÔ‡*×î¥K—`ccƒ† ÂÀÀ“'OFJJ à¿ÿþÖÏãñ`ccHHH)—$«8Ÿæ?xð`…Û¤££ƒ-Zàû￯tÝeË–aРßm:wî 333±Ë@zz:¾ùæ´mÛ]»vEPPP•ï•¸úÞ¼y#Ò'Ÿþ|úDiûHœ½{÷ŠÔêÔ©*ë—ö½ü|{Ë®·òööǃ“““H?à믿†¾¾>x<&L˜ \¿²}ìóþ¬î~, CCCáµ8{÷î–¯[·¥¥¥øæ›oÄ®gii‰üü|áï(..)+óðáÃ:ýƒª¨ªªÂÑÑ›6mB\\¢££1}út„‡‡£oß¾033÷ß~+¼Y`]õîÝ;ÄÆƲ›Ì2LÇ&0ÊIM ()á:çÞ¾}‹+VÀÈÈS§N…¦¦&~ýõW$''cÛ¶m°±±©73™ÚÏÔÔ ,À?ÿüƒû÷ïÃÉÉ Û¶mƒ™™ÆŒƒÈÈH®#2 #gìI’)--Å­[·ðÍ7ß ]»v"“ <<<¬¬,\½z~~~2d ¸ŽÍ0 Çzõêœ9s©©©ˆŽŽÆÚµkE&4nÜîîî"«Vf 4ÀŽ;pæÌ„……q‡aj¥ü111“s £LîÝ»www4oÞ ,€‘‘~ýõW¼yóÀ!CØ£¶™:K `È!8xð RRRpöìYbÞ¼y022ÂäÉ“qÿþ}®cÖZyȃÜÐ]ð¾ã:Ã0 Sׯ_cË–-èÖ­LMM±páB”””ÀßßÏŸ?Ç¿ÿþ‹mÛ¶ÁÙÙúúú\ÇeÞÔäÇÄãÇuëÖ¡¨¨ ,€‰‰ ºw[·âõë×\Ç•‰’’œ:u ýúõC»vípèÐ!Lš4 ÑÑш‰‰ŸŸ:vìÈuL¥—àÿ&@PPFŽ‰aÆ!%%aaa¸víF "Âœ9s›› àã…çC‡Err2&L˜€éÓ§cÆŒ2d^½z… &`æÌ™"“Ο?aÆaÀ€xùò%nÞ¼‰ÇÃÞÞ¹¹¹ÐÒÒBII Z´hwwwܾ}[˜155æææ(--•(«8ŸæW¶gÏ 8¯^½ÂÒ¥Káçç‡k×®U¸®¿¿?._¾ xôèˆ b—€µk×¢¸¸ÿý·p››bbb*}¯*ªoÀ€ "ᯯ/üüüªÝGâL:#FŒÀ®]»@D ú+«_SS>|€©©©Ø÷²S§NÂ÷²²í=pàˆ/^)ß²e ú÷ï—/_bçÎÂu«ÚÇ>¯¿:ûqu8;;ƒÇã!44TXvñâEXZZBOOO¢:zöì‰ÂÂB8;;ãúõëR½‡õM§N°zõjÄÄÄ ::'NÄÁƒѶm[ôïߧOŸFI¼òæÍ› "ôéÓ‡ë( ÃÈÕ@dd$EFFÖ¤ †ÏÝhÔ(®Sp&11‘æÌ™C 6¤¦M›Òúõë)55•ëX #sïß¿§3gÎP= <˜®^½*Ó6öíÛ'Óú*H5üX­’¢¶…aäe4¦ 4ëJéýû÷tñâEòöö¦fÍšjß¾=-]º”"""¨  €ëˆ ÃÔr¥¥¥M?þø#9::ŸÏ'@@#FŒ C‡QZZ×+åììL;v¤¢¢"®£(ĸqãhܸq iKÞcL(×6Ê°ñry‰‰‰¤¥¥E«W¯æ: Ã}øðÎ;Gööö€ºvíJ?ýôåääpa”BNN>|˜¬¬¬9::RPP}øðëhµÊWôé“>½ \Ga†Q*Ҝߗ×w¬œœ:rä 8ø|>éêê’——………Qaa¡\ÚdF1 èÒ¥K4mÚ4ÒÕÕ%>ŸOƒ ¢£GÖÊï|ïÞ½£Í›7“‰‰ ñù|=z4…„„Pqq1×Ñ”š‡‡ijj ÏËË£   ÒÐРQŸ]Ô¾}{²´´) !tåÊ""ÊÍÍ%4mÚ4á2¯^½"äîî.,{ýú5 “'O ËÌÍÍËÕÿàÁ@þþþ²åË—SÆ éÝ»w²M›6Ñ÷ß/UVqÊò8p \™§§§°¬¤¤„444hÍš5•®{ùòe@=ª²qFŒAÞÞÞ"eqqq€BCC%®ïúõëÄçóiÇŽ²êöѧíççç““““Øv%©ݺubßËmÛ¶UØ~eÛ[Vîåå%v=I÷±šìÇùüÿÚç ¨uëÖDD”MhèСUÖ[¦¨¨ˆ\]]  ###š5kݾ}[â:ê³ââb &âóùdjjJ[¶l¡ììl®£ÉÌŠ+ÈÜÜœëRSÄõ] #KŠ:ÿYÑùiv tF95hqBá²³³áãヶmÛ"$$7nÄ‹/°lÙ2rad®Aƒ;v,îܹƒ°°0£_¿~°··Ç£G¸ŽÇ0ŒŒ•¢|𹎡TnÞ¼‰iÓ¦ÁÐÐNNN¸{÷.fÏžèèh<~üþþþppp`woe¦Æx<:uê„yóæáêÕ«xóæ víÚ˜={6š5k†âĉ(((à8my?þø#ž?Ž={öp…aj4oÞK—.å: ÃÎ; ¸ººB[[W®\Áƒ0eÊ4jÔˆëx £5j„©S§"** áááÐÔÔÄèѣѱcGs¯V؉8Žã80ƒ×q†a˜ÿ/** žžžhÖ¬f̘---"%%À Aƒ ®®ÎuL†aj@ `ðàÁ§u:u ššš˜>}:š5k†iÓ¦ááÇ\ǬR~~>6lØ333¬^½NNNˆÅ¯¿þ ggg¨ªªrQéåååÇãÇãASS'NÄöíÛqöìYá2¯^½Â“'Oàèè(²n¯^½W®\)·´´þ»I“&åÊš6m HKKÖÿøñc888ˆÔÓµkWhkk#<<\X6uêTàÔ©S²£GÂÃãZY%Õ¹sgá¿ù|>7nŒ·oßV«.IéëëãÉ“'5ªãÝ»wøòË/1xð`øøøMåååaøðáÐÑÑ———Èk’Öïåå…ÒÒR‘÷200S¦L‘fËéÒ¥K¹2iö±2ÒîÇ5Abž< Í“nÕÔÔpöìY\½z_~ù%òòò°{÷nØØØ`È!x÷î]3Öeªªª1bÎ;‡§OŸb„ X³f Œ±lÙ2±O ©m®\¹Rîÿ$Ã0u›dÀ(']] +‹ë õË/¿ cÇŽ8}ú4öìÙƒ§OŸbÎœ9hØ°!×ÑF! „?þø·nÝBII ¬­­±lÙ2äççsaQ >à×18———‡]»vÁÂÂvvv¸wïüüü€{÷îaåÊ•èÔ©×1¦Î¸yó& Ëu¥Ò¸qcxzz"88iii8uê´´´0uêT4oÞóçÏÇ‹/¸Ž)Ô¦M,X°Y3]À IDAT~~~r?ÉÂ0uÅÙ³g‚={ö° ‹ çbcc1xð`¸ººâ‹/¾@ll,BBBп®£1JˆßþÏ€pþüyÄÆÆ¢{÷î5j† ‚¸¸8®£)­¿ñ7c1¾Ç÷p‚×q†a×®]ÀЭ[7ܹs›7oFJJ ~ûí7Œ3†}_aj6F•ž@ ÀرcñÛo¿!%%›7oÆßÿ +++ 8þù'×Ë!"ìß¿mÚ´Áºuë0þ|¼zõ ûöíCûöí¹ŽW«hjj‚ˆPZZŠ§OŸÂÜÜ~~~ÈÈÈ.“žžNHàñxÂp&%%‰Ô©¥¥%ü·ŠŠJ…e¥¥¥"õëëë—Ëg`` |Úµk;;;>|pûömÁÄĤZY%õi~àãEÝeùe!66®®®hÖ¬TTTÀãñpôèQdÕðz¬™3g¢°°GŽ–É¢||| ®®Ž_~ùÿý·Èk’ÖߤIŒ;Vä½´²²‚®®n¶Yܵ[Òìce¤Ý«+;;YYY055hkkCOO)))R×åèèˆcÇŽáíÛ· B¿~ý†µk×Ö(c}ÒªU+øûûãÅ‹X°`vïÞ sss8p@ìdÚ ==wïÞ…“;öÂ0u›dÀ('@Ì`«.JKKÈ#0~üx :qqq˜6mÔÔÔdÞÖ«W¯DÛŸþÃÃÃoÞ¼‘y»\X¶l™pÛ.^¼¨6<(lsïÞ½rmKÞÛÇEÿ•éÝ»7nܸíÛ·cïÞ½°´´Ä7šaùàƒRÈîÀXm“““???˜˜˜`É’%ppp@dd$¢¢¢°`ÁáAE¨hL `ii‰­[·¢¤¤Dayd)<<<ÑÑÑ5®‹‹~’e~ERæÜ¥¥¥ ¢Z{N´´´0vìXœ;wIIIX¾|9Î;‡víÚÁÍÍMi.b[±b6lˆuëÖq…a”^nn.æÏŸ)S¦°‹¸N½ÿ‹-‚••233ñçŸâĉ077—K{lœË]}²ÄÆoå™››ãäÉ“¸~ý:ÒÒÒ`ee…%K– ¨> ¸2oñc1ƒ0˱œë8 Ã0õÞÇ1hÐ 8::‚ÇãáòåËxôè¾þúk±ãÉZ‡*ôõõagg‡7"''§Âõcbbàáá–-[B]]ÆÆÆpppÀúõëËÝüÞ½{=z4Œ¡®®ŽÖ­[cÆŒ¸víšBÇÛúúúøúë¯ñèÑ#„……¡´´ööö>~~~ÐÑÑá:Z­Æãñжm[œ8q©©©ðõõ¾Ö¸qcÀ’%K„ÿ¿>ý9vìXÚ.«?33³ÜkÂ×ËL:wîÜAll,> OOO…e•‡ââb 8/_¾DDDŠ‹‹ADððð¨Ñÿÿ£G"00GŽ^àȦV®\‰àà`XYY᫯¾¹!¥4õÏž=[作5kVµ··2ÒîcŠ "°aÄeÆ CtttµŸ@ ®®Ž‘#G",, -[¶Ä;wd·ÞÐÕÕÅwß}‡øøxŒ7³fÍBÿþýÏu4©]ºt |>ýúõã: Ã0rÆ&0Ê©I@~Rv>D=‹ëׯãàÁƒr=¨fll,üÒP6sœˆ„wTþå—_àââ"Ó™Ñ\ñ÷÷WøÅP^^^(((PH[òÞ>.úïS***øúë¯KKK 0à,Ã0²¡•z9ÉàÇرcÚ´iƒ;wbþüùHLLÄÞ½{ammÍI¦ŠÆÉÉÉpwwÇ¢E‹ävÀ­6aýT7ôíÛ™™™ì !jÖ¬–,Y‚gÏžáĉø÷ßѹsgL›6ó'hiiaùòåØ·o^¾|Éi†Qv¾¾¾(((À† ¸ŽÂÔcIII°µµÅÁƒ±{÷nܹs¶¶¶rm“ßê6~«˜"##±sçNìÝ»vvvl\ôÿ•¢“1|ðqG¡ÂN1 Ãp¦°°‹-‚µµ5rssððp 8<O¡Y^¼x!nÚ´©\Ù¶mÛ$ª‡ëóf•açDC–cTqß[ŠŠŠðèÑ#xzzb×®]èÒ¥‹Ø‹ïO:kkk4kÖ ×¯_Gnn.îÝ»‡Y³faÏž=èÒ¥‹pÙ OŸ>022­[·››‹?þøZZZpttĽ{÷j¼-Òâñx§Û_½zÙÙÙ°¶¶Æ7ß|ƒ÷ïß+}úàСCؽ{7nݺ®c1ŒÒãóùhÑ¢<==…Ð6 ¹¹¹Âe¢££1eÊ,\¸6l@«V­Ð A4mÚnnnøý÷ßE¾wýôÓOPWWG@@LLLРA˜™™aëÖ­èܹ³Â·ñsŽŽŽø믿°sçN8p}úôÁóçÏžcíÚµ˜0a¼¼¼p÷î]tëÖMáê ???ðù|,\¸PX¶eË\¿~þþþHOOGzz:.\ˆ’’¸¸¸Ô¸ÍÍ›7#..¾¾¾ÈÈÈÀãÇáíí¶mÛâ믿YVKK ãÆÃÞ½{áìì @ òº¼³JÊÈÈðï¿ÿ"==FFFˆŒŒ,·œ™™š4i‚cÇŽ!66……… ChhhµÚ-))»»;LMM+¼±‰¬ú¨cÇŽX¿~=vïÞË—/W«þY³faïÞ½øòË/«µ½’’f“§> 99‡F= ¢¢‚ . Q£FÂe,,,püøqlݺË—/GBBŠ‹‹‘˜˜ˆ~ø+V¬À®]»DnŽ»}ûvœ8qiii(**‹/0wî\$$$(tûê*kkkDFFbêÔ©?~<Ö¯_Ïu$‰”––âòåË"“†©Ã¨"##)22²&U0Œx×®D))2¯zÑ¢EÔ´iSJKK£Q£F‘¦¦&µhÑ‚¶oß^nÙððp²µµ¥† ’¶¶61‚âââjœ!55•ŒiÀ€”ŸŸ_ãú¤åááAšššåÊׯ_OèìÙ³"å!!!dmmMêêêdhhHÞÞÞ”MDD;wî$€öìÙC>>>¤­­MÍ›7§ï¾û®\•Õ5`Àa]¨¸¸˜ˆˆÖ®]KÂ:|}}…ˬY³¦Â팋‹#*q"¢´´4Z¼x1µiÓ†ÔÕÕÉÊÊŠÎ;'¶€€211¡† ’££#=|øPØÕéÃ;wÒÌ™3IOO›››ÔÛwñâEêÕ« Ò×ק/¿ü’^¿~]nýO—366¦1cÆÐíÛ·+¬Ïž="ïÏÏ?ÿ\a6yØ´iñx< ’j½}ûöÉ)‘¨ÀÀ@ªáÇj•µ- #/“h¢Q2©ë×_%tÿþýr¯ 4ˆ¬­­…¿?|øFŽIººº$¨wïÞtíÚ5‘u’’’hâĉԬY3ÒÒÒ¢=zЩS§j”1,,ŒtuuÉÚÚš=zT£ºä¥¢1ÁW_}EèŸþ‘øsªªÏqc†F‘±±1|¸H_õêÕ‹ˆˆ^¼x!R®lrrrÈÛÛ›x<-[¶ŒJKK9ËràÀRSS£øøxÎ2ÈÛ¸qãhܸq iKÞcL(×6Ê°ñ2QII uïÞ8ýÊÔooÞ¼!SSS²±±¡ÌÌLN2°qnåã\6~«;ã·ŒŒ êÕ«™™™ÑÛ·o9É  NÒIâŽÐ™ÕYŽ;0 ÃT‡4ç÷¥ýŽõèÑ#jÖ¬uïÞ])¿³oÚ´‰Ð‹/DÊ%=/(Íy³ê‡”æüdUÙ$ÍQ[ωV§ïsss«=¦“tŒZÙ9rq*úÞBDtâÄ @7n–Mœ8‘ÔÔÔ$þžåååE€rss%ZžKÏž=£nݺ‘‘‘ÅÄÄ(¬ÝíÛ·“ŠŠ íÞ½[amÖe/_¾,÷]sÚ´i"ËÌ™3GøÚ¶mÛˆˆèÒ¥KdccCêêêÔ´iSš4i½|ù’ˆˆ~þùg‘úÜÝÝéòåË"ecÆŒ¡«W¯Š” 0@Øfhh(õìÙ“ÔÕÕIOOÜÝÝ+üŽ{ýúu@wïÞûzeYÅùôorpp»MŸ÷]›6mÄ®[fÖ¬Y¤££CÚÚÚ4kÖ¬ —½sçõíÛ—´´´¨eË–4cÆ ;v¬p¹”””rÇÜÝÝÅÖwæÌ™rïoÙ¹¹yµû¨ìó¹ìgöìÙ*Rfee%uýÉÉÉÔ¸qcáñˆÊ|¾½èÏ?ÿ,÷^ ¬¬¬rëWµÉb?þ”¸ÿk<ôôô¨OŸ>´aÆJÇ1114yòd222¢ P‹-ÈÕÕ•nܸ!²\nn.ýôÓOäääD¦¦¦¤¦¦Fzzz4hÐ «²_銊 p¥J·oß&ôðáC®£T‹"®ïbYRÔùÏŠÎO³IŒrŠ‰ù8É@F‹-"CCCrww§ëׯӻwïhÆ €nݺ%\.<<œø|>Í;—RRRèéÓ§4pà@Ò×קÄÄÄepuu¥6mÚ(݉Vooo@÷îÝ–;wŽx<­ZµŠ233)**Š:tè@ýû÷^°PvP¦K—.D999´}ûv@RÕ5~üxêܹ³H®ž={zöì™°lýúõ´wïÞJ·SÜ5I2Ì›7æÍ›Gééé”››KÇ'uuuŠŽŽ©ÿäÉ“|œè™™I÷ïß'''§rÔ¤éC333:}ú4ý÷ß´sçN©'„„„ŠŠ ­X±‚ÒÒÒ(..ŽzôèAmÛ¶¥œœœrË­\¹’ÒÓÓ)99™&Mš$²_|^aa!1‚víÚUi¿Ë“——éëëSFF†Äë°I £<¦Ñ4DƒdRWqq1ÑìÙ³EÊHEEEøEššš4qâDŠ§ŒŒ úî»ï¨Aƒ"ãX0`={öŒòóóéÞ½{4qâDzóæMµòýùçŸÔ ArssãdB¡¤*ôîÝ›TTT„o«úœ’ôó§¬+++úý÷ß)''‡/^L<¦L™BÁÁÁ”““CK–,!UUUJHH®+ieå>ŸØ!ÉçqMúIÒ|VVVäææF©©©”••EóæÍ©»¢üŸ;¢¢¢¨oß¾åÆU½W’ôCUÅ©(·¤ý"NÙX¤U«VtîÜ9ÊÎΦ_ý•444DþïWµÍe¯—ü´¬:û¢¤cÅÊr}øðLMMÉÝÝ]dÔÔTêÔ©“R_|êÔ)´xñbÎ2”””¹¹9yxxp–AÞØ$ƒêaãe¢Í›7“ºº:ÅÆÆr…©Ç†NmÛ¶¥ôôtÎ2°qnÕã\6~«;ã·´´4jݺ51‚³ \ºO÷Iƒ4h1Év|ªìdžaªK^“ Þ¾}KÍ›7'GGG¥½ ¹¢I’Ž$=oVÝñ™¤9Ä©oçD«Û÷5ÓU6F­êyE*›dM<úõë',ÓÓÓ£nݺUYo™ýû÷ /¾víšRs$úx“{{{jÑ¢¥¥¥É½½k×®ŸÏ'¹·Å0Œâíرƒ–,YÂu †‘ʺuëHUU•þüóO®£TjñâÅÔºuk®cT›dÀÔ6l’È“•õq’Áï¿Ë¼êE‹úý³º[µjE3fÌþnccC]ºtY&==½ÜI1iEFFºpáBµë¨©ÏXäååQPPihhШQ¢w˜nß¾=YZZŠ”…„„ºrå ýßOOOá2%%%¤¡¡!ò¤Iê*»+Ãóçωˆ(%%…Z·nM<O8“èãIáäääJ·SÜ5I2ˆ3bÄòöö)kß¾=uíÚU¤¬ìWŸP“¦½¼¼*ݦª¶ÏÜܼ\[< "HÌÍÍËMæÈÍÍ%}}}±õççç““““È3.dgg“¡¡!­X±BâuØ$†QKi)u§î2«Ï××—tuu©  @Xöí·ß’–––ðâ—Áƒ“………ðé8eìííÉÅÅ…ˆˆŠŠŠˆÇãÑáÇe’+++‹7nLãÇWúƒöŸ ÒÓÓiݺu@d\TÕ甤Ÿ?eõ|zçšW¯^ ïÚQæõë×€Nž<)u],UÝ1‘dý$I¾‚‚‚rÛ“¡¡¡Dù?w\¸p¡ÂI½WUõƒ$Å©(·¤ï›8ec‘o¾ùF¤ÜÇLJÔÔÔèÕ«Wmse'«³/Š#n¬XU®uëÖQÆ éÝ»w²M›6‰Œy•ÕÏ?ÿL<~ûí7Î2?~œø|>=~ü˜³ òÄ&TO}/'&&’––­^½šë(L=våÊâñxåîà­hlœ[õ8—ßêÖø­ìn‡W¯^å:ŠB¥P “1 ¦ÁTB%2¯_Y;0 ÃÔ„¼&L˜0Zµj%ò9©l*šd Ž¸±‚¤çÍjrR’âÔ§s¢5íûêŽé*£VuŽ¼"•M2 "200^À—MhèСUÖ[¦¨¨ˆ\]]…w¹622¢Y³f‰<Ù^Ùdee‘™™Mš4IîmõîÝ[ªþdFù­Y³†6nÜHIIIÔ®];‘0Lm1dȲµµå:F…JKKÉÌÌLªë¶” ›dÀÔ6\O2PÃ(#]] qcàÙ3¹TÏçó1hÐ ‘2 $$$ qçÎ8;;‹,c``[[[DDDT»íÀÀ@´oßÆ «v²——MMMLœ8Û·oÇÙ³g…˼zõ Ož<£££Èº½zõ\¹rE¤¼sçÎÂóù|4nÜoß¾•ª®aÆAMM AAA€|õÕWèÑ£BBBoÞ¼¡yóæRm³´Ûó)}}}4oÞÇÇæÍ›±k×®rËŠûœ’ôóçS–––Â7iÒ¤\YÓ¦MiiiÕnãóŒÕ”©¬Ÿ$Í'УG,_¾gÎœAAATUU‘ššZiÛ;¾ø⋠ש轪ªª›Qœš¾oe>ßξ}û¢¸¸÷ïß)¯Î8JÚ}±"Ÿ%Éååå…ÒÒRœ:uJXˆ)S¦H”K&LÀĉ±zõjN3´nÝ›7oæ,Ã(4oÞK—.å: S=z}úô)wÌ lœ+Ù8÷óúØø­vŽßÑ»wo=z”ë( SˆBŒÆhhB|™·¡¬Ç†a”MRRÎœ9ƒ7BGG‡ë82QÙX¡²óf²ŸU–£2uõœ¨,ú^cºÊΑו+“會ššΞ=‹«W¯âË/¿D^^vïÞ  2D)Ç$ºººØ°aNŸ>—/_Ê­û÷ï㯿þ‚ŸŸŸÜÚ`†K–,A—.]0gΘššr‡a¤æç燛7o"**Šë(bݾ} ?~<×Q†Q6É€Q^íÚOŸÊ¥jƒrý5j„ììlÀ»wïPZZ CCÃrë6mÚTì…Ø’zôèlmm«½¾¬hjj‚ˆPZZŠ§OŸÂÜÜ~~~"Û–žþñBЀ€áIY'ì—¤¤$‘:µ´´D~WSSCii©TuéêêÂÞÞ^d’‹‹ \\\pýúudgg#88#FŒz›%Í WWW4kÖ ***àñx8zô(²²²„u¥¤¤øx íSŸÿ.m6lØPêíú¼­Ï3÷ù²×+[N¨««ã—_~Áßÿ]í|²bgg‡øøxäååq…a)5Fc™N2055ÅàÁƒqèÐ!ÀåË—‘””„éÓ§²²²P\\ŒM›6‰ü æñxX³f 233…u;wíÛ·ÇW_}ôîÝ[ä$ƒ4®]»†1cÆ Q£F5ßH(Þ¿˜˜,Z´¨ÜX ÿ9%éçϧ>3¨¨¨TXöù8Bš6Äe”ôóXœÊúIš|—.]¨Q£0þ|èêêbøðáU~¾V4îÐÖÖ®pÊÞ«ªú¡:Å©éûVæóí400¼~ýZ¤¼:ã(i÷E@²±¢$¹š4i‚±cÇâðáÃ>”³²²‚®®®ÔÛÁ…©S§âŸþù[ªH|>‹-Âÿþ÷¿rûÃÔGgÏžEHHöìÙ@Àu¦‹ŠŠ*wQWØ8W²q®¸úØø­vŽßúõë‡pCaæ`b‹_ñ+t!Ÿ÷@Y;0 Ã(›;wî€ÇãaäÈ‘\G©iÇ •7«ÉøLÚ•©«çDeÑ÷òÓUvŽ¼º²³³‘••%¼@V[[zzzÂ÷CŽŽŽ8vìÞ¾}‹   ôë×aaaX»vm2Ê˨Q£|üÛ"/>„¦¦&zöì)·6†Q¼•+W‚ˆ••…¹sçr‡aª¥W¯^ÐÔÔÄǹŽ"Ö™3gо}{XYYq…aa“ å%ÇIUÍð×ÕÕ…ŠŠŠØ;^½}ûVxb¬: kt!¹¬ñx<´mÛ'Nœ@jj*|}}…¯5nÜÀÇ™¾e'e?ý9vì˜ÄíHS—‹‹ nܸääd<{ö ]»vÅÈ‘#QRR‚ÐÐPÁÅÅEêm•$Cqq1ˆ—/_"""ÅÅÅ "xxxˆÜ-ÂÈÈÊ]TU6Q¥:Û]Sem‰»Ð+##Cøzeˉ³råJÃÊÊ _}õòóóe”¸zÊ.˜)((à4Ã0Ò3€ò‘Èîÿ¯··7"""ƒÂÊÊJxPXGG|>ß}÷Ø¿ÁŸä777Çùóç‘••…ÐÐP´hÑ'NÄ… ¤Î”››[k&È‚¤Ÿ?\¶!ïÏciòéééaË–-HNNƵk×PXX{{{ÄÇÇWXEãi/î–´ª“±²öjºo|~"µlR¬´Oµ’IÇŠ’š={6îܹƒØØX>|³fÍ’Cjù(û;—››ËY†)S¦@__;wîä,Ã(ƒÜÜ\ÌŸ?S¦LAÿþý¹ŽÃÔsÊvÜ«&êã8—ߪ¦Ìã7 zs¼l#6âŽà4N£#:ʵ-e<îÀ0 £lòòò ®®555®£H­:c…ÊΛUw|&ë1K]='*«¾Wæ1]™àà`† &,6l¢££«ýuuuŒ9aaahÙ²¥\/⯠5554hÐÿý÷ŸÜÚøï¿ÿ ¥¥U+žFÍÔM7oÞ„bcc¹ŽÂ0Œ’QQQ¦¦&rrr¸ŽRáìÙ³pssã: Ã0 Ä&0ÊKŽ“ ª"гgÏrø333qóæÍr —†™™bbbjQæ,,,àááÇãÑ£GccctèÐwïÞ-·¼••%®_šº\\\PRR‚ `РA>>ö¼uëÖøù矑€N:I»‰exþüÎi-ó IDAT9RRRàææ†:€Ïÿø˜í÷ïß‹,¯¯¯öíÛ y]æóǾ˲«bll sssDDDˆ”GEE!''YîÚµk"˽~ý Ü“:Z·n 555;v IIIX²d‰Ì2WGLL ´µµk4ÙGÊBUç`/ÃÔññ ªÿD Ï9;;ÃÈÈ7nDpp0f̘!|M ÀÑÑAAAøðáƒDõihh`ðàÁ8}ú4ÔÕÕ«u÷ö®]»"<<¼Þü=ôóGQm”ݹôóõåùy,i¾7oÞˆ<:ÛÆÆ@QQ"##+Ì_ѸCÜöT•³ª~$£8õ»,öÏOvýùçŸPSSƒµµµDëË’¤cEIõîÝݺuC@@ѽ{wYÆ•«°°0ÀÄÄ„³ êêê˜3gvïÞ]îÄvmÇãñj|Ç;eÁãñêÍg"W|}}QPP€ 6p…aЦMáq¥Ú®®sÙø­îߢ¢¢Ð¶m[®cÈÝœÁr,ÇlÁ ‘{{ÊxÜaFÙtìØyyyˆŠŠâ:ŠÔª3V¨ì¼YuÇg²³ÔÕs¢²ê{eÓÞ°eË–ðöö–ûúú‚ÏçcóæÍb×Û¶mttt„BV®\‰U«V•[NUUjjjJw®µÌýû÷‘ŸŸ_­ë$eaaÔÔÔj=‚‘­ƒ Ÿ<²wï^…­ áááàñxˆŽŽ–z]IU”±´´T8 JѱÝÕuïÞ=ØÚÚBSS<ÆÆÆ\GªÔëׯáé鉖-[BMMMø^9rD¸Ì²eË„å/^ä.l”y¨¯’““ñöí[tì(ß›*Tǵk×””„ñãÇs¥FˆˆM4d1*—°IŒòj×HHŠŠ8iþûï¿Ç£G°`Á¤¦¦âùóç˜8q"ÔÔÔjt‘õˆ#pãÆ ©ï«~~~àóùX¸p¡°lË–-¸~ý:üýý‘žžŽôôt,\¸%%%R?M@ÒºLLLеkWœ9sF¤ÜÅÅÁÁÁprrªö6V•ÁÌÌ Mš4Á±cÇ‹ÂÂB„……!44´\]«W¯FTTÖ®]‹¬¬,<|ø7n¬övËÂæÍ›___dddàñãÇðööFÛ¶mñõ×_‹,‹U«V!##‰‰‰ðôô„‡‡G…”:vìˆõë×c÷îݸ|ù²LsKŠˆð¿ÿýÎÎÎJ7à+;ø*é E†© ññ±ÄoðFfuªªªÂÓÓû÷ª*ÜÝÝE^ß²e žîˆŠŠÂ¾}û¤Î)I?T•QœŠrËbßøã?pîÜ9äææâ·ß~áC‡0cÆ Nî„+ÍXQR³fÍÂÞ½{ñå—_Ê0©|=}ú[·nÅܹs9“Íš5 D„pšCÖ6lXí ”Mƒ PÄÑ÷úú 22»wïÆ–-[ФI®ã0 F¼~ýšë(2Q—ǹlüV·ÆoÉÉɸpá\]]¹Ž"WwpS030ó0O!m*ãq†aeÓ£GXYYáÛo¿­u“Ìk2V¨è¼YuÆgò³Ôås¢²è{eÓ}øðÉÉÉ8|ø0zôè\¸pAä©É8~ü8¶nÝŠåË—#!!ÅÅÅHLLÄ?ü€+V`×®]Ð××®³}ûvœ8qiii(**‹/0wî\$$$Èì;,¾ýö[tíÚ_|ñ…ÜÚ±µµE‹-°mÛ6¹µÁHÆËË«ÚOd«ÉºŠRQƾ}û"33S®“ij#www´hÑ©©©ˆ…††ב*åæ憇"44yyyHMMÅĉE–ñ÷÷G\\G ™ÚhëÖ­022BïÞ½¹ŽRÎþýûÑ«W/XZZr¥FÞ¿@Àu †Q:ÅÅÅhРAù¨"##)22²&U0LÅ¢¢ˆ¢GdZí¢E‹¨iÓ¦åÊÝÜܨW¯^"eaaaÔ§OÔ¨Q#rvv¦˜˜˜µ_\\L:t #FPiiiê’ÖË—/ €ÈÏ´iÓD–™3gŽðµm۶ѥK—ÈÆƆÔÕÕ©iÓ¦4iÒ$zùò%ýüóÏ"õ¹»»—k§M›6Âú+«ëS~~~¤««KÅÅŲˆˆ@UnëÒ¥KËå’4Ã;w¨oß¾¤¥¥E-[¶¤3fÐرc…u¥¤¤—ݵk™˜˜ºº:ÙØØЭ[·„Ë}º?IÓ‡(++«ÚÛJ={ö$uuuÒÓÓ#wwwzýúu¹:>]®yóæ´páB*(( "¢M›6‰Ô?{öl )³²²ªò}µŸ~ú‰TTTèÁƒ¯³oß>9&ú?AAA@؇ò ¨may)¦bR#5:A'dZï‹/M™2EìëqqqäææFMš4!@@:t E‹Qrr²p™óçÏÓ!CÈÀÀ€´µµÉÆƆΞ=[íL«W¯&555:sæLµë'qc[[[±ËJú9UÕç¸1ÃåË—EÊÆŒCW¯^)0`€Äm”™5kéè趶6Íš5KX.é8¤:ý$i¾óçÏÓ Aƒ„ûš]¹rE¢üeã@@}ûö¥{÷î:xð Ø>®è½ªª$É(NE¹%}ß>GèÌ™34yòdjÔ¨éééÑœ9s¨°°P¢mÞ¹s§Èk5Þ%+J3¾KNN¦Æ ·IÙ=þœÚµkG={ö¤üü|®ãÑ‚ ÈÄÄDä»Cm÷õ×_SÿþýÒ–¼Ç˜:::´ÿ~¹¶Q¦¾—KJJ¨{÷îäàà ðã S‘‚‚jݺ5¹ººr²_²q®dãܪêcã·Ú5~+--%jÓ¦\Kqí=£&Ô„œÉ™J¨D¡m+ãq†a˜ê’æü¾4ß±þüóORUU¥5kÖT7š\éèèˆ|Î{xx_“d¬ íy³êŒÏ¤9?ù©º~NT^}/͘NÒ1jeçÈ?%î{ Ç#===êÓ§mØ°²³³+ÌC“'O&###jРµhÑ‚\]]éÆ"ËåææÒO?ýDNNNdjjJjjj¤§§Gƒ ¢°°°*·› «W¯&UUUºyó¦ÜÛÚ¾}; ºuë–ÜÛb*WPP@hÏž= ]·ì{å#_“ô¹šd”Em·´Êú©¶ã}ÿþ=©¨¨P@@@•Ë–/ U@2é(ëþP_ݼy“ÔÕÕiçÎ\G)'==ð¼tm¶oß>ÒÓÓã:ÃHLQŸC‡{ü“M2`”Wq1‘¦&Qøpúܵk×HUU•~øᮣ0L­Iššš´hÑ"©ÖSÔíï¿ÿN(77WnmÔ–/Ô Ss2§oé[™ÖùÇú믿dZoM”––’©¨¨ŸŸq‰©ƒè*K;vì %K–pC"—.]"CCCêÖ­[…'¶¹ðâÅ âóùuê"­E‹‘BÚ’÷³iÓ¦ ; ^ßÆË›7o&uuuŠå: Ȉˆˆ 555Z±b×Qcã7n,[¶ŒÔÔÔêì÷"¢4J£vÔŽzPúþSxûÊxÜa¦ºä5É€ˆhÏž=ÄãñhÙ²eTR¢Ø a #-eÓÕw%%%´téRâñx ;¾óáÃ9r$5mÚ”þý÷_…´Y¥¥¥ÑâÅ‹©M›6¤®®NVVVtîÜ9‘e>ˆ³gÏòññ!mmmjÞ¼9}÷Ýwåê  jØ°!9::ÒÇ%¾_’u%Éüù-djj*ñú5Éøi8p \ÙÎ;iæÌ™¤§§GÈÍÍMXwHHY[[“ºº:’··w¹II/^¤^½z‘@ ccc3f ݾ}»Êíþ|]}}}úòË/En:PYÎáÇ‹Ô[6­lÒvÙ8ŸOæ@C† ‘¸_¤É]¶6jÔˆŒéàÁƒTTTD³gÏ&mmm211©ò"jqy?ýÑÔÔY¾¢ã%U½ŸÏž=£‘#G’iii‘‹‹ ýõ×_”››+q_WT‡$ûCeùdݧÌÇýÄÐÐFE>|à:N9[·n%---ÊÉÉá:Jmß¾ŒŒŒ¸ŽÁ0SÔøxàÀ4cÆŒrål’£Üúö%š>ër@<6lØÀu†Qz·oß&===2dˆÔw¨UÔmXX ÌÌL¹µQß.šbꦑ4’ÜÈ­ê%”žžN666doo/³:ei×®]Ô°aCêÖ­›ðÃTÇÈÙÙ™’’’èýû÷KöööÔµk×:;‰¥._¤¶fÍÚ¸q#%%%Q»ví(!!ëH•JMM%OOOâñxäææ&×I•Õ5|øp…Ýù_V­ZE]ºtQH[òcšššÒ¦M›äÚF™ú4^NLL$---Z½z5×QF¬Ÿ~ú‰ø|>Í›7¯N=i†©¿)Vqq1Í™3‡ø|>=z”ë8r“OùÔ›zSkjMoèÂÛWöã Ã0Ò’ç$¢ã@@@ýû÷§ÄÄD©×gyRÆ1C”@ŽŽŽ$èÈ‘# m;77—z÷îMzzz=Í·>š7oÍ›7ÒÓÓ)77—Ž?Nêêê-²\ÙEÏ]ºt¡   ÊÉÉ¡íÛ·Šˆˆ.wòäI@kÖ¬¡ÌÌLºÿ>999I4É@Òu%Í\ÑÜ%]¿&Ëú«l’Á§effftúôiúï¿ÿhç΋éÏ;G<V­ZE™™™E:t þýû Ÿ&B***´råJJOO§äädš4i’ÈEïmwÙº+V¬ ´´4Š‹‹£=zPÛ¶mE.*®,ç‡ÈÔÔTä©:DÏ1têÔ©Ò§^ŠëIúEÚÜVVVôûï¿SNN-^¼˜x<M™2…‚ƒƒ)''‡–,YBªªªU~FT”wÍš5M2äý´²²"777JMM¥¬¬,š7ož°nIûº²:ˆ*Þ$É'ë>­ÏÂÃÃIOOlmmé¿ÿsItêÔ‰fÎœÉu ™Ø°aµjÕŠë #1Eÿ´··§Ù³g—+g“ åöÍ7D º°‚ ;vì š2e åååq‡a”RÙiggçj=ò]Q´ew4{ûö­ÜÚ¨OM1u×ZB]©«Lê0`©©©QïÞ½)>>^&uÊÃãÇÉÁÁx<¹ºº²ÇM2Õ’——GþþþÔ¹sgÒÐÐ òôô¹K]RÙcÞë‚5kÖÒÕÕ¥íÛ·s§B™™™äççGZZZÔ¼ysúå—_¸ŽT¡ÐÐP@>ä:ŠLøûûSëÖ­Ò–¼Ç˜íÛ·§µk×ʵ2õi¼E—.]àää„°°0”–Ö¼˜úACCK—.ÅÇ‘——‡ÄÄD:tFFF\G“ ÐÇIñ "?~œëH2µråJ²²²0wî\®ã”óôéSÌ;&&&رc|}}ñôéSŒ3†ëh2dÌÍͱgÏ®£ÈD³fÍœœ "â:J ¼ÿžëuÊ/¿ü‚ìÙ³G©¿#1Ìøñãqûöm¼}ûX»v- ¹ŽÅÈ ¿)Faa!Ö¬Y dffâï¿ÿVê1ZM^ðB8„ ´C;…g¨-dža”M·nÝðÏ?ÿ`ùòåX³f Ú¶m‹]»v±± Ã9eÓÕw………@Û¶mñÃ?À××QQQèÚµ+'yNž<‰-[¶`ëÖ­èÞ½;®_¿ÎI–ÚB__Ož<ûZçÎ…ÿæóùhܸ1Þ¾} ÈÌÌÄ“'O`gg'²N¯^½ªl³&ëV•YVë×4c™.]º”+{õêž|îßç:‰Ü 6 ±±±èÙ³'† † & %%…ëX Ù’’lÞ¼–––HLLÄõë×áïï>ŸÏu´J5hÐPTTÄq†QnÐùÈG"¹Ž¢pÎÎÎøçŸpþüycÈ!hÛ¶-Ö¬Yƒ—/_ra˜z.??ÇG¿~ý`nnŽóçÏcíÚµHLLIJeË ¡¡ÁuÄJñx<Ìœ9GEVV×qjÌÄÄïß¿ž€«Í»D†rrr0þ|L™2ýû÷ç:ÃT©K—.ˆŠŠ‚¯¯/6lØ€N:áܹs\Çb˜Zé·ß~CÇŽ±iÓ&øùùáÁƒ"'íëšoð Nâ$Îà ú ×q†a)©««Ã××ñññ3f /^ SSSøùùáÍ›7\Çc†oÞ¼ŸŸLLLðÍ7ß`ܸqxöìV¬X!<ÏÊ^d>vìX>|pûömXYYAWWWê\UµWÜŸî£***–ÉóFq’¼ŸpéÒ%Œ5 óçχ®®.†Ž¿ÿþ[X$}]U5ÉWFú´6‰ŠŠÂ°aÃЯ_?˜™™!::sæÌÇã:šXOž«=­bÙÈ.v1žñJÇA6m"==^xO>ùGGG‚‚‚رcJG¡UUUNhh(ÎÎÎlß¾§žzŠ‹/òþûïcgg§tÄfºuëFTT{÷î%??___¦L™Bl'¯r3uuu“™™ILL uuuH’ÄO¡m8ÀäÉ“ñó󣨨ˆèèh~ùåùzPGµaúuëÆĉ•Žr׈"AhÙ•+W°°°¸á~Qd t|cÇÂ/¿(âž1b ¼óÎ;üúë¯<ðÀ<õÔSœ?^éh‚ÐnÊËËY·nÎÎÎ<û쳌3†””V®\Ùbu\G¥:Ñj©^„ÿ1À€~ô#–û³C¶)Ö®]Kff&°mÛ6¼½½ñôôäÕW_åøñã·ÕQ+‚p3™™™lݺ•àà`lllxüñÇ)**býúõäää°wï^ÂÂÂÐÖÖV:j›™˜˜0cÆ ¶nݪö3â8::¢©©Ù)Š LMMïÊ…4Ž?ÎöíÛÙ°aƒ¼ì´ ¨{{{vîÜɉ'èÕ«Ï=÷nnn¬Y³FœK Â_²zõj\\\xì±ÇèÙ³'ñññ|óÍ78;;+¯]mc«XÅv¶3iJÇAî–-[Fjj*_|ñ<÷Üs888ðøãó믿ÒÐРtLAî‚úúz¢¢¢xì±Ç°¶¶fÆŒhhh°k×.222X»v-]»vU:æ-sìØ1ÂÃÃÉËËcÈ! 0€ÿüç?÷U±|ZZ¹¹¹L›6 ´´´€ë³ÜÞŽ.]ºÐ£GŠ6Nžqâëׯ'%%…×^{ââbΟ?ÏܹséÞ½;óæÍksÞùóçóá‡òØcÝ•÷3w;÷½ÒšÏÄÄD6nÜHyy9¥¥¥|ôÑGèëëãïïßìX·ú¬ÿî7ûyhM>áæŽ?μyó°··göìÙ¸¹¹qäÈ:ÄC=„†††Ò[eË–-óÄO(å®ÉÊÊ¢´´///¥£B‡R[[˵k×Z,2@º'NœNœ8q'‡„¿WY)Iúú’ôùçJ'QDCCƒôÓO?I“'O–tuu%SSSiöìÙÒÁƒ¥ÆÆF¥ã B›\¾|YZ½zµäîî.R¿U³Âh IDAT~ý¤Í›7K¥¥¥íöš}ôQ»û¯¬­­¥-[¶´Ûñïå{„öôƒôƒ¤!iHÅR±ÒQ:¼Ó§OK6lBCC%sss ¬­­¥©S§J[¶l‘E{@„frrr¤¯¾úJzöÙg% ´µµ¥ÁƒKË—/—öìÙ#UWW+³]1Bš0a‚Ò1îؘ1c¤™3g¶ûë´wó•W^‘úõë×®¯¡Ò™ÛËï¾û®¤§§'%''+EîºÊÊJé_ÿú—4zôhIKKK211‘fÎœ)ýúë¯R]]Òñ¡]ÕÕÕIQQQÒã?.™˜˜HZZZÒرc¥Ï?ÿ\ºzõªÒñî©ÝÒnIGÒ‘–JK•Ž"‚Щµåúþ½<Ǻxñ¢ôÞ{ïIcÆŒ‘ôõõ% ©_¿~Ò«¯¾*œ%===iРAR\\œü¼Þ2OkömMf•ùóçKfff’©©©4þü6ï;·lÙÒì³6lØ Ÿ! •””Üpìß~ûM4h¤§§'ÙÚÚJ>ú¨”™™Ùì9QQQÒ€$===ÉÁÁAzá…¤ªªªfÏié}ÿu_ iÆŒRNNŽüxksJ’$eggKVVV­ºñ×Ï<Øê×kkî3fH{÷îmvßÃ?,ýñÇÍî5jT«ò6L’$IrqqivÿîÝ»¥eË–ÝðÚ*÷ï)=Z²´´”LMM¥   iß¾}mú¬[sŒ›ý<Ü*ßÝþL;ƒÓ§OK¯¿þº|mÎËËKZ¿~½”——§t´ÛRRR"YXXH«V­R:Ê]%RQQ‘ÒQ¡ÕîŹy^^žH1117<¦!I’Ô†‚…fâããðóó»ÝCBë<ø ˜šÂ7ß(DQ%%%„‡‡óÑGqòäIlll;v,aaaŒ3===¥#  ’’’ˆŒŒ$""‚¸¸8ÌÍÍ cîܹí¶,^S;vì`Îœ{3󙧧'Ó§Oç7Þh—ãßË÷"í©ˆ"l°!‚&0Aé8j£±±‘””:Dtt4ûöíãÊ•+˜˜˜Ð»woüüüäÍËËKmf@áö•••qöìYâããå-99---úöíK`` AAAŒ=sss¥ãÞ3ßÿ=aaa\¸pwww¥ãܶW^y…Ÿþ™³g϶ëë´wó½÷Þã½÷Þ»'³PvÖörFFÞÞÞ¼ôÒKív®!Eqq1ß}÷;wî$..CCCFŒAhh(ãÇËX Baa!111DDDIII ^^^Ìœ9“'žx;;;¥#Þs?ò#ðOó4[ÙŠâ|V¡½´åú¾RçXUUUrhDDÉÉÉÑ·o_‚‚‚ $00.]ºÜól‚p¿ª¬¬$!!C‡Kll,¥¥¥¸¹¹Lpp0>ø &&&JG½g>Ì×_Íwß}GVV={öäá‡&44” ©©©tDAPÔ–-[ÈÊÊbݺuJGéôÄg­Œ††Ž=Jdd$ß}÷.\ÀÉɉ‡~˜iÓ¦1hÐ ¥#Þ‘×_?ü´´4LMM•Žs׬_¿ž7’­tAhµ{qn~îÜ9<==9}ú4½{÷nö˜v»¾² Ü-&Àë¯Cm-èê*F1Ì™3‡9sæÀîÝ»‰ˆˆ`×®]˜šš2vìX&MšÄ˜1c°¶¶V:®pŸª®®æСCüôÓOüôÓO\¾|{{{BCCY¾|9cÆŒAGGGé˜íÂÊÊŠ¢¢"¥cB‡g…žx²Ÿý¢È  455ñööÆÛÛ›9sæÐÐÐÀéÓ§9zô(ñññìß¿ŸmÛ¶Q__……ýû÷—·>}úàêê*:µAræ̹˜àøñã\ºt '''ú÷ïÏŒ3ð÷÷gðàÁ+œX9'N¤k×®lÛ¶ 6(ç¶ùúú²aꪪ000P:Îm³±±¡  I’DÜmZ¸p!,[¶Lé(‚Ðî,--徯ÔÔT"""ˆŠŠbÑ¢E<ûì³øúúòàƒ2nÜ8üýýÅ„‚Z¨©©áرcüúë¯DEE‘€žžÆ ã7Þ 44777¥c*æ~aÓ˜ÉL¶°E‚ ȃ–×®]Ë… 8pà±±±|ÿý÷¬[·---zõêEPP 8777qÞ)w$I¤¦¦rôèQâââˆ%))‰††xàx÷Ýw6l<ð€Òq3xð`ÌÆ9|ø0ááá|õÕW¼õÖ[X[[3nÜ8&L˜@pp°(ŠîkÖ¬AOOéÓ§³eËöîÝ«t¤NK|ÖÊ(..&::šŸþ™¨¨(ŠŠŠpsscÒ¤I|þùç 4¨S´G‹ŠŠØ¼y3+V¬èTp½èºOŸ>JÇ„§´´ Å‰ E‘ BBàùçá÷߯¯j зo_úöí˪U«ÈÈÈà×_%""‚Ù³gSSSÓlÖ€#F`ee¥td¡“ª¯¯çôéÓDGGMll,ÕÕÕ¸¹¹1qâD¸/¶:99‘žž®t AP ãÏüÀ;¼£tµ¥¥¥E¿~ýš­ SWWÇ… äAÈ{öìáwÞ¡¡¡]]]ºw·7^^^ò­‡‡ZZZ ¾Aš*))!--¤¤$’““IJJ">>žÜÜ\ìííñóócæÌ™øùùáïï_Îv{+ÚÚÚÌ›7wÞy‡Õ«Wcdd¤t¤ÛÒ¯_?êëëILLÄßß_é8·ÍÆƆºº:JKK±°°P:ŽÚùöÛo‰ˆˆ ::}}}¥ãÂ=åîîÎâÅ‹Y¼xq³ÙlwïÞÍ[o½…¶¶6}úô‘Wî5j”¸!t•••9r„ØØX:Ä¡C‡¨ªªÂÕÕ•Ñ£G³|ùòûn–×›‰"Š)Láqç#>B“Îß(‚ ´]=èÑ£O?ý4ùùù;vLžQý“O>¡¦¦F^ñUÕïéççGÿþýŹ” ÜB}}=çÏŸ—WHMJJâðáÃËç\ÇçÅ_døðá8;;+¹ÃÑÐÐ €€€6nÜHZZš¼jÙã?Nmm­»!ÜW–.]Ê[o½ÅªU«pqqQ:N§&>ëöW^^αcÇäñP§NBCCƒ¾}û²`ÁBCC[µB˜ºyóÍ7166fÁ‚JG¹ë8ÀÂ… •Ž!Nnn.ØÚÚÞð˜†$IÒí¸-Ë)  GGøï•NÒ¡UTTpàÀbbbˆ‰‰áÔ©SH’D¯^½>|8ƒ ÂßߟîÝ»+UPSÅÅÅ?~œcÇŽqàÀââ⨪ªÂÅÅ…áÇ3bÄFŒÑa:™îår¾ÿüç?ùöÛoINNn—ã+µ4± ´‡#a0ƒI >ˆJñöTYYIrr2‰‰‰¤¤¤pöìYRRRÈÈÈÀÐÐOOO¼½½ñððÀÝÝwwwºw™™Âé¡sª­­åÒ¥K¤¦¦’ššÊùóçIII!11‘‚‚àú,ª‚ ¦›½½½ÂéÕCQQNNNlÚ´ImÛOXXX°víZæÍ›×n¯ÓÞmÌ„„|}}9wî={öl·×Î×^.//ÇËË‹1cÆðÙgŸ)G:”ÔÔTbcc9xð ‡âܹshjjÒ«W/† Bÿþýéׯ^^^hk‹yv„öSWWGrr2§NâøñãÄÆÆ’˜˜Hcc#Ä!CîëÕ ZE“™, A\ßW—s¬ªª*Îœ9éS§8uê œ={–ªª*tuuéÕ«¾¾¾x{{ãééIÏž=qqq¹/&Ç•ÆÆFÒÓÓ›õC&$$˜˜Hmm-øøøàë닯¯/}ûö¥wïÞj½ºfGPZZÊøã?Ø¿?§OŸn6vcØ°a :kkk¥£ ‚ \_U|ÿþýìß¿Ÿ˜˜’’’ÐÔÔ¤OŸ> 6Œ#F0dÈgúî,²³³yàX¿~=óçÏW:Î]uîÜ9<==9|ø0ƒ R:Ž ´Ú½87ß´io¿ý6yyy7<&®°ê㩧`Þ<(*QÙ}S&&&L˜0 &PVV&8p€íÛ·SWW‡……þþþÍ6…Ó Mee%'Ožäøñãò–––€««+AAA|ðÁ >WWW…Ó*¯G¤¥¥Q__/1ÂßÈ@œqæ;¾EíÌØؘ0`À€f÷———ËTE 33“††¬¬¬š4ýº¥ fAþ§²²’‹/Ê…ªíâÅ‹deeÉ¿gÖÖÖò*#!!!r1A×®]~êÍÊÊŠéÓ§³yófžyæµ\žVSS“ÀÀ@bbbڵȠ½ÙØØPPPÐîEÍk¯½Fuu5ëÖ­S:Š t8ªvéO<\ÿ'O†ð駟R]]¾¾¾ú÷ï/V A¸HII!>>žcÇŽ±ÿ~’““ÑÔÔÄ××—±cÇòÖ[o1tèÐûjb¾+V`gg'¯"Ö™ÄÄÄ`ll,&T„dffâääÔâcâ,APÓ¦ÁâÅðå—ðüóJ§Qfff„†† \Ÿ]ëÂ… ÄÇÇsèÐ!vïÞÍÛo¿-Ï’©š-UuÛ·o_±\ß} ¾¾žŒŒ ’’’ä¥0ããã9þ< ˜››Ó¿üqüüü8p }úPSSCRR}úˆAÓ‚p+h0…)|Ã7¬fµÒqîK¦¦¦7\|ƒëm…ÌÌLÒÒÒšm?ÿü3çÎãêÕ«èééѵkWìííqppÀÍÍ 777ù{ww÷N=‹ƒp«««£°°ÜÜ\ÒÒÒÈÉɹáëË—/ÓØØ€………ü;2}út¹­-V i_Ï=÷ýû÷çÀ 6Lé8·%88˜·Þz‹ÆÆFµaÑÚÚMMMy ‡Ð:Çgûöí|úé§bª ´‚ =ô=ôð¿Aeñññr?Çwß}Gqq1ÚÚÚ8;;ãææÖ¬¬wïÞ˜šš*üN„Ž ¦¦†‹/Ê?;ÉÉɤ¥¥‘˜˜HMM &&&ôîÝ›€€žyæ¼¼¼ð÷÷GOOOéèj!œpã1žà Q` ‚ ´mmmy"‡Ç{L¾¿´´”ÔÔTÒÒÒäÿç#""X»v-ÕÕÕ@ó~œ¿n...b௠¸’’’úïU[zz: ÍÎ{†ʼyópssÃÇÇGL ¤ ssófc7ÊÊÊ䢃ØØX>ùäjjj022Â××???ú÷={öTÛþAA¥566rþüyNœ8A||<'Nœ !!«W¯¢¯¯OŸ>}7nk×®eèС÷ma||<ÿþ÷¿ùꫯÐÕÕU:Î]÷Ã?ŒŽŽŽÒQ¡ÃEBç`daaðÉ'¢ÈàèèèÈj3gήŸ¼ÆÇÇsêÔ)yyï/¿ü’ÊÊJ\\\ä%DU³wïÞgggÑ‘¦f ät/^¼È… HJJâüùóÔÖÖ¢­­»»;>>>„……áãヿ¿?ÎÎÎJGW ^^^qâÄ Qd ­ð0ó>ï“DÞx+Gø?:::òE³¿’$‰¬¬,RSS¹|ù2ééédff’••EDDrû®w˜;::âââ"$ØØØààà€ vvvØÙÙahhx/ߢ Ü”ªx //¼¼< ÈÍÍ%??Ÿ¬¬,²²²ÈÌÌ$//O. ÐÑÑ¡k×®899áììÌ!Cptt¤[·n¸»»ãêê*œ)ÄÏÏÀÀ@6nܨÖE/¾ø"§NRÛ™Uttt°··çòåËJGQõõõÌ;—   ù¼]„¶i:¨L¥±±‘‹/’@rr2)))ÄÄÄðÑGQSS€““<ð&&& 6 WWWy»_/.vVeee\ºt‰Ë—/séÒ%ÒÒÒäÙ^³²²€ë…ÕxxxÊÒ¥Kñõõ¥{÷îj¹JRG°‹]<ÅSÌe.›Ù, A„{ÎÜÜ???üüü “ﯭ­åâÅ‹œ;wŽ´´4.]ºÄ¥K—øñǹ|ù²\€ ««‹‹‹ ®®®tëÖ®]»âè舽½=NNNØÛÛcii©ÔÛ:ââbrssÉÌÌ$77—¬¬,²³³åvkzz:µµµèëëÓ­[7\]]éÑ£cÇŽÅÍÍ OOOÜÝÝ;åàÀÎÆÌÌŒBBB€ëýÓ‰‰‰ò ØØØX¶oßNmm-&&&øúúÊEýû÷§{÷î¢ð@á/ùóÏ?å¿¥ñññœû âãAMXtDfffŒ9’‘#GÊ÷I’Ä¥K—HLL$))‰ÄÄDbbbøôÓO)--®éÖ­[³ÂÕ BGGGìììÄ·{¬¬¬Œ¬¬,222ÈÊÊ"--‹/’ššJjj*åååÀõ‹¤Ýºu£GL˜0eË–É…$bÜíÓÒÒÂÏϸ¸8fÏž­tAèðÀþËùü?¥ã­ ¡¡““ÓM+˜áú,JYYY¤§§Ëƒ²322øóÏ?‰%//+W®4ÛÇØØ{{{lmm±µµ•‹lmm±²²ÂÒÒ’.]º`ii‰¥¥¥ø¿JhµÆÆFŠ‹‹¹råJ³ÛÂÂBrrräÕòòò(,,¤   Ùþ†††r1Œƒƒƒfúôé8::âè舳³3vvvâBNöâ‹/òðÓ’’‚§§§ÒqÚÌÇÇ{{{¢££Õ¶ÈÀÍÍK—.)Cm¼ÿþûòâœZîMMMzôèA=šÝßÐÐÀ‘#GøòË/ùý÷ßÙ¿?9r¤Ù*,–––ÍŠºu놣££\Lkkk+ÚDcc#ùùùäää““Cff¦<(Kµ5='qppÀÕÕ•ž={2zôh<==ñôô¤[·nb‚•»h+[yžçYÊRÖ²Vé8‚ ‚ÐŒ®®.^^^xyyÝð˜$Iò –MÛ.\`ÿþýdffríÚ5ùùúúú78::bccƒµµ5¶¶¶X[[cee%´Ý'êêê(**’ûU[ÓB‚œœ²³³å‚¸Þ7éä䄃ƒݺu#((¨Ù9‰ƒƒƒ‚ïJh:::øúúâëëË3Ï<4_©/>>ž£G²mÛ6ª««ÑÕÕ¥{÷îò }ª[q.#Â}!''§ÙJ”IIIò ÚÚÚôèÑ???¦L™"héëë+»CúüóÏåÂŒÎx]"<<]]]¹°O„æ²²²˜2eJ‹‰"A½‚—lÝ ÿú—Òi:5 yã‰'6{¬¨¨H´®šÿÔ©S|ûí·äååÉÏÓÕÕÅÁÁGGGœœœä^íììäAƒªN4qöÖ***äÁoEEEäçç“MFFÙÙÙòΦ³G›ššâæ憻»;£GæÙgŸ• BœœœÄgÞNFÅŽ;$©S6¼ánÒD“YÌb;x×ÑC ï ,,,°°°ÀÇÇç¦Ï©­­m6C|AA999——ÇéÓ§å¯ËÊÊnØßÈÈH.8øëÖ¥Kºt邉‰ ¦¦¦˜››cjj*oíùö…vP__Oyy9¥¥¥”——S^^NEEååå”””P\\Lqq1çÏŸçÚµkÔÖÖÊ÷ýµ ÀÀÀ+++ºv튵µ5nnn`cc#T­¸all¬À;î¦I“&áááÁÆÙ±c‡ÒqÚLCCƒQ£FñÛo¿±lÙ2¥ãÜ6777ÒÒÒ”Ž¡222XµjË—/WËÂAP'IIIDFFA\\úúúŒ5Š%K–‚ƒƒÕÕÕ7 &»té{öìáòåËòDp}0ˆ­­­< Luëàà ÎZYYacc#VE¸MåååPTTDqq1EEEäääÈ3½æåå‘™™I~~>õõõò~ò ¯Ã‡çÉ'Ÿl68K\\nëXÇr–ó.ïò"/*GAÚDCC jñ9ªIÀ²³³oh›ìÝ»—ŒŒ ÊÊÊ$©Ù~]ºtÁÆÆF¾fª*@°¶¶ÆÒÒssó6±*¬²®]»Fiié ›jb“ÂÂBòóóåkº7ôQjkkccc#,÷îÝ›±cÇâè舃ƒ]»v¥k×®˜™™)ô.…Ž¤éJ}ª/kjj8sæ gÎœáܹs$&&òù矓žž\ï÷ôô” <==éÕ«®®®bŒ€ j§¡¡K—.5+$HII!%%…ªª*\\\ðôôdðàÁ<õÔSôîÝ1q]+•——óúë¯3oÞ<úôé£tœvñïÿ›I“&add¤tAèpªªªÈÎÎÆÍÍ­ÅÇE‘ ~^~æÌþ\]•Ns_²²²ÂÊÊŠÞðXMM ÙÙÙdgg“žžÞìë?þøƒììlòóó›u¢ijjÊÇ´¶¶–gò033ÃÌÌ ùkÕfjjŠ™™&&&÷ò­ß‘ššÊÊÊäMÕéTVVFyy¹|QQEEEÍŠ jjjšËÔÔT.ÞpttdðàÁÍ 9œœœÄk…Œ;–7ÞxƒÄÄÄ[°áº,à]Þå[¾e3”Ž#Ü#ºººòL𧾾^ž}¾¥éUŒ’““åç”––6›=¬)mmí‹LLL011ÁÜÜLLL000@__SSSttt033COOCCCŒÑÑÑÁÂÂ1ëmk×®QYYI]]%%%ÔÕÕQYYIUUÕÕÕ”——SWWGYYY³çWVVÊ-&¸Õ¿¥………\\òçŸRXXˆ¥¥%>>>L:•~ýú5+@±´´…&÷MMM-ZÄ¢E‹XµjöööJGj³I“&1}útòóó±µµU:ÎmquuåðáÃJÇP .ÄÁÁA­‹J¡£ª®®&66–ˆˆvïÞMff&666Œ;–E‹1~üø.2éëëßtF[¸>À(++K@–——GVV¹¹¹œ:uŠŸþ™ÜÜ\ù‚§ŠŽŽN³ÂÕfff†¹¹¹Ü6UµS›¶]MLLÔv`ÙµkךŒªÚ}ò}ªþ1UŸ˜ª_TTD]]]³ãÈ…]»veРAL:µY‘‡£££hÿ)HBâE^d3›ù˜™XùSAèœT×/½½½),,dïÞ½\¼x‘ãÇ“››‹••Ó¦M# €Q£FÉ훦׋ŠŠ8þ<±±±r;è¯í¸¾ZyKŪkªúúú4ûÚÜ܃¿îìfI’Dii)UUUTUUµúkյܒ’’fŵµµ7¼†ŽŽŽÜ¶·¶¶ÆÎÎ___y²=;;;ù1U‰ Ü ===üýýñ÷÷ovEE)))$&&Ê·~øa³âyb¦›£££(@A1dff’ššJZZZ³IoSRR¨®®FCCC.&9r$Ï=÷œ\H¥NcÇ:¢þóŸÔÕÕ±jÕ*¥£´‹¸¸8Ž=Ê{ｧtAèRRRhll¼éuQd ¨Ÿ3`õjx÷]ضMé4Â_èééÉ+ 2¤Åç466ÊfM—…T}ŸŸŸORR’|a±¤¤„²²2[<žªÌ ÌÍÍÑÔÔÄÌÌ ---y`à_ÿ©žÛZW¯^½¡ãH58®ººZîxª®®æÚµkÔÔÔȃì***šÍ Ö”ê"±ªÒÒÒ{{{úöí+Ï^"IŸ}ö`À€¬Y³†àààVgîºvíÊ7ß|#Š ¡p`2“ÙÈFQd ´H5«“M›ö»Õì÷”””4»¯¼¼œÂÂBÊÊʨ««£¼¼¼ÙÀø†††¿}MU»®R¨©©.æÁõ%®U³f¨ àz{àVËß ~ªvÉÍ4}Ÿ%%%Àõv›j5‰úúz***€ë«Q\½z@nµ†™™:::˜ššÊŸ‹‰‰ ÆÆƘššbcc#_m:°®5«R466Gdd$ß}÷o¿ý6ÎÎÎ<ôÐC„††âåå…¶¶8¾Íš5‹•+WòÁ°fÍ¥ã´Ùøñã100໘™ IDATï¾cþüùJǹ-îîî\¾|™††±Tû-|ûí·DDD-fÕ„»¤¨¨ˆ_~ù…ÈÈH~ûí7ÊËËñòòâÑG%$$„ÀÀÀ;jcÒ£GzôèqËç]½zUHÖtð¼ê¶°° .È™TmÔêêê›SÕ>433CSSSnOªnMLLÐÖÖnÖæTQõ›µEK}sª‚RU;QuÛÐÐ@yy¹Ü–T ìºUaoÓþ1+++¼½½åbŒ¦ƒ¶TEêZlq¿¨¥–§xŠpÂùš¯y˜‡•Ž$‚ í¢¾¾žÓ§OMDD‡FCCƒ¾}û2sæL‚ƒƒ>|ømõK]½zõ†óÿ:è]µ¥¥¥ÝtÐü­4mªú.›^CUMÄ×Wˆ‚æýÕ´ÿ³­nÕϨ꫅ÿõ]6}~ii)’$É×n¡å6ì_5-¸044D___.ÜprrÂÇǧŢŽ¦›˜Wè(LLL0` hv¿ªø@5ûwjj*QQQ¤¦¦RYY \çáêꊛ›îîîÍ \]]Ålà‚ Ü±šššfM·Ë—/Ë“¯ËFÍÂ… éÕ«¢˜ ÄÅűuëVvìØA—.]”ŽÓ.6nÜHÿþý P:Š tHÉÉÉèèèàîîÞâãb„… ~tt`éRX¼^{ ºvU:‘ÐFšššØÚÚ¶yοÎhVVV&ß×ÐÐ@ii)”––Ê3U;¯^½JAAA³ã©û«êêêt¨f.nJU¼`ff†½½½ÜÑfdd„®®®|!×ÔÔ##£fKUO­½˜=sæLÎœ9Ú5k=z4¬^½šQ£FµáSÚ›¦¦&ÿøÇ?صk«W¯îô3ÀÂÝ°… aG9Ê@n\%Gn‡¶¶6]ºt¹k!ªRª WÔÕÕɳW©Š[x¯*>„æý›êÏÈȸa¹t•›µYšºzõêß^Ìj©-ÓTÓh...ò@`U{¥éÅEmmm¹#OUP¡j©.$¶´*D{ÒÔÔ$((ˆ   Ö®]KRRááᄇ‡³yóf,--?~ ‚PRRBVVdff’MFF¤¦¦’-·lmmåB‚È_»¹¹©íŠÎꨦ¦†gžy†#Fðä“O*§]\¸p~ø;w*E:¬””zöìyÓó4Qd ¨§'Ÿ„5k`ÃKÙÜ7TKÅßiGš:ëÝ»7ß|ó GŽáÍ7ß$88˜ÀÀ@Ö¬YÃðáÕŽ'üŸ™3g²~ýzöîݢ1c”Ž#^Aô§?[Ø"Š „KSSSt’«ooo¼½½Y¹r¥<Ø0<<œI“&ahhȈ# còäÉbæ“ûÀóÏ?Ïûï¿ÏÆY¹r¥ÒqÚ,,,Œ°°0rrrpppP:N›¹¹¹×aˆ"ƒ–½öÚkTWW³nÝ:¥£‚ÚihhàðáÃDFFòÃ?pþüy¬¬¬7n_~ù%>ø Úþ_¯££#Ïâ/êâ—˜ÀÊ(#†|ñU:’ ‚ ܱÊÊJŽ9BDD?ýô—/_ÆÈȈÁƒ³|ùryÍŽÊÂÂBôm ‚Ð"Õä-Íî\YYÙ¬ø --ÌÌLNž>>Œ5Šwß}WB+-`s™ËÿãÿካÒqAèdÜÜÜX´h‹-"##ƒÝ»wÉìÙ³™;w.ÁÁÁ„††2yòd¬­­•Ž+´333/^̆ xþùçÕnÉ×qãÆallÌÿû_^xá¥ã´™ÆÆƤ¥¥1bÄ¥ãt8Çgûöí|öÙgâo ´Ò•+WØ·oDDDPZZŠ››!!!|òÉ' ©©©tLA¸ï㙈-¶áN8)IAnK}}=Çç·ß~cÏž=;v €þýûóøã3vìXØâ P‚ …±±1}úô¡OŸ>->~õêUÒÓÓåAƤ§§sþüyöîÝK~~¾¼‚ \ï£íÚµ+666888`ccƒ­­-öööX[[coo­­-666âï« ÜõõõŸŸOnn.äåå‘——Gaa!999MYY™¼Ÿ±±±\HäääD`` \Läè舋‹Ëß®¾.(ïèÑ£lܸ‘?ü°Í«¯ª‹Ã‡óÓO?%þ_„[HJJâÑG½éãâ·GP_Ï<ë×_/4øä¥Ó‚"† ƈåõ×_gÀ€²zõjFŽ©t¼ûÚË/¿Ìƒ>ȱcÇZ\zR„æã1ÞáV°‚/øBé8‚ tbÎÎÎrÁAqq1?ÿü3ááá,\¸ùóç3hÐ yÖxuœ1^¸¹%K–°eËÞ{ï=Ö¬Y£tœ6100àÑGeÇŽ,Y²D-g”qssãÏ?ÿT:F‡S__Ïܹs âñÇW:Ž thª•‰"##Ù¿? 4ˆW^y…É“'Ó£G¥# Â}m7»yŒÇ¾áL³” ‚ ꣡¡sçÎqèÐ!¢££Ù»w/¥¥¥ØÙÙ1dÈž}öYBBBÔnÒA„öTXXHjj*‰‰‰œ>}š³gÏráÂêëëÑÓÓÃÇLJž={âààÀ<@nn®<ˆùèÑ£äææ’ŸŸOUUU³cÛØØ`ccƒvvvX[[Ë… 666X[[ciiI—.]Ä ‚ÐDyy9W®\¡¨¨ˆ¢¢¢fE………òï\AAÍö500çlllððð`Ø°aØÛÛËÅŽŽŽ˜››+ôî„»¥ªªŠ'Ÿ|’‘#G2{öl¥ã´‹ºº:,XÀèÑ£;v¬Òq¡Ã*--%55??¿›>GêËÐÞy{ìzÁÁÀJ'ÅCll,o¼ñ£F"00eË–ªt¼ûÒرc:t(‹/æСCj9Lî%m´YÍjá–°?nÞ€A¸[,--™9s&3gΤ´´”½{÷ÁŠ+xá…ðõõ%$$„G}T \ìŒyá…xóÍ7yþùç±Q³ñæ͛LJ~Èþýû>|¸ÒqÚÌÛÛ›³gÏ*£Ãyÿý÷INN&!!Aœ3Â_444@DDááá$''Ó¥KFÅ'Ÿ|¤I“033S:¦ À;¼Ãr–3ylbZh)IAn©±±‘3gÎðÇðÇpðàAJKK±±±aØ°a¼õÖ[Œ1¥£ ‚ (®¼¼œ?ÿü“¤¤$âããINNæôéÓ`oo··7ÁÁÁ,[¶ ooozõê…žž^«_£ªªŠ’’rssÉÉÉ‘oU÷;vŒ’’²²²(//¿a}}}°··ÇÂÂâo·®]»ŠÒB‡V]]Í•+W())¹åÖôw¥¸¸˜ÚÚÚfÇÑÓÓ£K—.XXXààà€ƒƒýû÷—WšþÞØÛÛ‹>êûÄ’%KÈÍÍ%**ªÓþ›¯Y³†óçÏóõ×_+E:´Ã‡#Iƒ ºésD‘ ÞþñøøcX°Ž± ºpŸ bß¾}ÄÆƲnÝ:&NœH@@¯¼ò !!!¶qØQ½÷Þ{ 0€¯¾úê–Ë ‚pÝT¦2˜Á¼Ê«üÆoJÇá>cnn.¯`PUUEtt4‘‘‘lÛ¶U«VáååEXX¡¡¡·¬ä:¶… ²qãFÖ­[dž ”ŽÓ&½{÷fРA|ôÑGjYdàããÃöíÛ•ŽÑ¡ddd°jÕ*^}õU1pEþÏÕ«Wùý÷߉ŒŒäÇ$??777BBBØ´iÆ CGGG阂 üŸ«\åižæ[¾eXÌb¥# ‚ ÂM¥¥¥Mtt4¿ÿþ;ÅÅŘ˜˜0pà@^y傃ƒñõõES\oá>ÕÐÐ@zzz³b‚¤¤$Î;Gcc#¦¦¦<ðÀxyy‚··7}ûöÅÊÊêŽ_ÛÀÀþ¶ÿ½¼¼œÂÂBŠ‹‹¹råŠ|ÛôëÂÂBΟ?Oqq1ÅÅÅ”––ÞpCCC,,,055ÅÄÄSSS,,,011‘¿711ÁÌÌ 33³fÏ311ÁÜÜSSS´´D‘µð? ”——SZZJyy9TTTP^^NYYeeeò÷ªÛ¦Ï-//§¤¤„k×®Ýplsss¬¬¬èÒ¥‹¼’‡§§'AAAÍîSÝZ[[‹•>„|ûí·|ôÑG|õÕW¸¸¸(§]œ|˜îÝ»cmm}Óçˆ"AýmÙ¾¾ðÙgðôÓJ§„!((ˆ   âââxûí·™4i}úôáÕW_eêÔ©¢Øàñóóã™gžaÑ¢EŒ9;;;¥# B‡·–µ e({ÙËhF+G„û”¡¡¡„††²mÛ6>Lxx8ü1«V­ÂÕÕ•ÐÐP m+5bddÄo¼Á’%K˜3g={öT:R›Ì;—¹sçRPP v+1ôîÝ›ÌÌLJKKÅ,aÿgáÂ…888°téR¥£‚¢._¾Ìž={ˆˆˆ`Ïž=4440hÐ –,YÂĉñôôT:¢ -È$“ÉLæ2—ù…_Ä9¼ ‚Ðᨊ bccùý÷ßÉÎÎÆØؘAƒñòË/‹¢Aîkéé餤¤˜˜Hbb"gÏž%))‰šš´µµéÑ£>>>̘1|||èÖ­›Ò±055ÅÔÔww÷VïÓÐÐЬAu[RRrÃ`œ†766¶xl##£f… úúú`bb‚¶¶6èèè`llŒ¡¡!zzz˜™™¡­­™™zzzbllŒŽŽæææèèè`bb"ï'Ü=•••ÔÕÕQQQA]]¥¥¥ÔÕÕQYYɵkר©©¡¬¬Œºº:ÊËË©©©áÚµkò~%%%ò󫪪¨®®nVLpõêÕ_WSSó†‚ÕÏŒ›››üsmbb‚………\(дh@´w*--§Ÿ~š 0}út¥ã´‹ââbÂÂÂ:t( ,P:Ž tx‡fðàÁ·|Ž(2ÔŸ·7ÌŸ¯¼“'ƒ¥¥Ò‰¡Ã ""‚ãdzfͦM›FŸ>}xíµ×˜2eŠè8½Ö¯_Ott4O?ý4b¢ ü! !„^áF1 MÄß)A”¥¥¥%pnܸ‘S§NÁW_}ÅæÍ›qrrbܸq„„„ðàƒŠÙ•ÕÀܹsÙ±c .dÏž=JÇi“iÓ¦ñòË/³mÛ6V®\©tœ6ñññ 11‘   …Ó(ïÛo¿%""‚èèhôõõ•Ž#÷Tcc£üÿidd$'OžÄÐÐ#F°eËzè!µ+¤„ûM 1LcöØsœã¸âªt$AA --ØØX:DTT™™™1xð`.\H`` }7‚ Ü7$IâòåË$''Ë›je‚ŠŠ ìííéÕ«#GŽdÑ¢Eøøøàåå…žžžÂéï.---¬­­o9Kïß¹zõê-gŸ¯¬¬¤¬¬ì†Aéiii·ÄÞZªb¸>Q’ªOÑÈÈ]]]¹°ÀÌÌLbaaüo ûÍhiiýíÌ÷M_ûïTWWSUUuËç”——ÓÐÐpÓÇ›x”””×û–ÊÊʨ¯¯—žkkkåAþM_[õ¹·–©©):::-XXX`hhˆM³¢UÑÀÍVÅ022jõë B{¨««cÆŒ8;;óî»ï*§]ÔÕÕñÈ#ÐÐÐÀ—_~)Æg Âßhlläرc¬[·î–ÏEBç°jü÷¿ðÒKð¯)F:~üñGNŸ>Í›o¾É´iÓpww祗^bæÌ™bPK;266fçÎ >œ5kÖ°bÅ ¥# B‡·–µøâˇ|È|æ+GA¦©©‰ŸŸ~~~¬\¹’¤¤$ÂÃÉŒŒdÇŽXZZ2~üxÂÂÂ3fL§»ÓYhii±uëV†Jdd$!!!JGj5ž{î96oÞÌK/½¤V³X999annÎÙ³gïû"ƒòòr/^Ì“O>ÉÈ‘#•Ž#÷DUUÑÑÑDFFINN®®®Œ=š7Þxƒ±cÇÊÄAèØv°ƒ,`*Sù”O1ÄPéH‚ Â}êüùóÄÆÆÃüAvv6FFF±`Á†ŽŸŸŸ<ØR¡3ËÉÉ‘‹T·§OŸ¦²²¸>ÈÜËË ___f̘··7>>>ØÚÚ*œ\}add„½½ý]=®j0|Ó™õUƒå›”WÍš× jkk¨¨¨ ¾¾¸ù€ü´´4 ù€ü–4}›¹ÕªõwE €¼òÃÍ´T8¡©©‰››Û ¯¡­­‰‰ ºººòÀþ¦¯¡* Pg4]9¢-‚ n–.]JRR'Nœ¸å’$‰gŸ}–cÇŽqèÐ!1‰ ´BBBåååÜòyâŒZèÌÌ®L˜cÆÀ?þ¡t"AèúôéÃ7ß|CZZ›6mbñâŬX±‚yóæñüóÏÓ¥K¥#vJlÚ´‰çž{zè!¥# B‡æ7/ò"ËYÎD&∣ґAZäíí··7+W®$--ˆˆÂÃÙ4iŒ9’°°0&Ož,wl CPPS§NeñâÅŒ=Z­ B.\ÈúõëùøãY²d‰ÒqZMCCoooΞ=«tŽöÚkTWW³víZ¥£B»*((à×_%22’_~ù…ªª*|}}yæ™g ÅÏÏO鈂 ´Ae<Ã3ìf7ëXÇ ¼ t$Aá>RWWlj'ˆ‹‹ãàÁƒÄÅÅQXXˆ¡¡!ƒfÞ¼y >œˆ• Aè´HOO'--­Y1ABB‚<{»ª˜ÀÛÛ›°°0¼½½éÝ»·lÙéë룯¯/¯6 ‚p·íÚµ‹M›6ñŸÿü‡=z(箓$‰… òÅ_ðý÷ßÓ»wo¥# ‚ZˆŠŠÂÞÞ^^þfî¸ÈàçŸ&))éN#wÅÀ‘#q=›ÈÜ\*­¬”Ž#šŸŸ<ðûöícýúõ¬[·Ž!C†0nÜ8,--•Žw×$&&*€yóæqæÌ}ôQ¢¢¢6lX›‘˜˜ÈÎ;Û! t<îZîL4`béDÿ±Xé8‚ ­baaÁœ9s˜:u*gÎœáÔ©S<õÔS<ýôÓx{{ãëëK¿~ýþv¹aáÞ4h?þø#3gÎd„ JÇi“ÀÀ@Ö¬Yƒ™™ÙÏÈx/ÛË>>>íZd íåK—.±mÛ6ž~úi¢¢¢”Ž#w]vv6 œ:uŠ‹/¢££ƒ——<ò}ûöÅÜÜ€¤¤$ѧ,j$Õ*•íC·S«UË‹_Ä*ÏŠtìÿsA„–]¾|Y-Î+**8zô(±±±:tˆC‡QUU…  àÅ_$00µš8@¡5êêêHMM%))‰””ùöܹsÔÔÔ ¡¡‹‹ ^^^0{öl¼½½ñôô“Ý‚ ÍÄÇÇ3wî\–.]Ê?:á¤Í’$±dÉvìØÁ×_­ç:‚ÐQDEE1nÜ8444nù< I’¤Û}I’¨©©¹ÝÝáî«®FoèP$ccj÷ì±ü¥ ´Jee%Ÿþ9›6m"??ŸÐÐP^|ñEúõë§t´;¦©©‰®®®Ò1€ë³K<öØcDDDðÛo¿ئýkkk[½ô  t5ò îƒ|P÷³f)Gá¶\¹r…¨¨(¾ÿþ{öíÛG}}= `Ê”)L™2¥#Þ×Ö®]Ë;ï¼Ã‘#GÔjö–¬¬,¼¼¼Ø¼y3³fͺ£cÝËöò|À믿Ε+Wþ¶Ãêvtôör}}=AAA˜ššòÛo¿µËg ÷Zuu5qqqüòË/üðÃdggãääĘ1c7nÁÁÁbЗ ¨1 ‰´>à5×l 䳺ϰ“씎%‚ Ü!==½Vìرƒ9sæ´{I’8wîGåÈ‘#ÄÅÅ‘””Dcc#0dÈÔêÜ]áï””üöî;¾Êúîÿø‹ìMÈ^$dž°Ã^" <' ¸ ¶µø°µØûVÛªUëm{[ªÞ¶ÖQ-âূQ䀌°$,Y IÈ8„œ ²NrN~Ä– ë$|žçq=ΕpNò>ŽpNÎ÷ýýè{L%(((0ÜÒÒ€ŸŸ±±±æé*•ŠQ£Fáìì¬pz!„–N§Ó1~üx†ŽV«ÅÚÚZéH½ªµµ•Ÿÿüç|õÕW¬^½š{ï½WéHBÜ7âµ¹^¯ÇÛÛ›Ï>ûŒÄÄÄ+ÞöºJBX¤ÌL7~ÿ{øóŸ•N#DŸb0øì³ÏxõÕWÉÎÎfîܹ<ùä“Ìœ9SéhýF[[‹-bÏž=$''3}út¥# aÑ~Ïïy‡w8Æ1"ˆP:ŽB\—¦¦&¶oßNRRëׯ§±±‘Ñ£G£V«¹ÿþû‰ŠŠR:â-§½½É“'cmmMZZZŸúë£>ÊæÍ›ÉÍÍÅÁÁAé8Weß¾}L™2…S§N1tèP¥ãÜt¯½öúÓŸHOO'::Zé8BüdÕÕÕlÚ´ ­VËæÍ›ihh@¥R¡ÑhP«ÕL™2EJ4BôUTñ0³m<Çs¼À Xa¥t,!„7Ñ*ÔÔÔpàÀjkkqttd̘1Lš4‰©S§2yòd¼¼¼zýû !Äͤ×ëÉËË»èÈÏϧ±±777"""ˆŒŒ$**ŠÈÈH"##‰ŽŽfàÀ ?!„}Qss3·Ýv 8p777¥#õªêêj.\ÈÉ“'ùꫯd]›è×nÄkóµk×òàƒRUUÅ Aƒ®x[)ˆþéí·á‰'`ëV¿D„¸fhµZ^}õUöìÙÃðáÃY¾|9>ø ŽŽŽJÇëó K–,aÆ |úé§Ü}÷ÝJGÂb00 ØcÏ·|‹–1™D!®WKK Û¶mC«Õ²~ýzªªªP©T$&&¢Ñhˆ‹‹S:â-#;;›1cÆðâ‹/òÇ?þQé8W­¼¼œððp^~ùeþû¿ÿ[é8W¥¥¥WWW>úè#xà¥ãÜTÅÅÅÄÆÆòôÓOó /(Gˆk–™™‰V«%%%…ýû÷cooÏ”)SP«ÕÜsÏ=(QÑ‹¶³%,Á;>ã3&3YéHB!Ð ÚÚÚ8~üxRA^^L˜0 &0qâDFŽ‰­­moDBˆ›Ê`0PZZzÉ©ØÚÚDXX˜y*AXXaaa„††JY_!D¯1™LÜwß}ìرƒƒö»MŸöìÙÃ<€ Z­–ØØX¥# qC݈’Á’%K(**bÏž=?z[)ˆþ©£î¿¿³d°w/ÄÄ(Hˆ>ëرcüûßÿfõêÕØÙÙñ³ŸýŒ'Ÿ|’àà`¥£õiF£‘_ÿú׬ZµŠ×_åË—+I‹•O>ãǃ<ÈÛ¼­t!„èuF£‘ýû÷“””Ä—_~‰N§#44FCbb¢ì}¬X±‚_|‘C‡1|øp¥ã\µ?þñ¼ÿþûœ>}¥ã\•¸¸8¦OŸÎo¼¡t”›jþüùäää‘‘Ñg&Oˆ[[{{;@«Õ²nÝ:òòòðòòbÎœ9h4æÌ™ƒ‹‹‹Ò1…½¬‰&þ‡ÿáU^e! YÉJÜqW:–B…\ëB†öövrss9räˆù8zô(ÍÍ͸¸¸0bÄâââ˜:u*·ÝvÞÞÞ70½Bô®öövŠ‹‹ÉÏÏ'??ŸœœòóóÉËË£¸¸“É„••AAADFFÑc*AHHHŸš¢*„¢ïzâ‰'xÄê IDATï½÷øæ›o¸ýöÛ•ŽÓkÚÛÛùÛßþÆK/½Ä]wÝŪU«~tv!úƒÞ.455áëëË_ÿúW~ûÛßþèí¥d ú¯–˜=ÊÊàÀðñQ:‘}ZUU~ø!o½õ:ŽyóæñÄO0{öl¥£õYüíoãOú?ü0ï¼óöööJÇÂ"}É—Ü˽|Â'<ÄCJÇBˆÆd2qìØ1RRRX³f ¹¹¹æE‰‰‰Ì™3Gvõ»L&3fÌ ¡¡ï¾û®Ïü3®­­eèСüú׿æå—_V:ÎUùõ¯MFF{÷îU:ÊMóå—_rï½÷’šš*#{…E;wî;vì %%…äädêêêP©Th4Ôj5“'OÆÊÊJé˜Bˆd»ø%¿¤–ZÞäMäA¥# !„PØ•2 Nœ8Ñ£Lpüøq ÎÎÎŒ5Š¸¸8ó-Ï%…O¯×÷˜@Ðý(..¦½½ó‚îS ¢¢¢pvvVøQ!„¸•ýå/áÅ_äóÏ?çÞ{ïU:N¯9|ø0Ë–-#++‹¿ÿýïüö·¿• ÚÄ-£·KkÖ¬aÉ’%”––âskª¥d ú·sç`òdpu…]»ÀÉIéDBôyƒ 6ðüƒ}ûö1fÌ}ôQ–.]*;rþD)))<ôÐC¨T*Ö¬YCHHˆÒ‘„°HOñïð;ÙÉ&(G!nŠÌÌL’’’Ðjµ9r„Aƒq×]w¡Ñh˜7oNò§×äåå1zôhüqV¬X¡tœ«ö÷¿ÿ—^z‰ììì>1mì£>â±Ç£®®;;;¥ãÜpõõõ¨T*âããùàƒ”Ž#ÄE HIIA«Õ²{÷n:::˜0a‰‰‰,\¸°Oü\B\ŸîÓ æ0‡÷yŸ”Ž%„Ât-d¨®®&##ƒãÇ›“'O^¶P%»u !,R]]………—=š››°µµ%$$„ÐÐЋŽððpÙ5Y!„Eúä“Oxøá‡ùÇ?þÁòåË•ŽÓ+Î;ÇK/½Ä;ï¼Ã´iÓø÷¿ÿMTT”Ò±„¸©z»d Ñhhooç›o¾¹ªÛKÉ@ô§OäI0}:|ñÈ.Bôš½{÷òæ›oòõ×_ãååŲeËxä‘G T:ZŸ“ͽ÷ÞKYY|ð .T:’Ç„‰…,dû8À†2TéHBqS’œœLRRûöíÃÑÑ‘™3g’˜˜È‚ puuU:bŸ÷駟²dÉÖ®]Kbb¢Òq®ŠÁ``øðáŒ9’/¾øBé8?*++‹ØØX>L\\œÒqn¸ßþö·|þùçdggãåå¥t!0ìß¿­VKrr2ÙÙÙxzz2sæLÔjµü}"Ä-&4~Á/8ËYV°‚eôÞ›UB!úž¶¶6²³³9~ü8'Nœ`ãÆÔÔÔP^^€··7#FŒ`ĈŒ=š¸¸8"##¥P „°ƒÒÒÒ‹¦èt:ÊËË)((0߶û4‚ àà`lll|$B!ĵINNæî»ïæOú/¾ø¢Òq®[ss3ÿú׿xå•W°³³ãoû?ü°L/·¤Þ,ÔÔÔàççÇ|ÀC=tU÷‘’¸5ìÙwÜ¿øüë_ ¿ì¢W•””ðî»ï²jÕ*ª««™7oË–-cîܹòËåkÐÜÜÌO<ÁÊ•+yôÑGY±b…,îâ 40•©˜0±‹]xâ©t$!„PDii)›6m"%%…-[¶`mmÍÔ©SQ«ÕÜwß}W5ÚP\Úo~ó>ùä8@ll¬Òq®ÊÖ­[‰góæÍÄÇÇ+çŠ:::4hÿû¿ÿËc=¦tœêСCLš4‰U«V±téR¥ãˆ[Xcc#;vì0OÆÑëõ„……¡V«Ñh4̘1COq‹9Ç9žã9Þç}æ3Ÿwy_|•Ž%„â&ikkãÔ©Sdee‘Mff¦ù¼­­ ;;;T*666ÄÆÆJHHîîîJGBÜâêëë©®®æܹsœ={–sçÎQ]]ÍÙ³g©ªª¢¦¦†®%PÎÎÎx{{ãããƒÏE綶¶ ?!„¢w;vŒW^y…Ù³g÷ênçJhiiaëÖ­$''ÓØØHBB ,ÀÑÑQéhB(¦¨¨ˆßýîw½òµÞ|óMž{î9ÊËËqvv¾ªûHÉ@Ü:Ö¯‡„iÓàÿý?ÑÑÑJGB( vÞã=^àpà5^ã~îW:–Bˆ¤µµ•œœrrrÈÌÌ$;;›¬¬,òóóikkÃÚÚš!C† R©P©Tæ)QQQØÚÚÒØØHcc£ÒCq‹hll¤¬¬Œ²²2ÊËË)++£´´N‡N§£´´”––óí=== ÀÏÏ   ‚‚‚&88˜   >ó;=!„âz|üñÇ<÷ÜsÜsÏ=üßÿýVVVJGúI*++ùè£Xµjmmm<ôÐCüæ7¿‘MÕ„œœœprrºî¯ÓÑÑALL ³fÍâí·ß¾êûIÉ@ÜZŽ… àüyxþyxüq°ðÅBôU§NbåÊ•|øá‡TWW3sæL–-[ÆÂ… e‡Ä«P]]Í“O>É'Ÿ|Âm·ÝÆŠ+˜0a‚Ò±„°E1‹Y˜0‘L2ît$!„°MMMlß¾¤¤$6lØ@}}=*•ŠÄÄDî»ï>YTz•***ˆ‹‹cܸq¬[·®O,/(( 66–—_~™§žzJé8Wôì³Ï²~ýz²²²”Žrüúê«<ÿü󤧧Ëÿwâ¦0™L;vŒ””´Z-GŽÁÃÃٳg£V«IHHg…¸Åíf7ËYN6Ù<Æc¼Ì˸"4…¢?0 äçç“••ežJ™™Inn.F£‚ƒƒQ©TÄÆƆJ¥bôèѽ²PA!~ŒÁ` ººšòòr (((@§Ó™?ÖétTTT˜§888àïïOXX~~~GFFâââ¢ð£B!”SYYÉc=Ɔ xþùçyñÅûÄ{YÝutt°cÇÞ}÷]’““qwwç7¿ù ?þ8žžžJÇ¢ßÙ¸q#†ÌÌLbbb®ú~R2·žóçá•Wà7Àß–/‡‡WyCEˆÁ`0°~ýzÞ{ï=vîÜIPP<ò¿øÅ/dWÝ«°ÿ~þð‡?––FBBøØ4i’Ò±„°ç8ǽÜË>öñþÀs<‡-RBˆ.---¤¥¥‘’’ÂÚµk©¬¬D¥R¡ÑhP«ÕL™2¥ÏýÂñfúöÛo™={6/½ôÏ<óŒÒq®Ê_þò^yå222W:ÎemÚ´ µZMEEÞÞÞJÇéuÅÅÅÄÆÆòôÓOó /(Gôc]Å2­VKJJ ååå„……¡V«Ñh4ÜvÛm?ÙDqãe’É ¼À×|Í<æñþAJÇBñÔ×דŸŸOAAA2ANN&“ [[[‚‚‚Ìe‚î׎ŽŽJÇBôcz½þ¢â@÷Ïœ9ƒÑhÀÎÎOOOsqàrE!„B\¬µµ•þóŸüõ¯ÅÃÃ?þ˜Ûn»MéX×$33“¤¤$>ýôSN:E\\Ë–-cÉ’%òºEˆhÎœ9ttt°eË–kºŸ” Ä­ëÌøûßá“OÀʪ³hððÃ0z´ÒÉ„è·.œn0iÒ$–.]Êý÷ß/»Mü­VË_þò<È´iÓxꩧ¸ë®»°¶¶V:šŠj§·y›gy–"XÅ*Æ0FéXBaqŒF#û÷ï'))‰¯¾úŠ²²2† BBB‰‰‰Lž<¹ÏŽP½‘Þzë-–/_ÎÇÌ’%K”Žó£ÚÛÛ™0a...ìܹÓbK$õõõ 4ˆµk×r÷Ýw+§×%$$››KFFJÇýLqq1›7o&%%…mÛ¶ÑÞÞΨQ£ÌÅ‚¸¸8¥# !,Dy¼ÄK¬a ÃÆ+¼Â<æ)K!ÄUÐétdeeõ(PXXHGGvvv„‡‡÷(„……1lØ0ìíí•Ž/„èGÚÚÚ¨¨¨ ¤¤„²²2ÊÊÊ(..F§ÓQZZJii):Ž¶¶6¬¬¬ðõõ%$$„ÀÀ@ !((ˆÀÀ@‚‚‚ðõõµØßY !„–Ê`0ðÑGñòË/sîÜ9–/_Οþô'œ•Žö£:::8|ø06l`ýúõdffÂ}÷ÝÇC=Ä°aÔŽ(D¿—Mll,ÉÉɨÕêkº¯” „¨­…>€ÿN‚èh¸ï>¸ÿ~ˆŒT:ýRkk+7ndõêÕlÚ´ [[[-ZÄ’%K˜9s¦,œ¿‚Ý»w³bÅ 6oÞLpp0¿úÕ¯øå/‰¯¯¯ÒÑ„PT.¹ü’_òßñ;~ÇøžÈ=!„¸œ®]BÖ®]KNN^^^Ì™3‡ÄÄDæÌ™#;_wóì³Ïò÷¿ÿ/¿ü’ (çGedd0nÜ8þùÏòØc)ç²ÆŽËäÉ“yóÍ7•ŽÒ«’’’X¼x1©©©Ìœ9Sé8¢ŸÈÌÌ4O+Ø·oŽŽŽÌœ9FCBB‚¼BôPL1寬bá„ó Ïð bü¾M!,ISS¹¹¹äææ’““CNN¹¹¹äååÑÔÔÀàÁƒ‰ŽŽ&::š¨¨(bbbˆ‰‰aÈ!²Q€⺵´´˜KÅÅÅæAII :Ž’’*++1™L@gÀÇLJÀÀ@ " À\$èú¼ü^Q!„è=ÙÙÙ¬\¹’O>ù„¦¦&–-[ÆÓO?mñSt:;wîdçÎlÞ¼™²²2BBB˜?>‰‰‰2m]ˆ›ì ##ƒ'N\óï¤d Dw™™°zuçtƒòr ƒÙ³A­†øx°³S:¡ýNmm-ÉÉɬ^½šíÛ·ãççÇ=÷ÜÃÏ~ö3ÆŒ‘ÝÈ/'//÷ߟ>úˆúúzâããY¼x1óçÏ—©â–eÂÄ»¼ËK¼D -<Áü7ÿJGB‹Ö}ñêÞ½{ñðð0ïˆ=oÞ<œœœ”Ž¨¨ŽŽ–-[ƧŸ~ÊÖ­[™:uªÒ‘~Ô³Ï>Ë[o½Å‰' Q:Î%=ùä“lß¾ôôt¥£ôšúúzT*ñññ|ðÁJÇ}Xss3{÷î%%%Å<}&$$„øøxÔj5wÞy§ìP+„¸ÈINò:¯ó)ŸL0/ò"ð€” „Baz½¾Ç4‚®ó¢¢"L&666Öc*All,~~~JÇBôQ---èt:t:ååå\tÞ½@`kkËàÁƒñ÷÷ÇÏÏÂÂÂzœI@!„¸ šššHJJbåÊ•¤¥¥Æ/ùKyä¼½½•ŽwI•••ìÚµ‹;w²k×.rss±³³cüøñÜqÇ$$$0jÔ(¥c qK:yò$#GŽdíÚµÜsÏ=×|)q)F#|û-lÜZ-äæ‚›[gáàöÛ;•Jé”Bô;gÎœaÍš5¬\¹’S§N¡R©HLLäç?ÿ¹Å.ŽRZKK _ý5Ÿþ9[·nÅÚÚšyóæ±xñbÔj5ŽŽŽJGâ¦;ÏyÞâ-^ã5Œù/þ‹ßò[)!ÄU(**bÆ $%%±ÿ~ìíí™5k‰‰‰ÌŸ?777¥#*Âh4šw§ß½{7#GŽT:Òµ´´‡——;vì°È.7lØÀ¢E‹¨ªªÂÓ³LzüñÇY³f ÙÙÙxyy)Gô1gÏžå›o¾A«ÕòÍ7ßÐÔÔÄèѣͥ¯1cÆÈÎNBˆKJ%•×y-l!†žâ)â!l‘`Bq³455‘ŸŸO~~>yyy=&Ô××àîîNTTTÉÑÑÑ :;ÙäLq ªªª¨¨¨ ¤¤Ä<} ¸¸NGii)%%%444˜oïèèxÑô‚ƒƒ  ___yÍ)„B(¨¥¥…mÛ¶±~ýz¾úê+š››Y°`¿úÕ¯˜9s¦E½ÏÓÔÔDzz:GŽáèÑ£|÷ÝwdggcmmÍرc™1c·ß~;S¦L¹å72Â,X°€‚‚ÒÓÓÒÏ)q5NŸî,lÝ {ö@}=øøÀŒÇ”)¥kÙJˆÞÐÑÑAZZ«W¯&))‰††¦M›Æ¢E‹X¸p!JG´Hz½ž¯¿þšµkײcÇIHH`ñâÅÜyç888(Qˆ›ê<çy›·YÁ šh"þ‹ÿb“”Ž&„}Bׂפ¤$¶lÙ‚••Ó¦MC­V³xñb|}}•ŽxSµ´´Ï©S§Ø»w/C† Q:Ò}:...JGBtsèÐ!&L˜@rr2jµú'} )q­ŒFHO‡´4Ø»¶mƒÚZpv†‘#!.¦Ní,XÀ_öBôu---lܸ‘¯¾úŠ7ÒÐÐÀ„ X´hwß}7aaaJG´H555hµZ’’’ؼy3¶¶¶L™2…Ù³g£ÑhPÉ4q i Ïùœ·y›ã'Ž8–±Œ%,Á™ö!„WC¯×“’’‚V«eÓ¦M4773iÒ$4 ÷ÜsC‡U:âMQ[[Ëm·ÝFKK Û·o·øòë믿ÎÿøGÒÒÒ˜0a‚Òq.2räHî¸ã^{í5¥£\—öövÆ««+;wî”7îÅeµ´´––FJJ ëÖ­£¤¤oooâããÑh4Ì;ggg¥c !,˜«YÍÛ¼MeÌcÏñ™¨t4!„è7ºÑdeeQPP`>233iiiÀÃð°0ÂÂÂP©TÄÆÆFtt´ìÔ)„¸Hkk+åå唕•¡ÓéÐét”••™?W^^Niiiò€¾¾¾âççgž>àïïO@@~~~ÉkH!„¢éèèàäÉ“ìرƒ7²k×. ÀŒ3X¸p! Šmîç~&2Qv{Bˆ«ÔÜÜLjj*III$''SWWg~N¶xñbbbb”ŽxCUTT0{ölšššØ¾};¡¡¡JGº,“ÉÄœ9s8sæ Gµ¸Å.Ë—/gß¾}>|Xé(×åÕW_åùçŸ'==èèh¥ã sîÜ9vìØAJJ 6l ¾¾•J…F£A­V3eÊy#FqEFŒlbïñßð >øð ~Á2–L°Òñ„¢OÒëõ=Ê] irssÍ‹|ÝÝÝ:t¨¹LÐU(1b®®® ?!„¥Ðëõ—ø€G}”C‡1zôèŸüu¤d ÄP] vŽ£G¡´´óÏBB`Ì6¬óˆ…ÈH°µU6³}L{{;»wïæ믿fݺu”——Í¢E‹X°`qqqXYY)Óâ´··³wï^¶lÙÂæÍ›IOOÇÞÞžéӧό39r¤üâRô{:t|ȇ|Ægd‘EaÜÏý<À¨IBqµº—@¿øâ *** C­V“˜˜ØoÏêõzæÎË™3gضmÆ S:Òeét:FŽÉœ9sX½zµÒqzX¿~=wß}71ö÷§(..&66–§Ÿ~š^xAé8ÂB˜§¿ìÚµ ¦NŠZ­fÑ¢E)QÑSÌÇ|ÌJVRB ³™Í£ø o¼ñÆu}-)q³TUýP88v Nž„Ó§¡½½³`* Þy=l  ²ÐWˆe2™8vì)))|þùçäååáééÉÌ™3™={6sæÌ!8Xvv»”ŠŠ 6oÞlžrpîÜ9\\\˜2e S§NeúôéŒ7¥£ qä“Îg|ÆÖPB £Åb“@‚„âFöïßORR_ý5¥¥¥„„„0þ|4 3fÌÀÆÆF阽¦®®Ž»îº‹üü|¶lÙ¨Q£”ŽtYÛ·o'>>ž·ß~›G}Té8f xzz²zõj/^¬tœŸ$!!ÜÜ\222ä9ó-¬ëçŸV«eÆ äää0xð`æ΋F£!>>^v»B\•zêùŠ¯ø„Oø–oñÄ“‡y˜e,#œp¥ã !„Eéèè ´´”S§NqêÔ)NŸ>Ýã¼k"ƒƒááá :”ððpÂÃɈˆ ""‚   )q 1 æ¢@UU•¹(PVVFEEååå”——SUU…Ñh4ßÏÉÉÉ\¸Tq ëÚÑÑQÁG'„Bˆ›­¶¶–ƒràÀ<ÈÁƒ©©©ÁÁÁI“&qûí·3sæLÆß+%CƒÁ@QQ………—œHÐU¦¶¶¶&00Ð\$è^( ÅÇÇ纳!,Û²eËظq#ÙÙÙ×ýþ”” „PRk+dgCVVgé 3³óº¨L&pp€˜˜J±±Ç! ;´ qY¤¦¦’ššÊæÍ›ihh ,,ŒÙ³g›K...JÇ´H¤¥¥±wï^¶mÛFaa!666Œ9Ò\<˜5kƒ R:ª½Î„‰=ìá3>c=ë©¢ŠpÂI  ¦2úÏâX!„¸Ñ233IJJâ‹/¾ ;;Û¼à611‘øøxìì씎xÝšššX°`‡bÓ¦MLš4IéH—õâ‹/²bÅ ÒÒÒ;v¬Òq̦M›Fdd$|ðÒQ®YRR‹/&55•™3g*GÜdz½žÔÔTRRRHII¡¶¶Ö<É¥?«„7Ž#;ÙÉ'|Â×|M;íÜÁ,e)ó™}ÿ9“B\NGVVÖES rssÍE{{{.9‘`È!2õXˆ~®¥¥…ššÊËËÑét—½®¬¬Äd2™ïçàà€¿¿?~~~W¼öððPðÑ !„´¶¶ròäI>l.äääÐÑÑAXX'Nd„ Lœ8‘Ñ£GÿäRN§ëQ"è~]VVf~.ãááÑ£@нHÜ/ÞƒBü4›6mB­V³víZ¯ûëIÉ@Kd0@~~gù 3ó‡ëœœÎòBX؇JÕY@ ‘éBtÓÜÜÌž={غu+[·nåĉ8880uêTî¼óNî¼óN†.o2\Faa!{öìáÛo¿%--ÜÜ\¬­­>|8'NdܸqŒ7•J…µüìýˆ 8@ò÷—l²Ä æ1˜Íl<7„âj’’BRRûöíÃÝÝÙ³g£V«Y´hÎÎÎJGüÉZ[[¹ï¾ûضm6l`Ö¬YJGº$“ÉļyóÈÎÎæèÑ£xzz* €ÿùŸÿáý÷ߧ´´Té(פ¾¾•JE|||Ÿ,HˆŸ¦ëg™V«e÷îݘL&&NœˆF£aÁ‚DEE)QÑG1’F_ò%I$QES˜Â–p/÷⎻Ò…â¦ill¼¨@púôiNŸ>Maa!mmm@ç"š 't²¦!I] IDAT§ýSuu5•••TVV¢Ó騨¨0—ºO#hhh0ßÇÊÊ óÔ®koooó4|}}±··WðÑ !„ÂRÕ××süøqŽ9BVV™™™9r„––œ9r$qqqL:•éÓ§_Ó둆†siàRE‚–– ³HrQ‘ ëÜÝý‡ßµ··ÓÐÐ@cc#ƒá²ßÛÊÊ ww÷÷Bô/•••Œ1‚9sæðñÇ÷Ê×”’}I}}gá 7Nê,"œ:ÕyÔÕuÞÆÞ†…ˆÿá:<‚‚d‚¸åét:sá 55•³gÏâááÁäÉ“™<òˆÒqHLL¤©©‰7*åGµ··3~üx\]]Ù¹s§EIĵ;sæ [¶l!%%…mÛ¶ÑÞÞÎĉÑh4h4š>5eC¡<=zRI%åûK-µ¨P‘H"÷qŸ „}’Á` ¤¤„ÂÂBŠŠŠ(**êQ$8ûý{M Àßß¿Gy 44Ô|Þ—ŠäBÜê ÕÕÕ=& \nAEEEÅsæé—š<Ðõ9™: „Bˆ«ÕØØHkk+µµµ=þ·¶¶rþüyΟ?Occ#EEE”••™Ÿ«ÔÔÔP[[kÞíßÎÎ;;;¬­­ikkÃd2ÑÒÒò£ßßÖÖÖ\hoo7?÷±²²ÂÑÑOOOŒ——~~~øúú‚““¸»»coo““...ØÛÛãêêÊÀ-r‘þùóç)..¦¬¬Œ²²2óyAAùùùÓÑÑ••!!!DDDATTQQQ 6 ¥†·Œo¾ùFÊ+xòÉ'{õkKÉ@ˆ[]c#ugÎtÝÏ+*~¸­«ë¥ƒ !! ;Hˆ~¨¾¾žýû÷³oß>öíÛÇáÇ©­­ÅÆƆØØXâââ;v,cÇŽeĈùäßÔÖÖ’‘‘ÁñãÇ9~ü8ééédffÒÜÜŒ QQQŒ1‚‘#GšyÃIôE&L¤“ηß_ÒHã,gqÆ™ILbÓ˜ÎtÆ3G¤¨$„—ÓÜÜLjj*III$''SWW‡J¥2O8°ô¿{÷îeáÂ…øûû“œœl‘ÓœžyæÞxã víÚÅĉÍòŸÿü‡ßýîwœ;wìX‚W_}•çŸžôôt¢£e‘h_e2™8vì)))hµZŽ=ÊÀ¹ýöÛÑh4ÌŸ?¥c !ú<òH&-Zö²— `:ÓÑ| #LéˆBqEííí”––š EEEšKeeeæÝ8ÍÅî‚®-ý9½·²æææË–.<¯¬¬ì± ¯ƒƒÃ ]çÁÁÁ2•D!„¸Eu=×è*t]wÿܵüy×ùùóçikk»¦,6668;;ãâ₇‡žžžøøøàíí ---˜L&š››©¯¯G¯×sîÜ9Ξ=ËÙ³gÍ¥GGGBCC eÈ! 2¤Ç¹§§çøGiñ ¥¥¥˜§Ct•Ð éèèÀÍÍððp󤱱±Œ9R&T ÑËrrr˜4i’y¼Þ&%!Ä• PZ  ÓAyyçy×qæ |ÿÄ {ûÎiþþ~~Ÿƒ²IˆëÐÑÑÁéÓ§9|ø0GŽáðáÃ=z”úúzlmm>|¸¹xÇ°aäxpF£‘üü|222Ì„ŒŒ JKKðòòbĈÄÄÄk¾}š„„ôz=ëׯgüøñJGêÁd21þ|:Äþýû U,‹N§#0072wî\Årü˜ââbbccùýïÏóÏ?¯tqš››Ù»w/)))|ùå—èt:† ÂwÞ‰Z­&>>;;™<%„¸:M4±›Ýlþþ’GƒÄ\æ’@ñÄ㆛Ò1…¢½^ßcú@÷£¸¸˜öövìíí èQèšBàïïOhh¨E¾âVd2™Ì ᪪ª¨¨¨0Ÿ———›ÿ¬ë¼¹¹¹Çý=<<ðññÁËË |}}ñòòÂÛÛ___󵯯¯ENjB!ÄÕimm¥©©É¼X_¯×ÓÖÖfÞýß`0P[[K[[ æ@}}=ƒúúzóÂÿ††Z[[©¯¯§©©‰ÖÖVôzýf°³³»ä.þ]»ü;88àè舛›¶¶¶Fš››©­­¥¶¶Öüü¦¼¼ƒÁt>— '::šÈÈH"""ÆÑÑ‘ššJKK9sæ eeeæóÒÒRjkk͹œœœ&$$Ä\è^&ðöö¾aÿ^ú«ššNœ8ÁÉ“'9qâ„ù¼¾¾€àà`† ƈ#3f cÆŒ!,,L^g ñTWW3aÂüýýIMM½!k¥d „¸>­­P\Üy”–BIIg¡¤¤ócªª~¸½­-øúBPtÁÁ„ÀÀÎÏûùuÞNˆ>D§ÓqäÈó±oß>jjj°±±!88¸G3W¥R¡R©ä òeÔÔÔžžÎñãÇ9yò$™™™äää˜_è <Ø\:P©Tækµ&úŠbŠÙÃp€ƒ$tÚhà /&0¡Gñ@¥!DO&“‰}ûö¡Õjùꫯ8uêÁÁÁ,X°FÃŒ3°± Rs]]‹/æÛo¿å½÷ÞcÉ’%JGê¡©©‰Ûo¿ššöíÛ§èî1qqqL˜0wÞyG± ?&!!ÜÜ\Ž?.Eâ>¢ªªŠÍ›7£Õjùæ›ohjjbôèѨÕj4 qqqJGBô!'8Á–ï/i¤ÑJ+ÃN<ñÜÅ]Le*ÖX+Sq‹2 ”••QRR™3g(..î1• ¸¸Ø¼ÇÞÞþ’‹gºÎe¢“ÊjmmåܹsWœ2Ðõq÷‚PKN¸ð<00PŠÖB!„¨¯¯§­­ººº ùÛÚÚ¨­­Å`0ÐØØØ£ `08þ¼y‘]]mmm=ÊÝ¿îÕpuuÅÖÖ777ìíí8p ÎÎÎØÙÙáîîn. 899aooQ1ÀÞÞgggœ±··ÇÍÍ GGGs‰àÂõ1555=v¾ï~^\\lž\àââBDD„……áëë‹««+&“‰††ÊËËÍ÷Õét=¦1u¨» ÓÝËÓ]ç~~~²vç&)**"33Ó\<ÈÈÈ ''£Ñˆ››£Gf̘1æ먨(¬­åwmB\Ž^¯gÖ¬YÔÕÕ±ÿþVŠ’’âÆkm…²²Î£¸¸³xpa!¡²ò‡‰t;ËàãÓyîëÛYBðóoo™Š ,–Éd2/@ÊÈÈ0?I>sæ îîîŒ1‚áÇ3bÄFŽIll,ÎÎÎ '·\eeedgg“••EVVÙÙÙdffrîÜ9 óŸi×´ƒèèhs[}È!ØJqIX°Z8ÊQ¾ã;r ˆ"0€(¢ˆ#Ž1ß_F3ZŠBÑMff¦yÂAVVžžžÌ›7ÄÄD‹Ù•Üh4òûßÿž7Þxƒ¥K—òÖ[oYÔs¾òòr&MšDpp0[·nÅÁÁA‘þóŸùÏþCii©EþB?))‰Å‹“ššÊÌ™3•Ž#® 33­VKJJ ûöíÃÑÑ‘™3g¢ÑhÐh4øùù)QÑG”SÎNv’J*[Ø‚ƒÌÜÁÜI<ñø!?S„7ǹsç(..¦¤¤„¢¢"JJJ())¡¸¸˜3gÎPQQÑc!Í•vâ”E4BÜ\ƒ³gÏR]]MEEÕÕÕ=>î:¯®®¦¼¼œ†††÷8pà%§ tûùùáååe>„BÑ;ÚÛÛ{ìê©…üÝËWšÐU¸Ô´€cccƒ‹‹‹yÁ¾««+vvv¸ººšù»¸¸`kk{QÀÎÎlmmqvvfàÀ怭­-®®®æ¯ÛÛôz½¹Ýut/t•¬­­ "$$___s¶ÐÒÒÂÙ³g)--¥¬¬ŒŠŠ ó×·³³# €ÀÀ@BBB $00àà`ó¹<7²|MMM?~œ£GrìØ1Ž=ÊÉ“'1 8991bÄâââ?~<ãÆ#**J^Ï AgAíŽ;î ¼¼œÝ»wßЉõR2BX†öv¨¨ø¡„PVöC ¡¬¬sBYtÿÅÚ€åoïÎB‚·÷…„®"‚¯og9ÁÑQ¹Ç&D7µµµ=Z¹]cÁ°²²"44”aÆCtt4*•Šèèh\\\”Žn±ªªªÌÓ233ÍE„®˜¶¶¶ 2„ˆˆ¢¢¢Ì-÷®QyòDX¢J*9ÈA¾ã;Žr”#¡Š*0€pÂÍ¥ƒ®Ë )Y!WPP@JJ IIIìÛ·777î¸ãÔj5 .TüùÔ¶mÛXºt)nnn¬Y³†Q£F)š§»¬¬,¦L™ÂwÞÉ矎••ÕMÏpäÈÆŽËÑ£G=zôMÿþWR__J¥bÎœ9¬\¹Ré8â---¤¥¥‘’’Â×_Mii)ÞÞÞÄÇÇ“˜˜ÈwÞ)“'„W¥Š*v}ÙÉNrÈÁ[&2Ñ\*ˆ#+nþß“BˆþÍ`0˜wß³ëãS§NõØ}´k—ò®Ý7/ÜsÈ!Š<§âVѽ4PYYÉÙ³g¯X"¨¯¯ïq;;;ÌàÁƒñññÁÛÛÛü±¯¯/ÞÞÞxyyáããƒNNN =R!„Âr]i!÷2@×Bþî»úw/tŸ pá´€«YRÙµ¿kqÿåòw•.7-ÀÖÖ‹ —š` ŒF#:Î\è*@wMT+..îQœôôôÄÇÇóã5™L´´´P[[Kyyy ÐùºçÂ×:Ý? ‘]îû©¶¶6Nž|˜ôôtZ[[qsscüøñæcܸq²±¸åœ?ž9sæPXXÈ®]»ˆˆˆ¸¡ßOJBˆ¾¥©©³xPQååGEEçç*+($TUA÷o®®eŸ‹'#x{w^{yu2AÜd’‘‘ÁÉ“'Í æsrrhmm ((ˆèèhbbbÌÅ•J%Íë+¨¯¯'//üü|òóóÉËË3\[[ €ƒƒCÒAdd¤ùZÆ‚ KSJ)G¿¿áG9Š¡„2’‘Œøþ2’‘„&‹_„·¬ââbÖ­[‡V«e×®]ØØØ0{öl4 .Tì9Tee%K—.e÷îݬX±‚åË—[Ì»víbÎœ9<üðüûî»7=WGGÁÁÁüêW¿â…^¸©ßûÇ<þøã¬Y³†œœ¬tTWW³iÓ&´Z-›7o¦¡¡•J…F£A­V3eÊ‹ùKa¹ª©f{Øùý%“L¬±&Ž8f0ƒÛ¹©LÅ YØ'„¸>z½¾GiàÂÁ™3g0~?éÙÎÎŽÀÀÀË.¤‰ŒŒT¼@-DÔõÿ©^¯G¯×S^^~Ù/\?”<<<ðððÀßßÿ²ûúúJH!D¿d2™Ì üÏŸ?oÞí¿ká~]]F£‘ÚÚZó”€ oS__o¾¾Ô´€®2À±²²ºìBþ wõwwwÇÖÖö¢i]e€+M èL&””” Óé())¡¬¬Œ²²2s¡ ¬¬Œööv sÒ‚‡‡...ØÙÙammÁ` ©©‰ššš››Í_ÛÚÚÚ<‰ÉÏÏ___ðöö&00ooo‚ƒƒñõõ•èÁ`0‘‘Áwß}g>òòò0™L1~üx&L˜À¸q㈋‹“×͢ߪªªB­VS\\Ì®]»ˆŽŽ¾áßSJBˆþK¯ï,”—_úZ¯ïœœpþ|Ïû98tüüÀãçù…ûû+óØÄ-Áh4RTTDVVÙÙÙæúsrrÌ;ßxzzCLLŒy—þÈÈH°³³SøX.½^o~C¯  €ÌÌL²²²ÈËË37êííí 0¿‘×ýˆŽŽî·¿4}KæâAÆ÷—ӜƄ 'œÆ0sñ ë⎻ұ…â¦:wî7n$))‰­[·b4™8q"‰‰‰$&&â“ŸÓwttðæ›oòôÓO3oÞ{C¿wCCƒOOÏŸ?tè“&MâÃ?dÉ’%74ƒ¸²îÓJöïßS§NE­Vs÷Ýw¨tD!„…Ó¡c/{I#½ìåLj&š©Leö÷<N*„èK ¥¥¥—-tÿ}#üønœ2…@ˆë×ÜÜ|Ù²À¥>'¥!„ý™Ñh4/Üoll4ïößÐÐ@{{»yÁ÷Ûtíþß5- ¶¶£ÑØ£@pámêêê.úûôrÜÝݱ¶¶ÆÍÍͼX¿kq¿³³sÉîêßU¸Ò´ldsÑK2 èt:JKKͯcŠ‹‹)++3ŸWTT˜ ÎÎÎ8::bkkKGG‡ù¿©––ómìíí4hPçH]çݯƒƒƒåßè5uuu:t¨Gñ ¼¼kkkbbbzL<>|¸ü·'ú¼üü|æÎË€øæ›o¿)ßWJB¡×wNA8{¶sBEEçùÙ³…„îçŒ4eàÀÎé~~?LBè~Þ5)ÁË iÚŠ^RVVFvv6ÙÙÙdee‘““C^^:]ç®æÖÖÖ÷Ø¥¿ë2d¶¶¶ ?ËURR©S§z”ºŽêêj sçƒËÂÂÂðööVøQˆ[YMd’Iç8'8AèÑL0±Ä2ŒaÄc¾vÆYáäBqãÕÖÖ²mÛ6RRRX¿~=Œ=µZÍ<@ddäMËòí·ßòàƒbeeÅÇÌŒ3nÚ÷¾’/¿ü’ûþ?{o×yŸ÷fÃl˜ À 0ظ‰‹¢L‰Ú¼U–%²"¥8 ;éê,Nܦ±·MN#·‰}rj4I½Dù9iOÉ–Ö±lR–d[µe[»(Ê”IQ¤H‚Ä:Øfßg0¿?†ïË;ƒ$A¿Ÿ{ÞsßûÞ;÷΀ æ.Ïó}>ò>ûÙÏ^öDýû÷sß}÷ñío›ÿøÇ<öØcD"zzzº¤ÇÞ·oÿößþ[þâ/þ‚ý¯ÿ5&“‰R©ÄöíÛñz½üèG?’Êø—™r¹Ì‹/¾Èþýûyâ‰'xûí· ƒìرƒÝ»w³cÇ©F$BSf˜áü‚Ÿðžçy~ÊOf;vnâ&Þsvºƒ;ÄT Âœ4J!0š´˜J-if ¸îºëhm•û/‚p>d³Y¦¦¦˜œœ$‰è¾jLLL099©×EqPƵ··ÓÙÙIGGííítttÐÑÑA(¢££ƒ`0HGGøýR¤EA¸´4«ôŸH$j„ÿÉd’|>O&“©Ù¦\.FgmSo ˆF£ z?&“ ¿ßÕjÅãñèêÿJÈïõz±X,, ^¯wÖ6JØoܦÞ@ âÿKG<gdd„ññqFFFˆD"Œisåàà D£Q”TÔd2ióT/³Ù,F)©Ç㙕4ÐÓÓCgg§6tuu-›BJ‚044Ä+¯¼ÂË/¿Ì+¯¼ÂH&“8NÞýîw³}ûvn¾ùfn¹åÖ®]»ÔoWÌK/½Ä}÷ÝÇêÕ«Ù¿ÿeÕ¥‰É@á|ÈçajªjLˆFkSêûƒƒP,Ö¾ÞáhœŽÐl,²çA*•âwÞáøñã?~œwÞy‡cÇŽqüøqÆÇÇj\ÝêÕ«µéàÚk¯eݺu¬Y³†Õ«Wc·Û—øS,_‰DCóÁ‰'8}ú4ųÿç[[[Y½zµn«V­bÕªUô÷÷³jÕ*ººº–ø“W#ƒ ò&orˆCæ0G8Â[¼E–,&L¬b›ØÄf6빘AXÉd³Y~øòwï^öíÛG,cÓ¦MìÙ³‡Ý»w³mÛ¶Kþ¦¦¦ø­ßú-žxâ ~çw~‡/|á ËB4ý÷ÿ÷|ìcã _øÿé?ý§Yë_|ñEÚÚÚ¸îºëåx*±à±ÇãßøÓÓÓ´´´P(èëëãÌ™3‹r¬füÑý=ô•J…;ÿý¿ÿ7ßùÎwxðÁyã7.KܨÓÓÓ<ûì³ìÛ·ï~÷»Äãq6mÚÄîݻٵk·ß~»T¡!#ŒpÀ0½À L3M+­ÜÊ­ÜÁÚTàĹÔoW„e@±XdllŒÁÁAFFFfhhˆÑÑQÎœ9Ãèè(CCCº:§Éd¢««‹þþ~úúúèïï×÷úúúúèëë“Â#‚0¥R‰©©)Ýš™&''grr’t:]³›ÍVcƒÚ  Æc8Ž%úÄ‚ •D}c¥UÅ¿Y¥ÿxÄ7¿ùMÜn÷e=¾˜ A.%*ab&'«I ““Õ65UG"çúÙlíëíöjBGǹtµÜ–UT< IDATÞ^]…ª}5.ê…&Äãñ‚±MMMç*ô¯Y³FWÚ2öEßœr¹Ìàà 6 pæÌN:ÅéÓ§¡\.àp8´ù@=Œ\µj•6%„Ãa¹™#\Ê”9Å)m:ø¿à­³SŽ&LôÓϵg§ë¸N÷ûéÇ‚üž ‚°2PÓ÷îÝËÞ½{eíÚµìÚµ‹={öpÇw\Ò*ö{÷îåßÿûÍfãá‡æ¾ûî»dÇZ(_üâùä'?ÉÃ?ÌÇ?þq=þóŸÿœ÷¾÷½ÜsÏ=ìÝ»÷¢Žñúë¯óÈ#èÄ£±ÀÈš5k8yòäEk>n¿ýv^|ñE *Z™™™Án·ó™Ï|†?ù“?¹¤Ç¾Ú9yò$ûöícÿþý<÷ÜsT*n¹åvïÞ͇?üáËš0"•Aœ8¯ò*¯¦QF±`a›¸™›ÙÎvnã6ÞÅ»0#æ$A¸ÚˆÇã 1<<<Ë40<<¬E:Æ žôôôÐÝÝMoo/áp¸Æ@Ð××'ÅYÁ@6›%ÎjªZn£u‘Hd–ÈÍápº®Zn4ÖÙÙ)÷ÎA®”_UçWb}%þÇãÌÌÌ‹ÅjæJðŸL&µà¿Y¥•H$ôžšUúŸ«Šý6Jðߨҿ2øý~IU]Ád³Y}ÎdœŸ^5+(C‚jõÆ}¾s&„¶¶¹çÊœ W=ñx\‹ãO:U3Ðb/§ÓÙÐ|°zõjúûûÅÕ;Åb‘¡¡!Îœ9ÃéÓ§µù@-Ÿ9sFÿœm6½½½úAf?½½½ôööê¾D —’2eNr’Ãæ-Þâ8Ç9ÊQŽsœI&°cç®á:®c=ëkLA‚Kü A.œ™™<Ⱦ}ûøÆ7¾ÁñãÇéëëcçÎìÚµ‹;vèøàÅ$ò‡ø‡|õ«_eÏž=<üðô··/úq·ÿþßÿ;ŸûÜçxä‘Gøµ_û5Ž?Îm·Ý¦#•_ýu¶nÝzÁûöÙg¹ûî»g‰LêY¿~=ÇŽ»àãÌG±XÄãñÏçkÆ- ½½½üÝßýwÝu×%;þÕ†2õìß¿Ÿ}ûöqäÈÚÛÛ¹óÎ;Ùµk÷ß¿\W‚ )QâmÞæxžçù?ã(G™a†0a¶¦÷ð–ú- ‚p‰‰F£œk^ßÔxGG‡üßA¸Bh&ÚWÕú•! NS(´! ‘H耙™}_Q¥$ ½o• ö½PT%~%Ì:@ þ›Uú÷z½5£ ¾Ò ×›Bc²Ù¬Nhc``@'¦‰D´é2NÏ{?Ül6ÓÚÚJ   ÒÝÝM__ŸNP©MÁ`®®.Z[%•^.–ÑÑQ^yå^zé%^zé%^{í5R©n·›mÛ¶që­·rë­·rË-·ÐÝݽÔoW¸ xõÕWù7ÿæß099É׿þu>ô¡-Ù{“ •ÌÌÌlÓjÓÓÕu憨4¬Ösæ£Áدkk§ÄË_-¨‡wÚÀÀ€¾v8úáœjÆv«V­’*CM¨T*ŒŽŽ200ÀéÓ§µñ`pp3gÎ044Äôô´ÞÞår±jÕ*z{{éééÑýÞÞ^mLðx Õ€A`„m&8pvʑË—ë¹^› ÞË{éBÒa%‘ËåiXÍSõÏœ9C©TÒ¯ ³LÆ{‘áp˜p8,U`…«•.0Wš@}ê@4µcºÀ\)ÆÖÕÕµ$A®V.D´1É EUäWÕûUe~UÅßçóa6›ñûý5sŸÏ§EþÊ ÄþÊ ö©ÄÿjŸ‚°ØLOO311Áðð°6 166ÆÄÄÑh”D"A2™$—ËÍ2_ÖcµZq¹\x½^mP×-«V­¢³³“P(D0¤££CîA Â2 \.säÈ^zé%xpäÈfffèííÕ†ƒ[n¹…mÛ¶ár¹–ú- +„\.ÇŸüÉŸðçþç¼ÿýïçïÿþïéíí]Ò÷$&A„«‘D¢j6P) s¦¦ª­Áf\®s¦ƒ@üþÚù\}‡ãòná’Íf9uêTMEþ3gÎh±üÈȈ¾°¶Ûíô÷÷ë¶jÕ*‚ÐÓÓC__N1¯4%“Épúôi†††t*‚º©¡Ì©TJoïõzµá ÓÛÛKgg'½½½tuuÑÓÓCgg§Ü€… ä˜az›·9Æ1Nsš2eL˜è¥—k Óu\Ç:Ö±ŠUØY\¡® ÂbsòäIöíÛÇÞ½{yá…p:ÜyçìÙ³‡øËfð‹F£|êSŸâk_û{öìá/þâ/æ¼ôÀðçþçô÷÷/ÊñT*þÃøìÛ·±±1Šu†å矞Ûo¿ý¢öÿK¿ôK<õÔS³ö­¸þúë9tèÐc>¾øÅ/ò™Ï|¦éñ¡Pˆ×^{¾¾¾Kö^V|ÿûßgß¾}|ÿûߧ\.së­·²{÷nî»ï>6nܸÔoQ„%$C†Ãæ ÞàçüœCg§8q8ØÊV¶³›¹™ílg=ë1!"aA¸R‰F£ MõsEKK ííí MjlÕªUR±SX±(³@#S@3óÀøøø,[#³À|æ`0(é‚ çÉ…ˆö•!@%(CÀB“Â…ŠöÏ' ¥¥·Û­÷-Ë‘h4ªõ ÃÃÃŒŽŽêçë“““$“I‰„þ¿8—”Òl6c·Ûq:øý~ÎÔÕÕEgg'áp˜Õ«Wë4µžžžE/T$ÂÒH$xíµ×jŒ‘H«ÕÊõ×_¯7ß|36l¶pÞ¼ð üÖoýCCC<ôÐCüöoÿö²(œ!&Aaa¨Ô£ñ`zúÜr4 ±Xu^ß/fïÏá˜Ûˆ0—QÁë½üŸ_¸(%!¨‡‡ÇŽ#™Lêmi"®_¿¯ü4%‹144ÄéÓ§Ö}ã[c"‚Éd¢³³S›Œk{zzôx(3‚pÁ)2È`MòJB8Å)*T/ItúAý´šÕ˜‘ qA–ƒƒƒ<õÔSìÛ·§Ÿ~«ÕÊ]wÝÅîÝ»ù¥_ú%B¡ÐEã駟æ÷~ï÷å¿þ×ÿʧ?ýéY$¾ýíoóË¿üˬ_¿ž_|‘ööö‹>®‘D"ÁûÞ÷>Ž92K„oµZ¹ùæ›yá….ê±XŒn¸‘‘‘šÊ´Šw¿ûÝ8pࢎ1¿ú«¿Ê·¾õ­¦h­V+v»Çœ{ï½÷’½+™™<Ⱦ}ûtòG à®»îb×®]Üwß}øýþ¥~›‚ ,C i#Á¼Á!qœã”)ãÁÃõ\϶°•­lc7p6DÜ(ËB¡@$axx˜ññqFGGcllŒ‘‘ÆÇÇuµOã9^{{;áp¸¦(FOOÝÝÝôööÒÝÝ-•;…C.—czzzA‰Ê<055E¡Á3uß|®4£q »»‡{á*#‹éªþ•JE‹üÕ\ ôÕ\™Ô\ ùÕ\ üÕ\™ Ô\Šªâ0™Løý~-ÚW†€zÑþÅ$ÂJcjjŠm!‰099ÉÔÔ”NP†|>?g‚®Jp¹\x<ü~?mmm„B!ºººX·n×^{-¡Pˆ@ ÏËA˜ÅÀÀ@éààÁƒär9<7Þx#7Ýt“n×\sͲŒ ËÓ§Oó‡ø‡<þøãÜ}÷Ý|õ«_½$…å.1‚ —žlöœá@µ\®ñx}‹Dª‡zŒ&…óiÁ HžeG$ahhˆááa}S@UêWý|>¯·ooo§§§‡þþ~ý R=˜ìî«kÑv+‰|>ÏÔÔTÊqÆùØØXM¥†¹"èÕ¼¿¿_n®çEœ8'8Á)NiÓêŸæ4yªÿ÷8XÃÖ²¶áÜÃâTA¸¦¦¦xòÉ'Ù»w/?øÁ(•JÜzë­ìÙ³‡_ù•_¡§§ç‚ö{ðàA6oÞÌÃ?̃>Hgg'ù—É®]»€j\ë¦M›xçw°X,¼ë]ïâ¹çž[´D…l6Ë]wÝÅ«¯¾:g•ÿüàÜu×]u¬C‡±}ûöšs>ÅöíÛyùå—/jÿsÑÕÕE$i¸Îf³ÑÛÛË“O>)•÷ÉdxöÙgÙ¿?ûöíctt”5kÖð¡}ˆ]»vqÏ=÷ÐÒÒ²ÔoS„ËD‘"Ç8ÆŽp˜Ãà¯ò*ªcÄÙÌf6±‰mg§l3± ,3”à9‰h³Àðð0‘HD ”Ljßï'ÓÙÙ© V¨‚}}}ÚP ¢gáJ£YªÀ|-—ËmÝ(]`>ó@WW—TÁáŠÁ(ÊO§Ó5¢ýT*U#êO&“5¢ÿD"Qc ˆÇã5¦X, ÿ¾ÖÏŠê« þj®„üj®þjnì;ÎYÉjÚŸ:ŽJÒé4 044¤¯=ÆÇÇ™žž&‹i£@&“¡P(P,›šL&‹›Í†ÓéÄívãõzñûýtttèb{}}}¬^½šk®¹†öööE»‡.‚`¤X,rèÐ!^{í5Ý>L±XÄï÷³mÛ¶ãÁêÕ«—ú- KH"‘à _øù—I__=ô÷ßÿR¿­YˆÉ@AXþ,ČШML@ƒJ¨lPhk«¾VXšÅª«þàà`M"B}¬ºñaˆâ†˜„å@”¨N=¨Ÿ`†êMÅf)a¬e-NœKüIA¸ZP¢ë½{÷òÄON§¹ñÆÙµkýèG¹îºë¼¯k¯½–µk×òõ¯\.ÇýÑñÈ#p×]wñ¥/}‰—^z‰}ìcú{Øf³që­·òýïÿ¢Å[…B{ï½—çž{nNƒÅbaË–-8pà¢+°<úè£ü«õ¯fßqÇüìg?»¨}7cxx˜ÞÞÞ†ëÌf3÷Üs=öØŠ«þöÿðœ9s†?þã?>ï×F"žyæmª)‹úw|÷îÝlÛ¶í¼cA–#Œp˜Ã:™àçüœ£¥H'N6³™­le [¸ØÂüHš‰ ,%Æ{kƹ±ŠúÐЉD¢æuÍîóïµõõõIò¨°¬I&“5€X,ÖÐÐh¼Qª€Ëå"à÷ûšŒãííí´µµé¹ˆLA¸œ¨Jû—k®Ì ¥^´_?_È6ç;Á¿ \<Ñh”ññq®1 ’É$™L†\.G±X¤\.ÓL¦¨Ì---Ú¬ÓÚÚŠÏç# ÑÝÝM(Òæå¾¾>Ö­[Gkkëeþô‚ çG±Xäرc8p@·W_}•B¡€Ïçã]ïzÛ¶mÓmóæÍKý–…KL"‘àá‡桇¢R©ð_þËá“Ÿüä¬4û傘 A„•M,vÎtШ?×úp:«†¿¿Ú¼ÞjóûÁç;·¬šÏ7{.Ùlv^AüÈȈ®t¢Xˆ(¾¯¯›¤`4$“ÉèŸñðð0ccc çƇÔV«Ußêìì$ ÒÕÕ¥ûº 冯йšäƒú4„Õß9+Vúèc5«é£U¬¢ïìÔO?«X…÷AV"¹\Žüàìß¿Ÿ'žx‚ññq6mÚÄž={æc¿õÖ[lÚ´ ‹ÅB0äŸþ韸í¶ÛxöÙgùÿñ?ròäIœN§®ì¦°Z­ÜsÏ=<ñÄeèûÞ÷¾Ç¯ÿú¯“L&™™™iúðGñÄO,JußüÍßäk_û%ƒYøýï??þñ/zßøÇüG>ò‘Ô|>e–øÏÿù?ógög+ªbh"‘àãÿ8ßüæ7Y¿~=ÇŽ[Ðë>¬Ó ^xáœN'wÞy'»wïæ¾û«ë¿sA–ŠFt2ÁŽp’“âãŒU³¯1™`3›yïÂÎò| "+ ã½0£Y ~>>>N¹\Ö¯s8óæP©rOFX.\h¢ÀÔÔTC£@£D…´¶¶6IäaÁ¨ üªR¿á+Q~>Ÿ'“Éè*ÿªš?Ì®Òo¬æ_©TjªýÏÌÌÔ¤¿÷ç£Y•þújýjn¬üïv»±Ùlºbkk+V«Ç£ç‹E û½^/f³yÅs„+ uN•ËåˆD" ë¢rLMM1==M<'‘Hè¿UÙl–B¡Ps߶“É„Õj­1 x<|>mmm:Y ÓÓÓC__k×®% ^ÆŸ€ ÂÒ“J¥8xð ЉÇŽ£R©‡kÒnºé&B¡ÐR¿eaáK_ú_ùÊW°Ùl|ò“Ÿä÷~ï÷ðû—wq1‚ B3Òé¹ ñxµ%ç–‰s-n¼ß––s†ƒ@`¶)¡ÙºzãÂEVŠ½šQ7OηB¿ñl³¹TnkN&“ahhˆH$ÂÐÐ6Œ311ÁØؘî+ݘL&m6ƒ„Ãáš~(ª1&¸\®%ü”Ârb’É 2ÈiN3È 6!´Ñ¦MýôÏ2!„ cEÒ7A¸pÊå2/¾ø"{÷îåÿþßÿËÈÈkÖ¬a÷îÝìÙ³‡;& àóŸÿ<ú§J±XÄb±P©TxðÁùìg?K¹\æà»ßýnØh«ÕÊ<À#Ïã?Îç?ÿyŽ?ŽÕjmøÉb±pÍ5×päÈ‘‹äçr9n½õVŽ9¢Ï>øÁòÃþð¢öÛŒO}êSüõ_ÿµ©‡`>ú(þð‡/É1—ŠW^y…x€‘‘ý³}çwX·nݬm³Ù,Ï?ÿ<ûöíã[ßúÃÃìZµŠ{]»vq÷Ýw/ÛŠ*‚ œ?EŠ¼Ã;áG9Êasôì”% @=l<;mbØÀ»xt,ñ»„•G6›errRß'QŸññqFFFˆD"ŒŽŽ2::J6›Õ¯Sººº´A §§§¦ÈC8&ëÊÀ‚p¹¹P£ÀäädÃÊØjhoo—óYA¸JPýz‘&“!ŸÏë*üõ"%æ´ˆ?™LR*•H§Ó ýÚB¡@:¦T*éb ñxü¼Þ§àCµ0€ÏçÓ‚|³Ù\#Ô·X,5B~«ÕZ#ô·Ùl5F€–––£€Ýn×FA®r¹ñxœx<®¯ ”©xrr’ééi¢Ñ(ñxœd2I:Ö¦¦B¡0g’€Âl675 ¨ó(uÝÑÝÝMOOýýýtuuÉù• ÂEÇyýõ×µéàÕW_åÔ©Sô÷÷sÓM7±uëV¶lÙ–-[X³fÍ¿ca!T*~ô£ñðÃóï|‡@ À§?ýi>ñ‰Oàñx–úí-1‚ Â¥$›=gLˆF!—›=6׺éiÈçïÛ᨜ÎsýúÖl‡Å¬0¹\®Æp äk¡üøø8ããã57dü~?ÝÝÝtuuÑÕÕEGG‡Á‡B!½ÜÕÕ%†„&ÔWâ3šBêûÌ êa£ª|Æ~(º¨*Ï•M–,£Œròì4ÂHÍò ƒ9÷;@€µg§0aºé®YÆ„üMa~fff8xð ûöíã±ÇãرcƒAvìØÁž={رcÛ¶mãÍ7߬yÙlfÇŽüÍßü [·nezzºé1Ìf3¿û»¿Ë—¿üå‹~¿•J…gŸ}–ÿù?ÿ'Ï<ó V«u–ÀÇl6óè£òÑ~ô¢wæÌn¸á‰333ìرƒ§žzê¢÷Ûˆo¼‘7Þx›ÍFoo/û÷ïgÓ¦M—äxKA¥Rá‹_ü"ŸùÌg´QÄf³ñÐCñÉO~€‰‰ žzê)öïßÏÓO?M2™¬IÞx÷»ß]c„áÊ£@ã׉*à0‡É‘ L˜Ílf›ØÌfÖ²–-l!„TË„‹¡Yâ@£{Íîs4+x¡Öõ÷÷Ë=á’“H$H$ZܦZ,›e ¨«O—U¨ûx~¿¿©) Ùº•”:&+%òWBýf"%Ôo$òW•ûëEþõi‹!ò7™Lº’¨ßïÇd2i¿ë+q¾ð+¡¿±B¿2 (C€þ+¿û+S€ +›l6K2™$™L‹ÅH$LMM‰Dô³æ©©)b±étšt:M2™$•Jé¿‘Åb±aÁ™zÌf36›M§‘´¶¶ât:q¹\Ú$ ºººèé顧§G›A–!SSS:íàÀ¼ñÆœ:uŠJ¥‚ÏçÓ†ƒ-[¶°uëV6oÞŒÛí^ê·-ƒƒƒ|ó›ßäÿüŸÿÃÑ£G¹í¶ÛøÄ'>Áž={®8Sž˜ Aa¹“N×&$ÄãÕä„ú±fë H&ïÛj­¦#øýçRŒi õë%*x<Õý\Å‹EÆÇÇÖf„±±1FGG‰D"LNNêJý*æVa·Ûéèè  ÑÙÙI0l¸¬ rA0›l6;+ ÁØD"D"&&&˜˜˜¨‰ ¶Z­:A%!Ô÷C¡¡Pˆööv¹Ù•1à £ŒêäƒA9ÙšåI&õö.\¬fµNA襗z¦Ÿ~ºè"ˆD¾ ‚0›ƒòío›ú§âðáôµµ55Øl6ìv;™LfÞ‡J&“‰?ýÓ?åÿøí½:tˆ/ùË|ík_£R©h³Éd¢¿¿ŸãÇc³Ù.ú8O>ù$»wï¦R©°k×.öíÛwÑû¬'›Íâõz)•J˜ÍfvîÜÉ7¾ñe˜˜à_þËÉøÃY¿/&“‰›nº‰Ý»w³oß>^{í5œN'úЇؽ{7÷Þ{/]]]KôÎA¸PJ”`€wÎNÇÏNG9ÊiN3à 6l\Ã5:‘`3›Ùpvr"ÕÍa!¤R©š{*q@Ý{PëÔ=!ã÷°ÅbÑ÷"Œ÷Œéê^Pgg§Ü L&3ËÇuµÛ¹Z,Ó"ßz,Ë,@3S@£qA.J¸oì«ŠþÍDþJ¨ßHä¯^ÛLä¯Ò.D䯄úDþJ¨_/òWBý¹Dþ* ^ä¯Ä·"ò¡Êô‹Å´àßh ˜œœœu•L&Éf³d³Yòù<ù|žb±8oz€Âb±`³Ùôß:—ËEkk+^¯W›T1»P(D8¦··¿ßÏç[Q÷5A„Æ$“I:Ä¡C‡øùÏΡC‡xóÍ7I¥R˜Íf®¹æn¸á®¿þz6nÜȦM›X¿~ý¢<»æfbb‚o}ë[|ãßàùçŸÇçóñ«¿ú«|üãgëÖ­Kýö.1‚ ÂÕ‚1%á|ŒëšaLL¨OP8ŸåŽhi¹|?—ËL>Ÿgjjªa¥ºfËFšUéo´ÜÛÛ«o¨ 瘫z`}_UYRÌÁ^ÿóW-KÕßL–, hÓÁ ƒÚ„0Ì0ƒ ’&­··c§›nzÎNaÂôÑG˜0½ôêu¤RŒ \­;vŒOúÓ<óÌ3ºâüÅò•¯|…O|⋲/ÅØØý×Í—¿üe8P©TøÛ¿ý[~ó7sQŽñàƒòùÏž]»vñÿð ·Q¢@‹ šQ/øÙÏ~Æ{ßû^L&ÿí¿ý7>ûÙÏ®¨ïìgŸ}–|ä#Äb±¦¿Kf³™¶¶6vîÜÉîݻٹs§ˆ+á  Hq–‘@õÐi\í´s ×°žõÚD°™Í¬c6ä¡’ ÔF”40<<ʯýÚ¯÷{QÕc±˜®ÂeìOLLðÓŸþ”þð‡LNNâv»ùÈG>¢Ó›”Ð ÎU>TûˆÇãÌÌÌ芉K6Ū:¢Q\¡„ªª¢ÓéÄétÎÙw¹\8ü~¿Þ祦T*ñ¹Ï}ŽÏ}îs˜L¦97Ìf3>ú(ýèG/ùûáü(RdAN6˜s˜Õk¿Ö6˜6³™0á%þ‚°t”Ëe}^/¥>i ‰Ì*2`·Ûk’Õ5»JPëB¡Á`ðŠ‹9.?ÍÄÿsŒÛÔÿŽ*Œf€ó51½+’T*E±XÔÿ¿Te~8'ü_È6õÂÿ…l£„ÿ狺^T¢}ãõ©ú+ÿB¶itÛlù ‚°”¨ó ã9Q.—Ó‰“““Ú0 Ìétšl6K*•"ŸÏëd•B¡0oòk=*5@¨\.·ÛMkk«6„B!ÚÚÚðûýôõõiS€ÇãÁãñàpHá(Aaù‘Ïçyë­·8zô(GŽá­·ÞâÈ‘#?~œb±ˆÙlfõêÕlܸ‘k®¹†uëÖé¶fÍš!_LffføÅ/~ÁO~ò~úÓŸòÜsωDèééaçÎìܹ“»ï¾{Åé}Äd ‚ ÂÒ¡êÓÎwy!) “®`\îìËòõ¦R©ÓÑ  šz¨>555«š‘ÙlÖ¦%~7ŒÂxã6ân>Gýƒáúê‡õ놆†( 5û°Ûí´µµIbÂUD™2" 3Ì(£ 2È(£ 1Ä# Ÿâœ«–ÙB ]tÕ$ „ ÓEtÒC¡³“ ùÝ„+…H$Bww÷‚ˆ™ÍfZZZèììdttT§Øl6,‹6 Z,þê¯þŠ7F‰Åb³Zýx4Õføý~.— “ÉÄÔÔlÞ¼YÇ{«›iªj¡ªŽhqƒQø`¤P(ðä“Oò±}¬áñûW"‘f¤ÓéšïÜGy„|àø|>ÊårD EŒ"µ•žN§ÉårÄãq]鬾ªq=.— ¿ß¯[ ¨Y®SÆÐŽŽŽ‰OŸ>Íž={xýõ×TmÍjµò+¿ò+<öØcón+Ââ3Í4g§Sg'•HpšÓÚ„"¤ ®©›üø—øSÂåA]k7»Æ®Ÿõ]èp8”4 •Ù#¹\Žd2I2™$ê~<¯iêº~<75ÓºÝn¼^/>Ÿ¯¦YcõM/ʽ a9¡ªç¯£”Ù\])Ó¹º®2nS/ü_È6Žu>(½Ñ`®*ï+áÿB¶QÂÿ…lc¼þUÛ‚ ,wŒ÷òÙ,‰D‚D"¡“FFFôØôô´>J&“¤Ói}ï,ŸÏkãÖù$±X,ØívmhmmÅãñàóùp¹\¸Ýn}?M%´µµÕïyÏ{xßûÞÇÝwßÍ 7Ü°¢ïψÉ@A„•A¡PMHˆÅªs•¦‹]XÚÂ\7¶TzB³´Ÿ¯jJp¹ªÛ:Õ¦Ì NguÜåªö—0ma¾JiÞOLLP*•jö3_•´FïƒÁ 6›m‰>ùò"‘H099©Íõ© Ê0b\WoLP‰ ƥƦõcƹpå!ÓÔ€ RÆÓ•m¬X ¢‹.„ Ò„nºé¤“.ºè¦®%üt‚ üíßþ-ÿøÇÏ[a±Xøà?ˆßïgpp±±1-‚jôÀÎëõθ7[v¹\]åÐh,hÄØØ]]]ôù‘Ëå.I5°R©tIª¶f2r¹Ü¬ôõÐu.c‡SB#^¯W'T©¥Æå_üâ|éK_"ŸÏŸWÕ6ÇÃôô´T°„K@‚Ä,q9Aõÿº Ýt³†5³LëY—Ù&,A¸’‰Åb³®sU›.VëÔ¼þ»­µµU§Ö§Ö+q‘eã=¯Fi ›žžžU ÃÈù¦[{{»$`‹‚J‡S•ôÂüD"A¹\Ö¿Û333Ú­ŒÕJô_©TˆÅbÀ9ƒµ1qNUîWf£IûBÄý*!ª×:‹E§Ç)Áè\Û(áÿB¶1 ÿë·AXi¨ïƒF ʉD´ `zzšd2©ÊP(( Ú0Waf˜L&¬V«6lÙívêt:ñx<Úpé÷ûñù|5çò.—Kìp»Ý:]@A„KG"‘Іc;yò$ƒƒƒú™£Ãá §§‡îînúûû ‡Ãôööê±¾¾>ººº–½&hjjŠÁÁAŽ?αcÇ8zô¨ž' ¬V+×]w7Þx#7ß|3ï{ßû¸þú믪ûŒb2AAhD6[5$U#‚2!$“Õeed0ŒF†l2™êºl¶ÚæÂí®¼ÞªYÁé§ÚœÎê˜×[íÏed¨75\fff˜žžÖBÕ7ÎëñÓÓÓd2™YûRU›¥&¨èQ%rTsytΘ šñg­"c͉ç2 Ì5o&.–ž,Ym:ˆÕýúù8ã”9'>và @€nº ž5WëúéÇŠˆQáR°cÇžy晦ëM&Ó,ñ†z`g³ÙX³f 7Þxã,º×ë%NS,¹ÿþû¥J×2gff†X,¦Å–j®bâë«"‘ÈœÕßL&&“ ³Ù¬û333ºýèG?âøÀåû€‚°BÈ“g˜aNÖMê|맨Pý› ÀZäαֲ– lÀ{‰? \ MóSãÍLóç“ä×ÛÛ«Å¢ÂÊ"‘Hè¤cZ@*•š5fÜ.‘HÔŒ5º¥ðûýx<žšðx<º ®qLµÖÖÖš±Ka‚® –RÄß()à| œ½g¬ªëÓáT:†àSèTµgež1VãWÂ}eLo$êW¯3OájÅ(þÅbär92™ “““$ ¦¦¦ˆÇã:@ã¨DõÚ\.G±X$ŸÏS,Ï«èD=‹«ÕŠÝn×ÍétÒÚÚªÅþÆs¡`0¨Ÿµ··ãõzkL.—KΗAa…Q(`ppááa†††app‘‘†‡‡‰D"úœÄl6ÏY³~LéOÔu«ñšT%Ã5B]#“—òù¼Nœ¯Ñ/ 1<<Ìàà Nt·X,¬^½šk¯½– 6pÝu×±uëV¶lÙ‚Óé¼Ô?Úe˜ AA.Ù,ärÕy4º°þ|ÛÍ—ºàpœKQ˜¯¿íÚÛáDþÙlv–!A‰ã›™¦§§g¥&ú&e½ù`!sõðëje¾ôŠfÍx1h$4}ÌÕ$ÅbùP¤È8ãŒ2ÊcDˆhóÁ#DˆèuΉ4,Xºé&xv Öý.º"HZ–ðS Âò¢T*qæÌN:ÅÀÀ§Nªicccz[«ÕJ0¤§§‡¾¾>úúúèîîÖ@Âá0ÝÝÝú›põR,åĉ 100ÀÐБH„ÑÑQ}ž•H$j¾ÏN'.—‹k¯½–;３5kÖèÖ××'éÂUM… cŒ1È C 1È g8ÃCœæ4§8Å8ãzû AV×MkX£çD\!,oòùü,Ózý²Ñ覮éë v»½&A  ÍJPÉjY ìW>’P?699Ù´B®.×RÎw¬³³óªª4·R0Šó• ΉòÕï‘qì|…ú I0û|X ¡¾ÇãÁjµÎ¹N .Œ•ý[[[±Ùlº¢¿ ‚0?Í’r¹œ©E£QÒétÁQ•a P(h£Y±X¤X,R.—/(@a2™°ÙlºØˆÓéÔs·ÛÝnÇçóáv»q:„B!Ün7‡žž|>.— ŸÏ§Í­­­úûIAáb)‹ŒiãÁÄÄDMaÌF}e¾_(‹eÎÂ[õ Ÿ¶··ÓÝÝMOO«V­¢§§‡ÞÞ^Ö®]KK‹h!&AA„+cZB4z®‹UÓ²Ùsi ÙlÕ˜JUûÉdµe³Õ±Dâ\ŠÃ\8çÒŒ}c¢‚ËUíû|Õô…––ê6v{uÇSí«„‡£ú:“©æPó‰â=¤žO?_œ}³v5GÙ …¦éæÆ~£›æ>ŸO»ÒU ®qÞhÌçóéíMu¿'Â¥'I’a†kL jš`‚qÆu?GíCw?~:éœ×Œ"DmKô añ(—Ë pôèQÞzë-Þ~ûmNœ8Á©S§Ò:·Û]#ê^³f «W¯¦§§‡žžºººäŸ°¨”Ëe"‘ÃÃà k£ËÉ“'µñEÝ̵Z­ôõõ±fÍÖ®]Ëu×]ǦM›Ø°a«V­žpÅ3Î8Cg'ePýAa„Uñ´ ]tÑG½ô64´"y…åA2™œePF~£y ÞHШò»Ë墭­M·öövmPz3T§^þd³Y2™Œ®üŸÉdH§ÓD£Q2™ ™LFWÍUËÑh”d2Y“( Æ‡°Ùlx<ü~?^¯·&Àçó5óù|5‰jLΉ/Jlo¬°¯DöFÁ¾ú·o4fï«ŠýFѾ2«þ7Sf€ áBEü êŸO€ ‚°8¨ï£h4ª¿kÔwÅøø8ÙlVÏc±˜^ŸH$ôùÿ«¹jåry–yöB0›ÍX­VZZZ´9ÌápÐÒÒBkk+‡C‹üN'@@ þƒÁ >÷imm¥¥¥¿ß¯¿·äœHA„•ŠÒ¢¨ûêÞ„ñ^‚±À@.—«ILR×ðê:ßåréó'9‡ºxÄd ‚ ‚ 4&¯2™Å12 ÕmæÃfƒÖÖjki©”1Áë­Žy½Õe»ýÜz·»ú»½jlp8(ÚlD+b¹ÑR©:O¥ˆÅbD£ÑYóú±F…ÖÖÖ†) ^¯W7•˜ š×¯»ÚPâƒf†Sgœ«~3׺º \ˆA¡ÞÈà÷û%Iá“$É(£Lœ"Dg|–AMFZh¡ƒB„被©¡“N:èÀÉÕO(,-Ùl–£GòöÛoóÖ[oqôèQ½¬Ä0ÝÝÝlܸ‘uëÖi2„B¡%þ‚0›ñññY 'NœàèÑ£ŒŒŒÕ*Õ*.vÆ lܸQ÷¯öØXay0Í4à kÀJ"0ö¦È AzÏN«X¥ûýôÓGÝtK2“pÙ™Ëp?::ÊÈÈȬñ©©©†â¨ó1Úwww‡ikk«yX', ªú­2¨Š¹ÊN§‰Åb¤ÓiÒé4©TŠx<®×ÅãqR©™L†ÔÙû!s=t»Ý¸\.<^¯W/ûýþC€24SæWÏÍBÅ÷J<Ùhì|*ü«ûFcê÷ìB0 õ•¨¾Ñ˜Q´¯ pN¬¯”pN¤ßhL™à\*€Qü/âA„Ë‹±ê}õÿX,F*•bzzš\.§çJô¯ÎUŒÕþs¹Åb‘B¡ Ílår™R©4ç9ÌBP• Ìh.—K›#Õw”×ëÕÂ4e| …Bøý~Z[[ñz½´´´àõzõwÝÕøìGAAXùˆÉ@AA¸ü(³B.Wí«ù|c ]?*}Á8WI †~Ül&f2«TˆZ­ÄÊe¢Å"±b‘h.G¬T"šÍK§Iäó$Òi©ÑXLWÕi„Ñt`4"(sB}SByãØÕ"à+‹5„zSÂ\5o„I43(Ì•° „Ââ%Ê#DÏN£Œêec„bÄj^ëÀAÀ0uÓM˜p͘q΋łÅbÁf³a±Xp:5—˥¿w<---´··c³Ù…B: @‰ÿÕw“ÏçÃjµâóùjÒfAAA8?Äd ‚ ‚ ¬,J¥jrB&ù|5m!Ÿ¯¦,¤RÕ~<~ΘWSŒ¯‰F«cétm ƒ27Ì‡Û v;iŸ„ÕJ¼¥…„ÃAÂd"j³‘0™HT*$Ìf¥‰™âÅ"Ñ|žD¡@"—«¶L†ÔÙêrõØl6m@PÂ÷úäã:cSÕÝn÷UQY°YÐzñF£Ö¬(4s[³qµNUâNŠ”NF˜bŠÉ³Ó8㺯¦ &f™L˜è8;µÓ®û!Bº_?æFL%WSSS¼ð ¼öÚk¼ñÆŸ×ý‰…BA‹ýÓé4¥R‰l6K©T"ŸÏS*•jÄþårù‚¿ïê±X,ºâ¿ÿÛl6---ØívGáÌn·ãóùô}pUৣ£¯×+ÂAAA¸‚“ ‚ ‚ œ/ñxÕŒJU…BÕ˜ÏW ÉdµŸHœ3&ÄbÕíR©j+Î¥Óç ‰ÔUÊQCËY“‰¨ËEô¬q!j6W×™Íd+•ê¶Å"Ñb‘é|žü•…--<>N— ‡ÝN £ƒ@[NgN1½q¹½½}E>ôN&“5‚¢d2I"‘ÐcjÙØbgÓ,‰Éd’t:ÝpßV«¯×;§D™G”ÈÄh Qc‡ã2ÿT®Š™dRš™&˜ÐËyò5ûpâÔ†„ AÚÏNm´ÕÌëÇ„+‡ãÇóüóÏëvôèQÖ¯__c(¸ñÆ¥"» œcccÚppðàA<È;ï¼ÀƹýöÛyÏ{ÞÃí·ßÎúõë—øÝ ‹M‰ãg§QFgœÆÓccŒ1Ì0qΉ±­X ¢‡„é¡G›Œf‚Näï±0?ó™Šç2OLLP*•fís!fãfm%WŠ_lòù¼ôçr9R©©TŠ\.G"‘ N“Ïç‰ÅbZ¸§Dz™L†x<®Å|Éd’\.§«õ‹Å9­Ds>ŸO ¼@È_%Ü)ÛíÆívÓÚÚŠÏçÓë|>ߢW{O&“úwSUÄ/•J$“I2™ ù|¾¦‚q£±F•õÕÿ €X,F¥RÑ?Ófc#äWây“É„ßïo:¦þ¯ Vð¯ üv»—ËÕtL™3”éªi“JD)bGA„+›B¡ +ï+çúžJ&“Äb1òù<“““”Ëe=F£ZПÉdj*ù«y&“¡\.“Ëå(—ËZä_,©T*óž_œ/V«³Ù¬+ü·´´`µZ±ÛíZô¯ÿN§SŸ³¸\.Z[[q:ú|EÝ[öù|: Àív×ÌÅÐ&‚ ‚ B#Äd ‚ ‚ ËeNh4F›¯k²M&&‘N“ÌdHÄbÄóyR™ ér™$R@úìëcÛ árTÜ?Ÿ×H²Ÿ ÂÊA‰òmèSß]Ùl–ÉÉIFFFf% % J¥’6²ÅãqÊå2étšb±¨÷¥þ•JEÏ•À1+ù+¬V+&“©FìoµZ±ÙlØív¬VkMu—Ë…ÍfÃãñ`µZ X­VÚÛÛ±Z­„B!¬V+Øl6mšS߇j¾\ÓŠAAA¸:“ ‚ ‚ \Í(#B*Åb5]¡Tª&*¨d•ÖWSÎVŠŒNO“K§Éf2Dc1¢¹¹d’l©D4&Z(+É DgfÎ¥00;™¡³§ÙŒÃb!`³á´ÙpØlœv»NapÚí8Ün­­8½ÞjC0X1´µUSB!v;Áµk±9pšŠÅ¢NQPÂ*£)A¥+ÇTƒªÚ™N§› ®]½Ñï÷WM!g@ fY­WMUÂRÛ(cƒÅb¹Œ?¡¥#EJ”1¡Þˆ`\¯úõ¨Ô£ñ ƒ†É¿fYMW*oò&[Ø‚ wr'¿Ïïs/÷.ªÀ4Nó£ýˆ§žzŠ§Ÿ~š“'Oxÿû߯MÛ¶m“Ê‚° 8ÀóÏ?ÏÏ~ö3ž{î9b±ëÖ­cÇŽìܹ“öÏþ™®@|)y×páb›.ù±–Ò}Tò2¨D‚ &j^ëÄI'„ kã@] jÃ@ ÝtãÁ³DŸP¸ÜD£ÑYi_Fcmý:c ˜Z¯*²×£XÆ´¯fæ€fíj1 ¨j÷ª2o*•Ò"u£˜OUüU•îãñ8¥R‰x<>ošÀ|x½^ìv»¾NP×.— ³ÙŒÇãÁf³a6›µÉÜb±àv»)‹Øívìv;ù|‡Ã¡E„v»½&‘ Q5£ ¿Qå£òbÅûp®º¾ÕjÅã©þ½3VßW‚Cã¶ÆÊùÆmÕu”±’¿±ò¾±Ò°"šÍf|>и¿ ‚°r˜Ï¨¦’WÕý@@ßL¥R:‰F}ÿ+±?Tm” ?ŸÏë¹±Z¹\¦T*133£¿›[rb6›±X,˜Íf-ö·ÙlØl6]Ñ_‰û‡‹ÅRSÍ_ ÿÕ½KUÁ_?z<žY&Dõý©¾«AAA®vÄd ‚ ‚ Âå!«šâñªi!†L†|*Ezb‚XÞ€+%JtÓÍïóû|ŒÑAÇcbb‚Çœï~÷»üä'?¡P(pã7²sçNvìØÁm·ÝvÕ˜`áJ¢T*ñÒK/iSÐÁƒ±Ûí¼÷½ïåþûïç  .ÚñòäùGþ‘¿â¯x×ù;þŽßà7mÿ—›¥À2¨1e0 Ê”köáÇOˆthã@'„&LèìÔM7­,ŸsáâPÉ[êÑhV­7±Îeh&êuÙlvA)n·‹Å¢«ñªê¼f³Y›/[ZZ´HÝf³Õß57›Í”ËeŠÅ"f³™b±H¡PÀd2iÁ£ú Ã…TÒ?Ÿm/t_ʬV«¾—g³Ùèìì  éóeÊS‚~£!OAAAXˆÉ@AA„Ivt”\6K6!‹‘›šª&0D"ä2²‰ÑD¢šÆÍM$ˆ¦Räòy²¹\5™¡P¨&3äóäffH”J”繄rP508€€¡¯ÇÌæj"ƒÅBÀn¯öívœG5™¡µµšÌÐÚZóz ¸Ý8ý~QT ÐUÉ”ØLU&óz½zßï×}ephd`¸«{fÉÖ:M2I‘â¬ý9pÌiJh6uÐA çQý¿R!“ͲqãF~ã7~ƒ÷ïþûr?÷ÏÚÔŠ&îã>>ŧ¸ƒ;æÿ¹d³<ñÄ|ýë_ç™gžÁáppï½÷jcz¨-•ÃØØO?ý4O=õßûÞ÷ÈçóÜsÏ=üú¯ÿ:÷ßÿ›ÏNsš¿áoøÿøÿˆÇtvúüþ€?XäOqaäÉ74 M2Y3n4L3]³3f: S tÖŒ©å A:èÀ†ˆe¯²Ù,ÑhT ΣÑè/Çb±¦h@@‹«À¬Ölïìì\Öæ¾D"A¹\&333£ÑhTŸëf³Y¦§§ÉårÚ€‘Ëå´À_‰ …‚ÿg2ŠÅ"¹\Žb±¨Å‚óaù©óT‹Å¢†333º¯DÃ¥í/ƶ"àAXY(œK”QF»D"¡¿3ãñ8•JEß3*—˺â~¡P ŸÏ“N§)‹ú»W}ïÎÌÌËå( T*²Ù¬í+!¿þ«÷Q©T¨T*—LÈoüNW•ùìv;f³“É„ËåÒÕú&“‰ÖÖV, 6›M ò}>Ÿ®äÿÿ·wçQR•wþÇßµWWWWW³¨Í2âÊ®!" èÄ(Aç§& Ã€‰LÜwãqg@Tp<9ê œ£& Šè bY”ML”–­z﮽~ÏcUwuÓ´@ƒ~^œ>]uëÞ[Ͻu«›Ï÷ûtêÔ ÈÌZSPP€Ë墴´Ô.kâÏž5GúEDDDDD¤-Td """""rLÀȳÌíªÊJ"{÷ÒØÐ@My9±h”š}÷# T×Ô‹F©­¯§¾±‘Xná~Ëoê¸Ó¤[±&{Yö~=Íg *¦˜Îyþ´T@Ð….88º é¾­²?»˜BÌúúz{ßÌef2¡ºìûf6©ýu…/,,´³E™™£ sf’2÷M!fÓ™§²g8”ç$‰4»]UUe;ù›BÕx_Î6‡{†jlÑAU-$ä{,ß ¾zHÆ!QûkœíÄ @€S˜Â Éøpþ‡<þøã¬\¹’!C†på•WrÅWpÌ1Ç‚£‘#É®]»X°`/¼ð}ôC† ᦛnâ²Ë.kâ©¥–?ðåQ>ã3ܸIÐüzèÅË5\ÃLf’N§Y¶lüèG?Ú_¤UNy³çkkV7ºQJiûf‘vií³W³Ïam\wÏž=­dÀ[š µûét·ÛM8Æãñàtf®™¦;?`;ô9³dß®¬¬´c*//·ò+**ˆD"¶C*•¢¦¦†T*eþ&PØØØh†¦‹°Yï`2BóÝívÛÀn·Û†½^/.— ŸÏgC…æ³”×률 €ÂÂB\.ÅÅÅø|>»¬¸¸§ÓIii© .šÂ ŸÏg‹³;î³d‰ˆ|—$“IöîÝkƒú±XŒ;wÚÇwíÚe¯ÛUUUöç~óû³Mmm­ ×744ØîøÑh”X,F*•²aýt:müfSð–H$r‚ù溙Î7áýƒÍápØk–¹–^¯×>f®¥€½¶B¦®  Ànë÷ûm˜ßï÷Û뮹ff×¹Ýn{=öù|ƒAÜn·ýˆÃá ¸¸Ø.7×W§ÓI±ù½ˆˆˆˆˆˆˆR*2ù1SÓ›€˜ œUVVÚ²»Ü™P½ùôD"AUUÉd’ê½{IÄãÔÖÔDhØÎÆbÔ56oc¬ÀíÆïtt»ñ¤Ó„Ün\©%.Žd’0@2II: ©áx™Ùdfpp’™½ÁEföwÖ÷ ™ÿ/ ð:Âa|Ç×35˜ÚYÀbÑXŒˆßO4¥2‘ ’HÐJÑNÓ²;isH±­¡D£i81ßí¶.kú¸ ×}SÛ*·ñ/§ÿ ÎH…!TÃK áÿÁ‡£!æ;À¦Àñ'¿{1·]yÆ ûÆc<-[¶Œ‹/¾˜¿þõ¯ôëׯ£‡#H¯ßÁñá‡òÄOðÊ+¯pòÉ'sÏ=÷pùå—ó™ã3f3›yÌ#FŒ)Ò´ü«P'NúÅú‘ü<É?÷ü“x0Ž»«›@÷5Ôä]¿Ù¬-„ó tÚ÷ÇÌ<ÐtFi›D"Amm­½NšÏ&•••öó‹ù cºÞG"ª««ihh ±±ÑÞŽD"vÖ£H$’°oI0Ì ¥›]AAN§ŸÏ‡ßï·AwÀ3a÷ì `*•Âív“L&óvó7úS©”o:¶úÓé4‘Hä ÷³†f|úß&PhþæøM·`¿ßÇã±çËt6áB3s” šYÌì ~¿ŸcŽ9&g9øÌï "Y?¿F£Q"‘ˆ ÌG"{=2ëš`}"‘°ëš ½ù=d®15559ׯì¢8È\Íã&´o åLgýì.ù&¸oîgï+™Læ\¶ì€¾ À¶ƒ>|ÝQßÜ6Ei& oÖ/,,´û3?“öÚØæv8¶3ä„B!Û‘ß\W½^/~¿Ÿââb{ý430ŠˆˆˆˆˆˆˆìæÂù1âJJJËóå+bˆÅb6ô—]Ä`‚¶ˆ¡ºÚn“J¥Ø¼o_ÕÕÕ¤Óiªªª ΄ÓiªjjZ ÄãP]¹½/Dðxð¹\™"—ËÎÞàn7¾tš€Ã7•¢0Æ“JL¥2 É$îD‚¢höÍê||]äàz‘)~ðí{Ü„MaCÓï%%9Ë’ ‡xû,¿úÕ¯øôÓOyöÙgËxFŽÉÞ½{Ës W_}5“&MúΆTæë—ïïf2™dçμýöÛÜwß}<õÔSüùÏfàÀ9ÛΟ?ŸÉ“'sã7òÁн{w*++Y²d 7ß|3÷ÝwŸí®¾zõj†ÎÔ©SY¾|9ÇwÛ·ogæÌ™œsÎ9¬X±‚!CoñÑ«W/æ<5‡³>?‹»—ßM­«–tÿ4.\8p  õràÇ_ßm '­àÿüÃ!wKš^_Ìõ È)3×/ÈÌXT]]mæÃ,³·ãñ¸í(œ}4õÍm³]2™Ìél:›bvxò`r¹\¶ë¯Ó餠 À†÷M`À¤R);[@"‘ÀívÛ1;ÎœP§YnÎ[{ãp8lG`Èå™e&œh‚ù&i‚ùn·›¢¢"‡ /†B!\.—í4l‚þ.—˘ç0û9PÙ³¦æg#‰ÐÐÐ@mm­]f:Ø›e¦ È ß›eæg¥ì`¼éŽŸ]4f®YÙË’É$9éc±X³q›.÷Ù’ÉdÎ{~öµË0!üì€ýÁ*ülÓ?;ho¸Ýn{=„LqYö}ŸÏ—Sàlf‹æÝñ=½®˜}ƒA»½Çã±Eåæ¹ sún·Û^§ü~¿}¿ßßï·ÅnkF<é8*2‘o•¦y'S¸Ò1¡P5³3À×!QSÀ™î–€ }šÂ€íû &L„Ñ!ë¹ãñø~;-‹e¾ö…œC>.‡¿ÛMË…#&ìñ@*E¡Û7Æ“Nt8 ™¤8Æ™JQàpàO&q&”Äb”¤Rô<)ˆv7æG˜Ìo'Ò@rßíà~.\÷¼ Z˜ë¨å‹‘#¹û¹ÙÞ}?ŸÎ;sûí·óç?ÿ™çž{ŽéÓ§Ó©S§Ž–|ǹ\.ºwïΔ)SøáÈ÷¿ÿ}Ædž (**`ݺuLž<™›nº‰|Ðn{ì±Çrùå—Ó¿¾ÿýïÛåÏ?ÿ<>ŸY³fÙàX¯^½xì±ÇX¼xñá=À<ܸ™rÒ&Ÿ0™‡~˜»Î»‹3o=“ÒkJyÛ÷6µŽZ\IIW2ÿòü³­®®æÙgŸµïË€}rÞw‰{öì±ËkjjH$¤R)û> Ø®ü&|‹Ålà±=ÝìÛ£i 1_¸ÑtN§Óx<œN'ÉdÒ.O$6¨˜H$ìºÙÝ”Û{<¦¸ÁtÍÏo™ }úpß}÷1~üx»ïý»o¾×ï@Ïù8æ˜cxðÁ™8q"O?ý47ß|3>ø ©TÊÞojÀ€¶ 2ÿæãñ8 ÍÞó>ùä“vï`s:Üzë­ :”‹.ºˆe¿X–™êåLH^„§“)pJñõoI‹›ï«¢¢‚k¯½Ö5M¨°ÁÍö†8M°ßŒ°ÝøN§)À¬“Ýåù›0‰!78Ÿ~4A~€¢¢"Ûå8»3± ßg‡ý³»î›ŽÈ@Nwÿý=Oö>D53ûUKòpš÷A£¾¾>'ôÝý2!p³¬iwöD"a÷eŠŽŒH$b÷›J¥ìg3SÀš-‰Ø@zÓ@~"‘°…­M5•J‹ÅZ,jØ7+U>¦èµ)ó|M™nòM·1÷ÕÌ(‡J¾.íN§3'dn˜B­l.— Ç“s]q»Ý¸Ýîfáx‡Ãa3ë™ýù|>;Ë‹yn¯×kï›÷Ø@ `û>ŸÏÎœbÆmÞ›³ÃúÁ`P(”3Æp8l‹cŽ9ÆîË舢o9xé#©…‰ˆˆˆˆˆˆˆ|+­ZµŠ3Î8£ùC€@VsW7º…B쩫㩹s™2eÊáhMž<™—_~¹Yèpøðáüý識¼œ¿þõ¯L˜0»îº‹_þò—|ùå—\qÅtëÖÅ‹ãp8¨««£¨¨ˆ^½z1cÆ ÆÇóÏ?ÏÒ¥K™?>ƒ ¢OŸ>Ìœ9¯×˽÷Þ˼yóìóší³‹ ö·ÍâÅ‹9ï¼óX»ví~g…0û?í´Ó¸ÿþûù·û7žþyn¼ñF–,YÂÙgŸ döãÇç¶Ûnã—¿ü%\uÕUTVV²jÕ*DûÃþÀO~òî¿ÿ~®½öZ¶nÝÊwÜÁ[o½Åœ9s˜6m¯½öÚ~Ï]>í=ö_üâüæ7¿ÁçóñÚk¯1uêTV®\Iÿþýíz-mßÞñæ{ýÚzÎ[ÒÒßMÈth‡ÃœsÎ9¼÷Þ{têÔ‰^½z±jÕªV÷kÌ;—Ÿýìgœ}öÙüö·¿eäÈ‘-ß‘ ººšaÆñ駟6ðà|`ܾ¯b hC¶Ý:Mè3û¾ùûîr¹r:îû|>û˜Ïç³õ!7ÄŸÆÏïÈzùÂÿn·»Y(ôh”]×Ù3H@¦»y¾ÎæFuuu«#Ùáñìàx[Çbf3jMcc#©TŠh4Úbð;™L6ë–‰Dš…ÁMðÜhÚ9Ýì+{YÓm²·Í·~¶ì:Z Œ› üþŠfLG÷ÖÄãñV_/S¨ó]]´Ôô}Ù˳×É–NÏ÷Xvà=›ÏçË ¤n·»Yà2Áw¿ßßì¹LøÝï÷ç^™(·ÛmŸÇëõâóùì¶f@À¾?ší@Î{£ óà]»vmvŽB¡P‹çGDDDDDDDDDäÛJE"""""""rÈ}øᇜuÖYys),ˆ§tëÆO¦Neü„ Œ;–©S§òàƒΡ¶IÓ ÷ž={xöÙg¹ãŽ;øÙÏ~Æ3Ï¿ùŠ öwÎ[ÒZ‘@—.](..æóÏ?§¦¦†ââb.¸àþò—¿´º_#sÅWðꫯPZZÊ„ ¸êª«8óÌ3Û´Ã-™L2jÔ(þö·¿µFv’)zúpîC~¿ŸSN9¥YÀ4»SwKAðý1Aòö† MX»¥.ãû“ÝQü@÷‘oö†T*Õæ²×Ó¯¨|¡ò–è­m™ÂÓ9½¥ÀµÃá°Ûóuy7<>Ÿ¯Õ"¥–‚êM÷SXXØâ~;wnq{ŸÏGAAAÞ½Çã±3n˜|ö~³‹vÜn·]23tdÞÍ>\.~¿?g] ïL ùdÏ$"""""""""""r(¹÷¿ŠˆˆˆˆˆˆˆÈ7Ó4€ë “éM9 <é$&üä'Lœ4‰SN9€E‹±{÷n®¿þúÃ?Ø6ª¯¯·C¯×ËÉ'ŸÌ#<Â7ÞHYYŸ}ö×]w]Î6&„Ý4x~Úi§5Û¿ßïgèСÜ~ûí8~øÃRPP`óù´g›¶8p ½ír¹èÒ¥ »wï ¬¬ŒM›6qíµ×æl3hÐ B¡‹/æÖ[oeïÞ½­žã@Ïq°½S§N|öÙgû]¯½ãÝŸÖÎù7‘/Ì} 3x<^yå–,YÂsÏ=Ç믿ÎSO=ÅSO=ÅùçŸÏ‚ r:ô ¶oßN h=ÈžþoßWñxœ/¾ø"ïffö§ÓÙ,PÜVfƃö2¾ÛóüN§Óvw:-v+o‰ßïÏ <›Näm‘ݽé~Zãp8(,,ló9Ëî†n¶Úµ=»KzS^¯7gm>÷‡ƒîÝ»ï·#zö,­ ‡ÃGôL""""""""""""rôR‘ˆˆˆˆˆˆˆr¦ÈÀ àp0bð`®˜:• &PZZÚlýmÛ¶ …ò>v¤(,,l±[|EE³fÍbÖ¬YÍÿòË/sî7 Ào¿ý6<ð¿øÅ/˜4icÆŒáî»ïnµc|{¶ÙŸ`0˜sßãñØÎëæX;uêÔl»Î;ÛÇwìØ‘w½¦÷ôÜekï±oØ°»îº‹åË—³{÷nF4hP«Û}Óñ¶¦µsÞ^ÕÕÕTVVrúé§ …())±¯Í8çœs8çœsˆF£¼ýöÛ<ñÄ,Z´ˆx€Gyäó`ëÙ³'ï¼ó¡Pˆë®»ŽO>ù„wÞy§Í³ôèу­[·ÚAŠˆˆˆˆˆˆˆˆˆˆˆˆA4¯®ˆˆˆˆˆˆˆr‡ƒ ÇŽeîsϱ»¼œV®äç?ÿy‹E¥¥¥ÔÖÖ²wïÞÃ<Òƒ£K—.ÜrË-¤Óéf_/¼ðB›öSRR£>ÊW_}Åûï¿O$aÔ¨Q|þùçu›oÂk¾×jÏž=öqóZ7]¯ºº:ïþÚsîÚsìñxœ1cÆ°mÛ6–,YB<'Nóÿñ­w¾?ã=Ü^ýuÒé4ãƳËÆǺu먪ªj×>}>_|1‹-¢kÓ˜Œ IDATgÏžüßÿå™ àPQQAmm-#GŽä7Þ ±±‘W_}•Q£Fí·«¼ˆˆˆˆˆˆˆˆˆˆˆˆÈwŠ DDDDDDDä=z4o,\È”)Sèܹó~×9r$¡Pˆgžyæ0ŒîàëÑ£}úôaÅŠÍ;ýôÓY°`Á~÷±sçNhï6Œ¹sç‹Åøè£Ú½ÓypÔ£Gz÷îÍ’%Kr–¯Y³†ššÆŒdf,8õÔSYºtiÎz«V­j¶¿öœ»öûæ͛ٱc—_~9}úô±óh4ÚlÝ|ÛŒ×úpصkwÞy'={öäšk®±Ëï¼óN\.W‹³<þøãÛâ»îº‹ßüæ7ÍÖs»Ýx<ž6ýûîÏ<ó áp˜‘#G™×r„ ¼ÿþû4440gÎ €Ãáh¶m[ŠMDDDDDDDDDDDDD¾MTd """""""Gœ¢¢"~ýë_sÿý÷óñÇwôpÚåÑGåƒ>`úôéTTTPQQÁM7ÝD"‘`üøñmÚǺuëxüñÇ©©©¡ªªŠgžy¿ßÏСCÛ½™QàÓO?¥¢¢‚ÒÒÒ‹Úê‘GaãÆÜyçìÙ³‡M›6qÍ5×pòÉ'ó_ÿõ_v½{ï½—5kÖðÀPYYÉ'Ÿ|ÂŒ3ší¯½ç®=Ç^QQA×®]yá…Ø°a‘H„E‹±páÂfûoéÜŒ×úPH&“|õÕWüÏÿüC‡Åétòæ›oRTTd×éÛ·//¾ø"=ö·ß~;[·n%óÅ_ð»ßýŽ;ٳgÓ©S'»Í“O>ÉK/½Dyy9±XŒ-[¶pà 7°uëÖœ×ûH±zõjxàn¾ùf‚Á`³Ç½^/Ó¦McíÚµÔÕÕ1}útN:餩ˆˆˆˆˆˆˆˆˆˆˆˆÈ‘Á‘V+.9%“I~ðƒðÉ'Ÿ°hÑ" ÔÑC¢¬¬Œž={æ,1bD³îüÆ¢E‹¸çž{X½z5áp˜Ñ£GóÐCÑ£GæÏŸÏüãœõ+++ ‡Ãöþ›o¾É“O>ɪU«ˆÇãœvÚiÜwß}œ{î¹Ìš5‹ë¯¿Þ®{öÙg³dÉ’V·1®½öZ^zé%Òé4“&MböìÙÍÆÞt|'Ndúôé9ÇÒI'ñÏþ€·Þz‹{?þ˜@ À¸qãxøá‡m0ßxꩧx衇صkƒæ±Çcøðáœyæ™|øá‡û=w-iﱯX±‚_ýêW¬^½š’’.¸àöîÝËË/¿ ÀŽ;8î¸ãZ=w:Þ|¯ß´iÓèœgË÷wÓáp‡éÛ·/ãÇgÚ´i„B¡¼ãÙ°aÓ§OgñâÅìÙ³‡®]»ræ™grÓM71bÄ»^]]/¿ü2 ,`ãÆlß¾`0È!C¸ùæ›9ï¼óò¿8dÍš5œþù 4ˆ… Ú™*Ú¢²²’‡~˜Ý»w3oÞ¼C8J‘#‹Š DDDDDDDäˆU__ÏøñãùðÙ3gW^yeGIDŽ¿ÿýï¹öÚk9묳xíµ×=$‘£‚³£ """""""Ò’ÂÂBÞzë-~ýë_3yòd.ºè"¶nÝÚÑÑ#ØW_}Åe—]Æþç2uêTÞ|óMˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆÈÍívsï½÷²hÑ"þñпn¹å***:zh"r)//çæ›oæÔSOeíÚµ¼ûî»<ùä“x½ÞŽšˆˆˆˆˆˆˆˆˆˆˆˆÈQÅ‘N§Ó=‘¶ˆÅbÌž=›‡zˆúúz~úÓŸrýõ×s 'tôÐD¤ƒlÞ¼™™3gòÜsÏQXXÈm·ÝÆÏþsˆˆˆˆˆˆˆˆˆˆˆˆˆ´“Š DDDDDDDä¨S__ÏÓO?ÍÌ™3ùꫯ¸ð ¹êª«¸ð ñûý=<9Ä"‘o¼ñ/¼ðo¾ù&=zôà†n`Ú´iŽžˆˆˆˆˆˆˆˆˆˆˆˆÈQME"""""""rÔJ$¼úê«Ì›7÷Þ{¢¢".½ôR&MšÄ¨Q£p:=D9HR©ï¿ÿ>/¾ø"¯¼ò uuuŒ=š©S§rÉ%—àv»;zˆ"""""""""""""ß *2‘o…íÛ·3þ|^|ñEV¯^MÏž=¹ôÒK¹à‚ 5j”f89 E">øàþò—¿ðÊ+¯PVVÆ÷¾÷=&MšÄW\AiiiGQDDDDDDDDDDDDä[GE"""""""ò­³aÃ^zé%^ýuÖ­[G àÜsÏå‚ .`ìرœxâ‰=DiÁæÍ›Y¸p! .äÿ÷ihh`À€Œ?ž‰'Ò·oߎ¢ˆˆˆˆˆˆˆˆˆˆˆˆÈ·šŠ DDDDDDDä[mÛ¶m¼õÖ[,\¸wß}—ššz÷îÍèÑ£1b#FŒàøãïèaŠ|g}ñÅ,]º”åË—³xñb>ûì3B¡cÆŒaìرŒ;–ž={vô0EDDDDDDDDDDDD¾3Td """""""ßñxœeË–ñÖ[oñþûï³råJâñ8=zô°#FŒà´ÓNÃívwôpE¾u‰ü1Ë–-cùòå,]º”¯¾ú ¯×ËgœÁ¨Q£;v,#FŒÀãñtôpEDDDDDDDDDDDD¾“Td """""""ßY¬X±ÂvQ_¾|9•••ƒAÎ8ã Ì Aƒ™N8Á~uïÞ]rDJ¥R”••±e˶nÝÊ–-[øÇ?þÁ¦M›Ø´iuuutéÒ…>}úЧOz÷îMß¾}éׯ'œpBˆˆˆˆˆˆˆˆˆˆˆˆˆ|S*29ˆ***ظq#Ÿ~ú©->ؼy3[¶l!‰àõz9þøãmÑA¯^½ì÷ž={rì±Çâr¹:øHäÛ(™L²sçNÊÊÊlAö×—_~I,Àï÷s 'pâ‰'Ú‚óÕ¥K—>9TTd """""""r˜TVV²yóæ¼__~ù%‰D®[RRBii)ݺuãÄO´·›~1Ù±cÛ·oÏù¾yóf{;ßß³O<1ïW¯^½4ㆈˆˆˆˆˆˆˆˆˆˆˆÈwŠ DDDDDDDDŽñxœmÛ¶±}ûvÊÊÊرcÛ¶mcÇŽ”••±}ûv¶oßngC((( k×®{ì±tíÚ•®]»Ò¥KŽ;î8{»k×®wÜqtéÒ…@ ÐG(ª¡¡òòrvîÜIEEååå”——³k×.ÊËËí2s¿±±Ñnë÷ûéÖ­ݺu£G”––Ò³gOJKKéѣݻw§Gx<žhZˆ`¾;NŠ‹‹LHÞçóáp8‡Ã@¦ðÁï÷´ZĽM[UUUÑÒ¯¾¢Ñ¨-ˆD"v³MöãÕÕÕ¤R)»¬iA¾ÛÉd²Åqeq´öÕ¹sgºtéb‹ èøEDDDDDDDDDDDDDÚCE""""""""ÒŒ)8¨­­¥±±‘ššêëëííºº:Û¡?ûvmm-‰DÂ~7·êêêˆÇã$“Ijjj:øó …B¸\.<Á`€¢¢"Ün7n·ÛÞ.**²³<ƒAB¡~¿ßÞ.(( °°ââbü~?EEE¶x@DDDDDDDDDDDDDäH¦"é0õõõÄb1 õ™Ì좭3#˜™DDDDDDDDDDDDDDDE"""""""""""""""""""""""²³£ """""""""""""""""""""""Gˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆn`sGBDDDDDDDDDDDDDDDDDDDDDDD:Þÿ{ò9Cip6ŒIEND®B`‚patroni-4.0.4/docs/ha_multi_dc.rst000066400000000000000000000057241472010352700171360ustar00rootroot00000000000000.. _ha_multi_dc: =================== HA multi datacenter =================== The high availability of a PostgreSQL cluster deployed in multiple data centers is based on replication, which can be synchronous or asynchronous (`replication_modes `_). In both cases, it is important to be clear about the following concepts: - Postgres can run as primary or standby leader only when it owns the leading key and can update the leading key. - You should run the odd number of etcd, ZooKeeper or Consul nodes: 3 or 5! Synchronous Replication ----------------------- To have a multi DC cluster that can automatically tolerate a zone drop, a minimum of 3 is required. The architecture diagram would be the following: .. image:: _static/multi-dc-synchronous-replication.png We must deploy a cluster of etcd, ZooKeeper or Consul through the different DC, with a minimum of 3 nodes, one in each zone. Regarding postgres, we must deploy at least 2 nodes, in different DC. Then you have to set ``synchronous_mode: true`` in the global :ref:`dynamic configuration `. This enables sync replication and the primary node will choose one of the nodes as synchronous. Asynchronous Replication ------------------------ With only two data centers it would be better to have two independent etcd clusters and run Patroni :ref:`standby cluster ` in the second data center. If the first site is down, you can MANUALLY promote the ``standby_cluster``. The architecture diagram would be the following: .. image:: _static/multi-dc-asynchronous-replication.png Automatic promotion is not possible, because DC2 will never able to figure out the state of DC1. You should not use ``pg_ctl promote`` in this scenario, you need "manually promote" the healthy cluster by removing ``standby_cluster`` section from the :ref:`dynamic configuration `. .. warning:: If the source cluster is still up and running and you promote the standby cluster you create a split-brain. In case you want to return to the "initial" state, there are only two ways of resolving it: - Add the standby_cluster section back and it will trigger ``pg_rewind``; however, for ``pg_rewind`` to function properly, either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``, but there are still chances that ``pg_rewind`` might fail due to other factors. - Rebuild the standby cluster from scratch. Before promoting standby cluster one have to manually ensure that the source cluster is down (STONITH). When DC1 recovers, the cluster has to be converted to a standby cluster. Before doing that you may manually examine the database and extract all changes that happened between the time when network between DC1 and DC2 has stopped working and the time when you manually stopped the cluster in DC1. Once extracted, you may also manually apply these changes to the cluster in DC2. patroni-4.0.4/docs/index.rst000066400000000000000000000043111472010352700157640ustar00rootroot00000000000000.. Patroni documentation master file, created by sphinx-quickstart on Mon Dec 19 16:54:09 2016. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Introduction ============ Patroni is a template for high availability (HA) PostgreSQL solutions using Python. For maximum accessibility, Patroni supports a variety of distributed configuration stores like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in datacenters — or anywhere else — will hopefully find it useful. We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation `__. Currently supported PostgreSQL versions: 9.3 to 17. **Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus `__ database extension to Postgres. Please check the :ref:`Citus support page ` in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster. **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes ` chapter of the Patroni documentation. .. toctree:: :maxdepth: 2 :caption: Contents: README installation patroni_configuration rest_api patronictl replica_bootstrap replication_modes standby_cluster watchdog pause dcs_failsafe_mode kubernetes citus existing_data tools_integration security ha_multi_dc faq releases CONTRIBUTING Indices and tables ================== .. ifconfig:: builder == 'html' * :ref:`genindex` * :ref:`modindex` * :ref:`search` .. ifconfig:: builder != 'html' * :ref:`genindex` * :ref:`search` patroni-4.0.4/docs/installation.rst000066400000000000000000000141721472010352700173640ustar00rootroot00000000000000.. _installation: Installation ============ Pre-requirements for Mac OS --------------------------- To install requirements on a Mac, run the following: .. code-block:: shell brew install postgresql etcd haproxy libyaml python .. _psycopg2_install_options: Psycopg ------- Starting from `psycopg2-2.8`_ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages. Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it. There are a few options available: 1. Use the package manager from your distro .. code-block:: shell sudo apt-get install python3-psycopg2 # install psycopg2 module on Debian/Ubuntu sudo yum install python3-psycopg2 # install psycopg2 on RedHat/Fedora/CentOS 2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the :ref:`list of dependencies ` when installing Patroni with pip. .. _extras: General installation for pip ---------------------------- Patroni can be installed with pip: .. code-block:: shell pip install patroni[dependencies] where ``dependencies`` can be either empty, or consist of one or more of the following: etcd or etcd3 `python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS) consul `py-consul` module in order to use Consul as DCS zookeeper `kazoo` module in order to use Zookeeper as DCS exhibitor `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper) kubernetes `kubernetes` module in order to use Kubernetes as DCS in Patroni raft `pysyncobj` module in order to use python Raft implementation as DCS aws `boto3` in order to use AWS callbacks jsonlogger `python-json-logger` module in order to enable :ref:`logging ` in json format all all of the above (except psycopg family) psycopg `psycopg[binary]>=3.0.0` module psycopg2 `psycopg2>=2.5.4` module psycopg2-binary `psycopg2-binary` module For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is: .. code-block:: shell pip install patroni[psycopg3,etcd3,aws] Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed independently of Patroni. .. _package_installation: Package installation on Linux ----------------------------- Patroni packages may be available for your operating system, produced by the Postgres community for: * RHEL, RockyLinux, AlmaLinux; * Debian and Ubuntu; * SUSE Enterprise Linux. You can also find packages for direct dependencies of Patroni, like python modules that might not be available in the official operating system repositories. For more information see the `PGDG repository`_ documentation. If you are on a RedHat Enterprise Linux derivative operating system you may also require packages from EPEL, see `EPEL repository`_ documentation. Once you have installed the PGDG repository for your OS you can install patroni. .. note:: Patroni packages are not maintained by the Patroni developers, but rather by the Postgres community. If you require support please first try connecting on `Postgres slack`_. Installing on Debian derivatives ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ With PGDG repo installed, see :ref:`above `, install Patroni via apt run: .. code-block:: shell apt-get install patroni Installing on RedHat derivatives ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ With PGDG repo installed, see :ref:`above `, install patroni with an etcd DCS via dnf on RHEL 9 (and derivatives) run: .. code-block:: shell dnf install patroni patroni-etcd You can install etcd from PGDG if your RedHat derivative distribution does not provide packages. On the nodes that will host the DCS run: .. code-block:: shell dnf install 'dnf-command(config-manager)' dnf config-manager --enable pgdg-rhel9-extras dnf install etcd You can replace the version of RHEL with `8` in the repo to make `pgdg-rhel8-extras` if needed. The repo name is still `pgdg-rhelN-extras` on RockyLinux, AlmaLinux, Oracle Linux, etc... Installing on SUSE Enterprise Linux ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You might need to enable the SUSE PackageHub repositories for some dependencies. see `SUSE PackageHub`_ documentation. For SLES 15 with PGDG repo installed, see :ref:`above `, you can install patroni using: .. code-block:: shell zypper install patroni patroni-etcd With the SUSE PackageHub repo enabled you can also install etcd: .. code-block:: shell SUSEConnect -p PackageHub/15.5/x86_64 zypper install etcd Upgrading --------- Upgrading patroni is a very simple process, just update the software installation and restart the Patroni daemon on each node in the cluster. However, restarting the Patroni daemon will result in a Postgres database restart. In some situations this may cause a failover of the primary node in your cluster, therefore it is recommended to put the cluster into maintenance mode until the Patroni daemon restart has been completed. To put the cluster in maintenance mode, run the following command on one of the patroni nodes: .. code-block:: shell patronictl pause --wait Then on each node in the cluster, perform the package upgrade required for your OS: .. code-block:: shell apt-get update && apt-get install patroni patroni-etcd Restart the patroni daemon process on each node: .. code-block:: shell systemctl restart patroni Then finally resume monitoring of Postgres with patroni to take it out of maintenance mode: .. code-block:: shell patronictl resume --wait The cluster will now be full operational with the new version of Patroni. .. _psycopg2-2.8: http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/ .. _PGDG repository: https://www.postgresql.org/download/linux/ .. _EPEL repository: https://docs.fedoraproject.org/en-US/epel/ .. _SUSE PackageHub: https://packagehub.suse.com/how-to-use/ .. _Postgres slack: http://pgtreats.info/slack-invite patroni-4.0.4/docs/kubernetes.rst000066400000000000000000000103321472010352700170240ustar00rootroot00000000000000.. _kubernetes: Using Patroni with Kubernetes ============================= Patroni can use Kubernetes objects in order to store the state of the cluster and manage the leader key. That makes it capable of operating Postgres in Kubernetes environment without any consistency store, namely, one doesn't need to run an extra Etcd deployment. There are two different type of Kubernetes objects Patroni can use to store the leader and the configuration keys, they are configured with the `kubernetes.use_endpoints` or `PATRONI_KUBERNETES_USE_ENDPOINTS` environment variable. Use Endpoints ------------- Despite the fact that this is the recommended mode, it is turned off by default for compatibility reasons. When it is on, Patroni stores the cluster configuration and the leader key in the `metadata: annotations` fields of the respective `Endpoints` it creates. Changing the leader is safer than when using `ConfigMaps`, since both the annotations, containing the leader information, and the actual addresses pointing to the running leader pod are updated simultaneously in one go. Use ConfigMaps -------------- In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps. Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint. To direct the traffic to the Postgres leader you need to configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration). Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps. Configuration ------------- Patroni Kubernetes :ref:`settings ` and :ref:`environment variables ` are described in the general chapters of the documentation. .. _kubernetes_role_values: Customize role label ^^^^^^^^^^^^^^^^^^^^ By default, Patroni will set corresponding labels on the pod it runs in based on node's role, such as ``role=primary``. The key and value of label can be customized by `kubernetes.role_label`, `kubernetes.leader_label_value`, `kubernetes.follower_label_value` and `kubernetes.standby_leader_label_value`. Note that if you migrate from default role labels to custom ones, you can reduce downtime by following migration steps: 1. Add a temporary label using original role value for the pod with `kubernetes.tmp_role_label` (like ``tmp_role``). Once pods are restarted they will get following labels set by Patroni: .. code:: YAML labels: cluster-name: foo role: primary tmp_role: primary 2. After all pods have been updated, modify the service selector to select the temporary label. .. code:: YAML selector: cluster-name: foo tmp_role: primary 3. Add your custom role label (e.g., set `kubernetes.leader_label_value=primary`). Once pods are restarted they will get following new labels set by Patroni: .. code:: YAML labels: cluster-name: foo role: primary tmp_role: primary 4. After all pods have been updated again, modify the service selector to use new role value. .. code:: YAML selector: cluster-name: foo role: primary 5. Finally, remove the temporary label from your configuration and update all pods. .. code:: YAML labels: cluster-name: foo role: primary Examples -------- - The `kubernetes `__ folder of the Patroni repository contains examples of the Docker image, and the Kubernetes manifest to test Patroni Kubernetes setup. Note that in the current state it will not be able to use PersistentVolumes because of permission issues. - You can find the full-featured Docker image that can use Persistent Volumes in the `Spilo Project `_. - There is also a `Helm chart `_ to deploy the Spilo image configured with Patroni running using Kubernetes. - In order to run your database clusters at scale using Patroni and Spilo, take a look at the `postgres-operator `_ project. It implements the operator pattern to manage Spilo clusters. patroni-4.0.4/docs/patroni_configuration.rst000066400000000000000000000355701472010352700212730ustar00rootroot00000000000000.. _patroni_configuration: Patroni configuration ===================== .. toctree:: :hidden: dynamic_configuration yaml_configuration ENVIRONMENT There are 3 types of Patroni configuration: - Global :ref:`dynamic configuration `. These options are stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes. Dynamic configuration can be set at any time using :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API `. If the options changed are not part of the startup configuration, they are applied asynchronously (upon the next wake up cycle) to every node, which gets subsequently reloaded. If the node requires a restart to apply the configuration (for `PostgreSQL parameters `__ with context postmaster, if their values have changed), a special flag ``pending_restart`` indicating this is set in the members.data JSON. Additionally, the node status indicates this by showing ``"restart_pending": true``. - Local :ref:`configuration file ` (patroni.yml). These options are defined in the configuration file and take precedence over dynamic configuration. ``patroni.yml`` can be changed and reloaded at runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing :ref:`patronictl_reload`. Local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence. - :ref:`Environment configuration `. It is possible to set/override some of the "Local" configuration parameters with environment variables. Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``). .. _important_configuration_rules: Important rules --------------- PostgreSQL parameters controlled by Patroni ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some of the PostgreSQL parameters **must hold the same values on the primary and the replicas**. For those, **values set either in the local patroni configuration files or via the environment variables take no effect**. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values: - **max_connections**: 100 - **max_locks_per_transaction**: 64 - **max_worker_processes**: 8 - **max_prepared_transactions**: 0 - **wal_level**: hot_standby - **track_commit_timestamp**: off For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, **Patroni restricts setting their values to the** :ref:`dynamic configuration `. - **max_wal_senders**: 10 - **max_replication_slots**: 10 - **wal_keep_segments**: 8 - **wal_keep_size**: 128MB These parameters are validated to ensure they are sane, or meet a minimum value. There are some other Postgres parameters controlled by Patroni: - **listen_addresses** - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable - **port** - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable - **cluster_name** - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable - **hot_standby: on** To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM `__ There also are some parameters like **postgresql.listen**, **postgresql.data_dir** that **can be set only locally**, i.e. in the Patroni :ref:`config file ` or via :ref:`configuration ` variable. In most cases the local configuration will override the dynamic configuration. When applying the local or dynamic configuration options, the following actions are taken: - The node first checks if there is a `postgresql.base.conf` file or if the ``custom_conf`` parameter is set. - If the ``custom_conf`` parameter is set, the file it specifies is used as the base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`. - If the ``custom_conf`` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and is used as the base configuration. - If there is no ``custom_conf`` nor `postgresql.base.conf`, the original `postgresql.conf` is renamed to `postgresql.base.conf` and used as the base configuration. - The dynamic options (with the exceptions above) are dumped into the `postgresql.conf` and an include is set in `postgresql.conf` to the base configuration (either `postgresql.base.conf` or the file at ``custom_conf``). Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present or not. - Some parameters that are essential for Patroni to manage the cluster are overridden using the command line. - If an option that requires restart is changed (we should look at the context in pg_settings and at the actual values of those options), a pending_restart flag is set on that node. This flag is reset on any restart. The parameters would be applied in the following order (run-time are given the highest priority): 1. load parameters from file `postgresql.base.conf` (or from a ``custom_conf`` file, if set) 2. load parameters from file `postgresql.conf` 3. load parameters from file `postgresql.auto.conf` 4. run-time parameter using `-o --name=value` This allows configuration for all the nodes (2), configuration for a specific node using ``ALTER SYSTEM`` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1). .. _shared_memory_gucs: PostgreSQL parameters that touch shared memory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ PostgreSQL has some parameters that determine the size of the shared memory used by them: - **max_connections** - **max_prepared_transactions** - **max_locks_per_transaction** - **max_wal_senders** - **max_worker_processes** Changing these parameters require a PostgreSQL restart to take effect, and their shared memory structures cannot be smaller on the standby nodes than on the primary node. As explained before, Patroni restrict changing their values through :ref:`dynamic configuration `, which usually consists of: 1. Applying changes through :ref:`patronictl_edit_config` (or via REST API ``/config`` endpoint) 2. Restarting nodes through :ref:`patronictl_restart` (or via REST API ``/restart`` endpoint) **Note:** please keep in mind that you should perform a restart of the PostgreSQL nodes through :ref:`patronictl_restart` command, or via REST API ``/restart`` endpoint. An attempt to restart PostgreSQL by restarting the Patroni daemon, e.g. by executing ``systemctl restart patroni``, can cause a failover to occur in the cluster, if you are restarting the primary node. However, as those settings manage shared memory, some extra care should be taken when restarting the nodes: * If you want to **increase** the value of any of those settings: 1. Restart all standbys first 2. Restart the primary after that * If you want to **decrease** the value of any of those settings: 1. Restart the primary first 2. Restart all standbys after that **Note:** if you attempt to restart all nodes in one go after **decreasing** the value of any of those settings, Patroni will ignore the change and restart the standby with the original setting value, thus requiring that you restart the standbys again later. Patroni does that to prevent the standby to enter in an infinite crash loop, because PostgreSQL quits with a `FATAL` message if you attempt to set any of those parameters to a value lower than what is visible in ``pg_controldata`` on the Standby node. In other words, we can only decrease the setting on the standby once its ``pg_controldata`` is up-to-date with the primary in regards to these changes on the primary. More information about that can be found at `PostgreSQL Administrator's Overview `__. Patroni configuration parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Also the following Patroni configuration options **can be changed only dynamically**: - **ttl**: 30 - **loop_wait**: 10 - **retry_timeouts**: 10 - **maximum_lag_on_failover**: 1048576 - **max_timelines_history**: 0 - **check_timeline**: false - **postgresql.use_slots**: true Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its run-time values. Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the leader is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid. .. _validate_generate_config: Configuration generation and validation --------------------------------------- Patroni provides command-line interfaces for a Patroni :ref:`local configuration ` generation and validation. Using the ``patroni`` executable you can: - Create a sample local Patroni configuration; - Create a Patroni configuration file for the locally running PostgreSQL instance (e.g. as a preparation step for the :ref:`Patroni integration `); - Validate a given Patroni configuration file. .. _generate_sample_config: Sample Patroni configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code:: text patroni --generate-sample-config [configfile] Description """"""""""" Generate a sample Patroni configuration file in ``yaml`` format. Parameter values are defined using the :ref:`Environment configuration `, otherwise, if not set, the defaults used in Patroni or the ``#FIXME`` string for the values that should be later defined by the user. Some default values are defined based on the local setup: - **postgresql.listen**: the IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``5432`` port. - **postgresql.connect_address**: the IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``5432`` port. - **postgresql.authentication.rewind**: is only defined if the PostgreSQL version can be defined from the binary and the version is 11 or later. - **restapi.listen**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. - **restapi.connect_address**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. Parameters """""""""" ``configfile`` - full path to the configuration file used to store the result. If not provided, the result is sent to ``stdout``. .. _generate_config: Patroni configuration for a running instance ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code:: text patroni --generate-config [--dsn DSN] [configfile] Description """"""""""" Generate a Patroni configuration in ``yaml`` format for the locally running PostgreSQL instance. Either the provided DSN (takes precedence) or PostgreSQL `environment variables `__ will be used for the PostgreSQL connection. If the password is not provided, it should be entered via prompt. All the non-internal GUCs defined in the source Postgres instance, independently if they were set through a configuration file, through the postmaster command-line, or through environment variables, will be used as the source for the following Patroni configuration parameters: - **scope**: ``cluster_name`` GUC value; - **postgresql.listen**: ``listen_addresses`` and ``port`` GUC values; - **postgresql.datadir**: ``data_directory`` GUC value; - **postgresql.parameters**: ``archive_command``, ``restore_command``, ``archive_cleanup_command``, ``recovery_end_command``, ``ssl_passphrase_command``, ``hba_file``, ``ident_file``, ``config_file`` GUC values; - **bootstrap.dcs**: all other gathered PostgreSQL GUCs. If ``scope``, ``postgresql.listen`` or ``postgresql.datadir`` is not set from the Postgres GUCs, the respective :ref:`Environment configuration ` value is used. Other rules applied for the values definition: - **name**: ``PATRONI_NAME`` environment variable value if set, otherwise the current machine's hostname. - **postgresql.bin_dir**: path to the Postgres binaries gathered from the running instance. - **postgresql.connect_address**: the IP address returned by ``gethostname`` call for the current machine's hostname and the port used for the instance connection or the ``port`` GUC value. - **postgresql.authentication.superuser**: the configuration used for the instance connection; - **postgresql.pg_hba**: the lines gathered from the source instance's ``hba_file``. - **postgresql.pg_ident**: the lines gathered from the source instance's ``ident_file``. - **restapi.listen**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. - **restapi.connect_address**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. Other parameters defined using :ref:`Environment configuration ` are also included into the configuration. Parameters """""""""" ``configfile`` Full path to the configuration file used to store the result. If not provided, result is sent to ``stdout``. ``dsn`` Optional DSN string for the local PostgreSQL instance to get GUC values from. Validate Patroni configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code:: text patroni --validate-config [configfile] [--ignore-listen-port | -i] Description """"""""""" Validate the given Patroni configuration and print the information about the failed checks. Parameters """""""""" ``configfile`` Full path to the configuration file to check. If not given or file does not exist, will try to read from the ``PATRONI_CONFIG_VARIABLE`` environment variable or, if not set, from the :ref:`Patroni environment variables `. ``--ignore-listen-port | -i`` Optional flag to ignore bind failures for ``listen`` ports that are already in use when validating the ``configfile``. patroni-4.0.4/docs/patronictl.rst000066400000000000000000001662501472010352700170470ustar00rootroot00000000000000.. _patronictl: patronictl ========== Patroni has a command-line interface named ``patronictl``, which is used basically to interact with Patroni's REST API and with the DCS. It is intended to make it easier to perform operations in the cluster, and can easily be used by humans or scripts. .. _patronictl_configuration: Configuration ------------- ``patronictl`` uses 3 sections of the configuration: - **ctl**: how to authenticate against the Patroni REST API, and how to validate the server identity. Refer to :ref:`ctl settings ` for more details; - **restapi**: how to authenticate against the Patroni REST API, and how to validate the server identity. Only used if ``ctl`` configuration is not enough. ``patronictl`` is mainly interested in ``restapi.authentication`` section (in case ``ctl.authentication`` is missing) and ``restapi.cafile`` setting (in case ``ctl.cacert`` is missing). Refer to :ref:`REST API settings ` for more details; - DCS (e.g. **etcd**): how to contact and authenticate against the DCS used by Patroni. Those configuration options can come either from environment variables or from a configuration file. Look for the above sections in :ref:`Environment Configuration Settings ` or :ref:`YAML Configuration Settings ` to understand how you can set the options for them through environment variables or through a configuration file. If you opt for using environment variables, it's a straight forward approach. Patronictl will read the environment variables and use their values. If you opt for using a configuration file, you have different ways to inform ``patronictl`` about the file to be used. By default ``patronictl`` will attempt to load a configuration file named ``patronictl.yaml``, which is expected to be found under either of these paths, according to your system: - Mac OS X: ``~/Library/Application Support/patroni`` - Mac OS X (POSIX): ``~/.patroni`` - Unix: ``~/.config/patroni`` - Unix (POSIX): ``~/.patroni`` - Windows (roaming): ``C:\Users\\AppData\Roaming\patroni`` - Windows (not roaming): ``C:\Users\\AppData\Local\patroni`` You can override that behavior either by: - Setting the environment variable ``PATRONICTL_CONFIG_FILE`` with the path to a custom configuration file; - Using the ``-c`` / ``--config-file`` command-line argument of ``patronictl`` with the path to a custom configuration file. .. note:: If you are running ``patronictl`` in the same host as ``patroni`` daemon is running, you may just use the same configuration file if it contains all the configuration sections required by ``patronictl``. .. _patronictl_usage: Usage ----- ``patronictl`` exposes several handy operations. This section is intended to describe each of them. Before jumping into each of the sub-commands of ``patronictl``, be aware that ``patronictl`` itself has the following command-line arguments: ``-c`` / ``--config-file`` As explained before, used to provide a path to a configuration file for ``patronictl``. ``-d`` / ``--dcs-url`` / ``--dcs`` Provide a connection string to the DCS used by Patroni. This argument can be used either to override the DCS and ``namespace`` settings from the ``patronictl`` configuration, or to define it if it's missing in the configuration. The value should be in the format ``DCS://HOST:PORT/NAMESPACE``, e.g. ``etcd3://localhost:2379/service`` to connect to etcd v3 running on ``localhost`` with Patroni cluster stored under ``service`` namespace. Any part that is missing in the argument value will be replaced with the value present in the configuration or with its default. ``-k`` / ``--insecure`` Flag to bypass validation of REST API server SSL certificate. This is the synopsis for running a command from the ``patronictl``: .. code:: text patronictl [ { -c | --config-file } CONFIG_FILE ] [ { -d | --dcs-url | --dcs } DCS_URL ] [ { -k | --insecure } ] SUBCOMMAND .. note:: This is the syntax for the synopsis: - Options between square brackets are optional; - Options between curly brackets represent a "choose one of set" operation; - Options with ``[, ... ]`` can be specified multiple times; - Things written in uppercase represent a literal that should be given a value to. We will use this same syntax when describing ``patronictl`` sub-commands in the following sub-sections. Also, when describing sub-commands in the following sub-sections, the commands' synopsis should be seen as a replacement for the ``SUBCOMMAND`` in the above synopsis. In the following sub-sections you can find a description of each command implemented by ``patronictl``. For sake of example, we will use the configuration files present in the GitHub repository of Patroni (files ``postgres0.yml``, ``postgres1.yml`` and ``postgres2.yml``). .. _patronictl_dsn: patronictl dsn ^^^^^^^^^^^^^^ .. _patronictl_dsn_synopsis: Synopsis """""""" .. code:: text dsn [ CLUSTER_NAME ] [ { { -r | --role } { leader | primary | standby-leader | replica | standby | any } | { -m | --member } MEMBER_NAME } ] [ --group CITUS_GROUP ] .. _patronictl_dsn_description: Description """"""""""" ``patronictl dsn`` gets the connection string for one member of the Patroni cluster. If multiple members match the parameters of this command, one of them will be chosen, prioritizing the primary node. .. _patronictl_dsn_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``-r`` / ``--role`` Choose a member that has the given role. Role can be one of: - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or - ``primary``: the leader of a regular Patroni cluster; or - ``standby-leader``: the leader of a standby Patroni cluster; or - ``replica``: a replica of a Patroni cluster; or - ``standby``: same as ``replica``; or - ``any``: any role. Same as omitting this parameter; or ``-m`` / ``--member`` Choose a member of the cluster with the given name. ``MEMBER_NAME`` is the name of the member. ``--group`` Choose a member that is part of the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. .. _patronictl_dsn_examples: Examples """""""" Get DSN of the primary node: .. code:: bash $ patronictl -c postgres0.yml dsn batman -r primary host=127.0.0.1 port=5432 Get DSN of the node named ``postgresql1``: .. code:: bash $ patronictl -c postgres0.yml dsn batman --member postgresql1 host=127.0.0.1 port=5433 .. _patronictl_edit_config: patronictl edit-config ^^^^^^^^^^^^^^^^^^^^^^ .. _patronictl_edit_config_synopsis: Synopsis """""""" .. code:: text edit-config [ CLUSTER_NAME ] [ --group CITUS_GROUP ] [ { -q | --quiet } ] [ { -s | --set } CONFIG="VALUE" [, ... ] ] [ { -p | --pg } PG_CONFIG="PG_VALUE" [, ... ] ] [ { --apply | --replace } CONFIG_FILE ] [ --force ] .. _patronictl_edit_config_description: Description """"""""""" ``patronictl edit-config`` changes the dynamic configuration of the cluster and updates the DCS with that. .. note:: When invoked through a TTY the command attempts to show a diff of the dynamic configuration through a pager. By default, it attempts to use either ``less`` or ``more``. If you want a different pager, set the ``PAGER`` environment variable with the desired one. .. _patronictl_edit_config_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Change dynamic configuration of the given Citus group. If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. ``CITUS_GROUP`` is the ID of the Citus group. ``-q`` / ``--quiet`` Flag to skip showing the configuration diff. ``-s`` / ``--set`` Set a given dynamic configuration option with a given value. ``CONFIG`` is the name of the dynamic configuration path in the YAML tree, with levels joined by ``.`` . ``VALUE`` is the value for ``CONFIG``. If it is ``null``, then ``CONFIG`` will be removed from the dynamic configuration. ``-p`` / ``--pg`` Set a given dynamic Postgres configuration option with the given value. It is essentially a shorthand for ``--s`` / ``--set`` with ``CONFIG`` prepended with ``postgresql.parameters.``. ``PG_CONFIG`` is the name of the Postgres configuration to be set. ``PG_VALUE`` is the value for ``PG_CONFIG``. If it is ``null``, then ``PG_CONFIG`` will be removed from the dynamic configuration. ``--apply`` Apply dynamic configuration from the given file. It is similar to specifying multiple ``-s`` / ``--set`` options, one for each configuration from ``CONFIG_FILE``. ``CONFIG_FILE`` is the path to a file containing the dynamic configuration to be applied, in YAML format. Use ``-`` if you want to read from ``stdin``. ``--replace`` Replace the dynamic configuration in the DCS with the dynamic configuration specified in the given file. ``CONFIG_FILE`` is the path to a file containing the new dynamic configuration to take effect, in YAML format. Use ``-`` if you want to read from ``stdin``. ``--force`` Flag to skip confirmation prompts when changing the dynamic configuration. Useful for scripts. .. _patronictl_edit_config_examples: Examples """""""" Change ``max_connections`` Postgres GUC: .. code:: diff patronictl -c postgres0.yml edit-config batman --pg max_connections="150" --force --- +++ @@ -1,6 +1,8 @@ loop_wait: 10 maximum_lag_on_failover: 1048576 postgresql: + parameters: + max_connections: 150 pg_hba: - host replication replicator 127.0.0.1/32 md5 - host all all 0.0.0.0/0 md5 Configuration changed Change ``loop_wait`` and ``ttl`` settings: .. code:: diff patronictl -c postgres0.yml edit-config batman --set loop_wait="15" --set ttl="45" --force --- +++ @@ -1,4 +1,4 @@ -loop_wait: 10 +loop_wait: 15 maximum_lag_on_failover: 1048576 postgresql: pg_hba: @@ -6,4 +6,4 @@ - host all all 0.0.0.0/0 md5 use_pg_rewind: true retry_timeout: 10 -ttl: 30 +ttl: 45 Configuration changed Remove ``maximum_lag_on_failover`` setting from dynamic configuration: .. code:: diff patronictl -c postgres0.yml edit-config batman --set maximum_lag_on_failover="null" --force --- +++ @@ -1,5 +1,4 @@ loop_wait: 10 -maximum_lag_on_failover: 1048576 postgresql: pg_hba: - host replication replicator 127.0.0.1/32 md5 Configuration changed .. _patronictl_failover: patronictl failover ^^^^^^^^^^^^^^^^^^^ .. _patronictl_failover_synopsis: Synopsis """""""" .. code:: text failover [ CLUSTER_NAME ] [ --group CITUS_GROUP ] --candidate CANDIDATE_NAME [ --force ] .. _patronictl_failover_description: Description """"""""""" ``patronictl failover`` performs a manual failover in the cluster. It is designed to be used when the cluster is not healthy, e.g.: - There is no leader; or - There is no synchronous standby available in a synchronous cluster. It also allows to fail over to an asynchronous node if synchronous mode is enabled. .. note:: Nothing prevents you from running ``patronictl failover`` in a healthy cluster. However, we recommend using ``patronictl switchover`` in those cases. .. warning:: Triggering a failover can cause data loss depending on how up-to-date the promoted replica is in comparison to the primary. .. _patronictl_failover_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Perform a failover in the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``--candidate`` The node to be promoted on failover. ``CANDIDATE_NAME`` is the name of the node to be promoted. ``--force`` Flag to skip confirmation prompts when performing the failover. Useful for scripts. .. _patronictl_failover_examples: Examples """""""" Fail over to node ``postgresql2``: .. code:: bash $ patronictl -c postgres0.yml failover batman --candidate postgresql2 --force Current cluster topology + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 3 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 3 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 3 | 0 | +-------------+----------------+---------+-----------+----+-----------+ 2023-09-12 11:52:27.50978 Successfully failed over to "postgresql2" + Cluster: batman (7277694203142172922) -+---------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+---------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Replica | stopped | | unknown | | postgresql1 | 127.0.0.1:5433 | Replica | running | 3 | 0 | | postgresql2 | 127.0.0.1:5434 | Leader | running | 3 | | +-------------+----------------+---------+---------+----+-----------+ .. _patronictl_flush: patronictl flush ^^^^^^^^^^^^^^^^ .. _patronictl_flush_synopsis: Synopsis """""""" .. code:: text flush CLUSTER_NAME [ MEMBER_NAME [, ... ] ] { restart | switchover } [ --group CITUS_GROUP ] [ { -r | --role } { leader | primary | standby-leader | replica | standby | any } ] [ --force ] .. _patronictl_flush_description: Description """"""""""" ``patronictl flush`` discards scheduled events, if any. .. _patronictl_flush_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. ``MEMBER_NAME`` Discard scheduled events for the given Patroni member(s). Multiple members can be specified. If no members are specified, all of them are considered. .. note:: Only used if discarding scheduled restart events. ``restart`` Discard scheduled restart events. ``switchover`` Discard scheduled switchover event. ``--group`` Discard scheduled events from the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-r`` / ``--role`` Discard scheduled events for members that have the given role. Role can be one of: - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or - ``primary``: the leader of a regular Patroni cluster; or - ``standby-leader``: the leader of a standby Patroni cluster; or - ``replica``: a replica of a Patroni cluster; or - ``standby``: same as ``replica``; or - ``any``: any role. Same as omitting this parameter. .. note:: Only used if discarding scheduled restart events. ``--force`` Flag to skip confirmation prompts when performing the flush. Useful for scripts. .. _patronictl_flush_examples: Examples """""""" Discard a scheduled switchover event: .. code:: bash $ patronictl -c postgres0.yml flush batman switchover --force Success: scheduled switchover deleted Discard scheduled restart of all standby nodes: .. code:: bash $ patronictl -c postgres0.yml flush batman restart -r replica --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+---------------------------+ | Member | Host | Role | State | TL | Lag in MB | Scheduled restart | +-------------+----------------+---------+-----------+----+-----------+---------------------------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | 2023-09-12T17:17:00+00:00 | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | +-------------+----------------+---------+-----------+----+-----------+---------------------------+ Success: flush scheduled restart for member postgresql1 Success: flush scheduled restart for member postgresql2 Discard scheduled restart of nodes ``postgresql0`` and ``postgresql1``: .. code:: bash $ patronictl -c postgres0.yml flush batman postgresql0 postgresql1 restart --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+---------------------------+ | Member | Host | Role | State | TL | Lag in MB | Scheduled restart | +-------------+----------------+---------+-----------+----+-----------+---------------------------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | 2023-09-12T17:17:00+00:00 | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | +-------------+----------------+---------+-----------+----+-----------+---------------------------+ Success: flush scheduled restart for member postgresql0 Success: flush scheduled restart for member postgresql1 .. _patronictl_history: patronictl history ^^^^^^^^^^^^^^^^^^ .. _patronictl_history_synopsis: Synopsis """""""" .. code:: text history [ CLUSTER_NAME ] [ --group CITUS_GROUP ] [ { -f | --format } { pretty | tsv | json | yaml } ] .. _patronictl_history_description: Description """"""""""" ``patronictl history`` shows a history of failover and switchover events from the cluster, if any. The following information is included in the output: ``TL`` Postgres timeline at which the event occurred. ``LSN`` Postgres LSN at which the event occurred. ``Reason`` Reason fetched from the Postgres ``.history`` file. ``Timestamp`` Time when the event occurred. ``New Leader`` Patroni member that has been promoted during the event. .. _patronictl_history_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Show history of events from the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. ``-f`` / ``--format`` How to format the list of events in the output. Format can be one of: - ``pretty``: prints history as a pretty table; or - ``tsv``: prints history as tabular information, with columns delimited by ``\t``; or - ``json``: prints history in JSON format; or - ``yaml``: prints history in YAML format. The default is ``pretty``. ``--force`` Flag to skip confirmation prompts when performing the flush. Useful for scripts. .. _patronictl_history_examples: Examples """""""" Show the history of events: .. code:: bash $ patronictl -c postgres0.yml history batman +----+----------+------------------------------+----------------------------------+-------------+ | TL | LSN | Reason | Timestamp | New Leader | +----+----------+------------------------------+----------------------------------+-------------+ | 1 | 24392648 | no recovery target specified | 2023-09-11T22:11:27.125527+00:00 | postgresql0 | | 2 | 50331864 | no recovery target specified | 2023-09-12T11:34:03.148097+00:00 | postgresql0 | | 3 | 83886704 | no recovery target specified | 2023-09-12T11:52:26.948134+00:00 | postgresql2 | | 4 | 83887280 | no recovery target specified | 2023-09-12T11:53:09.620136+00:00 | postgresql0 | +----+----------+------------------------------+----------------------------------+-------------+ Show the history of events in YAML format: .. code:: bash $ patronictl -c postgres0.yml history batman -f yaml - LSN: 24392648 New Leader: postgresql0 Reason: no recovery target specified TL: 1 Timestamp: '2023-09-11T22:11:27.125527+00:00' - LSN: 50331864 New Leader: postgresql0 Reason: no recovery target specified TL: 2 Timestamp: '2023-09-12T11:34:03.148097+00:00' - LSN: 83886704 New Leader: postgresql2 Reason: no recovery target specified TL: 3 Timestamp: '2023-09-12T11:52:26.948134+00:00' - LSN: 83887280 New Leader: postgresql0 Reason: no recovery target specified TL: 4 Timestamp: '2023-09-12T11:53:09.620136+00:00' .. _patronictl_list: patronictl list ^^^^^^^^^^^^^^^ .. _patronictl_list_synopsis: Synopsis """""""" .. code:: text list [ CLUSTER_NAME [, ... ] ] [ --group CITUS_GROUP ] [ { -e | --extended } ] [ { -t | --timestamp } ] [ { -f | --format } { pretty | tsv | json | yaml } ] [ { -W | { -w | --watch } TIME } ] .. _patronictl_list_description: Description """"""""""" ``patronictl list`` shows information about Patroni cluster and its members. The following information is included in the output: ``Cluster`` Name of the Patroni cluster. ``Member`` Name of the Patroni member. ``Host`` Host where the member is located. ``Role`` Current role of the member. Can be one among: * ``Leader``: the current leader of a regular Patroni cluster; or * ``Standby Leader``: the current leader of a Patroni standby cluster; or * ``Sync Standby``: a synchronous standby of a Patroni cluster with synchronous mode enabled; or * ``Replica``: a regular standby of a Patroni cluster. ``State`` Current state of Postgres in the Patroni member. Some examples among the possible states: * ``running``: if Postgres is currently up and running; * ``streaming``: if a replica and Postgres is currently streaming WALs from the primary node; * ``in archive recovery``: if a replica and Postgres is currently fetching WALs from the archive; * ``stopped``: if Postgres had been shut down; * ``crashed``: if Postgres has crashed. ``TL`` Current Postgres timeline in the Patroni member. ``Lag in MB`` Amount worth of replication lag in megabytes between the Patroni member and its upstream. Besides that, the following information may be included in the output: ``System identifier`` Postgres system identifier. .. note:: Shown in the table header. Only shown if output format is ``pretty``. ``Group`` Citus group ID. .. note:: Shown in the table header. Only shown if a Citus cluster. ``Pending restart`` ``*`` indicates that the node needs a restart for some Postgres configuration to take effect. An empty value indicates the node does not require a restart. .. note:: Shown as a member attribute. Shown if: - Printing in ``pretty`` or ``tsv`` format and with extended output enabled; or - If node requires a restart. ``Scheduled restart`` Timestamp at which a restart has been scheduled for the Postgres instance managed by the Patroni member. An empty value indicates there is no scheduled restart for the member. .. note:: Shown as a member attribute. Shown if: - Printing in ``pretty`` or ``tsv`` format and with extended output enabled; or - If node has a scheduled restart. ``Tags`` Contains tags set for the Patroni member. An empty value indicates that either no tags have been configured, or that they have been configured with default values. .. note:: Shown as a member attribute. Shown if: - Printing in ``pretty`` or ``tsv`` format and with extended output enabled; or - If node has any custom tags, or any default tags with non-default values. ``Scheduled switchover`` Timestamp at which a switchover has been scheduled for the Patroni cluster, if any. .. note:: Shown in the table footer. Only shown if there is a scheduled switchover, and output format is ``pretty``. ``Maintenance mode`` If the cluster monitoring is currently paused. .. note:: Shown in the table footer. Only shown if the cluster is paused, and output format is ``pretty``. .. _patronictl_list_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Show information about members from the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-e`` / ``--extended`` Show extended information. Force showing ``Pending restart``, ``Scheduled restart`` and ``Tags`` attributes, even if their value is empty. .. note:: Only applies to ``pretty`` and ``tsv`` output formats. ``-t`` / ``--timestamp`` Print timestamp before printing information about the cluster and its members. ``-f`` / ``--format`` How to format the list of events in the output. Format can be one of: - ``pretty``: prints history as a pretty table; or - ``tsv``: prints history as tabular information, with columns delimited by ``\t``; or - ``json``: prints history in JSON format; or - ``yaml``: prints history in YAML format. The default is ``pretty``. ``-W`` Automatically refresh information every 2 seconds. ``-w`` / ``--watch`` Automatically refresh information at the specified interval. ``TIME`` is the interval between refreshes, in seconds. .. _patronictl_list_examples: Examples """""""" Show information about the cluster in pretty format: .. code:: bash $ patronictl -c postgres0.yml list batman + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Show information about the cluster in pretty format with extended columns: .. code:: bash $ patronictl -c postgres0.yml list batman -e + Cluster: batman (7277694203142172922) -+-----------+----+-----------+-----------------+-------------------+------+ | Member | Host | Role | State | TL | Lag in MB | Pending restart | Scheduled restart | Tags | +-------------+----------------+---------+-----------+----+-----------+-----------------+-------------------+------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | | | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | | | | +-------------+----------------+---------+-----------+----+-----------+-----------------+-------------------+------+ Show information about the cluster in YAML format, with timestamp of execution: .. code:: bash $ patronictl -c postgres0.yml list batman -f yaml -t 2023-09-12 13:30:48 - Cluster: batman Host: 127.0.0.1:5432 Member: postgresql0 Role: Leader State: running TL: 5 - Cluster: batman Host: 127.0.0.1:5433 Lag in MB: 0 Member: postgresql1 Role: Replica State: streaming TL: 5 - Cluster: batman Host: 127.0.0.1:5434 Lag in MB: 0 Member: postgresql2 Role: Replica State: streaming TL: 5 .. _patronictl_pause: patronictl pause ^^^^^^^^^^^^^^^^ .. _patronictl_pause_synopsis: Synopsis """""""" .. code:: text pause [ CLUSTER_NAME ] [ --group CITUS_GROUP ] [ --wait ] .. _patronictl_pause_description: Description """"""""""" ``patronictl pause`` temporarily puts the Patroni cluster in maintenance mode and disables automatic failover. .. _patronictl_pause_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Pause the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. ``--wait`` Wait until all Patroni members are paused before returning control to the caller. .. _patronictl_pause_examples: Examples """""""" Put the cluster in maintenance mode, and wait until all nodes have been paused: .. code:: bash $ patronictl -c postgres0.yml pause batman --wait 'pause' request sent, waiting until it is recognized by all nodes Success: cluster management is paused .. _patronictl_query: patronictl query ^^^^^^^^^^^^^^^^ .. _patronictl_query_synopsis: Synopsis """""""" .. code:: text query [ CLUSTER_NAME ] [ --group CITUS_GROUP ] [ { { -r | --role } { leader | primary | standby-leader | replica | standby | any } | { -m | --member } MEMBER_NAME } ] [ { -d | --dbname } DBNAME ] [ { -U | --username } USERNAME ] [ --password ] [ --format { pretty | tsv | json | yaml } ] [ { { -f | --file } FILE_NAME | { -c | --command } SQL_COMMAND } ] [ --delimiter ] [ { -W | { -w | --watch } TIME } ] .. _patronictl_query_description: Description """"""""""" ``patronictl query`` executes a SQL command or script against a member of the Patroni cluster. .. _patronictl_query_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Query the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-r`` / ``--role`` Choose a member that has the given role. Role can be one of: - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or - ``primary``: the leader of a regular Patroni cluster; or - ``standby-leader``: the leader of a standby Patroni cluster; or - ``replica``: a replica of a Patroni cluster; or - ``standby``: same as ``replica``; or - ``any``: any role. Same as omitting this parameter. ``-m`` / ``--member`` Choose a member that has the given name. ``MEMBER_NAME`` is the name of the member to be picked. ``-d`` / ``--dbname`` Database to connect and run the query. ``DBNAME`` is the name of the database. If not given, defaults to ``USERNAME``. ``-U`` / ``--username`` User to connect to the database. ``USERNAME`` name of the user. If not given, defaults to the operating system user running ``patronictl query``. ``--password`` Prompt for the password of the connecting user. As Patroni uses ``libpq``, alternatively you can create a ``~/.pgpass`` file or set the ``PGPASSWORD`` environment variable. ``--format`` How to format the output of the query. Format can be one of: - ``pretty``: prints query output as a pretty table; or - ``tsv``: prints query output as tabular information, with columns delimited by ``\t``; or - ``json``: prints query output in JSON format; or - ``yaml``: prints query output in YAML format. The default is ``tsv``. ``-f`` / ``--file`` Use a file as source of commands to run queries. ``FILE_NAME`` is the path to the source file. ``-c`` / ``--command`` Run the given SQL command in the query. ``SQL_COMMAND`` is the SQL command to be executed. ``--delimiter`` The delimiter when printing information in ``tsv`` format, or ``\t`` if omitted. ``-W`` Automatically re-run the query every 2 seconds. ``-w`` / ``--watch`` Automatically re-run the query at the specified interval. ``TIME`` is the interval between re-runs, in seconds. .. _patronictl_query_examples: Examples """""""" Run a SQL command as ``postgres`` user, and ask for its password: .. code:: bash $ patronictl -c postgres0.yml query batman -U postgres --password -c "SELECT now()" Password: now 2023-09-12 18:10:53.228084+00:00 Run a SQL command as ``postgres`` user, and take password from ``libpq`` environment variable: .. code:: bash $ PGPASSWORD=patroni patronictl -c postgres0.yml query batman -U postgres -c "SELECT now()" now 2023-09-12 18:11:37.639500+00:00 Run a SQL command and print in ``pretty`` format every 2 seconds: .. code:: bash $ patronictl -c postgres0.yml query batman -c "SELECT now()" --format pretty -W +----------------------------------+ | now | +----------------------------------+ | 2023-09-12 18:12:16.716235+00:00 | +----------------------------------+ +----------------------------------+ | now | +----------------------------------+ | 2023-09-12 18:12:18.732645+00:00 | +----------------------------------+ +----------------------------------+ | now | +----------------------------------+ | 2023-09-12 18:12:20.750573+00:00 | +----------------------------------+ Run a SQL command on database ``test`` and print the output in YAML format: .. code:: bash $ patronictl -c postgres0.yml query batman -d test -c "SELECT now() AS column_1, 'test' AS column_2" --format yaml - column_1: 2023-09-12 18:14:22.052060+00:00 column_2: test Run a SQL command on member ``postgresql2``: .. code:: bash $ patronictl -c postgres0.yml query batman -m postgresql2 -c "SHOW port" port 5434 Run a SQL command on any of the standbys: .. code:: bash $ patronictl -c postgres0.yml query batman -r replica -c "SHOW port" port 5433 .. _patronictl_reinit: patronictl reinit ^^^^^^^^^^^^^^^^^ .. _patronictl_reinit_synopsis: Synopsis """""""" .. code:: text reinit CLUSTER_NAME [ MEMBER_NAME [, ... ] ] [ --group CITUS_GROUP ] [ --wait ] [ --force ] .. _patronictl_reinit_description: Description """"""""""" ``patronictl reinit`` rebuilds a Postgres standby instance managed by a replica member of the Patroni cluster. .. _patronictl_reinit_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. ``MEMBER_NAME`` Name of the replica member for which the Postgres instance will be rebuilt. Multiple replica members can be specified. If no members are specified, the command does nothing. ``--group`` Rebuild a replica member of the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``--wait`` Wait until the reinitialization of the Postgres standby node(s) is finished. ``--force`` Flag to skip confirmation prompts when rebuilding Postgres standby instances. Useful for scripts. .. _patronictl_reinit_examples: Examples """""""" Request a rebuild of all replica members of the Patroni cluster and immediately return control to the caller: .. code:: bash $ patronictl -c postgres0.yml reinit batman postgresql1 postgresql2 --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Success: reinitialize for member postgresql1 Success: reinitialize for member postgresql2 Request a rebuild of ``postgresql2`` and wait for it to complete: .. code:: bash $ patronictl -c postgres0.yml reinit batman postgresql2 --wait --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Success: reinitialize for member postgresql2 Waiting for reinitialize to complete on: postgresql2 Reinitialize is completed on: postgresql2 .. _patronictl_reload: patronictl reload ^^^^^^^^^^^^^^^^^ .. _patronictl_reload_synopsis: Synopsis """""""" .. code:: text reload CLUSTER_NAME [ MEMBER_NAME [, ... ] ] [ --group CITUS_GROUP ] [ { -r | --role } { leader | primary | standby-leader | replica | standby | any } ] [ --force ] .. _patronictl_reload_description: Description """"""""""" ``patronictl reload`` requests a reload of local configuration for one or more Patroni members. It also triggers ``pg_ctl reload`` on the managed Postgres instance, even if nothing has changed. .. _patronictl_reload_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. ``MEMBER_NAME`` Request a reload of local configuration for the given Patroni member(s). Multiple members can be specified. If no members are specified, all of them are considered. ``--group`` Request a reload of members of the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-r`` / ``--role`` Select members that have the given role. Role can be one of: - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or - ``primary``: the leader of a regular Patroni cluster; or - ``standby-leader``: the leader of a standby Patroni cluster; or - ``replica``: a replica of a Patroni cluster; or - ``standby``: same as ``replica``; or - ``any``: any role. Same as omitting this parameter. ``--force`` Flag to skip confirmation prompts when requesting a reload of the local configuration. Useful for scripts. .. _patronictl_reload_examples: Examples """""""" Request a reload of the local configuration of all members of the Patroni cluster: .. code:: bash $ patronictl -c postgres0.yml reload batman --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Reload request received for member postgresql0 and will be processed within 10 seconds Reload request received for member postgresql1 and will be processed within 10 seconds Reload request received for member postgresql2 and will be processed within 10 seconds .. _patronictl_remove: patronictl remove ^^^^^^^^^^^^^^^^^ .. _patronictl_remove_synopsis: Synopsis """""""" .. code:: text remove CLUSTER_NAME [ --group CITUS_GROUP ] [ { -f | --format } { pretty | tsv | json | yaml } ] .. _patronictl_remove_description: Description """"""""""" ``patronictl remove`` removes information of the cluster from the DCS. It is an interactive action. .. warning:: This operation will destroy the information of the Patroni cluster from the DCS. .. _patronictl_remove_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. ``--group`` Remove information about the Patroni cluster related with the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-f`` / ``--format`` How to format the list of members in the output when prompting for confirmation. Format can be one of: - ``pretty``: prints members as a pretty table; or - ``tsv``: prints members as tabular information, with columns delimited by ``\t``; or - ``json``: prints members in JSON format; or - ``yaml``: prints members in YAML format. The default is ``pretty``. .. _patronictl_remove_examples: Examples """""""" Remove information about Patroni cluster ``batman`` from the DCS: .. code:: bash $ patronictl -c postgres0.yml remove batman + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Please confirm the cluster name to remove: batman You are about to remove all information in DCS for batman, please type: "Yes I am aware": Yes I am aware This cluster currently is healthy. Please specify the leader name to continue: postgresql0 .. _patronictl_restart: patronictl restart ^^^^^^^^^^^^^^^^^^ .. _patronictl_restart_synopsis: Synopsis """""""" .. code:: text restart CLUSTER_NAME [ MEMBER_NAME [, ...] ] [ --group CITUS_GROUP ] [ { -r | --role } { leader | primary | standby-leader | replica | standby | any } ] [ --any ] [ --pg-version PG_VERSION ] [ --pending ] [ --timeout TIMEOUT ] [ --scheduled TIMESTAMP ] [ --force ] .. _patronictl_restart_description: Description """"""""""" ``patronictl restart`` requests a restart of the Postgres instance managed by a member of the Patroni cluster. The restart can be performed immediately or scheduled for later. .. _patronictl_restart_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. ``--group`` Restart the Patroni cluster related with the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-r`` / ``--role`` Choose members that have the given role. Role can be one of: - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or - ``primary``: the leader of a regular Patroni cluster; or - ``standby-leader``: the leader of a standby Patroni cluster; or - ``replica``: a replica of a Patroni cluster; or - ``standby``: same as ``replica``; or - ``any``: any role. Same as omitting this parameter. ``--any`` Restart a single random node among the ones which match the given filters. ``--pg-version`` Select only members which version of the managed Postgres instance is older than the given version. ``PG_VERSION`` is the Postgres version to be compared. ``--pending`` Select only members which are flagged as ``Pending restart``. ``timeout`` Abort the restart if it takes more than the specified timeout, and fail over to a replica if the issue is on the primary. ``TIMEOUT`` is the amount of seconds to wait before aborting the restart. ``--scheduled`` Schedule a restart to occur at the given timestamp. ``TIMESTAMP`` is the timestamp when the restart should occur. Specify it in unambiguous format, preferably with time zone. You can also use the literal ``now`` for the restart to be executed immediately. ``--force`` Flag to skip confirmation prompts when requesting the restart operations. Useful for scripts. .. _patronictl_restart_examples: Examples """""""" Restart all members of the cluster immediately: .. code:: bash $ patronictl -c postgres0.yml restart batman --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Success: restart on member postgresql0 Success: restart on member postgresql1 Success: restart on member postgresql2 Restart a random member of the cluster immediately: .. code:: bash $ patronictl -c postgres0.yml restart batman --any --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Success: restart on member postgresql1 Schedule a restart to occur at ``2023-09-13T18:00-03:00``: .. code:: bash $ patronictl -c postgres0.yml restart batman --scheduled 2023-09-13T18:00-03:00 --force + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Success: restart scheduled on member postgresql0 Success: restart scheduled on member postgresql1 Success: restart scheduled on member postgresql2 .. _patronictl_resume: patronictl resume ^^^^^^^^^^^^^^^^^ .. _patronictl_resume_synopsis: Synopsis """""""" .. code:: text resume [ CLUSTER_NAME ] [ --group CITUS_GROUP ] [ --wait ] .. _patronictl_resume_description: Description """"""""""" ``patronictl resume`` takes the Patroni cluster out of maintenance mode and re-enables automatic failover. .. _patronictl_resume_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Resume the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. ``--wait`` Wait until all Patroni members are unpaused before returning control to the caller. .. _patronictl_resume_examples: Examples """""""" Put the cluster out of maintenance mode: .. code:: bash $ patronictl -c postgres0.yml resume batman --wait 'resume' request sent, waiting until it is recognized by all nodes Success: cluster management is resumed .. _patronictl_show_config: patronictl show-config ^^^^^^^^^^^^^^^^^^^^^^ .. _patronictl_show_config_synopsis: Synopsis """""""" .. code:: text show-config [ CLUSTER_NAME ] [ --group CITUS_GROUP ] .. _patronictl_show_config_description: Description """"""""""" ``patronictl show-config`` shows the dynamic configuration of the cluster that is stored in the DCS. .. _patronictl_show_config_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Show dynamic configuration of the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. .. _patronictl_show_config_examples: Examples """""""" Show dynamic configuration of cluster ``batman``: .. code:: bash $ patronictl -c postgres0.yml show-config batman loop_wait: 10 postgresql: parameters: max_connections: 250 pg_hba: - host replication replicator 127.0.0.1/32 md5 - host all all 0.0.0.0/0 md5 use_pg_rewind: true retry_timeout: 10 ttl: 30 .. _patronictl_switchover: patronictl switchover ^^^^^^^^^^^^^^^^^^^^^ .. _patronictl_switchover_synopsis: Synopsis """""""" .. code:: text switchover [ CLUSTER_NAME ] [ --group CITUS_GROUP ] [ { --leader | --primary } LEADER_NAME ] --candidate CANDIDATE_NAME [ --force ] .. _patronictl_switchover_description: Description """"""""""" ``patronictl switchover`` performs a switchover in the cluster. It is designed to be used when the cluster is healthy, e.g.: - There is a leader; - There are synchronous standbys available in a synchronous cluster. .. note:: If your cluster is unhealthy you might be interested in ``patronictl failover`` instead. .. _patronictl_switchover_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Perform a switchover in the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``--leader`` / ``--primary`` Indicate who is the leader to be demoted at switchover time. ``LEADER_NAME`` should match the name of the current leader in the cluster. ``--candidate`` The node to be promoted on switchover, and take the primary role. ``CANDIDATE_NAME`` is the name of the node to be promoted. ``--scheduled`` Schedule a switchover to occur at the given timestamp. ``TIMESTAMP`` is the timestamp when the switchover should occur. Specify it in unambiguous format, preferably with time zone. You can also use the literal ``now`` for the switchover to be executed immediately. ``--force`` Flag to skip confirmation prompts when performing the switchover. Useful for scripts. .. _patronictl_switchover_examples: Examples """""""" Switch over with node ``postgresql2``: .. code:: bash $ patronictl -c postgres0.yml switchover batman --leader postgresql0 --candidate postgresql2 --force Current cluster topology + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | +-------------+----------------+---------+-----------+----+-----------+ 2023-09-13 14:15:23.07497 Successfully switched over to "postgresql2" + Cluster: batman (7277694203142172922) -+---------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+---------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Replica | stopped | | unknown | | postgresql1 | 127.0.0.1:5433 | Replica | running | 6 | 0 | | postgresql2 | 127.0.0.1:5434 | Leader | running | 6 | | +-------------+----------------+---------+---------+----+-----------+ Schedule a switchover between ``postgresql0`` and ``postgresql2`` to occur at ``2023-09-13T18:00:00-03:00``: .. code:: bash $ patronictl -c postgres0.yml switchover batman --leader postgresql0 --candidate postgresql2 --scheduled 2023-09-13T18:00-03:00 --force Current cluster topology + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 8 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 8 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 8 | 0 | +-------------+----------------+---------+-----------+----+-----------+ 2023-09-13 14:18:11.20661 Switchover scheduled + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +-------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 8 | | | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 8 | 0 | | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 8 | 0 | +-------------+----------------+---------+-----------+----+-----------+ Switchover scheduled at: 2023-09-13T18:00:00-03:00 from: postgresql0 to: postgresql2 .. _patronictl_topology: patronictl topology ^^^^^^^^^^^^^^^^^^^ .. _patronictl_topology_synopsis: Synopsis """""""" .. code:: text topology [ CLUSTER_NAME [, ... ] ] [ --group CITUS_GROUP ] [ { -W | { -w | --watch } TIME } ] .. _patronictl_topology_description: Description """"""""""" ``patronictl topology`` shows information about the Patroni cluster and its members with a tree view approach. The following information is included in the output: ``Cluster`` Name of the Patroni cluster. .. note:: Shown in the table header. ``System identifier`` Postgres system identifier. .. note:: Shown in the table header. ``Member`` Name of the Patroni member. .. note:: Information in this column is shown as a tree view of members in terms of replication connections. ``Host`` Host where the member is located. ``Role`` Current role of the member. Can be one among: * ``Leader``: the current leader of a regular Patroni cluster; or * ``Standby Leader``: the current leader of a Patroni standby cluster; or * ``Sync Standby``: a synchronous standby of a Patroni cluster with synchronous mode enabled; or * ``Replica``: a regular standby of a Patroni cluster. ``State`` Current state of Postgres in the Patroni member. Some examples among the possible states: * ``running``: if Postgres is currently up and running; * ``streaming``: if a replica and Postgres is currently streaming WALs from the primary node; * ``in archive recovery``: if a replica and Postgres is currently fetching WALs from the archive; * ``stopped``: if Postgres had been shut down; * ``crashed``: if Postgres has crashed. ``TL`` Current Postgres timeline in the Patroni member. ``Lag in MB`` Amount worth of replication lag in megabytes between the Patroni member and its upstream. Besides that, the following information may be included in the output: ``Group`` Citus group ID. .. note:: Shown in the table header. Only shown if a Citus cluster. ``Pending restart`` ``*`` indicates the node needs a restart for some Postgres configuration to take effect. An empty value indicates the node does not require a restart. .. note:: Shown as a member attribute. Shown if node requires a restart. ``Scheduled restart`` Timestamp at which a restart has been scheduled for the Postgres instance managed by the Patroni member. An empty value indicates there is no scheduled restart for the member. .. note:: Shown as a member attribute. Shown if node has a scheduled restart. ``Tags`` Contains tags set for the Patroni member. An empty value indicates that either no tags have been configured, or that they have been configured with default values. .. note:: Shown as a member attribute. Shown if node has any custom tags, or any default tags with non-default values. ``Scheduled switchover`` Timestamp at which a switchover has been scheduled for the Patroni cluster, if any. .. note:: Shown in the table footer. Only shown if there is a scheduled switchover. ``Maintenance mode`` If the cluster monitoring is currently paused. .. note:: Shown in the table footer. Only shown if the cluster is paused. .. _patronictl_topology_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. ``--group`` Show information about members from the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. ``-W`` Automatically refresh information every 2 seconds. ``-w`` / ``--watch`` Automatically refresh information at the specified interval. ``TIME`` is the interval between refreshes, in seconds. .. _patronictl_topology_examples: Examples """""""" Show topology of the cluster ``batman`` -- ``postgresql1`` and ``postgresql2`` are replicating from ``postgresql0``: .. code:: bash $ patronictl -c postgres0.yml topology batman + Cluster: batman (7277694203142172922) ---+-----------+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +---------------+----------------+---------+-----------+----+-----------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 8 | | | + postgresql1 | 127.0.0.1:5433 | Replica | streaming | 8 | 0 | | + postgresql2 | 127.0.0.1:5434 | Replica | streaming | 8 | 0 | +---------------+----------------+---------+-----------+----+-----------+ .. _patronictl_version: patronictl version ^^^^^^^^^^^^^^^^^^ .. _patronictl_version_synopsis: Synopsis """""""" .. code:: text version [ CLUSTER_NAME [, ... ] ] [ MEMBER_NAME [, ... ] ] [ --group CITUS_GROUP ] .. _patronictl_version_description: Description """"""""""" ``patronictl version`` gets the version of ``patronictl`` application. Besides that it may also include version information about Patroni clusters and their members. .. _patronictl_version_parameters: Parameters """""""""" ``CLUSTER_NAME`` Name of the Patroni cluster. ``MEMBER_NAME`` Name of the member of the Patroni cluster. ``--group`` Consider a Patroni cluster with the given Citus group. ``CITUS_GROUP`` is the ID of the Citus group. .. _patronictl_version_examples: Examples """""""" Get version of ``patronictl`` only: .. code:: bash $ patronictl -c postgres0.yml version patronictl version 4.0.0 Get version of ``patronictl`` and of all members of cluster ``batman``: .. code:: bash $ patronictl -c postgres0.yml version batman patronictl version 4.0.0 postgresql0: Patroni 4.0.0 PostgreSQL 16.4 postgresql1: Patroni 4.0.0 PostgreSQL 16.4 postgresql2: Patroni 4.0.0 PostgreSQL 16.4 Get version of ``patronictl`` and of members ``postgresql1`` and ``postgresql2`` of cluster ``batman``: .. code:: bash $ patronictl -c postgres0.yml version batman postgresql1 postgresql2 patronictl version 4.0.0 postgresql1: Patroni 4.0.0 PostgreSQL 16.4 postgresql2: Patroni 4.0.0 PostgreSQL 16.4 patroni-4.0.4/docs/pause.rst000066400000000000000000000054001472010352700157720ustar00rootroot00000000000000.. _pause: Pause/Resume mode for the cluster ================================= The goal -------- Under certain circumstances Patroni needs to temporarily step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for reasons unknown to Patroni, some nodes can be even temporarily promoted, violating the assumption of running only one primary. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker. The implementation ------------------ When Patroni runs in a paused mode, it does not change the state of PostgreSQL, except for the following cases: - For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running. - For the Postgres primary with the leader lock Patroni updates the lock. If the node with the leader lock stops being the primary (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back. - Manual unscheduled restart, manual unscheduled failover/switchover and reinitialize are allowed. No scheduled action is allowed. Manual switchover is only allowed if the node to switch over to is specified. - If 'parallel' primaries are detected by Patroni, it emits a warning, but does not demote the primary without the leader lock. - If there is no leader lock in the cluster, the running primary acquires the lock. If there is more than one primary node, then the first primary to acquire the lock wins. If there are no primary altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old primary has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one. - When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not try to stop the Postgres instance it is managing. - Patroni will not try to remove replication slots that don't represent the other cluster member or are not listed in the configuration of the permanent slots. User guide ---------- ``patronictl`` supports :ref:`pause ` and :ref:`resume ` commands. One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}`` patroni-4.0.4/docs/releases.rst000066400000000000000000005537071472010352700165020ustar00rootroot00000000000000.. _releases: Release notes ============= Version 4.0.4 ------------- Released 2024-11-22 **Stability improvements** - Add compatibility with the ``py-consul`` module (Alexander Kukushkin) ``python-consul`` module is unmaintained for a long time, while ``py-consul`` is the official replacement. Backward compatibility with python-consul is retained. - Add compatibility with the ``prettytable>=3.12.0`` module (Alexander Kukushkin) Address deprecation warnings. - Compatibility with the ``ydiff==1.4.2`` module (Alexander Kukushkin) Fix compatibility issues for the latest version, constrain version in ``requirements.txt``, and introduce latest version compatibility test. **Bugfixes** - Run ``on_role_change`` callback after a failed primary recovery (Polina Bungina, Alexander Kukushkin) Additionally run ``on_role_change`` callback for a primary that failed to start after a crash to increase chances the callback is executed, even if the further start as a replica fails. - Fix a thread leak in ``patronictl list -W`` (Alexander Kukushkin) Cache DCS instance object to avoid thread leak. - Ensure only supported parameters are written to the connection string (Alexander Kukushkin) Patroni used to pass parameters introduced in newer versions to the connection string, which had been leading to connection errors. Version 4.0.3 ------------- Released 2024-10-18 **Bugfixes** - Disable ``pgaudit`` when creating users not to expose password (kviset) Patroni was logging ``superuser``, ``replication``, and ``rewind`` passwords on their creation when ``pgaudit`` extension was enabled. - Fix issue with mixed setups: primary on pre-Patroni v4 and replicas on v4+ (Alexander Kukushkin) Use ``xlog_location`` extracted from ``/members`` key instead of trying to get a member's slot position from ``/status`` key if Patroni version running on the leader is pre-4.0.0. Not doing so has been causing WALs accumulation on replicas. - Do not ignore valid PostgreSQL GUCs that don't have Patroni validator (Polina Bungina) Still check against ``postgres --describe-config`` if a GUC does not have a Patroni validator but is, in fact, a valid GUC. **Improvements** - Recheck annotations on 409 status code when reading leader object in K8s (Alexander Kukushkin) Avoid an additional update if ``PATCH`` request was canceled by Patroni, while the request successfully updated the target. - Add support of ``sslnegotiation`` client-side connection option (Alexander Kukushkin) ``sslnegotiation`` was added to the final PostgreSQL 17 release. Version 4.0.2 ------------- Released 2024-09-17 **Bugfixes** - Handle exceptions while discovering configuration validation files (Alexander Kukushkin) Skip directories for which Patroni does not have sufficient permissions to perform list operations. - Make sure inactive hot physical replication slots don't hold ``xmin`` (Alexander Kukushkin, Polina Bungina) Since version 3.2.0 Patroni creates physical replication slots for all members on replicas and periodically moves them forward using ``pg_replication_slot_advance()`` function. However if for any reason ``hot_standby_feedback`` is enabled and the primary is demoted to replica, the now inactive slots have ``NOT NULL`` ``xmin`` value propagated back to the new primary. This results in ``xmin`` horizon not being moved forward and vacuum not being able to clean up dead tuples. With this fix, Patroni recreates the physical replication slots that are supposed to be inactive but have ``NOT NULL`` ``xmin`` value. - Fix unhandled ``DCSError`` during the startup phase (Waynerv) Ensure DCS connectivity before trying to check the uniqueness of the node name. - Explicitly include ``CMDLINE_OPTIONS`` GUCs when querying ``pg_settings`` (Alexander Kukushkin) Make sure all GUCs that are passed to postmaster as command line parameters are restored when Patroni is joining a running standby. This is a follow-up for the bug fixed in Patroni 3.2.2. - Fix bug in ``synchronous_standby_names`` quotting logic (Alexander Kukushkin) According to PostgreSQL documentation, ``ANY`` and ``FIRST`` keywords are supposed to be double-quoted, which Patroni did not do before. - Fix keepalive connection out-of-range issue (hadizamani021) Ensure that ``keepalive`` option value calculated based on the ``ttl`` set does not exceed the maximum allowed value for the current platform. Version 4.0.1 ------------- Released 2024-08-30 **Bugfix** - Patroni was creating unnecessary replication slots for itself (Alexander Kukushkin) It was happening if ``name`` contains upper-case or special characters. Version 4.0.0 ------------- Released 2024-08-29 .. warning:: - This version completes work on getting rid of the "master" term, in favor of "primary". This means a couple of breaking changes, please read the release notes carefully. Upgrading to the Patroni 4+ will work reliably only if you run Patroni 3.1.0 or newer. Upgrading from an older version directly to 4+ is possible but may lead to unexpected behavior if the primary fails while the rest of the nodes are running on other Patroni versions. **Breaking changes** - The following breaking changes were introduced when getting rid of the non-inclusive "master" term in the Patroni code: - On Kubernetes, Patroni by default will set ``role`` label to ``primary``. In case if you want to keep the old behavior and avoid downtime or lengthy complex migrations, you can configure parameters ``kubernetes.leader_label_value`` and ``kubernetes.standby_leader_label_value`` to ``master``. Read more :ref:`here `. - Patroni role is written to DCS as ``primary`` instead of ``master``. - Patroni role returned by Patroni REST API has been changed from ``master`` to ``primary``. - Patroni REST API no longer accepts ``role=master`` in requests to ``/switchover``, ``/failover``, ``/restart`` endpoints. - ``/metrics`` REST API endpoint will no longer report ``patroni_master`` metric. - ``patronictl`` no longer accepts ``--master`` option for any command. ``--leader`` or ``--primary`` options should be used instead. - ``no_master`` option in the declarative configuration of custom replica creation methods is no longer treated as a special option, please use ``no_leader`` instead. - ``patroni_wale_restore`` script doesn't accept ``--no_master`` option anymore. - ``patroni_barman`` script doesn't accept ``--role=master`` option anymore. - All callback scripts are executed with ``role=primary`` option passed instead of ``role=master``. - ``patronictl failover`` does not accept ``--leader`` option that was deprecated since Patroni 3.2.0. - User creation functionality (``bootstrap.users`` configuration section) deprecated since Patroni 3.2.0 has been removed. **New features** - Quorum-based failover (Ants Aasma, Alexander Kukushkin) The feature implements quorum-based synchronous replication (available from PostgreSQL v10) which helps to reduce worst-case latencies, even during normal operation, as a higher latency of replicating to one standby can be compensated by other standbys. Patroni implements additional safeguards to prevent any user-visible data loss by choosing a failover candidate based on the latest transaction received. - Register Citus secondaries in ``pg_dist_node`` (Alexander Kukushkin) Patroni now maintains the list of nodes with ``role==replica``, ``state==running`` and without ``noloadbalance`` :ref:`tag ` in ``pg_dist_node``. - Configurable retention of members' replication slots (Alexander Kukushkin) Implements support of ``member_slots_ttl`` global configuration parameter that controls for how long member replication slots should be kept around when the member key is absent. - Make permissions of log files created by Patroni configurable (Alexander Kukushkin) Allows to set specific permissions for log files created by Patroni. If not specified, permissions are set based on the current ``umask`` value. - Compatibility with PostgreSQL 17 beta3 (Alexander Kukushkin) GUC's validator rules were extended. Patroni handles all the new auxiliary backends during shutdown and sets ``dbname`` in ``primary_conninfo``, as it is required for logical replication slots synchronization. - Implement ``--ignore-listen-port`` option for Patroni config validation (Sahil Naphade) Make it possible to ignore already bound ports when running ``patroni --validate-config``. **Improvements** - Make ``wal_log_hints`` configurable (Paul_Kim) Allows to avoid the overhead of ``wal_log_hints`` configuration being enabled in case ``use_pg_rewind`` is set to ``off``. - Log ``pg_basebackup`` command in ``DEBUG`` level (Waynerv) Facilitates failed initialization debugging. **Bugfixes** - Advance permanent slots for cascading nodes while in failsafe (Alexander Kukushkin) Ensure that slots for cascading replicas are properly advanced on the primary when failsafe mode is activated. It is done by extending replicas response on ``POST /failsafe`` REST API request with their ``xlog_location``. - Don't let the current node be chosen as synchronous (Alexander Kukushkin) There may be "something" streaming from the current primary node with ``application_name`` that matches the name of the current primary. Patroni was not properly handling this situation, which could end up in the primary being declared as a synchronous node and consequently was blocking switchovers. - Ignore ``restapi.allowlist_include_members`` for POST /failsafe (Alexander Kukushkin) - Improve GUCs validation (Polina Bungina) Due to additional validation through running ``postgres --describe-config`` command, it was previously not possible to set GUCs not listed there through Patroni configuration. This limitation is now removed. - Add line with ``localhost`` to ``.pgpass`` file when unix sockets are detected (Alexander Kukushkin) Patroni will add an additional line to ``.pgpass`` file if ``host`` parameter specified starts with ``/`` character. This allows to cover a corner case when ``host`` matches the default socket directory path. - Fix logging issues (Waynerv) Defined proper request URL in failsafe handling logs and fixed the order of timestamps in postmaster check log. Version 3.3.2 ------------- Released 2024-07-11 **Bugfixes** - Fix plain Postgres synchronous replication mode (Israel Barth Rubio) Since ``synchronous_mode`` was introduced to Patroni, the plain Postgres synchronous replication was not working. With this bugfix, Patroni sets the value of ``synchronous_standby_names`` as configured by the user, if that is the case, when ``synchronous_mode`` is disabled. - Handle logical slots invalidation on a standby (Polina Bungina) Since PG16 logical replication slots on a standby can be invalidated due to horizon: from now on, Patroni forces copy (i.e., recreation) of invalidated slots. - Fix race condition with logical slot advance and copy (Alexander Kukushkin) Due to this bug, it was a possible situation when an invalidated logical replication slot was copied with PostgreSQL restart more than once. Version 3.3.1 ------------- Released 2024-06-17 **Stability improvements** - Compatibility with Python 3.12 (Alexander Kukushkin) Handle a new attribute added to ``logging.LogRecord``. **Bugfixes** - Fix infinite recursion in ``replicatefrom`` tags handling (Alexander Kukushkin) As a part of this fix, also improve ``is_physical_slot()`` check and adjust documentation. - Fix wrong role reporting in standby clusters (Alexander Kukushkin) ``synchronous_standby_names`` and synchronous replication only work on a real primary node and in the case of cascading replication are simply ignored by Postgres. Before this fix, ``patronictl list`` and ``GET /cluster`` were falsely reporting some nodes as synchronous. - Fix availability of the ``allow_in_place_tablespaces`` GUC (Polina Bungina) ``allow_in_place_tablespaces`` was not only added to PostgreSQL 15 but also backpatched to PostgreSQL 10-14. Version 3.3.0 ------------- Released 2024-04-04 .. warning:: All older Partoni versions are not compatible with ``ydiff>=1.3``. There are the following options available to "fix" the problem: 1. upgrade Patroni to the latest version 2. install ``ydiff<1.3`` after installing Patroni 3. install ``cdiff`` module **New features** - Add ability to pass ``auth_data`` to Zookeeper client (Aras Mumcuyan) It allows to specify the authentication credentials to use for the connection. - Add a contrib script for ``Barman`` integration (Israel Barth Rubio) Provide an application ``patroni_barman`` that allows to perform ``Barman`` operations remotely and can be used as a custom bootstrap/custom replica method or as an ``on_role_change`` callback. Please check :ref:`here ` for more information. - Support ``JSON`` log format (alisalemmi) Apart from ``plain`` (default), Patroni now also supports ``json`` log format. Requires ``python-json-logger>=2.0.2`` library to be installed. - Show ``pending_restart_reason`` information (Polina Bungina) Provide extended information about the PostgreSQL parameters that caused ``pending_restart`` flag to be set. Both ``patronictl list`` and ``/patroni`` REST API endpoint now show the parameters names and their "diff" as ``pending_restart_reason``. - Implement ``nostream`` tag (Grigory Smolkin) If ``nostream`` tag is set to ``true``, the node will not use replication protocol to stream WAL but instead rely on archive recovery (if ``restore_command`` is configured). It also disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas. **Improvements** - Implement validation of the ``log`` section (Alexander Kukushkin) Until now validator was not checking the correctness of the logging configuration provided. - Improve logging for PostgreSQL parameters change (Polina Bungina) Convert old values to a human-readable format and log information about the ``pg_controldata`` vs Patroni global configuration mismatch. **Bugfixes** - Properly filter out not allowed ``pg_basebackup`` options (Israel Barth Rubio) Due to a bug, Patroni was not properly filtering out the not allowed options configured for the ``basebackup`` replica bootstrap method, when provided in the ``- setting: value`` format. - Fix ``etcd3`` authentication error handling (Alexander Kukushkin) Always retry one time on ``etcd3`` authentication error if authentication was not done right before executing the request. Also, do not restart watchers on reauthentication. - Improve logic of the validator files discovery (Waynerv) Use ``importlib`` library to discover the files with available configuration parameters when possible (for Python 3.9+). This implementation is more stable and doesn't break the Patroni distributions based on ``zip`` archives. - Use ``target_session_attrs`` only when multiple hosts are specified in the ``standby_cluster`` section (Alexander Kukushkin) ``target_session_attrs=read-write`` is now added to the ``primary_conninfo`` on the standby leader node only when ``standby_cluster.host`` section contains multiple hosts separated by commas. - Add compatibility code for ``ydiff`` library version 1.3+ (Alexander Kukushkin) Patroni is relying on some API from ``ydiff`` that is not public because it is supposed to be just a terminal tool rather than a python module. Unfortunately, the API change in 1.3 broke old Patroni versions. Version 3.2.2 ------------- Released 2024-01-17 **Bugfixes** - Don't let replica restore initialize key when DCS was wiped (Alexander Kukushkin) It was happening in the method where Patroni was supposed to take over a standalone PG cluster. - Use consistent read when fetching just updated sync key from Consul (Alexander Kukushkin) Consul doesn't provide any interface to immediately get ``ModifyIndex`` for the key that we just updated, therefore we have to perform an explicit read operation. Since stale reads are allowed by default, we sometimes used to get an outdated version of the key. - Reload Postgres config if a parameter that requires restart was reset to the original value (Polina Bungina) Previously Patroni wasn't updating the config, but only resetting the ``pending_restart``. - Fix erroneous inverted logic of the confirmation prompt message when doing a failover to an async candidate in synchronous mode (Polina Bungina) The problem existed only in ``patronictl``. - Exclude leader from failover candidates in ``patronictl`` (Polina Bungina) If the cluster is healthy, failing over to an existing leader is no-op. - Create Citus database and extension idempotently (Alexander Kukushkin, Zhao Junwang) It will allow to create them in the ``post_bootstrap`` script in case if there is a need to add some more dependencies to the Citus database. - Don't filter our contradictory ``nofailover`` tag (Polina Bungina) The configuration ``{nofailover: false, failover_priority: 0}`` set on a node didn't allow it to participate in the race, while it should, because ``nofailover`` tag should take precedence. - Fixed PyInstaller frozen issue (Sophia Ruan) The ``freeze_support()`` was called after ``argparse`` and as a result, Patroni wasn't able to start Postgres. - Fixed bug in the config generator for ``patronictl`` and ``Citus`` configuration (Israel Barth Rubio) It prevented ``patronictl`` and ``Citus`` configuration parameters set via environment variables from being written into the generated config. - Restore recovery GUCs and some Patroni-managed parameters when joining a running standby (Alexander Kukushkin) Patroni was failing to restart Postgres v12 onwards with an error about missing ``port`` in one of the internal structures. - Fixes around ``pending_restart`` flag (Polina Bungina) Don't expose ``pending_restart`` when in custom bootstrap with ``recovery_target_action = promote`` or when someone changed ``hot_standby`` or ``wal_log_hints`` using for example ``ALTER SYSTEM``. Version 3.2.1 ------------- Released 2023-11-30 **Bugfixes** - Limit accepted values for ``--format`` argument in ``patronictl`` (Alexander Kukushkin) It used to accept any arbitrary string and produce no output if the value wasn't recognized. - Verify that replica nodes received checkpoint LSN on shutdown before releasing the leader key (Alexander Kukushkin) Previously in some cases, we were using LSN of the SWITCH record that is followed by CHECKPOINT (if archiving mode is enabled). As a result the former primary sometimes had to do ``pg_rewind``, but there would be no data loss involved. - Do a real HTTP request when performing node name uniqueness check (Alexander Kukushkin) When running Patroni in containers it is possible that the traffic is routed using ``docker-proxy``, which listens on the port and accepts incoming connections. It was causing false positives. - Fixed Citus support with Etcd v2 (Alexander Kukushkin) Patroni was failing to deploy a new Citus cluster with Etcd v2. - Fixed ``pg_rewind`` behavior with Postgres v16+ (Alexander Kukushkin) The error message format of ``pg_waldump`` changed in v16 which caused ``pg_rewind`` to be called by Patroni even when it was not necessary. - Fixed bug with custom bootstrap (Alexander Kukushkin) Patroni was falsely applying ``--command`` argument, which is a bootstrap command itself. - Fixed the issue with REST API health check endpoints (Sophia Ruan) There were chances that after Postgres restart it could return ``unknown`` state for Postgres because connections were not properly closed. - Cache ``postgres --describe-config`` output results (Waynerv) They are used to figure out which GUCs are available to validate PostgreSQL configuration and we don't expect this list to change while Patroni is running. Version 3.2.0 ------------- Released 2023-10-25 **Deprecation notice** - The ``bootstrap.users`` support will be removed in version 4.0.0. If you need to create users after deploying a new cluster please use the ``bootstrap.post_bootstrap`` hook for that. **Breaking changes** - Enforce ``loop_wait + 2*retry_timeout <= ttl`` rule and hard-code minimal possible values (Alexander Kukushkin) Minimal values: ``loop_wait=2``, ``retry_timeout=3``, ``ttl=20``. In case values are smaller or violate the rule they are adjusted and a warning is written to Patroni logs. **New features** - Failover priority (Mark Pekala) With the help of ``tags.failover_priority`` it's now possible to make a node more preferred during the leader race. More details in the documentation (ref tags). - Implemented ``patroni --generate-config [--dsn DSN]`` and ``patroni --generate-sample-config`` (Polina Bungina) It allows to generate a config file for the running PostgreSQL cluster or a sample config file for the new Patroni cluster. - Use a dedicated connection to Postgres for Patroni REST API (Alexander Kukushkin) It helps to avoid blocking the main heartbeat loop if the system is under stress. - Enrich some endpoints with the ``name`` of the node (sskserk) For the monitoring endpoint ``name`` is added next to the ``scope`` and for metrics endpoint the ``name`` is added to tags. - Ensure strict failover/switchover difference (Polina Bungina) Be more precise in log messages and allow failing over to an asynchronous node in a healthy synchronous cluster. - Make permanent physical replication slots behave similarly to permanent logical slots (Alexander Kukushkin) Create permanent physical replication slots on all nodes that are allowed to become the leader and use ``pg_replication_slot_advance()`` function to advance ``restart_lsn`` for slots on standby nodes. - Add capability of specifying namespace through ``--dcs`` argument in ``patronictl`` (Israel Barth Rubio) It could be handy if ``patronictl`` is used without a configuration file. - Add support for additional parameters in custom bootstrap configuration (Israel Barth Rubio) Previously it was only possible to add custom arguments to the ``command`` and now one could list them as a mapping. **Improvements** - Set ``citus.local_hostname`` GUC to the same value which is used by Patroni to connect to the Postgres (Alexander Kukushkin) There are cases when Citus wants to have a connection to the local Postgres. By default it uses ``localhost``, which is not always available. **Bugfixes** - Ignore ``synchronous_mode`` setting in a standby cluster (Polina Bungina) Postgres doesn't support cascading synchronous replication and not ignoring ``synchronous_mode`` was breaking a switchover in a standby cluster. - Handle SIGCHLD for ``on_reload`` callback (Alexander Kukushkin) Not doing so results in a zombie process, which is reaped only when the next ``on_reload`` is executed. - Handle ``AuthOldRevision`` error when working with Etcd v3 (Alexander Kukushkin, Kenny Do) The error is raised if Etcd is configured to use JWT and when the user database in Etcd is updated. Version 3.1.2 ------------- Released 2023-09-26 **Bugfixes** - Fixed bug with ``wal_keep_size`` checks (Alexander Kukushkin) The ``wal_keep_size`` is a GUC that normally has a unit and Patroni was failing to cast its value to ``int``. As a result the value of ``bootstrap.dcs`` was not written to the ``/config`` key afterwards. - Detect and resolve inconsistencies between ``/sync`` key and ``synchronous_standby_names`` (Alexander Kukushkin) Normally, Patroni updates ``/sync`` and ``synchronous_standby_names`` in a very specific order, but in case of a bug or when someone manually reset ``synchronous_standby_names``, Patroni was getting into an inconsistent state. As a result it was possible that the failover happens to an asynchronous node. - Read GUC's values when joining running Postgres (Alexander Kukushkin) When restarted in ``pause``, Patroni was discarding the ``synchronous_standby_names`` GUC from the ``postgresql.conf``. To solve it and avoid similar issues, Patroni will read GUC's value if it is joining an already running Postgres. - Silenced annoying warnings when checking for node uniqueness (Alexander Kukushkin) ``WARNING`` messages are produced by ``urllib3`` if Patroni is quickly restarted. Version 3.1.1 ------------- Released 2023-09-20 **Bugfixes** - Reset failsafe state on promote (ChenChangAo) If switchover/failover happened shortly after failsafe mode had been activated, the newly promoted primary was demoting itself after failsafe becomes inactive. - Silence useless warnings in ``patronictl`` (Alexander Kukushkin) If ``patronictl`` uses the same patroni.yaml file as Patroni and can access ``PGDATA`` directory it might have been showing annoying warnings about incorrect values in the global configuration. - Explicitly enable synchronous mode for a corner case (Alexander Kukushkin) Synchronous mode effectively was never activated if there are no replicas streaming from the primary. - Fixed bug with ``0`` integer values validation (Israel Barth Rubio) In most cases, it didn't cause any issues, just warnings. - Don't return logical slots for standby cluster (Alexander Kukushkin) Patroni can't create logical replication slots in the standby cluster, thus they should be ignored if they are defined in the global configuration. - Avoid showing docstring in ``patronictl --help`` output (Israel Barth Rubio) The ``click`` module needs to get a special hint for that. - Fixed bug with ``kubernetes.standby_leader_label_value`` (Alexander Kukushkin) This feature effectively never worked. - Returned cluster system identifier to the ``patronictl list`` output (Polina Bungina) The problem was introduced while implementing the support for Citus, where we need to hide the identifier because it is different for coordinator and all workers. - Override ``write_leader_optime`` method in Kubernetes implementation (Alexander Kukushkin) The method is supposed to write shutdown LSN to the leader Endpoint/ConfigMap when there are no healthy replicas available to become the new primary. - Don't start stopped postgres in pause (Alexander Kukushkin) Due to a race condition, Patroni was falsely assuming that the standby should be restarted because some recovery parameters (``primary_conninfo`` or similar) were changed. - Fixed bug in ``patronictl query`` command (Israel Barth Rubio) It didn't work when only ``-m`` argument was provided or when none of ``-r`` or ``-m`` were provided. - Properly treat integer parameters that are used in the command line to start postgres (Polina Bungina) If values are supplied as strings and not casted to integer it was resulting in an incorrect calculation of ``max_prepared_transactions`` based on ``max_connections`` for Citus clusters. - Don't rely on ``pg_stat_wal_receiver`` when deciding on ``pg_rewind`` (Alexander Kukushkin) It could happen that ``received_tli`` reported by ``pg_stat_wal_receiver`` is ahead of the actual replayed timeline, while the timeline reported by ``DENTIFY_SYSTEM`` via replication connection is always correct. Version 3.1.0 ------------- Released 2023-08-03 **Breaking changes** - Changed semantic of ``restapi.keyfile`` and ``restapi.certfile`` (Alexander Kukushkin) Previously Patroni was using ``restapi.keyfile`` and ``restapi.certfile`` as client certificates as a fallback if there were no respective configuration parameters in the ``ctl`` section. .. warning:: If you enabled client certificates validation (``restapi.verify_client`` is set to ``required``), you also **must** provide **valid client certificates** in the ``ctl.certfile``, ``ctl.keyfile``, ``ctl.keyfile_password``. If not provided, Patroni will not work correctly. **New features** - Make Pod role label configurable (Waynerv) Values could be customized using ``kubernetes.leader_label_value``, ``kubernetes.follower_label_value`` and ``kubernetes.standby_leader_label_value`` parameters. This feature will be very useful when we change the ``master`` role to the ``primary``. You can read more about the feature and migration steps :ref:`here `. **Improvements** - Various improvements of ``patroni --validate-config`` (Alexander Kukushkin) Improved parameter validation for different DCS, ``bootstrap.dcs`` , ``ctl``, ``restapi``, and ``watchdog`` sections. - Start Postgres not in recovery if it crashed during recovery while Patroni is running (Alexander Kukushkin) It may reduce recovery time and will help to prevent unnecessary timeline increments. - Avoid unnecessary updates of ``/status`` key (Alexander Kukushkin) When there are no permanent logical slots Patroni was updating the ``/status`` on every heartbeat loop even when LSN on the primary didn't move forward. - Don't allow stale primary to win the leader race (Alexander Kukushkin) If Patroni was hanging during a significant time due to lack of resources it will additionally check that no other nodes promoted Postgres before acquiring the leader lock. - Implemented visibility of certain PostgreSQL parameters validation (Alexander Kukushkin, Feike Steenbergen) If validation of ``max_connections``, ``max_wal_senders``, ``max_prepared_transactions``, ``max_locks_per_transaction``, ``max_replication_slots``, or ``max_worker_processes`` failed Patroni was using some sane default value. Now in addition to that it will also show a warning. - Set permissions for files and directories created in ``PGDATA`` (Alexander Kukushkin) All files created by Patroni had only owner read/write permissions. This behaviour was breaking backup tools that run under a different user and relying on group read permissions. Now Patroni honors permissions on ``PGDATA`` and correctly sets permissions on all directories and files it creates inside ``PGDATA``. **Bugfixes** - Run ``archive_command`` through shell (Waynerv) Patroni might archive some WAL segments before doing crash recovery in a single-user mode or before ``pg_rewind``. If the archive_command contains some shell operators, like ``&&`` it didn't work with Patroni. - Fixed "on switchover" shutdown checks (Polina Bungina) It was possible that specified candidate is still streaming and didn't received shut down checking but the leader key was removed because some other nodes were healthy. - Fixed "is primary" check (Alexander Kukushkin) During the leader race replicas were not able to recognize that Postgres on the old leader is still running as a primary. - Fixed ``patronictl list`` (Alexander Kukushkin) The Cluster name field was missing in ``tsv``, ``json``, and ``yaml`` output formats. - Fixed ``pg_rewind`` behaviour after pause (Alexander Kukushkin) Under certain conditions, Patroni wasn't able to join the false primary back to the cluster with ``pg_rewind`` after coming out of maintenance mode. - Fixed bug in Etcd v3 implementation (Alexander Kukushkin) Invalidate internal KV cache if key update performed using ``create_revision``/``mod_revision`` field due to revision mismatch. - Fixed behaviour of replicas in standby cluster in pause (Alexander Kukushkin) When the leader key expires replicas in standby cluster will not follow the remote node but keep ``primary_conninfo`` as it is. Version 3.0.4 ------------- Released 2023-07-13 **New features** - Make the replication status of standby nodes visible (Alexander Kukushkin) For PostgreSQL 9.6+ Patroni will report the replication state as ``streaming`` when the standby is streaming from the other node or ``in archive recovery`` when there is no replication connection and ``restore_command`` is set. The state is visible in ``member`` keys in DCS, in the REST API, and in ``patronictl list`` output. **Improvements** - Improved error messages with Etcd v3 (Alexander Kukushkin) When Etcd v3 cluster isn't accessible Patroni was reporting that it can't access ``/v2`` endpoints. - Use quorum read in ``patronictl`` if it is possible (Alexander Kukushkin) Etcd or Consul clusters could be degraded to read-only, but from the ``patronictl`` view everything was fine. Now it will fail with the error. - Prevent splitbrain from duplicate names in configuration (Mark Pekala) When starting Patroni will check if node with the same name is registered in DCS, and try to query its REST API. If REST API is accessible Patroni exits with an error. It will help to protect from the human error. - Start Postgres not in recovery if it crashed while Patroni is running (Alexander Kukushkin) It may reduce recovery time and will help from unnecessary timeline increments. **Bugfixes** - REST API SSL certificate were not reloaded upon receiving a SIGHUP (Israel Barth Rubio) Regression was introduced in 3.0.3. - Fixed integer GUCs validation for parameters like ``max_connections`` (Feike Steenbergen) Patroni didn't like quoted numeric values. Regression was introduced in 3.0.3. - Fix issue with ``synchronous_mode`` (Alexander Kukushkin) Execute ``txid_current()`` with ``synchronous_commit=off`` so it doesn't accidentally wait for absent synchronous standbys when ``synchronous_mode_strict`` is enabled. Version 3.0.3 ------------- Released 2023-06-22 **New features** - Compatibility with PostgreSQL 16 beta1 (Alexander Kukushkin) Extended GUC's validator rules. - Make PostgreSQL GUC's validator extensible (Israel Barth Rubio) Validator rules are loaded from YAML files located in ``patroni/postgresql/available_parameters/`` directory. Files are ordered in alphabetical order and applied one after another. It makes possible to have custom validators for non-standard Postgres distributions. - Added ``restapi.request_queue_size`` option (Andrey Zhidenkov, Aleksei Sukhov) Sets request queue size for TCP socket used by Patroni REST API. Once the queue is full, further requests get a "Connection denied" error. The default value is 5. - Call ``initdb`` directly when initializing a new cluster (Matt Baker) Previously it was called via ``pg_ctl``, what required a special quoting of parameters passed to ``initdb``. - Added before stop hook (Le Duane) The hook could be configured via ``postgresql.before_stop`` and is executed right before ``pg_ctl stop``. The exit code doesn't impact shutdown process. - Added support for custom Postgres binary names (Israel Barth Rubio, Polina Bungina) When using a custom Postgres distribution it may be the case that the Postgres binaries are compiled with different names other than the ones used by the community Postgres distribution. Custom binary names could be configured using ``postgresql.bin_name.*`` and ``PATRONI_POSTGRESQL_BIN_*`` environment variables. **Improvements** - Various improvements of ``patroni --validate-config`` (Polina Bungina) - Make ``bootstrap.initdb`` optional. It is only required for new clusters, but ``patroni --validate-config`` was complaining if it was missing in the config. - Don't error out when ``postgresql.bin_dir`` is empty or not set. Try to first find Postgres binaries in the default PATH instead. - Make ``postgresql.authentication.rewind`` section optional. If it is missing, Patroni is using the superuser. - Improved error reporting in ``patronictl`` (Israel Barth Rubio) The ``\n`` symbol was rendered as it is, instead of the actual newline symbol. **Bugfixes** - Fixed issue in Citus support (Alexander Kukushkin) If the REST API call from the promoted worker to the coordinator failed during switchover it was leaving the given Citus group blocked during indefinite time. - Allow `etcd3` URL in `--dcs-url` option of `patronictl` (Israel Barth Rubio) If users attempted to pass a `etcd3` URL through `--dcs-url` option of `patronictl` they would face an exception. Version 3.0.2 ------------- Released 2023-03-24 .. warning:: Version 3.0.2 dropped support of Python older than 3.6. **New features** - Added sync standby replica status to ``/metrics`` endpoint (Thomas von Dein, Alexander Kukushkin) Before were only reporting ``primary``/``standby_leader``/``replica``. - User-friendly handling of ``PAGER`` in ``patronictl`` (Israel Barth Rubio) It makes pager configurable via ``PAGER`` environment variable, which overrides default ``less`` and ``more``. - Make K8s retriable HTTP status code configurable (Alexander Kukushkin) On some managed platforms it is possible to get status code ``401 Unauthorized``, which sometimes gets resolved after a few retries. **Improvements** - Set ``hot_standby`` to ``off`` during custom bootstrap only if ``recovery_target_action`` is set to ``promote`` (Alexander Kukushkin) It was necessary to make ``recovery_target_action=pause`` work correctly. - Don't allow ``on_reload`` callback to kill other callbacks (Alexander Kukushkin) ``on_start``/``on_stop``/``on_role_change`` are usually used to add/remove Virtual IP and ``on_reload`` should not interfere with them. - Switched to ``IMDSFetcher`` in aws callback example script (Polina Bungina) The ``IMDSv2`` requires a token to work with and the ``IMDSFetcher`` handles it transparently. **Bugfixes** - Fixed ``patronictl switchover`` on Citus cluster running on Kubernetes (Lukáš Lalinský) It didn't work for namespaces different from ``default``. - Don't write to ``PGDATA`` if major version is not known (Alexander Kukushkin) If right after the start ``PGDATA`` was empty (maybe wasn't yet mounted), Patroni was making a false assumption about PostgreSQL version and falsely creating ``recovery.conf`` file even if the actual major version is v10+. - Fixed bug with Citus metadata after coordinator failover (Alexander Kukushkin) The ``citus_set_coordinator_host()`` call doesn't cause metadata sync and the change was invisible on worker nodes. The issue is solved by switching to ``citus_update_node()``. - Use etcd hosts listed in the config file as a fallback when all etcd nodes "failed" (Alexander Kukushkin) The etcd cluster may change topology over time and Patroni tries to follow it. If at some point all nodes became unreachable Patroni will use a combination of nodes from the config plus the last known topology when trying to reconnect. Version 3.0.1 ------------- Released 2023-02-16 **Bugfixes** - Pass proper role name to an ``on_role_change`` callback script'. (Alexander Kukushkin, Polina Bungina) Patroni used to erroneously pass ``promoted`` role to an ``on_role_change`` callback script on promotion. The passed role name changed back to ``master``. This regression was introduced in 3.0.0. Version 3.0.0 ------------- Released 2023-01-30 This version adds integration with `Citus `__ and makes it possible to survive temporary DCS outages without demoting primary. .. warning:: - Version 3.0.0 is the last release supporting Python 2.7. Upcoming release will drop support of Python versions older than 3.7. - The RAFT support is deprecated. We will do our best to maintain it, but take neither guarantee nor responsibility for possible issues. - This version is the first step in getting rid of the "master", in favor of "primary". Upgrading to the next major release will work reliably only if you run at least 3.0.0. **New features** - DCS failsafe mode (Alexander Kukushkin, Polina Bungina) If the feature is enabled it will allow Patroni cluster to survive temporary DCS outages. You can find more details in the :ref:`documentation `. - Citus support (Alexander Kukushkin, Polina Bungina, Jelte Fennema) Patroni enables easy deployment and management of `Citus `__ clusters with HA. Please check :ref:`here ` page for more information. **Improvements** - Suppress recurring errors when dropping unknown but active replication slots (Michael Banck) Patroni will still write these logs, but only in DEBUG. - Run only one monitoring query per HA loop (Alexander Kukushkin) It wasn't the case if synchronous replication is enabled. - Keep only latest failed data directory (William Albertus Dembo) If bootstrap failed Patroni used to rename $PGDATA folder with timestamp suffix. From now on the suffix will be ``.failed`` and if such folder exists it is removed before renaming. - Improved check of synchronous replication connections (Alexander Kukushkin) When the new host is added to the ``synchronous_standby_names`` it will be set as synchronous in DCS only when it managed to catch up with the primary in addition to ``pg_stat_replication.sync_state = 'sync'``. **Removed functionality** - Remove ``patronictl scaffold`` (Alexander Kukushkin) The only reason for having it was a hacky way of running standby clusters. Version 2.1.7 ------------- Released 2023-01-04 **Bugfixes** - Fixed little incompatibilities with legacy python modules (Alexander Kukushkin) They prevented from building/running Patroni on Debian buster/Ubuntu bionic. Version 2.1.6 ------------- Released 2022-12-30 **Improvements** - Fix annoying exceptions on ssl socket shutdown (Alexander Kukushkin) The HAProxy is closing connections as soon as it got the HTTP Status code leaving no time for Patroni to properly shutdown SSL connection. - Adjust example Dockerfile for arm64 (Polina Bungina) Remove explicit ``amd64`` and ``x86_64``, don't remove ``libnss_files.so.*``. **Security improvements** - Enforce ``search_path=pg_catalog`` for non-replication connections (Alexander Kukushkin) Since Patroni is heavily relying on superuser connections, we want to protect it from the possible attacks carried out using user-defined functions and/or operators in ``public`` schema with the same name and signature as the corresponding objects in ``pg_catalog``. For that, ``search_path=pg_catalog`` is enforced for all connections created by Patroni (except replication connections). - Prevent passwords from being recorded in ``pg_stat_statements`` (Feike Steenbergen) It is achieved by setting ``pg_stat_statements.track_utility=off`` when creating users. **Bugfixes** - Declare ``proxy_address`` as optional (Denis Laxalde) As it is effectively a non-required option. - Improve behaviour of the insecure option (Alexander Kukushkin) Ctl's ``insecure`` option didn't work properly when client certificates were used for REST API requests. - Take watchdog configuration from ``bootstrap.dcs`` when the new cluster is bootstrapped (Matt Baker) Patroni used to initially configure watchdog with defaults when bootstrapping a new cluster rather than taking configuration used to bootstrap the DCS. - Fix the way file extensions are treated while finding executables in WIN32 (Martín Marqués) Only add ``.exe`` to a file name if it has no extension yet. - Fix Consul TTL setup (Alexander Kukushkin) We used ``ttl/2.0`` when setting the value on the HTTPClient, but forgot to multiply the current value by 2 in the class' property. It was resulting in Consul TTL off by twice. **Removed functionality** - Remove ``patronictl configure`` (Polina Bungina) There is no more need for a separate ``patronictl`` config creation. Version 2.1.5 ------------- Released 2022-11-28 This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 support as production ready. The Patroni on Raft remains in Beta. **New features** - Improve ``patroni --validate-config`` (Denis Laxalde) Exit with code 1 if config is invalid and print errors to stderr. - Don't drop replication slots in pause (Alexander Kukushkin) Patroni is automatically creating/removing physical replication slots when members are joining/leaving the cluster. In pause slots will no longer be removed. - Support the ``HEAD`` request method for monitoring endpoints (Robert Cutajar) If used instead of ``GET`` Patroni will return only the HTTP Status Code. - Support behave tests on Windows (Alexander Kukushkin) Emulate graceful Patroni shutdown (``SIGTERM``) on Windows by introduce the new REST API endpoint ``POST /sigterm``. - Introduce ``postgresql.proxy_address`` (Alexander Kukushkin) It will be written to the member key in DCS as the ``proxy_url`` and could be used/useful for service discovery. **Stability improvements** - Call ``pg_replication_slot_advance()`` from a thread (Alexander Kukushkin) On busy clusters with many logical replication slots the ``pg_replication_slot_advance()`` call was affecting the main HA loop and could result in the member key expiration. - Archive possibly missing WALs before calling ``pg_rewind`` on the old primary (Polina Bungina) If the primary crashed and was down during considerable time, some WAL files could be missing from archive and from the new primary. There is a chance that ``pg_rewind`` could remove these WAL files from the old primary making it impossible to start it as a standby. By archiving ``ready`` WAL files we not only mitigate this problem but in general improving continues archiving experience. - Ignore ``403`` errors when trying to create Kubernetes Service (Nick Hudson, Polina Bungina) Patroni was spamming logs by unsuccessful attempts to create the service, which in fact could already exist. - Improve liveness probe (Alexander Kukushkin) The liveness problem will start failing if the heartbeat loop is running longer than `ttl` on the primary or `2*ttl` on the replica. That will allow us to use it as an alternative for :ref:`watchdog ` on Kubernetes. - Make sure only sync node tries to grab the lock when switchover (Alexander Kukushkin, Polina Bungina) Previously there was a slim chance that up-to-date async member could become the leader if the manual switchover was performed without specifying the target. - Avoid cloning while bootstrap is running (Ants Aasma) Do not allow a create replica method that does not require a leader to be triggered while the cluster bootstrap is running. - Compatibility with kazoo-2.9.0 (Alexander Kukushkin) Depending on python version the ``SequentialThreadingHandler.select()`` method may raise ``TypeError`` and ``IOError`` exceptions if ``select()`` is called on the closed socket. - Explicitly shut down SSL connection before socket shutdown (Alexander Kukushkin) Not doing it resulted in ``unexpected eof while reading`` errors with OpenSSL 3.0. - Compatibility with `prettytable>=2.2.0` (Alexander Kukushkin) Due to the internal API changes the cluster name header was shown on the incorrect line. **Bugfixes** - Handle expired token for Etcd lease_grant (monsterxx03) In case of error get the new token and retry request. - Fix bug in the ``GET /read-only-sync`` endpoint (Alexander Kukushkin) It was introduced in previous release and effectively never worked. - Handle the case when data dir storage disappeared (Alexander Kukushkin) Patroni is periodically checking that the PGDATA is there and not empty, but in case of issues with storage the ``os.listdir()`` is raising the ``OSError`` exception, breaking the heart-beat loop. - Apply ``master_stop_timeout`` when waiting for user backends to close (Alexander Kukushkin) Something that looks like user backend could be in fact a background worker (e.g., Citus Maintenance Daemon) that is failing to stop. - Accept ``*:`` for ``postgresql.listen`` (Denis Laxalde) The ``patroni --validate-config`` was complaining about it being invalid. - Timeouts fixes in Raft (Alexander Kukushkin) When Patroni or patronictl are starting they try to get Raft cluster topology from known members. These calls were made without proper timeouts. - Forcefully update consul service if token was changed (John A. Lotoski) Not doing so results in errors "rpc error making call: rpc error making call: ACL not found". Version 2.1.4 ------------- Released 2022-06-01 **New features** - Improve ``pg_rewind`` behavior on typical Debian/Ubuntu systems (Gunnar "Nick" Bluth) On Postgres setups that keep `postgresql.conf` outside of the data directory (e.g. Ubuntu/Debian packages), ``pg_rewind --restore-target-wal`` fails to figure out the value of the ``restore_command``. - Allow setting ``TLSServerName`` on Consul service checks (Michael Gmelin) Useful when checks are performed by IP and the Consul ``node_name`` is not a FQDN. - Added ``ppc64le`` support in watchdog (Jean-Michel Scheiwiler) And fixed watchdog support on some non-x86 platforms. - Switched aws.py callback from ``boto`` to ``boto3`` (Alexander Kukushkin) ``boto`` 2.x is abandoned since 2018 and fails with python 3.9. - Periodically refresh service account token on K8s (Haitao Li) Since Kubernetes v1.21 service account tokens expire in 1 hour. - Added ``/read-only-sync`` monitoring endpoint (Dennis4b) It is similar to the ``/read-only`` but includes only synchronous replicas. **Stability improvements** - Don't copy the logical replication slot to a replica if there is a configuration mismatch in the logical decoding setup with the primary (Alexander Kukushkin) A replica won't copy a logical replication slot from the primary anymore if the slot doesn't match the ``plugin`` or ``database`` configuration options. Previously, the check for whether the slot matches those configuration options was not performed until after the replica copied the slot and started with it, resulting in unnecessary and repeated restarts. - Special handling of recovery configuration parameters for PostgreSQL v12+ (Alexander Kukushkin) While starting as replica Patroni should be able to update ``postgresql.conf`` and restart/reload if the leader address has changed by caching current parameters values instead of querying them from ``pg_settings``. - Better handling of IPv6 addresses in the ``postgresql.listen`` parameters (Alexander Kukushkin) Since the ``listen`` parameter has a port, people try to put IPv6 addresses into square brackets, which were not correctly stripped when there is more than one IP in the list. - Use ``replication`` credentials when performing divergence check only on PostgreSQL v10 and older (Alexander Kukushkin) If ``rewind`` is enabled, Patroni will again use either ``superuser`` or ``rewind`` credentials on newer Postgres versions. **Bugfixes** - Fixed missing import of ``dateutil.parser`` (Wesley Mendes) Tests weren't failing only because it was also imported from other modules. - Ensure that ``optime`` annotation is a string (Sebastian Hasler) In certain cases Patroni was trying to pass it as numeric. - Better handling of failed ``pg_rewind`` attempt (Alexander Kukushkin) If the primary becomes unavailable during ``pg_rewind``, ``$PGDATA`` will be left in a broken state. Following that, Patroni will remove the data directory even if this is not allowed by the configuration. - Don't remove ``slots`` annotations from the leader ``ConfigMap``/``Endpoint`` when PostgreSQL isn't ready (Alexander Kukushkin) If ``slots`` value isn't passed the annotation will keep the current value. - Handle concurrency problem with K8s API watchers (Alexander Kukushkin) Under certain (unknown) conditions watchers might become stale; as a result, ``attempt_to_acquire_leader()`` method could fail due to the HTTP status code 409. In that case we reset watchers connections and restart from scratch. Version 2.1.3 ------------- Released 2022-02-18 **New features** - Added support for encrypted TLS keys for ``patronictl`` (Alexander Kukushkin) It could be configured via ``ctl.keyfile_password`` or the ``PATRONI_CTL_KEYFILE_PASSWORD`` environment variable. - Added more metrics to the /metrics endpoint (Alexandre Pereira) Specifically, ``patroni_pending_restart`` and ``patroni_is_paused``. - Make it possible to specify multiple hosts in the standby cluster configuration (Michael Banck) If the standby cluster is replicating from the Patroni cluster it might be nice to rely on client-side failover which is available in ``libpq`` since PostgreSQL v10. That is, the ``primary_conninfo`` on the standby leader and ``pg_rewind`` setting ``target_session_attrs=read-write`` in the connection string. The ``pgpass`` file will be generated with multiple lines (one line per host), and instead of calling ``CHECKPOINT`` on the primary cluster nodes the standby cluster will wait for ``pg_control`` to be updated. **Stability improvements** - Compatibility with legacy ``psycopg2`` (Alexander Kukushkin) For example, the ``psycopg2`` installed from Ubuntu 18.04 packages doesn't have the ``UndefinedFile`` exception yet. - Restart ``etcd3`` watcher if all Etcd nodes don't respond (Alexander Kukushkin) If the watcher is alive the ``get_cluster()`` method continues returning stale information even if all Etcd nodes are failing. - Don't remove the leader lock in the standby cluster while paused (Alexander Kukushkin) Previously the lock was maintained only by the node that was running as a primary and not a standby leader. **Bugfixes** - Fixed bug in the standby-leader bootstrap (Alexander Kukushkin) Patroni was considering bootstrap as failed if Postgres didn't start accepting connections after 60 seconds. The bug was introduced in the 2.1.2 release. - Fixed bug with failover to a cascading standby (Alexander Kukushkin) When figuring out which slots should be created on cascading standby we forgot to take into account that the leader might be absent. - Fixed small issues in Postgres config validator (Alexander Kukushkin) Integer parameters introduced in PostgreSQL v14 were failing to validate because min and max values were quoted in the validator.py - Use replication credentials when checking leader status (Alexander Kukushkin) It could be that the ``remove_data_directory_on_diverged_timelines`` is set, but there is no ``rewind_credentials`` defined and superuser access between nodes is not allowed. - Fixed "port in use" error on REST API certificate replacement (Ants Aasma) When switching certificates there was a race condition with a concurrent API request. If there is one active during the replacement period then the replacement will error out with a port in use error and Patroni gets stuck in a state without an active API server. - Fixed a bug in cluster bootstrap if passwords contain ``%`` characters (Bastien Wirtz) The bootstrap method executes the ``DO`` block, with all parameters properly quoted, but the ``cursor.execute()`` method didn't like an empty list with parameters passed. - Fixed the "AttributeError: no attribute 'leader'" exception (Hrvoje Milković) It could happen if the synchronous mode is enabled and the DCS content was wiped out. - Fix bug in divergence timeline check (Alexander Kukushkin) Patroni was falsely assuming that timelines have diverged. For pg_rewind it didn't create any problem, but if pg_rewind is not allowed and the ``remove_data_directory_on_diverged_timelines`` is set, it resulted in reinitializing the former leader. Version 2.1.2 ------------- Released 2021-12-03 **New features** - Compatibility with ``psycopg>=3.0`` (Alexander Kukushkin) By default ``psycopg2`` is preferred. `psycopg>=3.0` will be used only if ``psycopg2`` is not available or its version is too old. - Add ``dcs_last_seen`` field to the REST API (Michael Banck) This field notes the last time (as unix epoch) a cluster member has successfully communicated with the DCS. This is useful to identify and/or analyze network partitions. - Release the leader lock when ``pg_controldata`` reports "shut down" (Alexander Kukushkin) To solve the problem of slow switchover/shutdown in case ``archive_command`` is slow/failing, Patroni will remove the leader key immediately after ``pg_controldata`` started reporting PGDATA as ``shut down`` cleanly and it verified that there is at least one replica that received all changes. If there are no replicas that fulfill this condition the leader key is not removed and the old behavior is retained, i.e. Patroni will keep updating the lock. - Add ``sslcrldir`` connection parameter support (Kostiantyn Nemchenko) The new connection parameter was introduced in the PostgreSQL v14. - Allow setting ACLs for ZNodes in Zookeeper (Alwyn Davis) Introduce a new configuration option ``zookeeper.set_acls`` so that Kazoo will apply a default ACL for each ZNode that it creates. **Stability improvements** - Delay the next attempt of recovery till next HA loop (Alexander Kukushkin) If Postgres crashed due to out of disk space (for example) and fails to start because of that Patroni is too eagerly trying to recover it flooding logs. - Add log before demoting, which can take some time (Michael Banck) It can take some time for the demote to finish and it might not be obvious from looking at the logs what exactly is going on. - Improve "I am" status messages (Michael Banck) ``no action. I am a secondary ({0})`` vs ``no action. I am ({0}), a secondary`` - Cast to int ``wal_keep_segments`` when converting to ``wal_keep_size`` (Jorge Solórzano) It is possible to specify ``wal_keep_segments`` as a string in the global :ref:`dynamic configuration ` and due to Python being a dynamically typed language the string was simply multiplied. Example: ``wal_keep_segments: "100"`` was converted to ``100100100100100100100100100100100100100100100100MB``. - Allow switchover only to sync nodes when synchronous replication is enabled (Alexander Kukushkin) In addition to that do the leader race only against known synchronous nodes. - Use cached role as a fallback when Postgres is slow (Alexander Kukushkin) In some extreme cases Postgres could be so slow that the normal monitoring query does not finish in a few seconds. The ``statement_timeout`` exception not being properly handled could lead to the situation where Postgres was not demoted on time when the leader key expired or the update failed. In case of such exception Patroni will use the cached ``role`` to determine whether Postgres is running as a primary. - Avoid unnecessary updates of the member ZNode (Alexander Kukushkin) If no values have changed in the members data, the update should not happen. - Optimize checkpoint after promote (Alexander Kukushkin) Avoid doing ``CHECKPOINT`` if the latest timeline is already stored in ``pg_control``. It helps to avoid unnecessary ``CHECKPOINT`` right after initializing the new cluster with ``initdb``. - Prefer members without ``nofailover`` when picking sync nodes (Alexander Kukushkin) Previously sync nodes were selected only based on the replication lag, hence the node with ``nofailover`` tag had the same chances to become synchronous as any other node. That behavior was confusing and dangerous at the same time because in case of a failed primary the failover could not happen automatically. - Remove duplicate hosts from the etcd machine cache (Michael Banck) Advertised client URLs in the etcd cluster could be misconfigured. Removing duplicates in Patroni in this case is a low-hanging fruit. **Bugfixes** - Skip temporary replication slots while doing slot management (Alexander Kukushkin) Starting from v10 ``pg_basebackup`` creates a temporary replication slot for WAL streaming and Patroni was trying to drop it because the slot name looks unknown. In order to fix it, we skip all temporary slots when querying ``pg_stat_replication_slots`` view. - Ensure ``pg_replication_slot_advance()`` doesn't timeout (Alexander Kukushkin) Patroni was using the default ``statement_timeout`` in this case and once the call failed there are very high chances that it will never recover, resulting in increased size of ``pg_wal`` and ``pg_catalog`` bloat. - The ``/status`` wasn't updated on demote (Alexander Kukushkin) After demoting PostgreSQL the old leader updates the last LSN in DCS. Starting from ``2.1.0`` the new ``/status`` key was introduced, but the optime was still written to the ``/optime/leader``. - Handle DCS exceptions when demoting (Alexander Kukushkin) While demoting the master due to failure to update the leader lock it could happen that DCS goes completely down and the ``get_cluster()`` call raises an exception. Not being handled properly it results in Postgres remaining stopped until DCS recovers. - The ``use_unix_socket_repl`` didn't work is some cases (Alexander Kukushkin) Specifically, if ``postgresql.unix_socket_directories`` is not set. In this case Patroni is supposed to use the default value from ``libpq``. - Fix a few issues with Patroni REST API (Alexander Kukushkin) The ``clusters_unlocked`` sometimes could be not defined, what resulted in exceptions in the ``GET /metrics`` endpoint. In addition to that the error handling method was assuming that the ``connect_address`` tuple always has two elements, while in fact there could be more in case of IPv6. - Wait for newly promoted node to finish recovery before deciding to rewind (Alexander Kukushkin) It could take some time before the actual promote happens and the new timeline is created. Without waiting replicas could come to the conclusion that rewind isn't required. - Handle missing timelines in a history file when deciding to rewind (Alexander Kukushkin) If the current replica timeline is missing in the history file on the primary the replica was falsely assuming that rewind isn't required. Version 2.1.1 ------------- Released 2021-08-19 **New features** - Support for ETCD SRV name suffix (David Pavlicek) Etcd allows to differentiate between multiple Etcd clusters under the same domain and from now on Patroni also supports it. - Enrich history with the new leader (huiyalin525) It adds the new column to the ``patronictl history`` output. - Make the CA bundle configurable for in-cluster Kubernetes config (Aron Parsons) By default Patroni is using ``/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`` and this new feature allows specifying the custom ``kubernetes.cacert``. - Support dynamically registering/deregistering as a Consul service and changing tags (Tommy Li) Previously it required Patroni restart. **Bugfixes** - Avoid unnecessary reload of REST API (Alexander Kukushkin) The previous release added a feature of reloading REST API certificates if changed on disk. Unfortunately, the reload was happening unconditionally right after the start. - Don't resolve cluster members when ``etcd.use_proxies`` is set (Alexander Kukushkin) When starting up Patroni checks the healthiness of Etcd cluster by querying the list of members. In addition to that, it also tried to resolve their hostnames, which is not necessary when working with Etcd via proxy and was causing unnecessary warnings. - Skip rows with NULL values in the ``pg_stat_replication`` (Alexander Kukushkin) It seems that the ``pg_stat_replication`` view could contain NULL values in the ``replay_lsn``, ``flush_lsn``, or ``write_lsn`` fields even when ``state = 'streaming'``. Version 2.1.0 ------------- Released 2021-07-06 This version adds compatibility with PostgreSQL v14, makes logical replication slots to survive failover/switchover, implements support of allowlist for REST API, and also reducing the number of logs to one line per heart-beat. **New features** - Compatibility with PostgreSQL v14 (Alexander Kukushkin) Unpause WAL replay if Patroni is not in a "pause" mode itself. It could be "paused" due to the change of certain parameters like for example ``max_connections`` on the primary. - Failover logical slots (Alexander Kukushkin) Make logical replication slots survive failover/switchover on PostgreSQL v11+. The replication slot if copied from the primary to the replica with restart and later the `pg_replication_slot_advance() `__ function is used to move it forward. As a result, the slot will already exist before the failover and no events should be lost, but, there is a chance that some events could be delivered more than once. - Implemented allowlist for Patroni REST API (Alexander Kukushkin) If configured, only IP's that matching rules would be allowed to call unsafe endpoints. In addition to that, it is possible to automatically include IP's of members of the cluster to the list. - Added support of replication connections via unix socket (Mohamad El-Rifai) Previously Patroni was always using TCP for replication connection what could cause some issues with SSL verification. Using unix sockets allows exempt replication user from SSL verification. - Health check on user-defined tags (Arman Jafari Tehrani) Along with :ref:`predefined tags: ` it is possible to specify any number of custom tags that become visible in the ``patronictl list`` output and in the REST API. From now on it is possible to use custom tags in health checks. - Added Prometheus ``/metrics`` endpoint (Mark Mercado, Michael Banck) The endpoint exposing the same metrics as ``/patroni``. - Reduced chattiness of Patroni logs (Alexander Kukushkin) When everything goes normal, only one line will be written for every run of HA loop. **Breaking changes** - The old ``permanent logical replication slots`` feature will no longer work with PostgreSQL v10 and older (Alexander Kukushkin) The strategy of creating the logical slots after performing a promotion can't guaranty that no logical events are lost and therefore disabled. - The ``/leader`` endpoint always returns 200 if the node holds the lock (Alexander Kukushkin) Promoting the standby cluster requires updating load-balancer health checks, which is not very convenient and easy to forget. To solve it, we change the behavior of the ``/leader`` health check endpoint. It will return 200 without taking into account whether the cluster is normal or the ``standby_cluster``. **Improvements in Raft support** - Reliable support of Raft traffic encryption (Alexander Kukushkin) Due to the different issues in the ``PySyncObj`` the encryption support was very unstable - Handle DNS issues in Raft implementation (Alexander Kukushkin) If ``self_addr`` and/or ``partner_addrs`` are configured using the DNS name instead of IP's the ``PySyncObj`` was effectively doing resolve only once when the object is created. It was causing problems when the same node was coming back online with a different IP. **Stability improvements** - Compatibility with ``psycopg2-2.9+`` (Alexander Kukushkin) In ``psycopg2`` the ``autocommit = True`` is ignored in the ``with connection`` block, which breaks replication protocol connections. - Fix excessive HA loop runs with Zookeeper (Alexander Kukushkin) Update of member ZNodes was causing a chain reaction and resulted in running the HA loops multiple times in a row. - Reload if REST API certificate is changed on disk (Michael Todorovic) If the REST API certificate file was updated in place Patroni didn't perform a reload. - Don't create pgpass dir if kerberos auth is used (Kostiantyn Nemchenko) Kerberos and password authentication are mutually exclusive. - Fixed little issues with custom bootstrap (Alexander Kukushkin) Start Postgres with ``hot_standby=off`` only when we do a PITR and restart it after PITR is done. **Bugfixes** - Compatibility with ``kazoo-2.7+`` (Alexander Kukushkin) Since Patroni is handling retries on its own, it is relying on the old behavior of ``kazoo`` that requests to a Zookeeper cluster are immediately discarded when there are no connections available. - Explicitly request the version of Etcd v3 cluster when it is known that we are connecting via proxy (Alexander Kukushkin) Patroni is working with Etcd v3 cluster via gPRC-gateway and it depending on the cluster version different endpoints (``/v3``, ``/v3beta``, or ``/v3alpha``) must be used. The version was resolved only together with the cluster topology, but since the latter was never done when connecting via proxy. Version 2.0.2 ------------- Released 2021-02-22 **New features** - Ability to ignore externally managed replication slots (James Coleman) Patroni is trying to remove any replication slot which is unknown to it, but there are certainly cases when replication slots should be managed externally. From now on it is possible to configure slots that should not be removed. - Added support for cipher suite limitation for REST API (Gunnar "Nick" Bluth) It could be configured via ``restapi.ciphers`` or the ``PATRONI_RESTAPI_CIPHERS`` environment variable. - Added support for encrypted TLS keys for REST API (Jonathan S. Katz) It could be configured via ``restapi.keyfile_password`` or the ``PATRONI_RESTAPI_KEYFILE_PASSWORD`` environment variable. - Constant time comparison of REST API authentication credentials (Alex Brasetvik) Use ``hmac.compare_digest()`` instead of ``==``, which is vulnerable to timing attack. - Choose synchronous nodes based on replication lag (Krishna Sarabu) If the replication lag on the synchronous node starts exceeding the configured threshold it could be demoted to asynchronous and/or replaced by the other node. Behaviour is controlled with ``maximum_lag_on_syncnode``. **Stability improvements** - Start postgres with ``hot_standby = off`` when doing custom bootstrap (Igor Yanchenko) During custom bootstrap Patroni is restoring the basebackup, starting Postgres up, and waiting until recovery finishes. Some PostgreSQL parameters on the standby can't be smaller than on the primary and if the new value (restored from WAL) is higher than the configured one, Postgres panics and stops. In order to avoid such behavior we will do custom bootstrap without ``hot_standby`` mode. - Warn the user if the required watchdog is not healthy (Nicolas Thauvin) When the watchdog device is not writable or missing in required mode, the member cannot be promoted. Added a warning to show the user where to search for this misconfiguration. - Better verbosity for single-user mode recovery (Alexander Kukushkin) If Patroni notices that PostgreSQL wasn't shutdown clearly, in certain cases the crash-recovery is executed by starting Postgres in single-user mode. It could happen that the recovery failed (for example due to the lack of space on disk) but errors were swallowed. - Added compatibility with ``python-consul2`` module (Alexander Kukushkin, Wilfried Roset) The good old ``python-consul`` is not maintained since a few years, therefore someone created a fork with new features and bug-fixes. - Don't use ``bypass_api_service`` when running ``patronictl`` (Alexander Kukushkin) When a K8s pod is running in a non-``default`` namespace it does not necessarily have enough permissions to query the ``kubernetes`` endpoint. In this case Patroni shows the warning and ignores the ``bypass_api_service`` setting. In case of ``patronictl`` the warning was a bit annoying. - Create ``raft.data_dir`` if it doesn't exists or make sure that it is writable (Mark Mercado) Improves user-friendliness and usability. **Bugfixes** - Don't interrupt restart or promote if lost leader lock in pause (Alexander Kukushkin) In pause it is allowed to run postgres as primary without lock. - Fixed issue with ``shutdown_request()`` in the REST API (Nicolas Limage) In order to improve handling of SSL connections and delay the handshake until thread is started Patroni overrides a few methods in the ``HTTPServer``. The ``shutdown_request()`` method was forgotten. - Fixed issue with sleep time when using Zookeeper (Alexander Kukushkin) There were chances that Patroni was sleeping up to twice longer between running HA code. - Fixed invalid ``os.symlink()`` calls when moving data directory after failed bootstrap (Andrew L'Ecuyer) If the bootstrap failed Patroni is renaming data directory, pg_wal, and all tablespaces. After that it updates symlinks so filesystem remains consistent. The symlink creation was failing due to the ``src`` and ``dst`` arguments being swapped. - Fixed bug in the post_bootstrap() method (Alexander Kukushkin) If the superuser password wasn't configured Patroni was failing to call the ``post_init`` script and therefore the whole bootstrap was failing. - Fixed an issues with pg_rewind in the standby cluster (Alexander Kukushkin) If the superuser name is different from Postgres, the ``pg_rewind`` in the standby cluster was failing because the connection string didn't contain the database name. - Exit only if authentication with Etcd v3 explicitly failed (Alexander Kukushkin) On start Patroni performs discovery of Etcd cluster topology and authenticates if it is necessarily. It could happen that one of etcd servers is not accessible, Patroni was trying to perform authentication on this server and failing instead of retrying with the next node. - Handle case with psutil cmdline() returning empty list (Alexander Kukushkin) Zombie processes are still postmasters children, but they don't have cmdline() - Treat ``PATRONI_KUBERNETES_USE_ENDPOINTS`` environment variable as boolean (Alexander Kukushkin) Not doing so was making impossible disabling ``kubernetes.use_endpoints`` via environment. - Improve handling of concurrent endpoint update errors (Alexander Kukushkin) Patroni will explicitly query the current endpoint object, verify that the current pod still holds the leader lock and repeat the update. Version 2.0.1 ------------- Released 2020-10-01 **New features** - Use ``more`` as pager in ``patronictl edit-config`` if ``less`` is not available (Pavel Golub) On Windows it would be the ``more.com``. In addition to that, ``cdiff`` was changed to ``ydiff`` in ``requirements.txt``, but ``patronictl`` still supports both for compatibility. - Added support of ``raft`` ``bind_addr`` and ``password`` (Alexander Kukushkin) ``raft.bind_addr`` might be useful when running behind NAT. ``raft.password`` enables traffic encryption (requires the ``cryptography`` module). - Added ``sslpassword`` connection parameter support (Kostiantyn Nemchenko) The connection parameter was introduced in PostgreSQL 13. **Stability improvements** - Changed the behavior in pause (Alexander Kukushkin) 1. Patroni will not call the ``bootstrap`` method if the ``PGDATA`` directory is missing/empty. 2. Patroni will not exit on sysid mismatch in pause, only log a warning. 3. The node will not try to grab the leader key in pause mode if Postgres is running not in recovery (accepting writes) but the sysid doesn't match with the initialize key. - Apply ``master_start_timeout`` when executing crash recovery (Alexander Kukushkin) If Postgres crashed on the leader node, Patroni does a crash-recovery by starting Postgres in single-user mode. During the crash-recovery the leader lock is being updated. If the crash-recovery didn't finish in ``master_start_timeout`` seconds, Patroni will stop it forcefully and release the leader lock. - Removed the ``secure`` extra from the ``urllib3`` requirements (Alexander Kukushkin) The only reason for adding it there was the ``ipaddress`` dependency for python 2.7. **Bugfixes** - Fixed a bug in the ``Kubernetes.update_leader()`` (Alexander Kukushkin) An unhandled exception was preventing demoting the primary when the update of the leader object failed. - Fixed hanging ``patronictl`` when RAFT is being used (Alexander Kukushkin) When using ``patronictl`` with Patroni config, ``self_addr`` should be added to the ``partner_addrs``. - Fixed bug in ``get_guc_value()`` (Alexander Kukushkin) Patroni was failing to get the value of ``restore_command`` on PostgreSQL 12, therefore fetching missing WALs for ``pg_rewind`` didn't work. Version 2.0.0 ------------- Released 2020-09-02 This version enhances compatibility with PostgreSQL 13, adds support of multiple synchronous standbys, has significant improvements in handling of ``pg_rewind``, adds support of Etcd v3 and Patroni on pure RAFT (without Etcd, Consul, or Zookeeper), and makes it possible to optionally call the ``pre_promote`` (fencing) script. **PostgreSQL 13 support** - Don't fire ``on_reload`` when promoting to ``standby_leader`` on PostgreSQL 13+ (Alexander Kukushkin) When promoting to ``standby_leader`` we change ``primary_conninfo``, update the role and reload Postgres. Since ``on_role_change`` and ``on_reload`` effectively duplicate each other, Patroni will call only ``on_role_change``. - Added support for ``gssencmode`` and ``channel_binding`` connection parameters (Alexander Kukushkin) PostgreSQL 12 introduced ``gssencmode`` and 13 ``channel_binding`` connection parameters and now they can be used if defined in the ``postgresql.authentication`` section. - Handle renaming of ``wal_keep_segments`` to ``wal_keep_size`` (Alexander Kukushkin) In case of misconfiguration (``wal_keep_segments`` on 13 and ``wal_keep_size`` on older versions) Patroni will automatically adjust the configuration. - Use ``pg_rewind`` with ``--restore-target-wal`` on 13 if possible (Alexander Kukushkin) On PostgreSQL 13 Patroni checks if ``restore_command`` is configured and tells ``pg_rewind`` to use it. **New features** - [BETA] Implemented support of Patroni on pure RAFT (Alexander Kukushkin) This makes it possible to run Patroni without 3rd party dependencies, like Etcd, Consul, or Zookeeper. For HA you will have to run either three Patroni nodes or two nodes with Patroni and one node with ``patroni_raft_controller``. For more information please check the :ref:`documentation `. - [BETA] Implemented support for Etcd v3 protocol via gPRC-gateway (Alexander Kukushkin) Etcd 3.0 was released more than four years ago and Etcd 3.4 has v2 disabled by default. There are also chances that v2 will be completely removed from Etcd, therefore we implemented support of Etcd v3 in Patroni. In order to start using it you have to explicitly create the ``etcd3`` section is the Patroni configuration file. - Supporting multiple synchronous standbys (Krishna Sarabu) It allows running a cluster with more than one synchronous replicas. The maximum number of synchronous replicas is controlled by the new parameter ``synchronous_node_count``. It is set to 1 by default and has no effect when the ``synchronous_mode`` is set to ``off``. - Added possibility to call the ``pre_promote`` script (Sergey Dudoladov) Unlike callbacks, the ``pre_promote`` script is called synchronously after acquiring the leader lock, but before promoting Postgres. If the script fails or exits with a non-zero exitcode, the current node will release the leader lock. - Added support for configuration directories (Floris van Nee) YAML files in the directory loaded and applied in alphabetical order. - Advanced validation of PostgreSQL parameters (Alexander Kukushkin) In case the specific parameter is not supported by the current PostgreSQL version or when its value is incorrect, Patroni will remove the parameter completely or try to fix the value. - Wake up the main thread when the forced checkpoint after promote completed (Alexander Kukushkin) Replicas are waiting for checkpoint indication via member key of the leader in DCS. The key is normally updated only once per HA loop. Without waking the main thread up, replicas will have to wait up to ``loop_wait`` seconds longer than necessary. - Use of ``pg_stat_wal_receiver`` view on 9.6+ (Alexander Kukushkin) The view contains up-to-date values of ``primary_conninfo`` and ``primary_slot_name``, while the contents of ``recovery.conf`` could be stale. - Improved handing of IPv6 addresses in the Patroni config file (Mateusz Kowalski) The IPv6 address is supposed to be enclosed into square brackets, but Patroni was expecting to get it plain. Now both formats are supported. - Added Consul ``service_tags`` configuration parameter (Robert Edström) They are useful for dynamic service discovery, for example by load balancers. - Implemented SSL support for Zookeeper (Kostiantyn Nemchenko) It requires ``kazoo>=2.6.0``. - Implemented ``no_params`` option for custom bootstrap method (Kostiantyn Nemchenko) It allows calling ``wal-g``, ``pgBackRest`` and other backup tools without wrapping them into shell scripts. - Move WAL and tablespaces after a failed init (Feike Steenbergen) When doing ``reinit``, Patroni was already removing not only ``PGDATA`` but also the symlinked WAL directory and tablespaces. Now the ``move_data_directory()`` method will do a similar job, i.e. rename WAL directory and tablespaces and update symlinks in PGDATA. **Improved in pg_rewind support** - Improved timeline divergence check (Alexander Kukushkin) We don't need to rewind when the replayed location on the replica is not ahead of the switchpoint or the end of the checkpoint record on the former primary is the same as the switchpoint. In order to get the end of the checkpoint record we use ``pg_waldump`` and parse its output. - Try to fetch missing WAL if ``pg_rewind`` complains about it (Alexander Kukushkin) It could happen that the WAL segment required for ``pg_rewind`` doesn't exist in the ``pg_wal`` directory anymore and therefore ``pg_rewind`` can't find the checkpoint location before the divergence point. Starting from PostgreSQL 13 ``pg_rewind`` could use ``restore_command`` for fetching missing WALs. For older PostgreSQL versions Patroni parses the errors of a failed rewind attempt and tries to fetch the missing WAL by calling the ``restore_command`` on its own. - Detect a new timeline in the standby cluster and trigger rewind/reinitialize if necessary (Alexander Kukushkin) The ``standby_cluster`` is decoupled from the primary cluster and therefore doesn't immediately know about leader elections and timeline switches. In order to detect the fact, the ``standby_leader`` periodically checks for new history files in ``pg_wal``. - Shorten and beautify history log output (Alexander Kukushkin) When Patroni is trying to figure out the necessity of ``pg_rewind``, it could write the content of the history file from the primary into the log. The history file is growing with every failover/switchover and eventually starts taking up too many lines, most of which are not so useful. Instead of showing the raw data, Patroni will show only 3 lines before the current replica timeline and 2 lines after. **Improvements on K8s** - Get rid of ``kubernetes`` python module (Alexander Kukushkin) The official python kubernetes client contains a lot of auto-generated code and therefore very heavy. Patroni uses only a small fraction of K8s API endpoints and implementing support for them wasn't hard. - Make it possible to bypass the ``kubernetes`` service (Alexander Kukushkin) When running on K8s, Patroni is usually communicating with the K8s API via the ``kubernetes`` service, the address of which is exposed in the ``KUBERNETES_SERVICE_HOST`` environment variable. Like any other service, the ``kubernetes`` service is handled by ``kube-proxy``, which in turn, depending on the configuration, is either relying on a userspace program or ``iptables`` for traffic routing. Skipping the intermediate component and connecting directly to the K8s master nodes allows us to implement a better retry strategy and mitigate risks of demoting Postgres when K8s master nodes are upgraded. - Sync HA loops of all pods of a Patroni cluster (Alexander Kukushkin) Not doing so was increasing failure detection time from ``ttl`` to ``ttl + loop_wait``. - Populate ``references`` and ``nodename`` in the subsets addresses on K8s (Alexander Kukushkin) Some load-balancers are relying on this information. - Fix possible race conditions in the ``update_leader()`` (Alexander Kukushkin) The concurrent update of the leader configmap or endpoint happening outside of Patroni might cause the ``update_leader()`` call to fail. In this case Patroni rechecks that the current node is still owning the leader lock and repeats the update. - Explicitly disallow patching non-existent config (Alexander Kukushkin) For DCS other than ``kubernetes`` the PATCH call is failing with an exception due to ``cluster.config`` being ``None``, but on Kubernetes it was happily creating the config annotation and preventing writing bootstrap configuration after the bootstrap finished. - Fix bug in ``pause`` (Alexander Kukushkin) Replicas were removing ``primary_conninfo`` and restarting Postgres when the leader key was absent, but they should do nothing. **Improvements in REST API** - Defer TLS handshake until worker thread has started (Alexander Kukushkin, Ben Harris) If the TLS handshake was done in the API thread and the client-side didn't send any data, the API thread was blocked (risking DoS). - Check ``basic-auth`` independently from client certificate in REST API (Alexander Kukushkin) Previously only the client certificate was validated. Doing two checks independently is an absolutely valid use-case. - Write double ``CRLF`` after HTTP headers of the ``OPTIONS`` request (Sergey Burladyan) HAProxy was happy with a single ``CRLF``, while Consul health-check complained about broken connection and unexpected EOF. - ``GET /cluster`` was showing stale members info for Zookeeper (Alexander Kukushkin) The endpoint was using the Patroni internal cluster view. For Patroni itself it didn't cause any issues, but when exposed to the outside world we need to show up-to-date information, especially replication lag. - Fixed health-checks for standby cluster (Alexander Kukushkin) The ``GET /standby-leader`` for a master and ``GET /master`` for a ``standby_leader`` were incorrectly responding with 200. - Implemented ``DELETE /switchover`` (Alexander Kukushkin) The REST API call deletes the scheduled switchover. - Created ``/readiness`` and ``/liveness`` endpoints (Alexander Kukushkin) They could be useful to eliminate "unhealthy" pods from subsets addresses when the K8s service is used with label selectors. - Enhanced ``GET /replica`` and ``GET /async`` REST API health-checks (Krishna Sarabu, Alexander Kukushkin) Checks now support optional keyword ``?lag=`` and will respond with 200 only if the lag is smaller than the supplied value. If relying on this feature please keep in mind that information about WAL position on the leader is updated only every ``loop_wait`` seconds! - Added support for user defined HTTP headers in the REST API response (Yogesh Sharma) This feature might be useful if requests are made from a browser. **Improvements in patronictl** - Don't try to call non-existing leader in ``patronictl pause`` (Alexander Kukushkin) While pausing a cluster without a leader on K8s, ``patronictl`` was showing warnings that member "None" could not be accessed. - Handle the case when member ``conn_url`` is missing (Alexander Kukushkin) On K8s it is possible that the pod doesn't have the necessary annotations because Patroni is not yet running. It was making ``patronictl`` to fail. - Added ability to print ASCII cluster topology (Maxim Fedotov, Alexander Kukushkin) It is very useful to get overview of the cluster with cascading replication. - Implement ``patronictl flush switchover`` (Alexander Kukushkin) Before that ``patronictl flush`` only supported cancelling scheduled restarts. **Bugfixes** - Attribute error during bootstrap of the cluster with existing PGDATA (Krishna Sarabu) When trying to create/update the ``/history`` key, Patroni was accessing the ``ClusterConfig`` object which wasn't created in DCS yet. - Improved exception handling in Consul (Alexander Kukushkin) Unhandled exception in the ``touch_member()`` method caused the whole Patroni process to crash. - Enforce ``synchronous_commit=local`` for the ``post_init`` script (Alexander Kukushkin) Patroni was already doing that when creating users (``replication``, ``rewind``), but missing it in the case of ``post_init`` was an oversight. As a result, if the script wasn't doing it internally on it's own the bootstrap in ``synchronous_mode`` wasn't able to finish. - Increased ``maxsize`` in the Consul pool manager (ponvenkates) With the default ``size=1`` some warnings were generated. - Patroni was wrongly reporting Postgres as running (Alexander Kukushkin) The state wasn't updated when for example Postgres crashed due to an out-of-disk error. - Put ``*`` into ``pgpass`` instead of missing or empty values (Alexander Kukushkin) If for example the ``standby_cluster.port`` is not specified, the ``pgpass`` file was incorrectly generated. - Skip physical replication slot creation on the leader node with special characters (Krishna Sarabu) Patroni appeared to be creating a dormant slot (when ``slots`` defined) for the leader node when the name contained special chars such as '-' (for e.g. "abc-us-1"). - Avoid removing non-existent ``pg_hba.conf`` in the custom bootstrap (Krishna Sarabu) Patroni was failing if ``pg_hba.conf`` happened to be located outside of the ``pgdata`` dir after custom bootstrap. Version 1.6.5 ------------- Released 2020-08-23 **New features** - Master stop timeout (Krishna Sarabu) The number of seconds Patroni is allowed to wait when stopping Postgres. Effective only when ``synchronous_mode`` is enabled. When set to value greater than 0 and the ``synchronous_mode`` is enabled, Patroni sends ``SIGKILL`` to the postmaster if the stop operation is running for more than the value set by ``master_stop_timeout``. Set the value according to your durability/availability tradeoff. If the parameter is not set or set to non-positive value, ``master_stop_timeout`` does not have an effect. - Don't create permanent physical slot with name of the primary (Alexander Kukushkin) It is a common problem that the primary recycles WAL segments while the replica is down. Now we have a good solution for static clusters, with a fixed number of nodes and names that never change. You just need to list the names of all nodes in the ``slots`` so the primary will not remove the slot when the node is down (not registered in DCS). - First draft of Config Validator (Igor Yanchenko) Use ``patroni --validate-config patroni.yaml`` in order to validate Patroni configuration. - Possibility to configure max length of timelines history (Krishna Sarabu) Patroni writes the history of failovers/switchovers into the ``/history`` key in DCS. Over time the size of this key becomes big, but in most cases only the last few lines are interesting. The ``max_timelines_history`` parameter allows to specify the maximum number of timeline history items to be kept in DCS. - Kazoo 2.7.0 compatibility (Danyal Prout) Some non-public methods in Kazoo changed their signatures, but Patroni was relying on them. **Improvements in patronictl** - Show member tags (Kostiantyn Nemchenko, Alexander Kukushkin) Tags are configured individually for every node and there was no easy way to get an overview of them - Improve members output (Alexander Kukushkin) The redundant cluster name won't be shown anymore on every line, only in the table header. .. code-block:: bash $ patronictl list + Cluster: batman (6813309862653668387) +---------+----+-----------+---------------------+ | Member | Host | Role | State | TL | Lag in MB | Tags | +-------------+----------------+--------+---------+----+-----------+---------------------+ | postgresql0 | 127.0.0.1:5432 | Leader | running | 3 | | clonefrom: true | | | | | | | | noloadbalance: true | | | | | | | | nosync: true | +-------------+----------------+--------+---------+----+-----------+---------------------+ | postgresql1 | 127.0.0.1:5433 | | running | 3 | 0.0 | | +-------------+----------------+--------+---------+----+-----------+---------------------+ - Fail if a config file is specified explicitly but not found (Kaarel Moppel) Previously ``patronictl`` was only reporting a ``DEBUG`` message. - Solved the problem of not initialized K8s pod breaking patronictl (Alexander Kukushkin) Patroni is relying on certain pod annotations on K8s. When one of the Patroni pods is stopping or starting there is no valid annotation yet and ``patronictl`` was failing with an exception. **Stability improvements** - Apply 1 second backoff if LIST call to K8s API server failed (Alexander Kukushkin) It is mostly necessary to avoid flooding logs, but also helps to prevent starvation of the main thread. - Retry if the ``retry-after`` HTTP header is returned by K8s API (Alexander Kukushkin) If the K8s API server is overwhelmed with requests it might ask to retry. - Scrub ``KUBERNETES_`` environment from the postmaster (Feike Steenbergen) The ``KUBERNETES_`` environment variables are not required for PostgreSQL, yet having them exposed to the postmaster will also expose them to backends and to regular database users (using pl/perl for example). - Clean up tablespaces on reinitialize (Krishna Sarabu) During reinit, Patroni was removing only ``PGDATA`` and leaving user-defined tablespace directories. This is causing Patroni to loop in reinit. The previous workarond for the problem was implementing the :ref:`custom bootstrap ` script. - Explicitly execute ``CHECKPOINT`` after promote happened (Alexander Kukushkin) It helps to reduce the time before the new primary is usable for ``pg_rewind``. - Smart refresh of Etcd members (Alexander Kukushkin) In case Patroni failed to execute a request on all members of the Etcd cluster, Patroni will re-check ``A`` or ``SRV`` records for changes of IPs/hosts before retrying the next time. - Skip missing values from ``pg_controldata`` (Feike Steenbergen) Values are missing when trying to use binaries of a version that doesn't match PGDATA. Patroni will try to start Postgres anyway, and Postgres will complain that the major version doesn't match and abort with an error. **Bugfixes** - Disable SSL verification for Consul when required (Julien Riou) Starting from a certain version of ``urllib3``, the ``cert_reqs`` must be explicitly set to ``ssl.CERT_NONE`` in order to effectively disable SSL verification. - Avoid opening replication connection on every cycle of HA loop (Alexander Kukushkin) Regression was introduced in 1.6.4. - Call ``on_role_change`` callback on failed primary (Alexander Kukushkin) In certain cases it could lead to the virtual IP remaining attached to the old primary. Regression was introduced in 1.4.5. - Reset rewind state if postgres started after successful pg_rewind (Alexander Kukushkin) As a result of this bug Patroni was starting up manually shut down postgres in the pause mode. - Convert ``recovery_min_apply_delay`` to ``ms`` when checking ``recovery.conf`` Patroni was indefinitely restarting replica if ``recovery_min_apply_delay`` was configured on PostgreSQL older than 12. - PyInstaller compatibility (Alexander Kukushkin) PyInstaller freezes (packages) Python applications into stand-alone executables. The compatibility was broken when we switched to the ``spawn`` method instead of ``fork`` for ``multiprocessing``. Version 1.6.4 ------------- Released 2020-01-27 **New features** - Implemented ``--wait`` option for ``patronictl reinit`` (Igor Yanchenko) Patronictl will wait for ``reinit`` to finish is the ``--wait`` option is used. - Further improvements of Windows support (Igor Yanchenko, Alexander Kukushkin) 1. All shell scripts which are used for integration testing are rewritten in python 2. The ``pg_ctl kill`` will be used to stop postgres on non posix systems 3. Don't try to use unix-domain sockets **Stability improvements** - Make sure ``unix_socket_directories`` and ``stats_temp_directory`` exist (Igor Yanchenko) Upon the start of Patroni and Postgres make sure that ``unix_socket_directories`` and ``stats_temp_directory`` exist or try to create them. Patroni will exit if failed to create them. - Make sure ``postgresql.pgpass`` is located in the place where Patroni has write access (Igor Yanchenko) In case if it doesn't have a write access Patroni will exit with exception. - Disable Consul ``serfHealth`` check by default (Kostiantyn Nemchenko) Even in case of little network problems the failing ``serfHealth`` leads to invalidation of all sessions associated with the node. Therefore, the leader key is lost much earlier than ``ttl`` which causes unwanted restarts of replicas and maybe demotion of the primary. - Configure tcp keepalives for connections to K8s API (Alexander Kukushkin) In case if we get nothing from the socket after TTL seconds it can be considered dead. - Avoid logging of passwords on user creation (Alexander Kukushkin) If the password is rejected or logging is configured to verbose or not configured at all it might happen that the password is written into postgres logs. In order to avoid it Patroni will change ``log_statement``, ``log_min_duration_statement``, and ``log_min_error_statement`` to some safe values before doing the attempt to create/update user. **Bugfixes** - Use ``restore_command`` from the ``standby_cluster`` config on cascading replicas (Alexander Kukushkin) The ``standby_leader`` was already doing it from the beginning the feature existed. Not doing the same on replicas might prevent them from catching up with standby leader. - Update timeline reported by the standby cluster (Alexander Kukushkin) In case of timeline switch the standby cluster was correctly replicating from the primary but ``patronictl`` was reporting the old timeline. - Allow certain recovery parameters be defined in the custom_conf (Alexander Kukushkin) When doing validation of recovery parameters on replica Patroni will skip ``archive_cleanup_command``, ``promote_trigger_file``, ``recovery_end_command``, ``recovery_min_apply_delay``, and ``restore_command`` if they are not defined in the patroni config but in files other than ``postgresql.auto.conf`` or ``postgresql.conf``. - Improve handling of postgresql parameters with period in its name (Alexander Kukushkin) Such parameters could be defined by extensions where the unit is not necessarily a string. Changing the value might require a restart (for example ``pg_stat_statements.max``). - Improve exception handling during shutdown (Alexander Kukushkin) During shutdown Patroni is trying to update its status in the DCS. If the DCS is inaccessible an exception might be raised. Lack of exception handling was preventing logger thread from stopping. Version 1.6.3 ------------- Released 2019-12-05 **Bugfixes** - Don't expose password when running ``pg_rewind`` (Alexander Kukushkin) Bug was introduced in the `#1301 `__ - Apply connection parameters specified in the ``postgresql.authentication`` to ``pg_basebackup`` and custom replica creation methods (Alexander Kukushkin) They were relying on url-like connection string and therefore parameters never applied. Version 1.6.2 ------------- Released 2019-12-05 **New features** - Implemented ``patroni --version`` (Igor Yanchenko) It prints the current version of Patroni and exits. - Set the ``user-agent`` http header for all http requests (Alexander Kukushkin) Patroni is communicating with Consul, Etcd, and Kubernetes API via the http protocol. Having a specifically crafted ``user-agent`` (example: ``Patroni/1.6.2 Python/3.6.8 Linux``) might be useful for debugging and monitoring. - Make it possible to configure log level for exception tracebacks (Igor Yanchenko) If you set ``log.traceback_level=DEBUG`` the tracebacks will be visible only when ``log.level=DEBUG``. The default behavior remains the same. **Stability improvements** - Avoid importing all DCS modules when searching for the module required by the config file (Alexander Kukushkin) There is no need to import modules for Etcd, Consul, and Kubernetes if we need only e.g. Zookeeper. It helps to reduce memory usage and solves the problem of having INFO messages ``Failed to import smth``. - Removed python ``requests`` module from explicit requirements (Alexander Kukushkin) It wasn't used for anything critical, but causing a lot of problems when the new version of ``urllib3`` is released. - Improve handling of ``etcd.hosts`` written as a comma-separated string instead of YAML array (Igor Yanchenko) Previously it was failing when written in format ``host1:port1, host2:port2`` (the space character after the comma). **Usability improvements** - Don't force users to choose members from an empty list in ``patronictl`` (Igor Yanchenko) If the user provides a wrong cluster name, we will raise an exception rather than ask to choose a member from an empty list. - Make the error message more helpful if the REST API cannot bind (Igor Yanchenko) For an inexperienced user it might be hard to figure out what is wrong from the Python stacktrace. **Bugfixes** - Fix calculation of ``wal_buffers`` (Alexander Kukushkin) The base unit has been changed from 8 kB blocks to bytes in PostgreSQL 11. - Use ``passfile`` in ``primary_conninfo`` only on PostgreSQL 10+ (Alexander Kukushkin) On older versions there is no guarantee that ``passfile`` will work, unless the latest version of ``libpq`` is installed. Version 1.6.1 ------------- Released 2019-11-15 **New features** - Added ``PATRONICTL_CONFIG_FILE`` environment variable (msvechla) It allows configuring the ``--config-file`` argument for ``patronictl`` from the environment. - Implement ``patronictl history`` (Alexander Kukushkin) It shows the history of failovers/switchovers. - Pass ``-c statement_timeout=0`` in ``PGOPTIONS`` when doing ``pg_rewind`` (Alexander Kukushkin) It protects from the case when ``statement_timeout`` on the server is set to some small value and one of the statements executed by pg_rewind is canceled. - Allow lower values for PostgreSQL configuration (Soulou) Patroni didn't allow some of the PostgreSQL configuration parameters be set smaller than some hardcoded values. Now the minimal allowed values are smaller, default values have not been changed. - Allow for certificate-based authentication (Jonathan S. Katz) This feature enables certificate-based authentication for superuser, replication, rewind accounts and allows the user to specify the ``sslmode`` they wish to connect with. - Use the ``passfile`` in the ``primary_conninfo`` instead of password (Alexander Kukushkin) It allows to avoid setting ``600`` permissions on postgresql.conf - Perform ``pg_ctl reload`` regardless of config changes (Alexander Kukushkin) It is possible that some config files are not controlled by Patroni. When somebody is doing a reload via the REST API or by sending SIGHUP to the Patroni process, the usual expectation is that Postgres will also be reloaded. Previously it didn't happen when there were no changes in the ``postgresql`` section of Patroni config. - Compare all recovery parameters, not only ``primary_conninfo`` (Alexander Kukushkin) Previously the ``check_recovery_conf()`` method was only checking whether ``primary_conninfo`` has changed, never taking into account all other recovery parameters. - Make it possible to apply some recovery parameters without restart (Alexander Kukushkin) Starting from PostgreSQL 12 the following recovery parameters could be changed without restart: ``archive_cleanup_command``, ``promote_trigger_file``, ``recovery_end_command``, and ``recovery_min_apply_delay``. In future Postgres releases this list will be extended and Patroni will support it automatically. - Make it possible to change ``use_slots`` online (Alexander Kukushkin) Previously it required restarting Patroni and removing slots manually. - Remove only ``PATRONI_`` prefixed environment variables when starting up Postgres (Cody Coons) It will solve a lot of problems with running different Foreign Data Wrappers. **Stability improvements** - Use LIST + WATCH when working with K8s API (Alexander Kukushkin) It allows to efficiently receive object changes (pods, endpoints/configmaps) and makes less stress on K8s master nodes. - Improve the workflow when PGDATA is not empty during bootstrap (Alexander Kukushkin) According to the ``initdb`` source code it might consider a PGDATA empty when there are only ``lost+found`` and ``.dotfiles`` in it. Now Patroni does the same. If ``PGDATA`` happens to be non-empty, and at the same time not valid from the ``pg_controldata`` point of view, Patroni will complain and exit. - Avoid calling expensive ``os.listdir()`` on every HA loop (Alexander Kukushkin) When the system is under IO stress, ``os.listdir()`` could take a few seconds (or even minutes) to execute, badly affecting the HA loop of Patroni. This could even cause the leader key to disappear from DCS due to the lack of updates. There is a better and less expensive way to check that the PGDATA is not empty. Now we check the presence of the ``global/pg_control`` file in the PGDATA. - Some improvements in logging infrastructure (Alexander Kukushkin) Previously there was a possibility to loose the last few log lines on shutdown because the logging thread was a ``daemon`` thread. - Use ``spawn`` multiprocessing start method on python 3.4+ (Maciej Kowalczyk) It is a known `issue `__ in Python that threading and multiprocessing do not mix well. Switching from the default method ``fork`` to the ``spawn`` is a recommended workaround. Not doing so might result in the Postmaster starting process hanging and Patroni indefinitely reporting ``INFO: restarting after failure in progress``, while Postgres is actually up and running. **Improvements in REST API** - Make it possible to check client certificates in the REST API (Alexander Kukushkin) If the ``verify_client`` is set to ``required``, Patroni will check client certificates for all REST API calls. When it is set to ``optional``, client certificates are checked for all unsafe REST API endpoints. - Return the response code 503 for the ``GET /replica`` health check request if Postgres is not running (Alexander Anikin) Postgres might spend significant time in recovery before it starts accepting client connections. - Implement ``/history`` and ``/cluster`` endpoints (Alexander Kukushkin) The ``/history`` endpoint shows the content of the ``history`` key in DCS. The ``/cluster`` endpoint shows all cluster members and some service info like pending and scheduled restarts or switchovers. **Improvements in Etcd support** - Retry on Etcd RAFT internal error (Alexander Kukushkin) When the Etcd node is being shut down, it sends ``response code=300, data='etcdserver: server stopped'``, which was causing Patroni to demote the primary. - Don't give up on Etcd request retry too early (Alexander Kukushkin) When there were some network problems, Patroni was quickly exhausting the list of Etcd nodes and giving up without using the whole ``retry_timeout``, potentially resulting in demoting the primary. **Bugfixes** - Disable ``synchronous_commit`` when granting execute permissions to the ``pg_rewind`` user (kremius) If the bootstrap is done with ``synchronous_mode_strict: true`` the `GRANT EXECUTE` statement was waiting indefinitely due to the non-synchronous nodes being available. - Fix memory leak on python 3.7 (Alexander Kukushkin) Patroni is using ``ThreadingMixIn`` to process REST API requests and python 3.7 made threads spawn for every request non-daemon by default. - Fix race conditions in asynchronous actions (Alexander Kukushkin) There was a chance that ``patronictl reinit --force`` could be overwritten by the attempt to recover stopped Postgres. This ended up in a situation when Patroni was trying to start Postgres while basebackup was running. - Fix race condition in ``postmaster_start_time()`` method (Alexander Kukushkin) If the method is executed from the REST API thread, it requires a separate cursor object to be created. - Fix the problem of not promoting the sync standby that had a name containing upper case letters (Alexander Kukushkin) We converted the name to the lower case because Postgres was doing the same while comparing the ``application_name`` with the value in ``synchronous_standby_names``. - Kill all children along with the callback process before starting the new one (Alexander Kukushkin) Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time. - Fix 'start failed' issue (Alexander Kukushkin) Under certain conditions the Postgres state might be set to 'start failed' despite Postgres being up and running. Version 1.6.0 ------------- Released 2019-08-05 This version adds compatibility with PostgreSQL 12, makes is possible to run pg_rewind without superuser on PostgreSQL 11 and newer, and enables IPv6 support. **New features** - Psycopg2 was removed from requirements and must be installed independently (Alexander Kukushkin) Starting from 2.8.0 ``psycopg2`` was split into two different packages, ``psycopg2``, and ``psycopg2-binary``, which could be installed at the same time into the same place on the filesystem. In order to decrease dependency hell problem, we let a user choose how to install it. There are a few options available, please consult the :ref:`documentation `. - Compatibility with PostgreSQL 12 (Alexander Kukushkin) Starting from PostgreSQL 12 there is no ``recovery.conf`` anymore and all former recovery parameters are converted into `GUC `_. In order to protect from ``ALTER SYSTEM SET primary_conninfo`` or similar, Patroni will parse ``postgresql.auto.conf`` and remove all standby and recovery parameters from there. Patroni config remains backward compatible. For example despite ``restore_command`` being a GUC, one can still specify it in the ``postgresql.recovery_conf.restore_command`` section and Patroni will write it into ``postgresql.conf`` for PostgreSQL 12. - Make it possible to use ``pg_rewind`` without superuser on PostgreSQL 11 and newer (Alexander Kukushkin) If you want to use this feature please define ``username`` and ``password`` in the ``postgresql.authentication.rewind`` section of Patroni configuration file. For an already existing cluster you will have to create the user manually and ``GRANT EXECUTE`` permission on a few functions. You can find more details in the PostgreSQL `documentation `__. - Do a smart comparison of actual and desired ``primary_conninfo`` values on replicas (Alexander Kukushkin) It might help to avoid replica restart when you are converting an already existing primary-standby cluster to one managed by Patroni - IPv6 support (Alexander Kukushkin) There were two major issues. Patroni REST API service was listening only on ``0.0.0.0`` and IPv6 IP addresses used in the ``api_url`` and ``conn_url`` were not properly quoted. - Kerberos support (Ajith Vilas, Alexander Kukushkin) It makes possible using Kerberos authentication between Postgres nodes instead of defining passwords in Patroni configuration file - Manage ``pg_ident.conf`` (Alexander Kukushkin) This functionality works similarly to ``pg_hba.conf``: if the ``postgresql.pg_ident`` is defined in the config file or DCS, Patroni will write its value to ``pg_ident.conf``, however, if ``postgresql.parameters.ident_file`` is defined, Patroni will assume that ``pg_ident`` is managed from outside and not update the file. **Improvements in REST API** - Added ``/health`` endpoint (Wilfried Roset) It will return an HTTP status code only if PostgreSQL is running - Added ``/read-only`` and ``/read-write`` endpoints (Julien Riou) The ``/read-only`` endpoint enables reads balanced across replicas and the primary. The ``/read-write`` endpoint is an alias for ``/primary``, ``/leader`` and ``/master``. - Use ``SSLContext`` to wrap the REST API socket (Julien Riou) Usage of ``ssl.wrap_socket()`` is deprecated and was still allowing soon-to-be-deprecated protocols like TLS 1.1. **Logging improvements** - Two-step logging (Alexander Kukushkin) All log messages are first written into the in-memory queue and later they are asynchronously flushed into the stderr or file from a separate thread. The maximum queue size is limited (configurable). If the limit is reached, Patroni will start losing logs, which is still better than blocking the HA loop. - Enable debug logging for GET/OPTIONS API calls together with latency (Jan Tomsa) It will help with debugging of health-checks performed by HAProxy, Consul or other tooling that decides which node is the primary/replica. - Log exceptions caught in Retry (Daniel Kucera) Log the final exception when either the number of attempts or the timeout were reached. It will hopefully help to debug some issues when communication to DCS fails. **Improvements in patronictl** - Enhance dialogues for scheduled switchover and restart (Rafia Sabih) Previously dialogues did not take into account scheduled actions and therefore were misleading. - Check if config file exists (Wilfried Roset) Be verbose about configuration file when the given filename does not exists, instead of ignoring silently (which can lead to misunderstanding). - Add fallback value for ``EDITOR`` (Wilfried Roset) When the ``EDITOR`` environment variable was not defined, ``patronictl edit-config`` was failing with `PatroniCtlException`. The new strategy is to try ``editor`` and than ``vi``, which should be available on most systems. **Improvements in Consul support** - Allow to specify Consul consistency mode (Jan Tomsa) You can read more about consistency mode `here `__. - Reload Consul config on SIGHUP (Cameron Daniel Kucera, Alexander Kukushkin) It is especially useful when somebody is changing the value of ``token``. **Bugfixes** - Fix corner case in switchover/failover (Sharoon Thomas) The variable ``scheduled_at`` may be undefined if REST API is not accessible and we are using DCS as a fallback. - Open trust to localhost in ``pg_hba.conf`` during custom bootstrap (Alexander Kukushkin) Previously it was open only to unix_socket, which was causing a lot of errors: ``FATAL: no pg_hba.conf entry for replication connection from host "127.0.0.1", user "replicator"`` - Consider synchronous node as healthy even when the former leader is ahead (Alexander Kukushkin) If the primary loses access to the DCS, it restarts Postgres in read-only, but it might happen that other nodes can still access the old primary via the REST API. Such a situation was causing the synchronous standby not to promote because the old primary was reporting WAL position ahead of the synchronous standby. - Standby cluster bugfixes (Alexander Kukushkin) Make it possible to bootstrap a replica in a standby cluster when the standby_leader is not accessible and a few other minor fixes. Version 1.5.6 ------------- Released 2019-08-03 **New features** - Support work with etcd cluster via set of proxies (Alexander Kukushkin) It might happen that etcd cluster is not accessible directly but via set of proxies. In this case Patroni will not perform etcd topology discovery but just round-robin via proxy hosts. Behavior is controlled by `etcd.use_proxies`. - Changed callbacks behavior when role on the node is changed (Alexander Kukushkin) If the role was changed from `master` or `standby_leader` to `replica` or from `replica` to `standby_leader`, `on_restart` callback will not be called anymore in favor of `on_role_change` callback. - Change the way how we start postgres (Alexander Kukushkin) Use `multiprocessing.Process` instead of executing itself and `multiprocessing.Pipe` to transmit the postmaster pid to the Patroni process. Before that we were using pipes, what was leaving postmaster process with stdin closed. **Bug fixes** - Fix role returned by REST API for the standby leader (Alexander Kukushkin) It was incorrectly returning `replica` instead of `standby_leader` - Wait for callback end if it could not be killed (Julien Tachoires) Patroni doesn't have enough privileges to terminate the callback script running under `sudo` what was cancelling the new callback. If the running script could not be killed, Patroni will wait until it finishes and then run the next callback. - Reduce lock time taken by dcs.get_cluster method (Alexander Kukushkin) Due to the lock being held DCS slowness was affecting the REST API health checks causing false positives. - Improve cleaning of PGDATA when `pg_wal`/`pg_xlog` is a symlink (Julien Tachoires) In this case Patroni will explicitly remove files from the target directory. - Remove unnecessary usage of os.path.relpath (Ants Aasma) It depends on being able to resolve the working directory, what will fail if Patroni is started in a directory that is later unlinked from the filesystem. - Do not enforce ssl version when communicating with Etcd (Alexander Kukushkin) For some unknown reason python3-etcd on debian and ubuntu are not based on the latest version of the package and therefore it enforces TLSv1 which is not supported by Etcd v3. We solved this problem on Patroni side. Version 1.5.5 ------------- Released 2019-02-15 This version introduces the possibility of automatic reinit of the former master, improves patronictl list output and fixes a number of bugs. **New features** - Add support of `PATRONI_ETCD_PROTOCOL`, `PATRONI_ETCD_USERNAME` and `PATRONI_ETCD_PASSWORD` environment variables (Étienne M) Before it was possible to configure them only in the config file or as a part of `PATRONI_ETCD_URL`, which is not always convenient. - Make it possible to automatically reinit the former master (Alexander Kukushkin) If the pg_rewind is disabled or can't be used, the former master could fail to start as a new replica due to diverged timelines. In this case, the only way to fix it is wiping the data directory and reinitializing. This behavior could be changed by setting `postgresql.remove_data_directory_on_diverged_timelines`. When it is set, Patroni will wipe the data directory and reinitialize the former master automatically. - Show information about timelines in patronictl list (Alexander Kukushkin) It helps to detect stale replicas. In addition to that, `Host` will include ':{port}' if the port value isn't default or there is more than one member running on the same host. - Create a headless service associated with the $SCOPE-config endpoint (Alexander Kukushkin) The "config" endpoint keeps information about the cluster-wide Patroni and Postgres configuration, history file, and last but the most important, it holds the `initialize` key. When the Kubernetes master node is restarted or upgraded, it removes endpoints without services. The headless service will prevent it from being removed. **Bug fixes** - Adjust the read timeout for the leader watch blocking query (Alexander Kukushkin) According to the Consul documentation, the actual response timeout is increased by a small random amount of additional wait time added to the supplied maximum wait time to spread out the wake up time of any concurrent requests. It adds up to `wait / 16` additional time to the maximum duration. In our case we are adding `wait / 15` or 1 second depending on what is bigger. - Always use replication=1 when connecting via replication protocol to the postgres (Alexander Kukushkin) Starting from Postgres 10 the line in the pg_hba.conf with database=replication doesn't accept connections with the parameter replication=database. - Don't write primary_conninfo into recovery.conf for wal-only standby cluster (Alexander Kukushkin) Despite not having neither `host` nor `port` defined in the `standby_cluster` config, Patroni was putting the `primary_conninfo` into the `recovery.conf`, which is useless and generating a lot of errors. Version 1.5.4 ------------- Released 2019-01-15 This version implements flexible logging and fixes a number of bugs. **New features** - Improvements in logging infrastructure (Alexander Kukushkin, Lucas Capistrant, Alexander Anikin) Logging configuration could be configured not only from environment variables but also from Patroni config file. It makes it possible to change logging configuration in runtime by updating config and doing reload or sending SIGHUP to the Patroni process. By default Patroni writes logs to stderr, but now it becomes possible to write logs directly into the file and rotate when it reaches a certain size. In addition to that added support of custom dateformat and the possibility to fine-tune log level for each python module. - Make it possible to take into account the current timeline during leader elections (Alexander Kukushkin) It could happen that the node is considering itself as a healthiest one although it is currently not on the latest known timeline. In some cases we want to avoid promoting of such node, which could be achieved by setting `check_timeline` parameter to `true` (default behavior remains unchanged). - Relaxed requirements on superuser credentials Libpq allows opening connections without explicitly specifying neither username nor password. Depending on situation it relies either on pgpass file or trust authentication method in pg_hba.conf. Since pg_rewind is also using libpq, it will work the same way. - Implemented possibility to configure Consul Service registration and check interval via environment variables (Alexander Kukushkin) Registration of service in Consul was added in the 1.5.0, but so far it was only possible to turn it on via patroni.yaml. **Stability Improvements** - Set archive_mode to off during the custom bootstrap (Alexander Kukushkin) We want to avoid archiving wals and history files until the cluster is fully functional. It really helps if the custom bootstrap involves pg_upgrade. - Apply five seconds backoff when loading global config on start (Alexander Kukushkin) It helps to avoid hammering DCS when Patroni just starting up. - Reduce amount of error messages generated on shutdown (Alexander Kukushkin) They were harmless but rather annoying and sometimes scary. - Explicitly secure rw perms for recovery.conf at creation time (Lucas Capistrant) We don't want anybody except patroni/postgres user reading this file, because it contains replication user and password. - Redirect HTTPServer exceptions to logger (Julien Riou) By default, such exceptions were logged on standard output messing with regular logs. **Bug fixes** - Removed stderr pipe to stdout on pg_ctl process (Cody Coons) Inheriting stderr from the main Patroni process allows all Postgres logs to be seen along with all patroni logs. This is very useful in a container environment as Patroni and Postgres logs may be consumed using standard tools (docker logs, kubectl, etc). In addition to that, this change fixes a bug with Patroni not being able to catch postmaster pid when postgres writing some warnings into stderr. - Set Consul service check deregister timeout in Go time format (Pavel Kirillov) Without explicitly mentioned time unit registration was failing. - Relax checks of standby_cluster cluster configuration (Dmitry Dolgov, Alexander Kukushkin) It was accepting only strings as valid values and therefore it was not possible to specify the port as integer and create_replica_methods as a list. Version 1.5.3 ------------- Released 2018-12-03 Compatibility and bugfix release. - Improve stability when running with python3 against zookeeper (Alexander Kukushkin) Change of `loop_wait` was causing Patroni to disconnect from zookeeper and never reconnect back. - Fix broken compatibility with postgres 9.3 (Alexander Kukushkin) When opening a replication connection we should specify replication=1, because 9.3 does not understand replication='database' - Make sure we refresh Consul session at least once per HA loop and improve handling of consul sessions exceptions (Alexander Kukushkin) Restart of local consul agent invalidates all sessions related to the node. Not calling session refresh on time and not doing proper handling of session errors was causing demote of the primary. Version 1.5.2 ------------- Released 2018-11-26 Compatibility and bugfix release. - Compatibility with kazoo-2.6.0 (Alexander Kukushkin) In order to make sure that requests are performed with an appropriate timeout, Patroni redefines create_connection method from python-kazoo module. The last release of kazoo slightly changed the way how create_connection method is called. - Fix Patroni crash when Consul cluster loses the leader (Alexander Kukushkin) The crash was happening due to incorrect implementation of touch_member method, it should return boolean and not raise any exceptions. Version 1.5.1 ------------- Released 2018-11-01 This version implements support of permanent replication slots, adds support of pgBackRest and fixes number of bugs. **New features** - Permanent replication slots (Alexander Kukushkin) Permanent replication slots are preserved on failover/switchover, that is, Patroni on the new primary will create configured replication slots right after doing promote. Slots could be configured with the help of `patronictl edit-config`. The initial configuration could be also done in the :ref:`bootstrap.dcs `. - Add pgbackrest support (Yogesh Sharma) pgBackrest can restore in existing $PGDATA folder, this allows speedy restore as files which have not changed since last backup are skipped, to support this feature new parameter `keep_data` has been introduced. See :ref:`replica creation method ` section for additional examples. **Bug fixes** - A few bugfixes in the "standby cluster" workflow (Alexander Kukushkin) Please see https://github.com/patroni/patroni/pull/823 for more details. - Fix REST API health check when cluster management is paused and DCS is not accessible (Alexander Kukushkin) Regression was introduced in https://github.com/patroni/patroni/commit/90cf930036a9d5249265af15d2b787ec7517cf57 Version 1.5.0 ------------- Released 2018-09-20 This version enables Patroni HA cluster to operate in a standby mode, introduces experimental support for running on Windows, and provides a new configuration parameter to register PostgreSQL service in Consul. **New features** - Standby cluster (Dmitry Dolgov) One or more Patroni nodes can form a standby cluster that runs alongside the primary one (i.e. in another datacenter) and consists of standby nodes that replicate from the master in the primary cluster. All PostgreSQL nodes in the standby cluster are replicas; one of those replicas elects itself to replicate directly from the remote master, while the others replicate from it in a cascading manner. More detailed description of this feature and some configuration examples can be found at :ref:`here `. - Register Services in Consul (Pavel Kirillov, Alexander Kukushkin) If `register_service` parameter in the consul :ref:`configuration ` is enabled, the node will register a service with the name `scope` and the tag `master`, `replica` or `standby-leader`. - Experimental Windows support (Pavel Golub) From now on it is possible to run Patroni on Windows, although Windows support is brand-new and hasn't received as much real-world testing as its Linux counterpart. We welcome your feedback! **Improvements in patronictl** - Add patronictl -k/--insecure flag and support for restapi cert (Wilfried Roset) In the past if the REST API was protected by the self-signed certificates `patronictl` would fail to verify them. There was no way to disable that verification. It is now possible to configure `patronictl` to skip the certificate verification altogether or provide CA and client certificates in the :ref:`ctl: ` section of configuration. - Exclude members with nofailover tag from patronictl switchover/failover output (Alexander Anikin) Previously, those members were incorrectly proposed as candidates when performing interactive switchover or failover via patronictl. **Stability improvements** - Avoid parsing non-key-value output lines of pg_controldata (Alexander Anikin) Under certain circuimstances pg_controldata outputs lines without a colon character. That would trigger an error in Patroni code that parsed pg_controldata output, hiding the actual problem; often such lines are emitted in a warning shown by pg_controldata before the regular output, i.e. when the binary major version does not match the one of the PostgreSQL data directory. - Add member name to the error message during the leader election (Jan Mussler) During the leader election, Patroni connects to all known members of the cluster and requests their status. Such status is written to the Patroni log and includes the name of the member. Previously, if the member was not accessible, the error message did not indicate its name, containing only the URL. - Immediately reserve the WAL position upon creation of the replication slot (Alexander Kukushkin) Starting from 9.6, `pg_create_physical_replication_slot` function provides an additional boolean parameter `immediately_reserve`. When it is set to `false`, which is also the default, the slot doesn't reserve the WAL position until it receives the first client connection, potentially losing some segments required by the client in a time window between the slot creation and the initial client connection. - Fix bug in strict synchronous replication (Alexander Kukushkin) When running with `synchronous_mode_strict: true`, in some cases Patroni puts `*` into the `synchronous_standby_names`, changing the sync state for most of the replication connections to `potential`. Previously, Patroni couldn't pick a synchronous candidate under such curcuimstances, as it only considered those with the state `async`. Version 1.4.6 ------------- Released 2018-08-14 **Bug fixes and stability improvements** This release fixes a critical issue with Patroni API /master endpoint returning 200 for the non-master node. This is a reporting issue, no actual split-brain, but under certain circumstances clients might be directed to the read-only node. - Reset is_leader status on demote (Alexander Kukushkin, Oleksii Kliukin) Make sure demoted cluster member stops responding with code 200 on the /master API call. - Add new "cluster_unlocked" field to the API output (Dmitry Dolgov) This field indicates whether the cluster has the master running. It can be used when it is not possible to query any other node but one of the replicas. Version 1.4.5 ------------- Released 2018-08-03 **New features** - Improve logging when applying new postgres configuration (Don Seiler) Patroni logs changed parameter names and values. - Python 3.7 compatibility (Christoph Berg) async is a reserved keyword in python3.7 - Set state to "stopped" in the DCS when a member is shut down (Tony Sorrentino) This shows the member state as "stopped" in "patronictl list" command. - Improve the message logged when stale postmaster.pid matches a running process (Ants Aasma) The previous one was beyond confusing. - Implement patronictl reload functionality (Don Seiler) Before that it was only possible to reload configuration by either calling REST API or by sending SIGHUP signal to the Patroni process. - Take and apply some parameters from controldata when starting as a replica (Alexander Kukushkin) The value of `max_connections` and some other parameters set in the global configuration may be lower than the one actually used by the primary; when this happens, the replica cannot start and should be fixed manually. Patroni takes care of that now by reading and applying the value from `pg_controldata`, starting postgres and setting `pending_restart` flag. - If set, use LD_LIBRARY_PATH when starting postgres (Chris Fraser) When starting up Postgres, Patroni was passing along PATH, LC_ALL and LANG env vars if they are set. Now it is doing the same with LD_LIBRARY_PATH. It should help if somebody installed PostgreSQL to non-standard place. - Rename create_replica_method to create_replica_methods (Dmitry Dolgov) To make it clear that it's actually an array. The old name is still supported for backward compatibility. **Bug fixes and stability improvements** - Fix condition for the replica start due to pg_rewind in paused state (Oleksii Kliukin) Avoid starting the replica that had already executed pg_rewind before. - Respond 200 to the master health-check only if update_lock has been successful (Alexander Kukushkin) Prevent Patroni from reporting itself a master on the former (demoted) master if DCS is partitioned. - Fix compatibility with the new consul module (Alexander Kukushkin) Starting from v1.1.0 python-consul changed internal API and started using `list` instead of `dict` to pass query parameters. - Catch exceptions from Patroni REST API thread during shutdown (Alexander Kukushkin) Those uncaught exceptions kept PostgreSQL running at shutdown. - Do crash recovery only when Postgres runs as the master (Alexander Kukushkin) Require `pg_controldata` to report 'in production' or 'shutting down' or 'in crash recovery'. In all other cases no crash recovery is necessary. - Improve handling of configuration errors (Henning Jacobs, Alexander Kukushkin) It is possible to change a lot of parameters in runtime (including `restapi.listen`) by updating Patroni config file and sending SIGHUP to Patroni process. This fix eliminates obscure exceptions from the 'restapi' thread when some of the parameters receive invalid values. Version 1.4.4 ------------- Released 2018-05-22 **Stability improvements** - Fix race condition in poll_failover_result (Alexander Kukushkin) It didn't affect directly neither failover nor switchover, but in some rare cases it was reporting success too early, when the former leader released the lock, producing a 'Failed over to "None"' instead of 'Failed over to "desired-node"' message. - Treat Postgres parameter names as case insensitive (Alexander Kukushkin) Most of the Postgres parameters have snake_case names, but there are three exceptions from this rule: DateStyle, IntervalStyle and TimeZone. Postgres accepts those parameters when written in a different case (e.g. timezone = 'some/tzn'); however, Patroni was unable to find case-insensitive matches of those parameter names in pg_settings and ignored such parameters as a result. - Abort start if attaching to running postgres and cluster not initialized (Alexander Kukushkin) Patroni can attach itself to an already running Postgres instance. It is imperative to start running Patroni on the master node before getting to the replicas. - Fix behavior of patronictl scaffold (Alexander Kukushkin) Pass dict object to touch_member instead of json encoded string, DCS implementation will take care of encoding it. - Don't demote master if failed to update leader key in pause (Alexander Kukushkin) During maintenance a DCS may start failing write requests while continuing to responds to read ones. In that case, Patroni used to put the Postgres master node to a read-only mode after failing to update the leader lock in DCS. - Sync replication slots when Patroni notices a new postmaster process (Alexander Kukushkin) If Postgres has been restarted, Patroni has to make sure that list of replication slots matches its expectations. - Verify sysid and sync replication slots after coming out of pause (Alexander Kukushkin) During the `maintenance` mode it may happen that data directory was completely rewritten and therefore we have to make sure that `Database system identifier` still belongs to our cluster and replication slots are in sync with Patroni expectations. - Fix a possible failure to start not running Postgres on a data directory with postmaster lock file present (Alexander Kukushkin) Detect reuse of PID from the postmaster lock file. More likely to hit such problem if you run Patroni and Postgres in the docker container. - Improve protection of DCS being accidentally wiped (Alexander Kukushkin) Patroni has a lot of logic in place to prevent failover in such case; it can also restore all keys back; however, until this change an accidental removal of /config key was switching off pause mode for 1 cycle of HA loop. - Do not exit when encountering invalid system ID (Oleksii Kliukin) Do not exit when the cluster system ID is empty or the one that doesn't pass the validation check. In that case, the cluster most likely needs a reinit; mention it in the result message. Avoid terminating Patroni, as otherwise reinit cannot happen. **Compatibility with Kubernetes 1.10+** - Added check for empty subsets (Cody Coons) Kubernetes 1.10.0+ started returning `Endpoints.subsets` set to `None` instead of `[]`. **Bootstrap improvements** - Make deleting recovery.conf optional (Brad Nicholson) If `bootstrap..keep_existing_recovery_conf` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file. This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate `recovery.conf` for you. - Allow options to the basebackup built-in method (Oleksii Kliukin) It is now possible to supply options to the built-in basebackup method by defining the `basebackup` section in the configuration, similar to how those are defined for custom replica creation methods. The difference is in the format accepted by the `basebackup` section: since pg_basebackup accepts both `--key=value` and `--key` options, the contents of the section could be either a dictionary of key-value pairs, or a list of either one-element dictionaries or just keys (for the options that don't accept values). See :ref:`replica creation method ` section for additional examples. Version 1.4.3 ------------- Released 2018-03-05 **Improvements in logging** - Make log level configurable from environment variables (Andy Newton, Keyvan Hedayati) `PATRONI_LOGLEVEL` - sets the general logging level `PATRONI_REQUESTS_LOGLEVEL` - sets the logging level for all HTTP requests e.g. Kubernetes API calls See `the docs for Python logging ` to get the names of possible log levels **Stability improvements and bug fixes** - Don't rediscover etcd cluster topology when watch timed out (Alexander Kukushkin) If we have only one host in etcd configuration and exactly this host is not accessible, Patroni was starting discovery of cluster topology and never succeeding. Instead it should just switch to the next available node. - Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander Kukushkin) Now it behaves similarly to the usual bootstrap with `initdb` - Single user mode was waiting for user input and never finish (Alexander Kukushkin) Regression was introduced in https://github.com/patroni/patroni/pull/576 Version 1.4.2 ------------- Released 2018-01-30 **Improvements in patronictl** - Rename scheduled failover to scheduled switchover (Alexander Kukushkin) Failover and switchover functions were separated in version 1.4, but `patronictl list` was still reporting `Scheduled failover` instead of `Scheduled switchover`. - Show information about pending restarts (Alexander Kukushkin) In order to apply some configuration changes sometimes it is necessary to restart postgres. Patroni was already giving a hint about that in the REST API and when writing node status into DCS, but there were no easy way to display it. - Make show-config to work with cluster_name from config file (Alexander Kukushkin) It works similar to the `patronictl edit-config` **Stability improvements** - Avoid calling pg_controldata during bootstrap (Alexander Kukushkin) During initdb or custom bootstrap there is a time window when pgdata is not empty but pg_controldata has not been written yet. In such case pg_controldata call was failing with error messages. - Handle exceptions raised from psutil (Alexander Kukushkin) cmdline is read and parsed every time when `cmdline()` method is called. It could happen that the process being examined has already disappeared, in that case `NoSuchProcess` is raised. **Kubernetes support improvements** - Don't swallow errors from k8s API (Alexander Kukushkin) A call to Kubernetes API could fail for a different number of reasons. In some cases such call should be retried, in some other cases we should log the error message and the exception stack trace. The change here will help debug Kubernetes permission issues. - Update Kubernetes example Dockerfile to install Patroni from the master branch (Maciej Szulik) Before that it was using `feature/k8s`, which became outdated. - Add proper RBAC to run patroni on k8s (Maciej Szulik) Add the Service account that is assigned to the pods of the cluster, the role that holds only the necessary permissions, and the rolebinding that connects the Service account and the Role. Version 1.4.1 ------------- Released 2018-01-17 **Fixes in patronictl** - Don't show current leader in suggested list of members to failover to. (Alexander Kukushkin) patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to. - Make patronictl switchover compatible with the old Patroni api (Alexander Kukushkin) In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint. Version 1.4 ----------- Released 2018-01-10 This version adds support for using Kubernetes as a DCS, allowing to run Patroni as a cloud-native agent in Kubernetes without any additional deployments of Etcd, Zookeeper or Consul. **Upgrade notice** Installing Patroni via pip will no longer bring in dependencies for (such as libraries for Etcd, Zookeper, Consul or Kubernetes, or support for AWS). In order to enable them one need to list them in pip install command explicitly, for instance `pip install patroni[etcd,kubernetes]`. **Kubernetes support** Implement Kubernetes-based DCS. The endpoints meta-data is used in order to store the configuration and the leader key. The meta-data field inside the pods definition is used to store the member-related data. In addition to using Endpoints, Patroni supports ConfigMaps. You can find more information about this feature in the :ref:`Kubernetes chapter of the documentation ` **Stability improvements** - Factor out postmaster process into a separate object (Ants Aasma) This object identifies a running postmaster process via pid and start time and simplifies detection (and resolution) of situations when the postmaster was restarted behind our back or when postgres directory disappeared from the file system. - Minimize the amount of SELECT's issued by Patroni on every loop of HA cycle (Alexander Kukushkin) On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master. - Remove leader key on shutdown only when we have the lock (Ants Aasma) Unconditional removal was generating unnecessary and misleading exceptions. **Improvements in patronictl** - Add version command to patronictl (Ants Aasma) It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified). - Make optional specifying cluster_name argument for some of patronictl commands (Alexander Kukushkin, Ants Aasma) It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined. - Show information about scheduled switchover and maintenance mode (Alexander Kukushkin) Before that it was possible to get this information only from Patroni logs or directly from DCS. - Improve ``patronictl reinit`` (Alexander Kukushkin) Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancels other long-running actions before proceeding with reinit. - Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander Kukushkin) It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster. Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API. - Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander Kukushkin) The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader. - Alter the behavior of ``patronictl failover`` (Alexander Kukushkin) It will work even if there is no leader, but in that case you will have to explicitly specify a node which should become the new leader. **Expose information about timeline and history** - Expose current timeline in DCS and via API (Alexander Kukushkin) Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS - Store promotion history in the /history key in DCS (Alexander Kukushkin) In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote. **Add endpoints for getting synchronous and asynchronous replicas** - Add new /sync and /async endpoints (Alexander Kukushkin, Oleksii Kliukin) Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchronous replicas correspondingly (excluding those marked as `noloadbalance`). **Allow multiple hosts for Etcd** - Add a new `hosts` parameter to Etcd configuration (Alexander Kukushkin) This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter. Version 1.3.6 ------------- Released 2017-11-10 **Stability improvements** - Verify process start time when checking if postgres is running. (Ants Aasma) After a crash that doesn't clean up postmaster.pid there could be a new process with the same pid, resulting in a false positive for is_running(), which will lead to all kinds of bad behavior. - Shutdown postgresql before bootstrap when we lost data directory (ainlolcat) When data directory on the master is forcefully removed, postgres process can still stay alive for some time and prevent the replica created in place of that former master from starting or replicating. The fix makes Patroni cache the postmaster pid and its start time and let it terminate the old postmaster in case it is still running after the corresponding data directory has been removed. - Perform crash recovery in a single user mode if postgres master dies (Alexander Kukushkin) It is unsafe to start immediately as a standby and not possible to run ``pg_rewind`` if postgres hasn't been shut down cleanly. The single user crash recovery only kicks in if ``pg_rewind`` is enabled or there is no master at the moment. **Consul improvements** - Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander Kukushkin) Before that Patroni was always communicating with datacenter of the host it runs on. - Always send a token in X-Consul-Token http header (Alexander Kukushkin) If ``consul.token`` is defined in Patroni configuration, we will always send it in the 'X-Consul-Token' http header. python-consul module tries to be "consistent" with Consul REST API, which doesn't accept token as a query parameter for `session API `__, but it still works with 'X-Consul-Token' header. - Adjust session TTL if supplied value is smaller than the minimum possible (Stas Fomin, Alexander Kukushkin) It could happen that the TTL provided in the Patroni configuration is smaller than the minimum one supported by Consul. In that case, Consul agent fails to create a new session. Without a session Patroni cannot create member and leader keys in the Consul KV store, resulting in an unhealthy cluster. **Other improvements** - Define custom log format via environment variable ``PATRONI_LOGFORMAT`` (Stas Fomin) Allow disabling timestamps and other similar fields in Patroni logs if they are already added by the system logger (usually when Patroni runs as a service). Version 1.3.5 ------------- Released 2017-10-12 **Bugfix** - Set role to 'uninitialized' if data directory was removed (Alexander Kukushkin) If the node was running as a master it was preventing from failover. **Stability improvement** - Try to run postmaster in a single-user mode if we tried and failed to start postgres (Alexander Kukushkin) Usually such problem happens when node running as a master was terminated and timelines were diverged. If ``recovery.conf`` has ``restore_command`` defined, there are really high chances that postgres will abort startup and leave controldata unchanged. It makes impossible to use ``pg_rewind``, which requires a clean shutdown. **Consul improvements** - Make it possible to specify health checks when creating session (Alexander Kukushkin) If not specified, Consul will use "serfHealth". From one side it allows fast detection of isolated master, but from another side it makes it impossible for Patroni to tolerate short network lags. **Bugfix** - Fix watchdog on Python 3 (Ants Aasma) A misunderstanding of the ioctl() call interface. If mutable=False then fcntl.ioctl() actually returns the arg buffer back. This accidentally worked on Python2 because int and str comparison did not return an error. Error reporting is actually done by raising IOError on Python2 and OSError on Python3. Version 1.3.4 ------------- Released 2017-09-08 **Different Consul improvements** - Pass the consul token as a header (Andrew Colin Kissa) Headers are now the preferred way to pass the token to the consul `API `__. - Advanced configuration for Consul (Alexander Kukushkin) possibility to specify ``scheme``, ``token``, client and ca certificates :ref:`details `. - compatibility with python-consul-0.7.1 and above (Alexander Kukushkin) new python-consul module has changed signature of some methods - "Could not take out TTL lock" message was never logged (Alexander Kukushkin) Not a critical bug, but lack of proper logging complicates investigation in case of problems. **Quote synchronous_standby_names using quote_ident** - When writing ``synchronous_standby_names`` into the ``postgresql.conf`` its value must be quoted (Alexander Kukushkin) If it is not quoted properly, PostgreSQL will effectively disable synchronous replication and continue to work. **Different bugfixes around pause state, mostly related to watchdog** (Alexander Kukushkin) - Do not send keepalives if watchdog is not active - Avoid activating watchdog in a pause mode - Set correct postgres state in pause mode - Do not try to run queries from API if postgres is stopped Version 1.3.3 ------------- Released 2017-08-04 **Bugfixes** - synchronous replication was disabled shortly after promotion even when synchronous_mode_strict was turned on (Alexander Kukushkin) - create empty ``pg_ident.conf`` file if it is missing after restoring from the backup (Alexander Kukushkin) - open access in ``pg_hba.conf`` to all databases, not only postgres (Franco Bellagamba) Version 1.3.2 ------------- Released 2017-07-31 **Bugfix** - patronictl edit-config didn't work with ZooKeeper (Alexander Kukushkin) Version 1.3.1 ------------- Released 2017-07-28 **Bugfix** - failover via API was broken due to change in ``_MemberStatus`` (Alexander Kukushkin) Version 1.3 ----------- Released 2017-07-27 Version 1.3 adds custom bootstrap possibility, significantly improves support for pg_rewind, enhances the synchronous mode support, adds configuration editing to patronictl and implements watchdog support on Linux. In addition, this is the first version to work correctly with PostgreSQL 10. **Upgrade notice** There are no known compatibility issues with the new version of Patroni. Configuration from version 1.2 should work without any changes. It is possible to upgrade by installing new packages and either restarting Patroni (will cause PostgreSQL restart), or by putting Patroni into a :ref:`pause mode ` first and then restarting Patroni on all nodes in the cluster (Patroni in a pause mode will not attempt to stop/start PostgreSQL), resuming from the pause mode at the end. **Custom bootstrap** - Make the process of bootstrapping the cluster configurable (Alexander Kukushkin) Allow custom bootstrap scripts instead of ``initdb`` when initializing the very first node in the cluster. The bootstrap command receives the name of the cluster and the path to the data directory. The resulting cluster can be configured to perform recovery, making it possible to bootstrap from a backup and do point in time recovery. Refer to the :ref:`documentation page ` for more detailed description of this feature. **Smarter pg_rewind support** - Decide on whether to run pg_rewind by looking at the timeline differences from the current master (Alexander Kukushkin) Previously, Patroni had a fixed set of conditions to trigger pg_rewind, namely when starting a former master, when doing a switchover to the designated node for every other node in the cluster or when there is a replica with the nofailover tag. All those cases have in common a chance that some replica may be ahead of the new master. In some cases, pg_rewind did nothing, in some other ones it was not running when necessary. Instead of relying on this limited list of rules make Patroni compare the master and the replica WAL positions (using the streaming replication protocol) in order to reliably decide if rewind is necessary for the replica. **Synchronous replication mode strict** - Enhance synchronous replication support by adding the strict mode (James Sewell, Alexander Kukushkin) Normally, when ``synchronous_mode`` is enabled and there are no replicas attached to the master, Patroni will disable synchronous replication in order to keep the master available for writes. The ``synchronous_mode_strict`` option changes that, when it is set Patroni will not disable the synchronous replication in a lack of replicas, effectively blocking all clients writing data to the master. In addition to the synchronous mode guarantee of preventing any data loss due to automatic failover, the strict mode ensures that each write is either durably stored on two nodes or not happening altogether if there is only one node in the cluster. **Configuration editing with patronictl** - Add configuration editing to patronictl (Ants Aasma, Alexander Kukushkin) Add the ability to patronictl of editing dynamic cluster configuration stored in DCS. Support either specifying the parameter/values from the command-line, invoking the $EDITOR, or applying configuration from the yaml file. **Linux watchdog support** - Implement watchdog support for Linux (Ants Aasma) Support Linux software watchdog in order to reboot the node where Patroni is not running or not responding (e.g because of the high load) The Linux software watchdog reboots the non-responsive node. It is possible to configure the watchdog device to use (`/dev/watchdog` by default) and the mode (on, automatic, off) from the watchdog section of the Patroni configuration. You can get more information from the :ref:`watchdog documentation `. **Add support for PostgreSQL 10** - Patroni is compatible with all beta versions of PostgreSQL 10 released so far and we expect it to be compatible with the PostgreSQL 10 when it will be released. **PostgreSQL-related minor improvements** - Define pg_hba.conf via the Patroni configuration file or the dynamic configuration in DCS (Alexander Kukushkin) Allow to define the contents of ``pg_hba.conf`` in the ``pg_hba`` sub-section of the ``postgresql`` section of the configuration. This simplifies managing ``pg_hba.conf`` on multiple nodes, as one needs to define it only ones in DCS instead of logging to every node, changing it manually and reload the configuration. When defined, the contents of this section will replace the current ``pg_hba.conf`` completely. Patroni ignores it if ``hba_file`` PostgreSQL parameter is set. - Support connecting via a UNIX socket to the local PostgreSQL cluster (Alexander Kukushkin) Add the ``use_unix_socket`` option to the ``postgresql`` section of Patroni configuration. When set to true and the PostgreSQL ``unix_socket_directories`` option is not empty, enables Patroni to use the first value from it to connect to the local PostgreSQL cluster. If ``unix_socket_directories`` is not defined, Patroni will assume its default value and omit the ``host`` parameter in the PostgreSQL connection string altogether. - Support change of superuser and replication credentials on reload (Alexander Kukushkin) - Support storing of configuration files outside of PostgreSQL data directory (@jouir) Add the new configuration ``postgresql`` configuration directive ``config_dir``. It defaults to the data directory and must be writable by Patroni. **Bug fixes and stability improvements** - Handle EtcdEventIndexCleared and EtcdWatcherCleared exceptions (Alexander Kukushkin) Faster recovery when the watch operation is ended by Etcd by avoiding useless retries. - Remove error spinning on Etcd failure and reduce log spam (Ants Aasma) Avoid immediate retrying and emitting stack traces in the log on the second and subsequent Etcd connection failures. - Export locale variables when forking PostgreSQL processes (Oleksii Kliukin) Avoid the `postmaster became multithreaded during startup` fatal error on non-English locales for PostgreSQL built with NLS. - Extra checks when dropping the replication slot (Alexander Kukushkin) In some cases Patroni is prevented from dropping the replication slot by the WAL sender. - Truncate the replication slot name to 63 (NAMEDATALEN - 1) characters to comply with PostgreSQL naming rules (Nick Scott) - Fix a race condition resulting in extra connections being opened to the PostgreSQL cluster from Patroni (Alexander Kukushkin) - Release the leader key when the node restarts with an empty data directory (Alex Kerney) - Set asynchronous executor busy when running bootstrap without a leader (Alexander Kukushkin) Failure to do so could have resulted in errors stating the node belonged to a different cluster, as Patroni proceeded with the normal business while being bootstrapped by a bootstrap method that doesn't require a leader to be present in the cluster. - Improve WAL-E replica creation method (Joar Wandborg, Alexander Kukushkin). - Use csv.DictReader when parsing WAL-E base backup, accepting ISO dates with space-delimited date and time. - Support fetching current WAL position from the replica to estimate the amount of WAL to restore. Previously, the code used to call system information functions that were available only on the master node. Version 1.2 ----------- Released 2016-12-13 This version introduces significant improvements over the handling of synchronous replication, makes the startup process and failover more reliable, adds PostgreSQL 9.6 support and fixes plenty of bugs. In addition, the documentation, including these release notes, has been moved to https://patroni.readthedocs.io. **Synchronous replication** - Add synchronous replication support. (Ants Aasma) Adds a new configuration variable ``synchronous_mode``. When enabled, Patroni will manage ``synchronous_standby_names`` to enable synchronous replication whenever there are healthy standbys available. When synchronous mode is enabled, Patroni will automatically fail over only to a standby that was synchronously replicating at the time of the master failure. This effectively means that no user visible transaction gets lost in such a case. See the :ref:`feature documentation ` for the detailed description and implementation details. **Reliability improvements** - Do not try to update the leader position stored in the ``leader optime`` key when PostgreSQL is not 100% healthy. Demote immediately when the update of the leader key failed. (Alexander Kukushkin) - Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander Kukushkin) - Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander Kukushkin) - Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander Kukushkin) - Write all postgres parameters into postgresql.conf. (Alexander Kukushkin) It allows starting PostgreSQL configured by Patroni with just ``pg_ctl``. - Avoid exceptions when there are no users in the config. (Kirill Pushkin) - Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander Kukushkin) - Improve the leader watch functionality. (Alexander Kukushkin) Previously the replicas were always watching the leader key (sleeping until the timeout or the leader key changes). With this change, they only watch when the replica's PostgreSQL is in the ``running`` state and not when it is stopped/starting or restarting PostgreSQL. - Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander Kukushkin) Previously a race condition could occur when running inside the Docker containers, since the same process inside Patroni both spawned new processes and handled SIGCHILD from them. This change uses fork/execs for Patroni and leaves the original PID 1 process responsible for handling signals from children. - Fix WAL-E restore. (Oleksii Kliukin) Previously WAL-E restore used the ``no_master`` flag to avoid consulting with the master altogether, making Patroni always choose restoring from WAL over the ``pg_basebackup``. This change reverts it to the original meaning of ``no_master``, namely Patroni WAL-E restore may be selected as a replication method if the master is not running. The latter is checked by examining the connection string passed to the method. In addition, it makes the retry mechanism more robust and handles other minutia. - Implement asynchronous DNS resolver cache. (Alexander Kukushkin) Avoid failing when DNS is temporary unavailable (for instance, due to an excessive traffic received by the node). - Implement starting state and master start timeout. (Ants Aasma, Alexander Kukushkin) Previously ``pg_ctl`` waited for a timeout and then happily trodded on considering PostgreSQL to be running. This caused PostgreSQL to show up in listings as running when it was actually not and caused a race condition that resulted in either a failover, or a crash recovery, or a crash recovery interrupted by failover and a missed rewind. This change adds a ``master_start_timeout`` parameter and introduces a new state for the main HA loop: ``starting``. When ``master_start_timeout`` is 0 we will failover immediately when the master crashes as soon as there is a failover candidate. Otherwise, Patroni will wait after attempting to start PostgreSQL on the master for the duration of the timeout; when it expires, it will failover if possible. Manual failover requests will be honored during the crash of the master even before the timeout expiration. Introduce the ``timeout`` parameter to the ``restart`` API endpoint and ``patronictl``. When it is set and restart takes longer than the timeout, PostgreSQL is considered unhealthy and the other nodes becomes eligible to take the leader lock. - Fix ``pg_rewind`` behavior in a pause mode. (Ants Aasma) Avoid unnecessary restart in a pause mode when Patroni thinks it needs to rewind but rewind is not possible (i.e. ``pg_rewind`` is not present). Fallback to default ``libpq`` values for the ``superuser`` (default OS user) if ``superuser`` authentication is missing from the ``pg_rewind`` related Patroni configuration section. - Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander Kukushkin) - Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander Kukushkin) This avoids the issue of a current master continuing to keep its role when it is partitioned together with the minority of nodes in Etcd and other DCSs that allow "inconsistent reads". **Miscellaneous** - Add ``post_init`` configuration option on bootstrap. (Alejandro Martínez) Patroni will call the script argument of this option right after running ``initdb`` and starting up PostgreSQL for a new cluster. The script receives a connection URL with ``superuser`` and sets ``PGPASSFILE`` to point to the ``.pgpass`` file containing the password. If the script fails, Patroni initialization fails as well. It is useful for adding new users or creating extensions in the new cluster. - Implement PostgreSQL 9.6 support. (Alexander Kukushkin) Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander Kukushkin) **Documentation improvements** - Add a Patroni main `loop workflow diagram `__. (Alejandro Martínez, Alexander Kukushkin) - Improve README, adding the Helm chart and links to release notes. (Lauri Apple) - Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii Kliukin) Makes the documentation easily viewable from different devices (including smartphones) and searchable. - Move the package to the semantic versioning. (Oleksii Kliukin) Patroni will follow the major.minor.patch version schema to avoid releasing the new minor version on small but critical bugfixes. We will only publish the release notes for the minor version, which will include all patches. Version 1.1 ----------- Released 2016-09-07 This release improves management of Patroni cluster by bring in pause mode, improves maintenance with scheduled and conditional restarts, makes Patroni interaction with Etcd or Zookeeper more resilient and greatly enhances patronictl. **Upgrade notice** When upgrading from releases below 1.0 read about changing of credentials and configuration format at 1.0 release notes. **Pause mode** - Introduce pause mode to temporary detach Patroni from managing PostgreSQL instance (Murat Kabilov, Alexander Kukushkin, Oleksii Kliukin). Previously, one had to send SIGKILL signal to Patroni to stop it without terminating PostgreSQL. The new pause mode detaches Patroni from PostgreSQL cluster-wide without terminating Patroni. It is similar to the maintenance mode in Pacemaker. Patroni is still responsible for updating member and leader keys in DCS, but it will not start, stop or restart PostgreSQL server in the process. There are a few exceptions, for instance, manual failovers, reinitializes and restarts are still allowed. You can read :ref:`a detailed description of this feature `. In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle the pause mode. **Scheduled and conditional restarts** - Add conditions to the restart API command (Oleksii Kliukin) This change enhances Patroni restarts by adding a couple of conditions that can be verified in order to do the restart. Among the conditions are restarting when PostgreSQL role is either a master or a replica, checking the PostgreSQL version number or restarting only when restart is necessary in order to apply configuration changes. - Add scheduled restarts (Oleksii Kliukin) It is now possible to schedule a restart in the future. Only one scheduled restart per node is supported. It is possible to clear the scheduled restart if it is not needed anymore. A combination of scheduled and conditional restarts is supported, making it possible, for instance, to scheduled minor PostgreSQL upgrades in the night, restarting only the instances that are running the outdated minor version without adding postgres-specific logic to administration scripts. - Add support for conditional and scheduled restarts to patronictl (Murat Kabilov). patronictl restart supports several new options. There is also patronictl flush command to clean the scheduled actions. **Robust DCS interaction** - Set Kazoo timeouts depending on the loop_wait (Alexander Kukushkin) Originally, ping_timeout and connect_timeout values were calculated from the negotiated session timeout. Patroni loop_wait was not taken into account. As a result, a single retry could take more time than the session timeout, forcing Patroni to release the lock and demote. This change set ping and connect timeout to half of the value of loop_wait, speeding up detection of connection issues and leaving enough time to retry the connection attempt before losing the lock. - Update Etcd topology only after original request succeed (Alexander Kukushkin) Postpone updating the Etcd topology known to the client until after the original request. When retrieving the cluster topology, implement the retry timeouts depending on the known number of nodes in the Etcd cluster. This makes our client prefer to get the results of the request to having the up-to-date list of nodes. Both changes make Patroni connections to DCS more robust in the face of network issues. **Patronictl, monitoring and configuration** - Return information about streaming replicas via the API (Feike Steenbergen) Previously, there was no reliable way to query Patroni about PostgreSQL instances that fail to stream changes (for instance, due to connection issues). This change exposes the contents of pg_stat_replication via the /patroni endpoint. - Add patronictl scaffold command (Oleksii Kliukin) Add a command to create cluster structure in Etcd. The cluster is created with user-specified sysid and leader, and both leader and member keys are made persistent. This command is useful to create so-called master-less configurations, where Patroni cluster consisting of only replicas replicate from the external master node that is unaware of Patroni. Subsequently, one may remove the leader key, promoting one of the Patroni nodes and replacing the original master with the Patroni-based HA cluster. - Add configuration option ``bin_dir`` to locate PostgreSQL binaries (Ants Aasma) It is useful to be able to specify the location of PostgreSQL binaries explicitly when Linux distros that support installing multiple PostgreSQL versions at the same time. - Allow configuration file path to be overridden using ``custom_conf`` of (Alejandro Martínez) Allows for custom configuration file paths, which will be unmanaged by Patroni, :ref:`details `. **Bug fixes and code improvements** - Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike Steenbergen) Make sure that Patroni understand 2-digits version numbers when doing conditional restarts based on the PostgreSQL version. - Use pkgutil to find DCS modules (Alexander Kukushkin) Use the dedicated python module instead of traversing directories manually in order to find DCS modules. - Always call on_start callback when starting Patroni (Alexander Kukushkin) Previously, Patroni did not call any callbacks when attaching to the already running node with the correct role. Since callbacks are often used to route client connections that could result in the failure to register the running node in the connection routing scheme. With this fix, Patroni calls on_start callback even when attaching to the already running node. - Do not drop active replication slots (Murat Kabilov, Oleksii Kliukin) Avoid dropping active physical replication slots on master. PostgreSQL cannot drop such slots anyway. This change makes possible to run non-Patroni managed replicas/consumers on the master. - Close Patroni connections during start of the PostgreSQL instance (Alexander Kukushkin) Forces Patroni to close all former connections when PostgreSQL node is started. Avoids the trap of reusing former connections if postmaster was killed with SIGKILL. - Replace invalid characters when constructing slot names from member names (Ants Aasma) Make sure that standby names that do not comply with the slot naming rules don't cause the slot creation and standby startup to fail. Replace the dashes in the slot names with underscores and all other characters not allowed in slot names with their unicode codepoints. Version 1.0 ----------- Released 2016-07-05 This release introduces the global dynamic configuration that allows dynamic changes of the PostgreSQL and Patroni configuration parameters for the entire HA cluster. It also delivers numerous bugfixes. **Upgrade notice** When upgrading from v0.90 or below, always upgrade all replicas before the master. Since we don't store replication credentials in DCS anymore, an old replica won't be able to connect to the new master. **Dynamic Configuration** - Implement the dynamic global configuration (Alexander Kukushkin) Introduce new REST API endpoint /config to provide PostgreSQL and Patroni configuration parameters that should be set globally for the entire HA cluster (master and all the replicas). Those parameters are set in DCS and in many cases can be applied without disrupting PostgreSQL or Patroni. Patroni sets a special flag called "pending restart" visible via the API when some of the values require the PostgreSQL restart. In that case, restart should be issued manually via the API. Patroni SIGHUP or POST to /reload will make it re-read the configuration file. See the :ref:`Patroni configuration ` for the details on which parameters can be changed and the order of processing difference configuration sources. The configuration file format *has changed* since the v0.90. Patroni is still compatible with the old configuration files, but in order to take advantage of the bootstrap parameters one needs to change it. Users are encourage to update them by referring to the :ref:`dynamic configuration documentation page `. **More flexible configuration*** - Make postgresql configuration and database name Patroni connects to configurable (Misja Hoebe) Introduce `database` and `config_base_name` configuration parameters. Among others, it makes possible to run Patroni with PipelineDB and other PostgreSQL forks. - Implement possibility to configure some Patroni configuration parameters via environment (Alexander Kukushkin) Those include the scope, the node name and the namespace, as well as the secrets and makes it easier to run Patroni in a dynamic environment, i.e. Kubernetes Please, refer to the :ref:`supported environment variables ` for further details. - Update the built-in Patroni docker container to take advantage of environment-based configuration (Feike Steenbergen). - Add Zookeeper support to Patroni docker image (Alexander Kukushkin) - Split the Zookeeper and Exhibitor configuration options (Alexander Kukushkin) - Make patronictl reuse the code from Patroni to read configuration (Alexander Kukushkin) This allows patronictl to take advantage of environment-based configuration. - Set application name to node name in primary_conninfo (Alexander Kukushkin) This simplifies identification and configuration of synchronous replication for a given node. **Stability, security and usability improvements** - Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander Kukushkin) This change reduces the amount of noise generated by Patroni API health checks during the lengthy initialization of this node from the backup. - Fix a bunch of pg_rewind corner-cases (Alexander Kukushkin) Avoid running pg_rewind if the source cluster is not the master. In addition, avoid removing the data directory on an unsuccessful rewind, unless the new parameter *remove_data_directory_on_rewind_failure* is set to true. By default it is false. - Remove passwords from the replication connection string in DCS (Alexander Kukushkin) Previously, Patroni always used the replication credentials from the Postgres URL in DCS. That is now changed to take the credentials from the patroni configuration. The secrets (replication username and password) and no longer exposed in DCS. - Fix the asynchronous machinery around the demote call (Alexander Kukushkin) Demote now runs totally asynchronously without blocking the DCS interactions. - Make patronictl always send the authorization header if it is configured (Alexander Kukushkin) This allows patronictl to issue "protected" requests, i.e. restart or reinitialize, when Patroni is configured to require authorization on those. - Handle the SystemExit exception correctly (Alexander Kukushkin) Avoids the issues of Patroni not stopping properly when receiving the SIGTERM - Sample haproxy templates for confd (Alexander Kukushkin) Generates and dynamically changes haproxy configuration from the patroni state in the DCS using confide - Improve and restructure the documentation to make it more friendly to the new users (Lauri Apple) - API must report role=master during pg_ctl stop (Alexander Kukushkin) Makes the callback calls more reliable, particularly in the cluster stop case. In addition, introduce the `pg_ctl_timeout` option to set the timeout for the start, stop and restart calls via the `pg_ctl`. - Fix the retry logic in etcd (Alexander Kukushkin) Make retries more predictable and robust. - Make Zookeeper code more resilient against short network hiccups (Alexander Kukushkin) Reduce the connection timeouts to make Zookeeper connection attempts more frequent. Version 0.90 ------------ Released 2016-04-27 This releases adds support for Consul, includes a new *noloadbalance* tag, changes the behavior of the *clonefrom* tag, improves *pg_rewind* handling and improves *patronictl* control program. **Consul support** - Implement Consul support (Alexander Kukushkin) Patroni runs against Consul, in addition to Etcd and Zookeeper. the connection parameters can be configured in the YAML file. **New and improved tags** - Implement *noloadbalance* tag (Alexander Kukushkin) This tag makes Patroni always return that the replica is not available to the load balancer. - Change the implementation of the *clonefrom* tag (Alexander Kukushkin) Previously, a node name had to be supplied to the *clonefrom*, forcing a tagged replica to clone from the specific node. The new implementation makes *clonefrom* a boolean tag: if it is set to true, the replica becomes a candidate for other replicas to clone from it. When multiple candidates are present, the replicas picks one randomly. **Stability and security improvements** - Numerous reliability improvements (Alexander Kukushkin) Removes some spurious error messages, improves the stability of the failover, addresses some corner cases with reading data from DCS, shutdown, demote and reattaching of the former leader. - Improve systems script to avoid killing Patroni children on stop (Jan Keirse, Alexander Kukushkin) Previously, when stopping Patroni, *systemd* also sent a signal to PostgreSQL. Since Patroni also tried to stop PostgreSQL by itself, it resulted in sending to different shutdown requests (the smart shutdown, followed by the fast shutdown). That resulted in replicas disconnecting too early and a former master not being able to rejoin after demote. Fix by Jan with prior research by Alexander. - Eliminate some cases where the former master was unable to call pg_rewind before rejoining as a replica (Oleksii Kliukin) Previously, we only called *pg_rewind* if the former master had crashed. Change this to always run pg_rewind for the former master as long as pg_rewind is present in the system. This fixes the case when the master is shut down before the replicas managed to get the latest changes (i.e. during the "smart" shutdown). - Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander Kukushkin). - Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander Kukushkin) Both unit and acceptance tests run automatically against Etcd, Zookeeper and Consul on each commit or pull-request. - Clear environment variables before calling PostgreSQL commands from Patroni (Feike Steenbergen) This prevents a possibility of reading system environment variables by connecting to the PostgreSQL cluster managed by Patroni. **Configuration and control changes** - Unify patronictl and Patroni configuration (Feike Steenbergen) patronictl can use the same configuration file as Patroni itself. - Enable Patroni to read the configuration from the environment variables (Oleksii Kliukin) This simplifies generating configuration for Patroni automatically, or merging a single configuration from different sources. - Include database system identifier in the information returned by the API (Feike Steenbergen) - Implement *delete_cluster* for all available DCSs (Alexander Kukushkin) Enables support for DCSs other than Etcd in patronictl. Version 0.80 ------------ Released 2016-03-14 This release adds support for *cascading replication* and simplifies Patroni management by providing *scheduled failovers*. One may use older versions of Patroni (in particular, 0.78) combined with this one in order to migrate to the new release. Note that the scheduled failover and cascading replication related features will only work with Patroni 0.80 and above. **Cascading replication** - Add support for the *replicatefrom* and *clonefrom* tags for the patroni node (Oleksii Kliukin). The tag *replicatefrom* allows a replica to use an arbitrary node a source, not necessary the master. The *clonefrom* does the same for the initial backup. Together, they enable Patroni to fully support cascading replication. - Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii Kliukin). This is useful in order to create replicas from the snapshots stored on S3 or FTP. A replication method that does not require a running replication connection should supply *no_master: true* in the yaml configuration. Those scripts will still be called in order if the replication connection is present. **Patronictl, API and DCS improvements** - Implement scheduled failovers (Feike Steenbergen). Failovers can be scheduled to happen at a certain time in the future, using either patronictl, or API calls. - Add support for *dbuser* and *password* parameters in patronictl (Feike Steenbergen). - Add PostgreSQL version to the health check output (Feike Steenbergen). - Improve Zookeeper support in patronictl (Oleksandr Shulgin) - Migrate to python-etcd 0.43 (Alexander Kukushkin) **Configuration** - Add a sample systems configuration script for Patroni (Jan Keirse). - Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections (Alexander Kukushkin). - Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander Kukushkin). **Tests** - Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander Kukushkin, Oleksii Kliukin). The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits. Release notes for some older versions can be found on `project's github page `__. patroni-4.0.4/docs/replica_bootstrap.rst000066400000000000000000000241321472010352700203740ustar00rootroot00000000000000.. _replica_imaging_and_bootstrap: Replica imaging and bootstrap ============================= Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize`` key is present in DCS for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the first node that takes the initialize key lock. .. _custom_bootstrap: Bootstrap --------- PostgreSQL provides ``initdb`` command to initialize a new cluster and Patroni calls it by default. In certain cases, particularly when creating a new cluster as a copy of an existing one, it is necessary to replace a built-in method with custom actions. Patroni supports executing user-defined scripts to bootstrap new clusters, supplying some required arguments to them, i.e. the name of the cluster and the path to the data directory. This is configured in the ``bootstrap`` section of the Patroni configuration. For example: .. code:: YAML bootstrap: method: : command: [param1 [, ...]] keep_existing_recovery_conf: False no_params: False recovery_conf: recovery_target_action: promote recovery_target_timeline: latest restore_command: Each bootstrap method must define at least a ``name`` and a ``command``. A special ``initdb`` method is available to trigger the default behavior, in which case ``method`` parameter can be omitted altogether. The ``command`` can be specified using either an absolute path, or the one relative to the ``patroni`` command location. In addition to the fixed parameters defined in the configuration files, Patroni supplies two cluster-specific ones: --scope Name of the cluster to be bootstrapped --datadir Path to the data directory of the cluster instance to be bootstrapped Passing these two additional flags can be disabled by setting a special ``no_params`` parameter to ``True``. If the bootstrap script returns ``0``, Patroni tries to configure and start the PostgreSQL instance produced by it. If any of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed, cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap. If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a ``recovery.conf`` before starting the newly bootstrapped instance (or set the recovery settings on Postgres configuration if running PostgreSQL >= 12). Typically, such recovery configuration should contain at least one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``. If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists (PostgreSQL <= 11). Similarly, in that case Patroni will not remove the existing ``recovery.signal`` or ``standby.signal`` if either exists, nor will it override the configured recovery settings (PostgreSQL >= 12). This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate recovery configuration for you. Besides that, any additional key/value pairs informed in the custom bootstrap method configuration will be passed as arguments to ``command`` in the format ``--name=value``. For example: .. code:: YAML bootstrap: method: : command: arg1: value1 arg2: value2 Makes the configured ``command`` to be called additionally with ``--arg1=value1 --arg2=value2`` command-line arguments. .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails As an example, you are able to bootstrap a fresh Patroni cluster from a Barman backup with a configuration like this: .. code:: YAML bootstrap: method: barman barman: keep_existing_recovery_conf: true command: patroni_barman --api-url https://barman-host:7480 recover barman-server: my_server ssh-command: ssh postgres@patroni-host .. note:: ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API. The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``. .. _custom_replica_creation: Building replicas ----------------- Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires a running leader node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block: .. code:: YAML postgresql: create_replica_methods: - : command: keep_data: True no_params: True no_leader: 1 example: wal_e .. code:: YAML postgresql: create_replica_methods: - wal_e - basebackup wal_e: command: patroni_wale_restore no_leader: 1 envdir: {{WALE_ENV_DIR}} use_iam: 1 basebackup: max-rate: '100M' example: pgbackrest .. code:: YAML postgresql: create_replica_methods: - pgbackrest - basebackup pgbackrest: command: /usr/bin/pgbackrest --stanza= --delta restore keep_data: True no_params: True basebackup: max-rate: '100M' example: Barman .. code:: YAML postgresql: create_replica_methods: - barman - basebackup barman: command: patroni_barman --api-url https://barman-host:7480 recover barman-server: my_server ssh-command: ssh postgres@patroni-host basebackup: max-rate: '100M' .. note:: ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API. The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``. The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command to execute and any custom parameters that should be passed to that command. All parameters will be passed in a ``--name=value`` format. Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones: --scope Which cluster this replica belongs to --datadir Path to the data directory of the replica --role Always 'replica' --connstring Connection string to connect to the cluster member to clone from (primary or other replica). The user in the connection string can execute SQL and replication protocol commands. A special ``no_leader`` parameter, if defined, allows Patroni to call the replica creation method even if there is no running leader or replicas. In that case, an empty string will be passed in a connection string. This is useful for restoring the formerly running cluster from the binary backup. A special ``keep_data`` parameter, if defined, will instruct Patroni to not clean PGDATA folder before calling restore. A special ``no_params`` parameter, if defined, restricts passing parameters to custom command. A ``basebackup`` method is a special case: it will be used if ``create_replica_methods`` is empty, although it is possible to list it explicitly among the ``create_replica_methods`` methods. This method initializes a new replica with the ``pg_basebackup``, the base backup is taken from the leader unless there are replicas with ``clonefrom`` tag, in which case one of such replicas will be used as the origin for pg_basebackup. It works without any configuration; however, it is possible to specify a ``basebackup`` configuration section. Same rules as with the other method configuration apply, namely, only long (with --) options should be specified there. Not all parameters make sense, if you override a connection string or provide an option to created tar-ed or compressed base backups, patroni won't be able to make a replica out of it. There is no validation performed on the names or values of the parameters passed to the ``basebackup`` section. Also note that in case symlinks are used for the WAL folder it is up to the user to specify the correct ``--waldir`` path as an option, so that after replica buildup or re-initialization the symlink would persist. This option is supported only since v10 though. You can specify basebackup parameters as either a map (key-value pairs) or a list of elements, where each element could be either a key-value pair or a single key (for options that does not receive any values, for instance, ``--verbose``). Consider those 2 examples: .. code:: YAML postgresql: basebackup: max-rate: '100M' checkpoint: 'fast' and .. code:: YAML postgresql: basebackup: - verbose - max-rate: '100M' - waldir: /pg-wal-mount/external-waldir If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle. patroni-4.0.4/docs/replication_modes.rst000066400000000000000000000312321472010352700203570ustar00rootroot00000000000000.. _replication_modes: ================= Replication modes ================= Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation `__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you. Asynchronous mode durability ============================ In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When the primary server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to primary. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the primary, and are effectively unrecoverable [1]_. The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because the primary transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second. By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former primary become the new leader by changing the value of ``check_timeline`` parameter to ``true``. PostgreSQL synchronous replication ================================== You can use Postgres's `synchronous replication `__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: increased latency and reduced throughput on writes. This throughput will be entirely based on network performance. In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only. To enable a simple synchronous replication test, add the following lines to the ``parameters`` section of your YAML configuration files: .. code:: YAML synchronous_commit: "on" synchronous_standby_names: "*" When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails. Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When the primary and the secondary that is currently acting as a synchronous replica fail simultaneously a third node that might not contain all transactions will be promoted. .. _synchronous_mode: Synchronous mode ================ For use cases where losing committed transactions is not permissible you can turn on Patroni's ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commands to promote a standby even if it results in transaction loss. Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, primary server will still accept writes, but does not guarantee their replication. When the primary fails in this mode no standby will be promoted. When the host that used to be the primary comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters. When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches the primary to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the primary to release itself from synchronous standby duties before PostgreSQL shutdown is initiated. When it is absolutely necessary to guarantee that each write is stored durably on at least two nodes, enable ``synchronous_mode_strict`` in addition to the ``synchronous_mode``. This parameter prevents Patroni from switching off the synchronous replication on the primary when no synchronous standby candidates are available. As a downside, the primary is not be available for writes (unless the Postgres transaction explicitly turns off ``synchronous_mode``), blocking all client write requests until at least one synchronous replica comes up. You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby. Setting tag ``nostream`` to true will also have the same effect. Synchronous mode can be switched on and off using ``patronictl edit-config`` command or via Patroni REST interface. See :ref:`dynamic configuration ` for instructions. Note: Because of the way synchronous replication is implemented in PostgreSQL it is still possible to lose transactions even when using ``synchronous_mode_strict``. If the PostgreSQL backend is cancelled while waiting to acknowledge replication (as a result of packet cancellation due to client timeout or backend failure) transaction changes become visible for other backends. Such changes are not yet replicated and may be lost in case of standby promotion. Synchronous Replication Factor ============================== The parameter ``synchronous_node_count`` is used by Patroni to manage the number of synchronous standby databases. It is set to ``1`` by default. It has no effect when ``synchronous_mode`` is set to ``off``. When enabled, Patroni manages the precise number of synchronous standby databases based on parameter ``synchronous_node_count`` and adjusts the state in DCS & ``synchronous_standby_names`` in PostgreSQL as members join and leave. If the parameter is set to a value higher than the number of eligible nodes it will be automatically reduced by Patroni. Maximum lag on synchronous node =============================== By default Patroni sticks to nodes that are declared as ``synchronous``, according to the ``pg_stat_replication`` view, even when there are other nodes ahead of it. This is done to minimize the number of changes of ``synchronous_standby_names``. To change this behavior one may use ``maximum_lag_on_syncnode`` parameter. It controls how much lag the replica can have to still be considered as "synchronous". Patroni utilizes the max replica LSN if there is more than one standby, otherwise it will use leader's current wal LSN. The default is ``-1``, and Patroni will not take action to swap a synchronous unhealthy standby when the value is set to ``0`` or less. Please set the value high enough so that Patroni won't swap synchronous standbys frequently during high transaction volume. Synchronous mode implementation =============================== When in synchronous mode Patroni maintains synchronization state in the DCS (``/sync`` key), containing the latest primary and current synchronous standby databases. This state is updated with strict ordering constraints to ensure the following invariants: - A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant. - A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby in the ``/sync`` key in DCS.. - A node that is not the leader or current synchronous standby is not allowed to promote itself automatically. Patroni will only assign one or more synchronous standby nodes based on ``synchronous_node_count`` parameter to ``synchronous_standby_names``. On each HA loop iteration Patroni re-evaluates synchronous standby nodes choice. If the current list of synchronous standby nodes are connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster members available for sync that are furthest ahead in replication are picked. Example: --------- ``/config`` key in DCS ^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: YAML synchronous_mode: on synchronous_node_count: 2 ... ``/sync`` key in DCS ^^^^^^^^^^^^^^^^^^^^ .. code-block:: JSON { "leader": "node0", "sync_standby": "node1,node2" } postgresql.conf ^^^^^^^^^^^^^^^ .. code-block:: INI synchronous_standby_names = 'FIRST 2 (node1,node2)' In the above examples only nodes ``node1`` and ``node2`` are known to be synchronous and allowed to be automatically promoted if the primary (``node0``) fails. .. _quorum_mode: Quorum commit mode ================== Starting from PostgreSQL v10 Patroni supports quorum-based synchronous replication. In this mode, Patroni maintains synchronization state in the DCS, containing the latest known primary, the number of nodes required for quorum, and the nodes currently eligible to vote on quorum. In steady state, the nodes voting on quorum are the leader and all synchronous standbys. This state is updated with strict ordering constraints, with regards to node promotion and ``synchronous_standby_names``, to ensure that at all times any subset of voters that can achieve quorum includes at least one node with the latest successful commit. On each iteration of HA loop, Patroni re-evaluates synchronous standby choices and quorum, based on node availability and requested cluster configuration. In PostgreSQL versions above 9.6 all eligible nodes are added as synchronous standbys as soon as their replication catches up to leader. Quorum commit helps to reduce worst case latencies, even during normal operation, as a higher latency of replicating to one standby can be compensated by other standbys. The quorum-based synchronous mode could be enabled by setting ``synchronous_mode`` to ``quorum`` using ``patronictl edit-config`` command or via Patroni REST interface. See :ref:`dynamic configuration ` for instructions. Other parameters, like ``synchronous_node_count``, ``maximum_lag_on_syncnode``, and ``synchronous_mode_strict`` continue to work the same way as with ``synchronous_mode=on``. Example: --------- ``/config`` key in DCS ^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: YAML synchronous_mode: quorum synchronous_node_count: 2 ... ``/sync`` key in DCS ^^^^^^^^^^^^^^^^^^^^ .. code-block:: JSON { "leader": "node0", "sync_standby": "node1,node2,node3", "quorum": 1 } postgresql.conf ^^^^^^^^^^^^^^^ .. code-block:: INI synchronous_standby_names = 'ANY 2 (node1,node2,node3)' If the primary (``node0``) failed, in the above example two of the ``node1``, ``node2``, ``node3`` will have the latest transaction received, but we don't know which ones. To figure out whether the node ``node1`` has received the latest transaction, we need to compare its LSN with the LSN on **at least** one node (``quorum=1`` in the ``/sync`` key) among ``node2`` and ``node3``. If ``node1`` isn't behind of at least one of them, we can guarantee that there will be no user visible data loss if ``node1`` is promoted. .. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed primary with the cluster. However, for ``use_pg_rewind`` to function properly, either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``. .. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays. patroni-4.0.4/docs/rest_api.rst000066400000000000000000000662221472010352700164740ustar00rootroot00000000000000.. _rest_api: Patroni REST API ================ Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the :ref:`patronictl` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints. Health check endpoints ---------------------- For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``HEAD`` or ``OPTIONS`` method instead of ``GET``. - The following requests to Patroni REST API will return HTTP status code **200** only when the Patroni node is running as the primary with leader lock: - ``GET /`` - ``GET /primary`` - ``GET /read-write`` - ``GET /standby-leader``: returns HTTP status code **200** only when the Patroni node is running as the leader in a :ref:`standby cluster `. - ``GET /leader``: returns HTTP status code **200** when the Patroni node has the leader lock. The major difference from the two previous endpoints is that it doesn't take into account whether PostgreSQL is running as the ``primary`` or the ``standby_leader``. - ``GET /replica``: replica health check endpoint. It returns HTTP status code **200** only when the Patroni node is in the state ``running``, the role is ``replica`` and ``noloadbalance`` tag is not set. - ``GET /replica?lag=``: replica check endpoint. In addition to checks from ``replica``, it also checks replication latency and returns status code **200** only when it is below specified value. The key cluster.last_leader_operation from DCS is used for Leader wal position and compute latency on replica for performance reasons. max-lag can be specified in bytes (integer) or in human readable values, for e.g. 16kB, 64MB, 1GB. - ``GET /replica?lag=1048576`` - ``GET /replica?lag=1024kB`` - ``GET /replica?lag=10MB`` - ``GET /replica?lag=1GB`` - ``GET /replica?tag_key1=value1&tag_key2=value2``: replica check endpoint. In addition, It will also check for user defined tags ``key1`` and ``key2`` and their respective values in the **tags** section of the yaml configuration management. If the tag isn't defined for an instance, or if the value in the yaml configuration doesn't match the querying value, it will return HTTP Status Code 503. In the following requests, since we are checking for the leader or standby-leader status, Patroni doesn't apply any of the user defined tags and they will be ignored. - ``GET /?tag_key1=value1&tag_key2=value2`` - ``GET /leader?tag_key1=value1&tag_key2=value2`` - ``GET /primary?tag_key1=value1&tag_key2=value2`` - ``GET /read-write?tag_key1=value1&tag_key2=value2`` - ``GET /standby_leader?tag_key1=value1&tag_key2=value2`` - ``GET /standby-leader?tag_key1=value1&tag_key2=value2`` - ``GET /read-only``: like the above endpoint, but also includes the primary. - ``GET /synchronous`` or ``GET /sync``: returns HTTP status code **200** only when the Patroni node is running as a synchronous standby. - ``GET /read-only-sync``: like the above endpoint, but also includes the primary. - ``GET /quorum``: returns HTTP status code **200** only when this Patroni node is listed as a quorum node in ``synchronous_standby_names`` on the primary. - ``GET /read-only-quorum``: like the above endpoint, but also includes the primary. - ``GET /asynchronous`` or ``GET /async``: returns HTTP status code **200** only when the Patroni node is running as an asynchronous standby. - ``GET /asynchronous?lag=`` or ``GET /async?lag=``: asynchronous standby check endpoint. In addition to checks from ``asynchronous`` or ``async``, it also checks replication latency and returns status code **200** only when it is below specified value. The key cluster.last_leader_operation from DCS is used for Leader wal position and compute latency on replica for performance reasons. max-lag can be specified in bytes (integer) or in human readable values, for e.g. 16kB, 64MB, 1GB. - ``GET /async?lag=1048576`` - ``GET /async?lag=1024kB`` - ``GET /async?lag=10MB`` - ``GET /async?lag=1GB`` - ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running. - ``GET /liveness``: returns HTTP status code **200** if Patroni heartbeat loop is properly running and **503** if the last run was more than ``ttl`` seconds ago on the primary or ``2*ttl`` on the replica. Could be used for ``livenessProbe``. - ``GET /readiness``: returns HTTP status code **200** when the Patroni node is running as the leader or when PostgreSQL is up and running. The endpoint could be used for ``readinessProbe`` when it is not possible to use Kubernetes endpoints for leader elections (OpenShift). Both, ``readiness`` and ``liveness`` endpoints are very light-weight and not executing any SQL. Probes should be configured in such a way that they start failing about time when the leader key is expiring. With the default value of ``ttl``, which is ``30s`` example probes would look like: .. code-block:: yaml readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 livenessProbe: httpGet: scheme: HTTP path: /liveness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 Monitoring endpoint ------------------- The ``GET /patroni`` is used by Patroni during the leader race. It also could be used by your monitoring system. The JSON document produced by this endpoint has the same structure as the JSON produced by the health check endpoints. **Example:** A healthy cluster .. code-block:: bash $ curl -s http://localhost:8008/patroni | jq . { "state": "running", "postmaster_start_time": "2024-08-28 19:39:26.352526+00:00", "role": "primary", "server_version": 160004, "xlog": { "location": 67395656 }, "timeline": 1, "replication": [ { "usename": "replicator", "application_name": "patroni2", "client_addr": "10.89.0.6", "state": "streaming", "sync_state": "async", "sync_priority": 0 }, { "usename": "replicator", "application_name": "patroni3", "client_addr": "10.89.0.2", "state": "streaming", "sync_state": "async", "sync_priority": 0 } ], "dcs_last_seen": 1692356718, "tags": { "clonefrom": true }, "database_system_identifier": "7268616322854375442", "patroni": { "version": "4.0.0", "scope": "demo", "name": "patroni1" } } **Example:** An unlocked cluster .. code-block:: bash $ curl -s http://localhost:8008/patroni | jq . { "state": "running", "postmaster_start_time": "2024-08-28 19:39:26.352526+00:00", "role": "replica", "server_version": 160004, "xlog": { "received_location": 67419744, "replayed_location": 67419744, "replayed_timestamp": null, "paused": false }, "timeline": 1, "replication": [ { "usename": "replicator", "application_name": "patroni2", "client_addr": "10.89.0.6", "state": "streaming", "sync_state": "async", "sync_priority": 0 }, { "usename": "replicator", "application_name": "patroni3", "client_addr": "10.89.0.2", "state": "streaming", "sync_state": "async", "sync_priority": 0 } ], "cluster_unlocked": true, "dcs_last_seen": 1692356928, "tags": { "clonefrom": true }, "database_system_identifier": "7268616322854375442", "patroni": { "version": "4.0.0", "scope": "demo", "name": "patroni1" } } **Example:** An unlocked cluster with :ref:`DCS failsafe mode ` enabled .. code-block:: bash $ curl -s http://localhost:8008/patroni | jq . { "state": "running", "postmaster_start_time": "2024-08-28 19:39:26.352526+00:00", "role": "replica", "server_version": 160004, "xlog": { "location": 67420024 }, "timeline": 1, "replication": [ { "usename": "replicator", "application_name": "patroni2", "client_addr": "10.89.0.6", "state": "streaming", "sync_state": "async", "sync_priority": 0 }, { "usename": "replicator", "application_name": "patroni3", "client_addr": "10.89.0.2", "state": "streaming", "sync_state": "async", "sync_priority": 0 } ], "cluster_unlocked": true, "failsafe_mode_is_active": true, "dcs_last_seen": 1692356928, "tags": { "clonefrom": true }, "database_system_identifier": "7268616322854375442", "patroni": { "version": "4.0.0", "scope": "demo", "name": "patroni1" } } **Example:** A cluster with the :ref:`pause mode ` enabled .. code-block:: bash $ curl -s http://localhost:8008/patroni | jq . { "state": "running", "postmaster_start_time": "2024-08-28 19:39:26.352526+00:00", "role": "replica", "server_version": 160004, "xlog": { "location": 67420024 }, "timeline": 1, "replication": [ { "usename": "replicator", "application_name": "patroni2", "client_addr": "10.89.0.6", "state": "streaming", "sync_state": "async", "sync_priority": 0 }, { "usename": "replicator", "application_name": "patroni3", "client_addr": "10.89.0.2", "state": "streaming", "sync_state": "async", "sync_priority": 0 } ], "pause": true, "dcs_last_seen": 1724874295, "tags": { "clonefrom": true }, "database_system_identifier": "7268616322854375442", "patroni": { "version": "4.0.0", "scope": "demo", "name": "patroni1" } } Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` endpoint. .. code-block:: bash $ curl http://localhost:8008/metrics # HELP patroni_version Patroni semver without periods. \ # TYPE patroni_version gauge patroni_version{scope="batman",name="patroni1"} 040000 # HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise. # TYPE patroni_postgres_running gauge patroni_postgres_running{scope="batman",name="patroni1"} 1 # HELP patroni_postmaster_start_time Epoch seconds since Postgres started. # TYPE patroni_postmaster_start_time gauge patroni_postmaster_start_time{scope="batman",name="patroni1"} 1724873966.352526 # HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise. # TYPE patroni_primary gauge patroni_primary{scope="batman",name="patroni1"} 1 # HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader. # TYPE patroni_xlog_location counter patroni_xlog_location{scope="batman",name="patroni1"} 22320573386952 # HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise. # TYPE patroni_standby_leader gauge patroni_standby_leader{scope="batman",name="patroni1"} 0 # HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise. # TYPE patroni_replica gauge patroni_replica{scope="batman",name="patroni1"} 0 # HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise. # TYPE patroni_sync_standby gauge patroni_sync_standby{scope="batman",name="patroni1"} 0 # HELP patroni_quorum_standby Value is 1 if this node is a quorum standby replica, 0 otherwise. # TYPE patroni_quorum_standby gauge patroni_quorum_standby{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica. # TYPE patroni_xlog_received_location counter patroni_xlog_received_location{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica. # TYPE patroni_xlog_replayed_location counter patroni_xlog_replayed_location{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null. # TYPE patroni_xlog_replayed_timestamp gauge patroni_xlog_replayed_timestamp{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise. # TYPE patroni_xlog_paused gauge patroni_xlog_paused{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise. # TYPE patroni_postgres_streaming gauge patroni_postgres_streaming{scope="batman",name="patroni1"} 1 # HELP patroni_postgres_in_archive_recovery Value is 1 if Postgres is replicating from archive, 0 otherwise. # TYPE patroni_postgres_in_archive_recovery gauge patroni_postgres_in_archive_recovery{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise. # TYPE patroni_postgres_server_version gauge patroni_postgres_server_version{scope="batman",name="patroni1"} 160004 # HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked. # TYPE patroni_cluster_unlocked gauge patroni_cluster_unlocked{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise. # TYPE patroni_postgres_timeline counter patroni_failsafe_mode_is_active{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise. # TYPE patroni_postgres_timeline counter patroni_postgres_timeline{scope="batman",name="patroni1"} 24 # HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni. # TYPE patroni_dcs_last_seen gauge patroni_dcs_last_seen{scope="batman",name="patroni1"} 1724874235 # HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise. # TYPE patroni_pending_restart gauge patroni_pending_restart{scope="batman",name="patroni1"} 1 # HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise. # TYPE patroni_is_paused gauge patroni_is_paused{scope="batman",name="patroni1"} 1 Cluster status endpoints ------------------------ - The ``GET /cluster`` endpoint generates a JSON document describing the current cluster topology and state: .. code-block:: bash $ curl -s http://localhost:8008/cluster | jq . { "members": [ { "name": "patroni1", "role": "leader", "state": "running", "api_url": "http://10.89.0.4:8008/patroni", "host": "10.89.0.4", "port": 5432, "timeline": 5, "tags": { "clonefrom": true } }, { "name": "patroni2", "role": "replica", "state": "streaming", "api_url": "http://10.89.0.6:8008/patroni", "host": "10.89.0.6", "port": 5433, "timeline": 5, "tags": { "clonefrom": true }, "lag": 0 } ], "scope": "demo", "scheduled_switchover": { "at": "2023-09-24T10:36:00+02:00", "from": "patroni1", "to": "patroni3" } } - The ``GET /history`` endpoint provides a view on the history of cluster switchovers/failovers. The format is very similar to the content of history files in the ``pg_wal`` directory. The only difference is the timestamp field showing when the new timeline was created. .. code-block:: bash $ curl -s http://localhost:8008/history | jq . [ [ 1, 25623960, "no recovery target specified", "2019-09-23T16:57:57+02:00" ], [ 2, 25624344, "no recovery target specified", "2019-09-24T09:22:33+02:00" ], [ 3, 25624752, "no recovery target specified", "2019-09-24T09:26:15+02:00" ], [ 4, 50331856, "no recovery target specified", "2019-09-24T09:35:52+02:00" ] ] .. _config_endpoint: Config endpoint --------------- ``GET /config``: Get the current version of the dynamic configuration: .. code-block:: bash $ curl -s http://localhost:8008/config | jq . { "ttl": 30, "loop_wait": 10, "retry_timeout": 10, "maximum_lag_on_failover": 1048576, "postgresql": { "use_slots": true, "use_pg_rewind": true, "parameters": { "hot_standby": "on", "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5, "max_connections": "100" } } } ``PATCH /config``: Change the existing configuration. .. code-block:: bash $ curl -s -XPATCH -d \ '{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \ http://localhost:8008/config | jq . { "ttl": 20, "loop_wait": 5, "maximum_lag_on_failover": 1048576, "retry_timeout": 10, "postgresql": { "use_slots": true, "use_pg_rewind": true, "parameters": { "hot_standby": "on", "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5, "max_connections": "101" } } } The above REST API call patches the existing configuration and returns the new configuration. Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "pending_restart" flag should be exposed: .. code-block:: bash $ curl -s http://localhost:8008/patroni | jq . { "database_system_identifier": "6287881213849985952", "postmaster_start_time": "2024-08-28 19:39:26.352526+00:00", "xlog": { "location": 2197818976 }, "timeline": 1, "dcs_last_seen": 1724874545, "database_system_identifier": "7408277255830290455", "pending_restart": true, "pending_restart_reason": { "max_connections": { "old_value": "100", "new_value": "101" } }, "patroni": { "version": "4.0.0", "scope": "batman", "name": "patroni1" }, "state": "running", "role": "primary", "server_version": 160004 } Removing parameters: If you want to remove (reset) some setting just patch it with ``null``: .. code-block:: bash $ curl -s -XPATCH -d \ '{"postgresql":{"parameters":{"max_connections":null}}}' \ http://localhost:8008/config | jq . { "ttl": 20, "loop_wait": 5, "retry_timeout": 10, "maximum_lag_on_failover": 1048576, "postgresql": { "use_slots": true, "use_pg_rewind": true, "parameters": { "hot_standby": "on", "unix_socket_directories": ".", "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5 } } } The above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration. ``PUT /config``: It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally: .. code-block:: bash $ curl -s -XPUT -d \ '{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \ http://localhost:8008/config | jq . { "ttl": 20, "maximum_lag_on_failover": 1048576, "retry_timeout": 10, "postgresql": { "use_slots": true, "parameters": { "hot_standby": "on", "unix_socket_directories": ".", "wal_level": "hot_standby", "max_wal_senders": 5 }, "use_pg_rewind": true }, "loop_wait": 3 } Switchover and failover endpoints --------------------------------- .. _switchover_api: Switchover ^^^^^^^^^^ ``/switchover`` endpoint only works when the cluster is healthy (there is a leader). It also allows to schedule a switchover at a given time. When calling ``/switchover`` endpoint a candidate can be specified but is not required, in contrast to ``/failover`` endpoint. If a candidate is not provided, all the eligible nodes of the cluster will participate in the leader race after the leader stepped down. In the JSON body of the ``POST`` request you must specify the ``leader`` field. The ``candidate`` and the ``scheduled_at`` fields are optional and can be used to schedule a switchover at a specific time. Depending on the situation, requests might return different HTTP status codes and bodies. Status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412**, or **503**) will be returned with some details in the response body. ``DELETE /switchover`` can be used to delete the currently scheduled switchover. **Example:** perform a switchover to any healthy standby .. code-block:: bash $ curl -s http://localhost:8008/switchover -XPOST -d '{"leader":"postgresql1"}' Successfully switched over to "postgresql2" **Example:** perform a switchover to a specific node .. code-block:: bash $ curl -s http://localhost:8008/switchover -XPOST -d \ '{"leader":"postgresql1","candidate":"postgresql2"}' Successfully switched over to "postgresql2" **Example:** schedule a switchover from the leader to any other healthy standby in the cluster at a specific time. .. code-block:: bash $ curl -s http://localhost:8008/switchover -XPOST -d \ '{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}' Switchover scheduled Failover ^^^^^^^^ ``/failover`` endpoint can be used to perform a manual failover when there are no healthy nodes (e.g. to an asynchronous standby if all synchronous standbys are not healthy enough to promote). However there is no requirement for a cluster not to have leader - failover can also be run on a healthy cluster. In the JSON body of the ``POST`` request you must specify the ``candidate`` field. If the ``leader`` field is specified, a switchover is triggered instead. **Example:** .. code-block:: bash $ curl -s http://localhost:8008/failover -XPOST -d '{"candidate":"postgresql1"}' Successfully failed over to "postgresql1" .. warning:: :ref:`Be very careful ` when using this endpoint, as this can cause data loss in certain situations. In most cases, :ref:`the switchover endpoint ` satisfies the administrator's needs. ``POST /switchover`` and ``POST /failover`` endpoints are used by :ref:`patronictl_switchover` and :ref:`patronictl_failover`, respectively. ``DELETE /switchover`` is used by :ref:`patronictl flush cluster-name switchover `. .. list-table:: Failover/Switchover comparison :widths: 25 25 25 :header-rows: 1 * - - Failover - Switchover * - Requires leader specified - no - yes * - Requires candidate specified - yes - no * - Can be run in pause - yes - yes (only to a specific candidate) * - Can be scheduled - no - yes (if not in pause) .. _failover_healthcheck: Healthy standby ^^^^^^^^^^^^^^^ There are a couple of checks that a member of a cluster should pass to be able to participate in the leader race during a switchover or to become a leader as a failover/switchover candidate: - be reachable via Patroni API; - not have ``nofailover`` tag set to ``true``; - have watchdog fully functional (if required by the configuration); - in case of a switchover in a healthy cluster or an automatic failover, not exceed maximum replication lag (``maximum_lag_on_failover`` :ref:`configuration parameter `); - in case of a switchover in a healthy cluster or an automatic failover, not have a timeline number smaller than the cluster timeline if ``check_timeline`` :ref:`configuration parameter ` is set to ``true``; - in :ref:`synchronous mode `: - In case of a switchover (both with and without a candidate): be listed in the ``/sync`` key members; - For a failover in both healthy and unhealthy clusters, this check is omitted. .. warning:: In case of a manual failover in a cluster without a leader, a candidate will be allowed to promote even if: - it is not in the ``/sync`` key members when synchronous mode is enabled; - its lag exceeds the maximum replication lag allowed; - it has the timeline number smaller than the last known cluster timeline. .. _restart_endpoint: Restart endpoint ---------------- - ``POST /restart``: You can restart Postgres on the specific node by performing the ``POST /restart`` call. In the JSON body of ``POST`` request it is possible to optionally specify some restart conditions: - **restart_pending**: boolean, if set to ``true`` Patroni will restart PostgreSQL only when restart is pending in order to apply some changes in the PostgreSQL config. - **role**: perform restart only if the current role of the node matches with the role from the POST request. - **postgres_version**: perform restart only if the current version of postgres is smaller than specified in the POST request. - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``primary_start_timeout``. - **schedule**: timestamp with time zone, schedule the restart somewhere in the future. - ``DELETE /restart``: delete the scheduled restart ``POST /restart`` and ``DELETE /restart`` endpoints are used by :ref:`patronictl_restart` and :ref:`patronictl flush cluster-name restart ` respectively. .. _reload_endpoint: Reload endpoint --------------- The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of :ref:`patronictl_restart`. The reload endpoint is used by :ref:`patronictl_reload`. Reinitialize endpoint --------------------- ``POST /reinitialize``: reinitialize the PostgreSQL data directory on the specified node. It is allowed to be executed only on replicas. Once called, it will remove the data directory and start ``pg_basebackup`` or some alternative :ref:`replica creation method `. The call might fail if Patroni is in a loop trying to recover (restart) a failed Postgres. In order to overcome this problem one can specify ``{"force":true}`` in the request body. The reinitialize endpoint is used by :ref:`patronictl_reinit`. patroni-4.0.4/docs/security.rst000066400000000000000000000052531472010352700165320ustar00rootroot00000000000000.. _security: ======================= Security Considerations ======================= A Patroni cluster has two interfaces to be protected from unauthorized access: the distributed configuration storage (DCS) and the Patroni REST API. Protecting DCS ============== Patroni and :ref:`patronictl` both store and retrieve data to/from the DCS. Despite DCS doesn't contain any sensitive information, it allows changing some of Patroni/Postgres configuration. Therefore the very first thing that should be protected is DCS itself. The details of protection depend on the type of DCS used. The authentication and encryption parameters (tokens/basic-auth/client certificates) for the supported types of DCS are covered in :ref:`settings `. The general recommendation is to enable TLS for all DCS communication. Protecting the REST API ======================= Protecting the REST API is a more complicated task. The Patroni REST API is used by Patroni itself during the leader race, by the :ref:`patronictl` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. From the point of view of security, REST API contains safe (``GET`` requests, only retrieve information) and unsafe (``PUT``, ``POST``, ``PATCH`` and ``DELETE`` requests, change the state of nodes) endpoints. The unsafe endpoints can be protected with HTTP basic-auth by setting the ``restapi.authentication.username`` and ``restapi.authentication.password`` parameters. There is no way to protect the safe endpoints without enabling TLS. When TLS for the REST API is enabled and a PKI is established, mutual authentication of the API server and API client is possible for all endpoints. The ``restapi`` section parameters enable TLS client authentication to the server. Depending on the value of the ``verify_client`` parameter, the API server requires a successful client certificate verification for both safe and unsafe API calls (``verify_client: required``), or only for unsafe API calls (``verify_client: optional``), or for no API calls (``verify_client: none``). The ``ctl`` section parameters enable TLS server authentication to the client (the :ref:`patronictl` tool which uses the same config as patroni). Set ``insecure: true`` to disable the server certificate verification by the client. See :ref:`settings ` for a detailed description of the TLS client parameters. Protecting the PostgreSQL database proper from unauthorized access is beyond the scope of this document and is covered in https://www.postgresql.org/docs/current/client-authentication.html patroni-4.0.4/docs/standby_cluster.rst000066400000000000000000000074031472010352700200670ustar00rootroot00000000000000.. _standby_cluster: Standby cluster --------------- Patroni also support running cascading replication to a remote datacenter (region) using a feature that is called "standby cluster". This type of clusters has: * "standby leader", that behaves pretty much like a regular cluster leader, except it replicates from a remote node. * cascade replicas, that are replicating from standby leader. Standby leader holds and updates a leader lock in DCS. If the leader lock expires, cascade replicas will perform an election to choose another leader from the standbys. There is no further relationship between the standby cluster and the primary cluster it replicates from, in particular, they must not share the same DCS scope if they use the same DCS. They do not know anything else from each other apart from replication information. Also, the standby cluster is not being displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the primary cluster. For the sake of flexibility, you can specify methods of creating a replica and recovery WAL records when a cluster is in the "standby mode" by providing :ref:`create_replica_methods ` key in `standby_cluster` section. It is distinct from creating replicas, when cluster is detached and functions as a normal cluster, which is controlled by `create_replica_methods` in `postgresql` section. Both "standby" and "normal" `create_replica_methods` reference keys in `postgresql` section. To configure such cluster you need to specify the section ``standby_cluster`` in a patroni configuration: .. code:: YAML bootstrap: dcs: standby_cluster: host: 1.2.3.4 port: 5432 primary_slot_name: patroni create_replica_methods: - basebackup Note, that these options will be applied only once during cluster bootstrap, and the only way to change them afterwards is through DCS. Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA of the remote primary and will not start if it does not find it after a basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is your responsibility to copy it to PGDATA. If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster. It will not be done automatically by the standby cluster implementation. You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided. In case the remote site doesn't provide a single endpoint that connects to a primary, one could list all hosts of the source cluster in the ``standby_cluster.host`` section. When ``standby_cluster.host`` contains multiple hosts separated by commas, Patroni will: * add ``target_session_attrs=read-write`` to the ``primary_conninfo`` on the standby leader node. * use ``target_session_attrs=read-write`` when trying to determine whether we need to run ``pg_rewind`` or when executing ``pg_rewind`` on all nodes of the standby cluster. * It is important to note that for ``pg_rewind`` to operate successfully, either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``. Otherwise, ``pg_rewind`` will not function properly. There is also a possibility to replicate the standby cluster from another standby cluster or from a standby member of the primary cluster: for that, you need to define a single host in the ``standby_cluster.host`` section. However, you need to beware that in this case ``pg_rewind`` will fail to execute on the standby cluster. patroni-4.0.4/docs/tools_integration.rst000066400000000000000000000050031472010352700204170ustar00rootroot00000000000000.. _tools_integration: Integration with other tools ============================ Patroni is able to integrate with other tools in your stack. In this section you will find a list of examples, which although not an exhaustive list, might provide you with ideas on how Patroni can integrate with other tools. Barman ------ Patroni delivers an application named ``patroni_barman`` which has logic to communicate with ``pg-backup-api``, so you are able to perform Barman operations remotely. This application currently has a couple of sub-commands: ``recover`` and ``config-switch``. patroni_barman recover ^^^^^^^^^^^^^^^^^^^^^^ The ``recover`` sub-command can be used as a custom bootstrap or custom replica creation method. You can find more information about that in :ref:`replica_imaging_and_bootstrap`. patroni_barman config-switch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``config-switch`` sub-command is designed to be used as an ``on_role_change`` callback in Patroni. As an example, assume you are streaming WALs from your current primary to your Barman host. In the event of a failover in the cluster you might want to start streaming WALs from the new primary. You can accomplish this by using ``patroni_barman config-switch`` as the ``on_role_change`` callback. .. note:: That sub-command relies on the ``barman config-switch`` command, which is in charge of overriding the configuration of a Barman server by applying a pre-defined model on top of it. This command is available since Barman 3.10. Please consult the Barman documentation for more details. This is an example of how you can configure Patroni to apply a configuration model in case this Patroni node is promoted to primary: .. code:: YAML postgresql: callbacks: on_role_change: > patroni_barman --api-url YOUR_API_URL config-switch --barman-server YOUR_BARMAN_SERVER_NAME --barman-model YOUR_BARMAN_MODEL_NAME --switch-when promoted .. note:: ``patroni_barman config-switch`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman config-switch`` through the backup API. Also, it requires that you have pre-configured Barman models to be applied. The above example uses a subset of the available parameters. You can get more information running ``patroni_barman config-switch --help``, and by consulting the Barman documentation. patroni-4.0.4/docs/watchdog.rst000066400000000000000000000076261472010352700164710ustar00rootroot00000000000000.. _watchdog: Watchdog support ================ Having multiple PostgreSQL servers running as primary can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons: - Patroni has crashed due to a bug, out-of-memory condition or by being accidentally killed by a system administrator. - Shutting down PostgreSQL is too slow. - Patroni does not get to run due to high load on the system, the VM being paused by the hypervisor, or other infrastructure issues. To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail. Patroni will try to activate the watchdog before promoting PostgreSQL to primary. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become leader. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state. By default Patroni will set up the watchdog to expire 5 seconds before TTL expires. With the default setup of ``loop_wait=10`` and ``ttl=30`` this gives HA loop at least 15 seconds (``ttl`` - ``safety_margin`` - ``loop_wait``) to complete before the system gets forcefully reset. By default accessing DCS is configured to time out after 10 seconds. This means that when DCS is unavailable, for example due to network issues, Patroni and PostgreSQL will have at least 5 seconds (``ttl`` - ``safety_margin`` - ``loop_wait`` - ``retry_timeout``) to come to a state where all client connections are terminated. Safety margin is the amount of time that Patroni reserves for time between leader key update and watchdog keepalive. Patroni will try to send a keepalive immediately after confirmation of leader key update. If Patroni process is suspended for extended amount of time at exactly the right moment the keepalive may be delayed for more than the safety margin without triggering the watchdog. This results in a window of time where watchdog will not trigger before leader key expiration, invalidating the guarantee. To be absolutely sure that watchdog will trigger under all circumstances set up the watchdog to expire after half of TTL by setting ``safety_margin`` to -1 to set watchdog timeout to ``ttl // 2``. If you need this guarantee you probably should increase ``ttl`` and/or reduce ``loop_wait`` and ``retry_timeout``. Currently watchdogs are only supported using Linux watchdog device interface. Setting up software watchdog on Linux ------------------------------------- Default Patroni configuration will try to use ``/dev/watchdog`` on Linux if it is accessible to Patroni. For most use cases using software watchdog built into the Linux kernel is secure enough. To enable software watchdog issue the following commands as root before starting Patroni: .. code-block:: bash modprobe softdog # Replace postgres with the user you will be running patroni under chown postgres /dev/watchdog For testing it may be helpful to disable rebooting by adding ``soft_noboot=1`` to the modprobe command line. In this case the watchdog will just log a line in kernel ring buffer, visible via `dmesg`. Patroni will log information about the watchdog when it is successfully enabled. patroni-4.0.4/docs/yaml_configuration.rst000066400000000000000000001261171472010352700205570ustar00rootroot00000000000000.. _yaml_configuration: ============================ YAML Configuration Settings ============================ Global/Universal ---------------- - **name**: the name of the host. Must be unique for the cluster. - **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service" - **scope**: cluster name .. _log_settings: Log --- - **type**: sets the format of logs. Can be either **plain** or **json**. To use **json** format, you must have the :ref:`jsonlogger ` installed. The default value is **plain**. - **level**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging `_) - **traceback\_level**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **log.level=DEBUG**. - **format**: sets the log formatting string. If the log type is **plain**, the log format should be a string. Refer to `the LogRecord attributes `_ for available attributes. If the log type is **json**, the log format can be a list in addition to a string. Each list item should correspond to LogRecord attributes. Be cautious that only the field name is required, and the **%(** and **)** should be omitted. If you wish to print a log field with a different key name, use a dictionary where the dictionary key is the log field, and the value is the name of the field you want to be printed in the log. Default value is **%(asctime)s %(levelname)s: %(message)s** - **dateformat**: sets the datetime formatting string. (see the `formatTime() documentation `_) - **static_fields**: add additional fields to the log. This option is only available when the log type is set to **json**. - **max\_queue\_size**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m. - **dir**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this value, the application will retain 4 25MB logs by default. You can tune those retention values with `file_num` and `file_size` (see below). - **mode**: Permissions for log files (for example, ``0644``). If not specified, permissions will be set based on the current umask value. - **file\_num**: The number of application logs to retain. - **file\_size**: Size of patroni.log file (in bytes) that triggers a log rolling. - **loggers**: This section allows redefining logging level per python module - **patroni.postmaster: WARNING** - **urllib3: DEBUG** Here is an example of how to config patroni to log in json format. .. code:: YAML log: type: json format: - message - module - asctime: '@timestamp' - levelname: level static_fields: app: patroni .. _bootstrap_settings: Bootstrap configuration ----------------------- .. note:: Once Patroni has initialized the cluster for the first time and settings have been stored in the DCS, all future changes to the ``bootstrap.dcs`` section of the YAML configuration will not take any effect! If you want to change them please use either :ref:`patronictl_edit_config` or the Patroni :ref:`REST API `. - **bootstrap**: - **dcs**: This section will be written into `///config` of the given configuration store after initializing the new cluster. The global dynamic configuration for the cluster. You can put any of the parameters described in the :ref:`Dynamic Configuration settings ` under ``bootstrap.dcs`` and after Patroni has initialized (bootstrapped) the new cluster, it will write this section into `///config` of the configuration store. - **method**: custom script to use for bootstrapping this cluster. See :ref:`custom bootstrap methods documentation ` for details. When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method`` parameter is present in the configuration file. - **initdb**: (optional) list options to be passed on to initdb. - **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3. - **- encoding: UTF8**: default encoding for new databases. - **- locale: UTF8**: default locale for new databases. - **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file. .. _citus_settings: Citus ----- Enables integration Patroni with `Citus `__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here `. - **group**: the Citus group id, integer. Use ``0`` for coordinator and ``1``, ``2``, etc... for workers - **database**: the database where ``citus`` extension should be created. Must be the same on the coordinator and all workers. Currently only one database is supported. .. _consul_settings: Consul ------ Most of the parameters are optional, but you have to specify one of the **host** or **url** - **host**: the host:port for the Consul local agent. - **url**: url for the Consul local agent, in format: http(s)://host:port. - **port**: (optional) Consul port. - **scheme**: (optional) **http** or **https**, defaults to **http**. - **token**: (optional) ACL token. - **verify**: (optional) whether to verify the SSL certificate for HTTPS requests. - **cacert**: (optional) The ca certificate. If present it will enable validation. - **cert**: (optional) file with the client certificate. - **key**: (optional) file with the client key. Can be empty if the key is part of **cert**. - **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used. - **consistency**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference `__) - **checks**: (optional) list of Consul health checks used for the session. By default an empty list is used. - **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, primary, replica, or standby-leader depending on the node's role. Defaults to **false**. - **service\_tags**: (optional) additional static tags to add to the Consul service apart from the role (``primary``/``replica``/``standby-leader``). By default an empty list is used. - **service\_check\_interval**: (optional) how often to perform health check against registered url. Defaults to '5s'. - **service\_check\_tls\_server\_name**: (optional) override SNI host when connecting via TLS, see also `consul agent check API reference `__. The ``token`` needs to have the following ACL permissions: :: service_prefix "${scope}" { policy = "write" } key_prefix "${namespace}/${scope}" { policy = "write" } session_prefix "" { policy = "write" } Etcd ---- Most of the parameters are optional, but you have to specify one of the **host**, **hosts**, **url**, **proxy** or **srv** - **host**: the host:port for the etcd endpoint. - **hosts**: list of etcd endpoint in format host1:port1,host2:port2,etc... Could be a comma separated string or an actual yaml list. - **use\_proxies**: If this parameter is set to true, Patroni will consider **hosts** as a list of proxies and will not perform a topology discovery of etcd cluster. - **url**: url for the etcd. - **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**. - **srv**: Domain to search the SRV record(s) for cluster autodiscovery. Patroni will try to query these SRV service names for specified domain (in that order until first success): ``_etcd-client-ssl``, ``_etcd-client``, ``_etcd-ssl``, ``_etcd``, ``_etcd-server-ssl``, ``_etcd-server``. If SRV records for ``_etcd-server-ssl`` or ``_etcd-server`` are retrieved then ETCD peer protocol is used do query ETCD for available members. Otherwise hosts from SRV records will be used. - **srv\_suffix**: Configures a suffix to the SRV name that is queried during discovery. Use this flag to differentiate between multiple etcd clusters under the same domain. Works only with conjunction with **srv**. For example, if ``srv_suffix: foo`` and ``srv: example.org`` are set, the following DNS SRV query is made:``_etcd-client-ssl-foo._tcp.example.com`` (and so on for every possible ETCD SRV service name). - **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them. - **username**: (optional) username for etcd authentication. - **password**: (optional) password for etcd authentication. - **cacert**: (optional) The ca certificate. If present it will enable validation. - **cert**: (optional) file with the client certificate. - **key**: (optional) file with the client key. Can be empty if the key is part of **cert**. Etcdv3 ------ If you want that Patroni works with Etcd cluster via protocol version 3, you need to use the ``etcd3`` section in the Patroni configuration file. All configuration parameters are the same as for ``etcd``. .. warning:: Keys created with protocol version 2 are not visible with protocol version 3 and the other way around, therefore it is not possible to switch from ``etcd`` to ``etcd3`` just by updating Patroni config file. In addition, Patroni uses Etcd's gRPC-gateway (proxy) to communicate with the V3 API, which means that TLS common name authentication is not possible. ZooKeeper ---------- - **hosts**: List of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...']. - **use_ssl**: (optional) Whether SSL is used or not. Defaults to ``false``. If set to ``false``, all SSL specific parameters are ignored. - **cacert**: (optional) The CA certificate. If present it will enable validation. - **cert**: (optional) File with the client certificate. - **key**: (optional) File with the client key. - **key_password**: (optional) The client key password. - **verify**: (optional) Whether to verify certificate or not. Defaults to ``true``. - **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value. Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``. For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``. - **auth_data**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary. .. note:: It is required to install ``kazoo>=2.6.0`` to support SSL. Exhibitor --------- - **hosts**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes. - **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor. - **port**: Exhibitor port. .. _kubernetes_settings: Kubernetes ---------- - **bypass\_api\_service**: (optional) When communicating with the Kubernetes API, Patroni is usually relying on the `kubernetes` service, the address of which is exposed in the pods via the `KUBERNETES_SERVICE_HOST` environment variable. If `bypass_api_service` is set to ``true``, Patroni will resolve the list of API nodes behind the service and connect directly to them. - **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`. - **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates. - **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`. - **role\_label**: (optional) name of the label containing role (`primary`, `replica`, or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``. - **leader\_label\_value**: (optional) value of the pod label when Postgres role is ``primary``. Default value is ``primary``. - **follower\_label\_value**: (optional) value of the pod label when Postgres role is ``replica``. Default value is ``replica``. - **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``primary``. - **tmp_\role\_label**: (optional) name of the temporary label containing role (`primary` or `replica`). Value of this label will always use the default of corresponding role. Set only when necessary. - **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state. - **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted. - **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set. - **cacert**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret. - **retriable\_http\_codes**: (optional) list of HTTP status codes from K8s API to retry on. By default Patroni is retrying on ``500``, ``503``, and ``504``, or if K8s API response has ``retry-after`` HTTP header. .. _raft_settings: Raft (deprecated) ----------------- - **self\_addr**: ``ip:port`` to listen on for Raft connections. The ``self_addr`` must be accessible from other nodes of the cluster. If not set, the node will not participate in consensus. - **bind\_addr**: (optional) ``ip:port`` to listen on for Raft connections. If not specified the ``self_addr`` will be used. - **partner\_addrs**: list of other Patroni nodes in the cluster in format: ['ip1:port', 'ip2:port', 'etc...'] - **data\_dir**: directory where to store Raft log and snapshot. If not specified the current working directory is used. - **password**: (optional) Encrypt Raft traffic with a specified password, requires ``cryptography`` python module. Short FAQ about Raft implementation - Q: How to list all the nodes providing consensus? A: ``syncobj_admin -conn host:port -status`` where the host:port is the address of one of the cluster nodes - Q: Node that was a part of consensus and has gone and I can't reuse the same IP for other node. How to remove this node from the consensus? A: ``syncobj_admin -conn host:port -remove host2:port2`` where the ``host2:port2`` is the address of the node you want to remove from consensus. - Q: Where to get the ``syncobj_admin`` utility? A: It is installed together with ``pysyncobj`` module (python RAFT implementation), which is Patroni dependency. - Q: it is possible to run Patroni node without adding in to the consensus? A: Yes, just comment out or remove ``raft.self_addr`` from Patroni configuration. - Q: It is possible to run Patroni and PostgreSQL only on two nodes? A: Yes, on the third node you can run ``patroni_raft_controller`` (without Patroni and PostgreSQL). In such a setup, one can temporarily lose one node without affecting the primary. .. _postgresql_settings: PostgreSQL ---------- - **postgresql**: - **authentication**: - **superuser**: - **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. - **password**: password for the superuser, set during initialization (initdb). - **sslmode**: (optional) maps to the `sslmode `__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation `__. The default mode is ``prefer``. - **sslkey**: (optional) maps to the `sslkey `__ connection parameter, which specifies the location of the secret key used with the client's certificate. - **sslpassword**: (optional) maps to the `sslpassword `__ connection parameter, which specifies the password for the secret key specified in ``sslkey``. - **sslcert**: (optional) maps to the `sslcert `__ connection parameter, which specifies the location of the client certificate. - **sslrootcert**: (optional) maps to the `sslrootcert `__ connection parameter, which specifies the location of a file containing one or more certificate authorities (CA) certificates that the client will use to verify a server's certificate. - **sslcrl**: (optional) maps to the `sslcrl `__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **sslcrldir**: (optional) maps to the `sslcrldir `__ connection parameter, which specifies the location of a directory with files containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **sslnegotiation**: (optional) maps to the `sslnegotiation `__ connection parameter, which controls how SSL encryption is negotiated with the server, if SSL is used. - **gssencmode**: (optional) maps to the `gssencmode `__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server - **channel_binding**: (optional) maps to the `channel_binding `__ connection parameter, which controls the client's use of channel binding. - **replication**: - **username**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication - **password**: replication password; the user will be created during initialization. - **sslmode**: (optional) maps to the `sslmode `__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation `__. The default mode is ``prefer``. - **sslkey**: (optional) maps to the `sslkey `__ connection parameter, which specifies the location of the secret key used with the client's certificate. - **sslpassword**: (optional) maps to the `sslpassword `__ connection parameter, which specifies the password for the secret key specified in ``sslkey``. - **sslcert**: (optional) maps to the `sslcert `__ connection parameter, which specifies the location of the client certificate. - **sslrootcert**: (optional) maps to the `sslrootcert `__ connection parameter, which specifies the location of a file containing one or more certificate authorities (CA) certificates that the client will use to verify a server's certificate. - **sslcrl**: (optional) maps to the `sslcrl `__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **sslcrldir**: (optional) maps to the `sslcrldir `__ connection parameter, which specifies the location of a directory with files containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **sslnegotiation**: (optional) maps to the `sslnegotiation `__ connection parameter, which controls how SSL encryption is negotiated with the server, if SSL is used. - **gssencmode**: (optional) maps to the `gssencmode `__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server - **channel_binding**: (optional) maps to the `channel_binding `__ connection parameter, which controls the client's use of channel binding. - **rewind**: - **username**: (optional) name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions `__ will be granted. - **password**: (optional) password for the user for ``pg_rewind``; the user will be created during initialization. - **sslmode**: (optional) maps to the `sslmode `__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation `__. The default mode is ``prefer``. - **sslkey**: (optional) maps to the `sslkey `__ connection parameter, which specifies the location of the secret key used with the client's certificate. - **sslpassword**: (optional) maps to the `sslpassword `__ connection parameter, which specifies the password for the secret key specified in ``sslkey``. - **sslcert**: (optional) maps to the `sslcert `__ connection parameter, which specifies the location of the client certificate. - **sslrootcert**: (optional) maps to the `sslrootcert `__ connection parameter, which specifies the location of a file containing one or more certificate authorities (CA) certificates that the client will use to verify a server's certificate. - **sslcrl**: (optional) maps to the `sslcrl `__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **sslcrldir**: (optional) maps to the `sslcrldir `__ connection parameter, which specifies the location of a directory with files containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list. - **sslnegotiation**: (optional) maps to the `sslnegotiation `__ connection parameter, which controls how SSL encryption is negotiated with the server, if SSL is used. - **gssencmode**: (optional) maps to the `gssencmode `__ connection parameter, which determines whether or with what priority a secure GSS TCP/IP connection will be negotiated with the server - **channel_binding**: (optional) maps to the `channel_binding `__ connection parameter, which controls the client's use of channel binding. - **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.) - **on\_reload**: run this script when configuration reload is triggered. - **on\_restart**: run this script when the postgres restarts (without changing role). - **on\_role\_change**: run this script when the postgres is being promoted or demoted. - **on\_start**: run this script when the postgres starts. - **on\_stop**: run this script when the postgres stops. - **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications. - **proxy\_address**: IP address + port through which a connection pool (e.g. pgbouncer) running next to Postgres is accessible. The value is written to the member key in DCS as ``proxy_url`` and could be used/useful for service discovery. - **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item. See :ref:`custom replica creation methods documentation ` for further explanation. - **data\_dir**: The location of the Postgres data directory, either :ref:`existing ` or to be initialized by Patroni. - **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni. - **bin\_dir**: (optional) Path to PostgreSQL binaries (pg_ctl, initdb, pg_controldata, pg_basebackup, postgres, pg_isready, pg_rewind). If not provided or is an empty string, PATH environment variable will be used to find the executables. - **bin\_name**: (optional) Make it possible to override Postgres binary names, if you are using a custom Postgres distribution: - **pg\_ctl**: (optional) Custom name for ``pg_ctl`` binary. - **initdb**: (optional) Custom name for ``initdb`` binary. - **pg\controldata**: (optional) Custom name for ``pg_controldata`` binary. - **pg\_basebackup**: (optional) Custom name for ``pg_basebackup`` binary. - **postgres**: (optional) Custom name for ``postgres`` binary. - **pg\_isready**: (optional) Custom name for ``pg_isready`` binary. - **pg\_rewind**: (optional) Custom name for ``pg_rewind`` binary. - **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. - **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is defined, Patroni will use the first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that the default value should be used and omit ``host`` from the connection parameters. - **use\_unix\_socket\_repl**: specifies that Patroni should prefer to use unix sockets for replication user cluster connection. Default value is ``false``. If ``unix_socket_directories`` is defined, Patroni will use the first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that the default value should be used and omit ``host`` from the connection parameters. - **pgpass**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni. - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. - **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overridden by Patroni's own configuration facilities - see :ref:`dynamic configuration ` for details. - **parameters**: configuration parameters (GUCs) for Postgres in format ``{ssl: "on", ssl_cert_file: "cert_file"}``. - **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. Patroni ignores this parameter if ``hba_file`` PostgreSQL parameter is set to a non-default value. Together with :ref:`dynamic configuration ` this parameter simplifies management of ``pg_hba.conf``. - **- host all all 0.0.0.0/0 md5** - **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication. - **pg\_ident**: list of lines that Patroni will use to generate ``pg_ident.conf``. Patroni ignores this parameter if ``ident_file`` PostgreSQL parameter is set to a non-default value. Together with :ref:`dynamic configuration ` this parameter simplifies management of ``pg_ident.conf``. - **- mapname1 systemname1 pguser1** - **- mapname1 systemname2 pguser2** - **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds. - **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica. Either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``, or ``pg_rewind`` will not work. - **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove the PostgreSQL data directory and recreate the replica. Otherwise it will try to follow the new leader. Default value is **false**. - **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove the PostgreSQL data directory and recreate the replica if it notices that timelines are diverging and the former primary can not start streaming from the new primary. This option is useful when ``pg_rewind`` can not be used. While performing timelines divergence check on PostgreSQL v10 and older Patroni will try to connect with replication credential to the "postgres" database. Hence, such access should be allowed in the pg_hba.conf. Default value is **false**. - **replica\_method**: for each create_replica_methods other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value". - **pre\_promote**: a fencing script that executes during a failover after acquiring the leader lock but before promoting the replica. If the script exits with a non-zero code, Patroni does not promote the replica and removes the leader key from DCS. - **before\_stop**: a script that executes immediately prior to stopping postgres. As opposed to a callback, this script runs synchronously, blocking shutdown until it has completed. The return code of this script does not impact whether shutdown proceeds afterwards. .. _restapi_settings: REST API -------- - **restapi**: - **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API `. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the leader is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API. - **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks). - **authentication**: (optional) - **username**: Basic-auth username to protect unsafe REST API endpoints. - **password**: Basic-auth password to protect unsafe REST API endpoints. - **certfile**: (optional): Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. - **keyfile**: (optional): Specifies the file with the secret key in the PEM format. - **keyfile\_password**: (optional): Specifies a password for decrypting the keyfile. - **cafile**: (optional): Specifies the file with the CA_BUNDLE with certificates of trusted CAs to use while verifying client certs. - **ciphers**: (optional): Specifies the permitted cipher suites (e.g. "ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:!SSLv1:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1") - **verify\_client**: (optional): ``none`` (default), ``optional`` or ``required``. When ``none`` REST API will not check client certificates. When ``required`` client certificates are required for all REST API calls. When ``optional`` client certificates are required for all unsafe REST API endpoints. When ``required`` is used, then client authentication succeeds, if the certificate signature verification succeeds. For ``optional`` the client cert will only be checked for ``PUT``, ``POST``, ``PATCH``, and ``DELETE`` requests. - **allowlist**: (optional): Specifies the set of hosts that are allowed to call unsafe REST API endpoints. The single element could be a host name, an IP address or a network address using CIDR notation. By default ``allow all`` is used. In case if ``allowlist`` or ``allowlist_include_members`` are set, anything that is not included is rejected. - **allowlist\_include\_members**: (optional): If set to ``true`` it allows accessing unsafe REST API endpoints from other cluster members registered in DCS (IP address or hostname is taken from the members ``api_url``). Be careful, it might happen that OS will use a different IP for outgoing connections. - **http\_extra\_headers**: (optional): HTTP headers let the REST API server pass additional information with an HTTP response. - **https\_extra\_headers**: (optional): HTTPS headers let the REST API server pass additional information with an HTTP response when TLS is enabled. This will also pass additional information set in ``http_extra_headers``. - **request_queue_size**: (optional): Sets request queue size for TCP socket used by Patroni REST API. Once the queue is full, further requests get a "Connection denied" error. The default value is 5. Here is an example of both **http_extra_headers** and **https_extra_headers**: .. code:: YAML restapi: listen: connect_address: authentication: username: password: http_extra_headers: 'X-Frame-Options': 'SAMEORIGIN' 'X-XSS-Protection': '1; mode=block' 'X-Content-Type-Options': 'nosniff' cafile: certfile: keyfile: https_extra_headers: 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains' .. warning:: - The ``restapi.connect_address`` must be accessible from all nodes of a given Patroni cluster. Internally Patroni is using it during the leader race to find nodes with minimal replication lag. - If you enabled client certificates validation (``restapi.verify_client`` is set to ``required``), you also **must** provide **valid client certificates** in the ``ctl.certfile``, ``ctl.keyfile``, ``ctl.keyfile_password``. If not provided, Patroni will not work correctly. .. _patronictl_settings: CTL --- - **ctl**: (optional) - **authentication**: - **username**: Basic-auth username for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "username" parameter. - **password**: Basic-auth password for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "password" parameter. - **insecure**: Allow connections to REST API without verifying SSL certs. - **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided :ref:`patronictl` will use the value provided for REST API "cafile" parameter. - **certfile**: Specifies the file with the client certificate in the PEM format. - **keyfile**: Specifies the file with the client secret key in the PEM format. - **keyfile\_password**: Specifies a password for decrypting the client keyfile. Watchdog -------- - **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be successfully enabled. - **device**: Path to watchdog device. Defaults to ``/dev/watchdog``. - **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration. .. _tags_settings: Tags ---- - **clonefrom**: ``true`` or ``false``. If set to ``true`` other nodes might prefer to use this node for bootstrap (take ``pg_basebackup`` from). If there are several nodes with ``clonefrom`` tag set to ``true`` the node to bootstrap from will be chosen randomly. The default value is ``false``. - **noloadbalance**: ``true`` or ``false``. If set to ``true`` the node will return HTTP Status Code 503 for the ``GET /replica`` REST API health-check and therefore will be excluded from the load-balancing. Defaults to ``false``. - **replicatefrom**: The name of another replica to replicate from. Used to support cascading replication. - **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica. - **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``, meaning this node _can_ participate in leader races. - **failover_priority**: integer, controls the priority that this node should have during failover. Nodes with higher priority will be preferred over lower priority nodes if they received/replayed the same amount of WAL. However, nodes with higher values of receive/replay LSN are preferred regardless of their priority. If the ``failover_priority`` is 0 or negative - such node is not allowed to participate in the leader race and to become a leader (similar to ``nofailover: true``). - **nostream**: ``true`` or ``false``. If set to ``true`` the node will not use replication protocol to stream WAL. It will rely instead on archive recovery (if ``restore_command`` is configured) and ``pg_wal``/``pg_xlog`` polling. It also disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas. Setting this tag on primary node has no effect. .. warning:: Provide only one of ``nofailover`` or ``failover_priority``. Providing ``nofailover: true`` is the same as ``failover_priority: 0``, and providing ``nofailover: false`` will give the node priority 1. In addition to these predefined tags, you can also add your own ones: - **key1**: ``true`` - **key2**: ``false`` - **key3**: ``1.4`` - **key4**: ``"RandomString"`` Tags are visible in the :ref:`REST API ` and :ref:`patronictl_list` You can also check for an instance health using these tags. If the tag isn't defined for an instance, or if the respective value doesn't match the querying value, it will return HTTP Status Code 503. patroni-4.0.4/extras/000077500000000000000000000000001472010352700145025ustar00rootroot00000000000000patroni-4.0.4/extras/README.md000066400000000000000000000012271472010352700157630ustar00rootroot00000000000000### confd `confd` directory contains haproxy and pgbouncer template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool You need to copy content of `confd` directory into /etcd/confd and run confd service: ```bash $ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_URL -interval=10 ``` It will periodically update haproxy.cfg and pgbouncer.ini with the actual list of Patroni nodes from `etcd` and "reload" haproxy and pgbouncer.ini when it is necessary. ### startup-scripts `startup-scripts` directory contains startup scripts for various OSes and management tools for Patroni. patroni-4.0.4/extras/confd/000077500000000000000000000000001472010352700155735ustar00rootroot00000000000000patroni-4.0.4/extras/confd/conf.d/000077500000000000000000000000001472010352700167425ustar00rootroot00000000000000patroni-4.0.4/extras/confd/conf.d/haproxy.toml000066400000000000000000000004651472010352700213360ustar00rootroot00000000000000[template] #prefix = "/service/batman" #owner = "haproxy" #mode = "0644" src = "haproxy.tmpl" dest = "/etc/haproxy/haproxy.cfg" check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}" reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)" keys = [ "/", ] patroni-4.0.4/extras/confd/conf.d/pgbouncer.toml000066400000000000000000000003241472010352700216220ustar00rootroot00000000000000[template] prefix = "/service/batman" owner = "postgres" mode = "0644" src = "pgbouncer.tmpl" dest = "/etc/pgbouncer/pgbouncer.ini" reload_cmd = "systemctl reload pgbouncer" keys = [ "/members/","/leader" ]patroni-4.0.4/extras/confd/templates/000077500000000000000000000000001472010352700175715ustar00rootroot00000000000000patroni-4.0.4/extras/confd/templates/haproxy-citus.tmpl000066400000000000000000000024121472010352700233050ustar00rootroot00000000000000global maxconn 100 defaults log global mode tcp retries 2 timeout client 30m timeout connect 4s timeout server 30m timeout check 5s listen stats mode http bind *:7000 stats enable stats uri / listen coordinator bind *:5000 option httpchk HEAD /primary http-check expect status 200 default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions {{range gets "/0/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check check-ssl port {{index (split (index (split $data.api_url "/") 2) ":") 1}} verify required ca-file /etc/ssl/certs/ssl-cert-snakeoil.pem crt /etc/ssl/private/ssl-cert-snakeoil.crt {{end}} listen workers bind *:5001 option httpchk HEAD /primary http-check expect status 200 default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions {{range gets "/*/members/*"}}{{$group := index (split .Key "/") 1}}{{if ne $group "0"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check check-ssl port {{index (split (index (split $data.api_url "/") 2) ":") 1}} verify required ca-file /etc/ssl/certs/ssl-cert-snakeoil.pem crt /etc/ssl/private/ssl-cert-snakeoil.crt {{end}}{{end}} patroni-4.0.4/extras/confd/templates/haproxy.tmpl000066400000000000000000000017361472010352700221700ustar00rootroot00000000000000global maxconn 100 defaults log global mode tcp retries 2 timeout client 30m timeout connect 4s timeout server 30m timeout check 5s listen stats mode http bind *:7000 stats enable stats uri / listen primary bind *:5000 option httpchk HEAD /primary http-check expect status 200 default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions {{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}} {{end}} listen replicas bind *:5001 option httpchk HEAD /replica http-check expect status 200 default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions {{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}} {{end}} patroni-4.0.4/extras/confd/templates/pgbouncer.tmpl000066400000000000000000000014071472010352700224550ustar00rootroot00000000000000[databases] {{with get "/leader"}}{{$leader := .Value}}{{$leadkey := printf "/members/%s" $leader}}{{with get $leadkey}}{{$data := json .Value}}{{$hostport := base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}}{{ $host := base (index (split $hostport ":") 0)}}{{ $port := base (index (split $hostport ":") 1)}}* = host={{ $host }} port={{ $port }} pool_size=10{{end}}{{end}} [pgbouncer] logfile = /var/log/postgresql/pgbouncer.log pidfile = /var/run/postgresql/pgbouncer.pid listen_addr = * listen_port = 6432 unix_socket_dir = /var/run/postgresql auth_type = trust auth_file = /etc/pgbouncer/userlist.txt auth_hba_file = /etc/pgbouncer/pg_hba.txt admin_users = pgbouncer stats_users = pgbouncer pool_mode = session max_client_conn = 100 default_pool_size = 20 patroni-4.0.4/extras/startup-scripts/000077500000000000000000000000001472010352700176715ustar00rootroot00000000000000patroni-4.0.4/extras/startup-scripts/README.md000066400000000000000000000021441472010352700211510ustar00rootroot00000000000000# startup scripts for Patroni This directory contains sample startup scripts for various OSes and management tools for Patroni. Scripts supplied: ### patroni.upstart.conf Upstart job for Ubuntu 12.04 or 14.04. Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker. ### patroni.service Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. ### patroni Init.d service file for Debian-like distributions. Copy it to /etc/init.d/, make executable: ```chmod 755 /etc/init.d/patroni``` and run with ```service patroni start```, or make it starting on boot with ```update-rc.d patroni defaults```. Also you might edit some configuration variables in it: PATRONI for patroni.py location CONF for configuration file LOGFILE for log (script creates it if does not exist) Note. If you have several versions of Postgres installed, please add to POSTGRES_VERSION the release number which you wish to run. Script uses this value to append PATH environment with correct path to Postgres bin. patroni-4.0.4/extras/startup-scripts/patroni000066400000000000000000000065311472010352700212750ustar00rootroot00000000000000#!/bin/sh # ### BEGIN INIT INFO # Provides: patroni # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Patroni init script # Description: Runners to orchestrate a high-availability PostgreSQL ### END INIT INFO ### BEGIN USER CONFIGURATION CONF="/etc/patroni/postgres.yml" LOGFILE="/var/log/patroni.log" USER="postgres" GROUP="postgres" NAME=patroni PATRONI="/opt/patroni/$NAME.py" PIDFILE="/var/run/$NAME.pid" # Set this parameter, if you have several Postgres versions installed # POSTGRES_VERSION="9.4" POSTGRES_VERSION="" ### END USER CONFIGURATION . /lib/lsb/init-functions # Loading this library for get_versions() function if test ! -e /usr/share/postgresql-common/init.d-functions; then log_failure_msg "Probably postgresql-common does not installed." exit 1 else . /usr/share/postgresql-common/init.d-functions fi # Is there Patroni executable? if test ! -e $PATRONI; then log_failure_msg "Patroni executable $PATRONI does not exist." exit 1 fi # Is there Patroni configuration file? if test ! -e $CONF; then log_failure_msg "Patroni configuration file $CONF does not exist." exit 1 fi # Create logfile if doesn't exist if test ! -e $LOGFILE; then log_action_msg "Creating logfile for Patroni..." touch $LOGFILE chown $USER:$GROUP $LOGFILE fi prepare_pgpath() { if [ "$POSTGRES_VERSION" != "" ]; then if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin" else log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable." exit 0 fi else get_versions if echo $versions | grep -q -e "\s"; then log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment." else versions=`echo $versions | sed -e 's/^[ \t]*//'` PGPATH="/usr/lib/postgresql/$versions/bin" fi fi } get_pid() { if test -e $PIDFILE; then PID=`cat $PIDFILE` CHILDPID=`ps --ppid $PID -o %p --no-headers` else log_failure_msg "Could not find PID file. Patroni probably down." exit 1 fi } case "$1" in start) prepare_pgpath PGPATH=$PATH:$PGPATH log_success_msg "Starting Patroni\n" exec start-stop-daemon --start --quiet \ --background \ --pidfile $PIDFILE --make-pidfile \ --chuid $USER:$GROUP \ --chdir `eval echo ~$USER` \ --exec $PATRONI \ --startas /bin/sh -- \ -c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1" ;; stop) log_success_msg "Stopping Patroni" get_pid start-stop-daemon --stop --pid $CHILDPID start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet ;; reload) log_success_msg "Reloading Patroni configuration" get_pid kill -HUP $CHILDPID ;; status) get_pid if start-stop-daemon -T --pid $CHILDPID; then log_success_msg "Patroni is running\n" exit 0 else log_warning_msg "Patroni in not running\n" fi ;; restart) $0 stop $0 start ;; *) echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}" exit 1 ;; esac if [ $? -eq 0 ]; then echo . exit 0 else echo " failed" exit 1 fi patroni-4.0.4/extras/startup-scripts/patroni.service000066400000000000000000000024351472010352700227330ustar00rootroot00000000000000# This is an example systemd config file for Patroni # You can copy it to "/etc/systemd/system/patroni.service", [Unit] Description=Runners to orchestrate a high-availability PostgreSQL After=syslog.target network.target [Service] Type=simple User=postgres Group=postgres # Read in configuration file if it exists, otherwise proceed EnvironmentFile=-/etc/patroni_env.conf # The default is the user's home directory, and if you want to change it, you must provide an absolute path. # WorkingDirectory=/home/sameuser # Where to send early-startup messages from the server # This is normally controlled by the global default set by systemd #StandardOutput=syslog # Pre-commands to start watchdog device # Uncomment if watchdog is part of your patroni setup #ExecStartPre=-/usr/bin/sudo /sbin/modprobe softdog #ExecStartPre=-/usr/bin/sudo /bin/chown postgres /dev/watchdog # Start the patroni process ExecStart=/bin/patroni /etc/patroni.yml # Send HUP to reload from patroni.yml ExecReload=/bin/kill -s HUP $MAINPID # Only kill the patroni process, not it's children, so it will gracefully stop postgres KillMode=process # Give a reasonable amount of time for the server to start up/shut down TimeoutSec=30 # Restart the service if it crashed Restart=on-failure [Install] WantedBy=multi-user.target patroni-4.0.4/extras/startup-scripts/patroni.upstart.conf000066400000000000000000000014641472010352700237220ustar00rootroot00000000000000# patroni - patroni daemon # # controls startup/shutdown of postgres # you should disable any postgres start jobs # # assumes that patroni has been installed into the # pythonpath by using setup.py install description "patroni start daemon" start on net-device-up stop on runlevel [06] respawn respawn limit 5 10 # set location of patroni env PATRONI=/usr/local/bin/patroni # virtualenv example # env PATRONI=/var/lib/postgresql/patronienv/bin/patroni # set location of config file env PATRONICONF=/etc/patroni/patroni.yml # set log dir for patroni logs # postgres user must have write permission env POSTGRESLOGDIR=/var/log/postgresql setuid postgres setgid postgres script exec start-stop-daemon --start \ --exec $PATRONI -- $PATRONICONF \ >> $POSTGRESLOGDIR/patroni.log 2>&1 end script patroni-4.0.4/features/000077500000000000000000000000001472010352700150125ustar00rootroot00000000000000patroni-4.0.4/features/Dockerfile000066400000000000000000000052771472010352700170170ustar00rootroot00000000000000# syntax = docker/dockerfile:1.5 # Used only for running tests using tox, see ../tox.ini ARG PG_MAJOR ARG PGHOME=/home/postgres ARG LC_ALL=C.UTF-8 ARG LANG=C.UTF-8 ARG BASE_IMAGE=postgres FROM ${BASE_IMAGE}:${PG_MAJOR} ARG PGHOME ARG LC_ALL ARG LANG ENV PGHOME="$PGHOME" ENV PG_USER="${PG_USER:-postgres}" ENV PG_GROUP="${PG_GROUP:-$PG_USER}" ENV LC_ALL="$LC_ALL" ENV LANG="$LANG" ARG ETCDVERSION=3.3.13 ENV ETCDVERSION="$ETCDVERSION" ARG ETCDURL="https://github.com/coreos/etcd/releases/download/v$ETCDVERSION" USER root RUN set -ex \ && apt-get update \ && apt-get reinstall init-system-helpers \ && apt-get install -y \ python3-dev \ python3-venv \ rsync \ curl \ gcc \ golang \ jq \ locales \ sudo \ busybox \ net-tools \ iputils-ping \ && rm -rf /var/cache/apt \ \ && python3 -m venv /tox \ && /tox/bin/pip install --no-cache-dir tox>=4 \ \ && mkdir -p "$PGHOME" \ && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \ && chown -R "$PG_USER:$PG_GROUP" /var/log /home/postgres \ \ # Download etcd \ && curl -sL "$ETCDURL/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \ | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl ENV PATH="/tox/bin:$PATH" # This Dockerfile syntax only works with docker buildx and the syntax # line at the top of this file. COPY </dev/null \\ | sed 's|^./||' >/tmp/copy_exclude.lst \\ || true runuser -u "\$PG_USER" -- \\ rsync -a \\ --exclude=.tox \\ --exclude="features/output*" \\ --exclude-from="/tmp/copy_exclude.lst" \\ . "\$PGHOME/src/" cd "\$PGHOME/src" runuser -u "\$PG_USER" -w ETCD_UNSUPPORTED_ARCH -- "\$@" & wait $! # SIGINT whilst child proc is running is not seen by trap so we run a copy here instead of using # trap copy_output SIGINT EXIT copy_output EOF RUN chmod +x /tox-wrapper.sh VOLUME /src ENTRYPOINT ["/tox-wrapper.sh"] patroni-4.0.4/features/archive-restore.py000066400000000000000000000013501472010352700204650ustar00rootroot00000000000000#!/usr/bin/env python import argparse import os import shutil if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--dirname", required=True) parser.add_argument("--pathname", required=True) parser.add_argument("--filename", required=True) parser.add_argument("--mode", required=True, choices=("archive", "restore")) args, _ = parser.parse_known_args() full_filename = os.path.join(args.dirname, args.filename) if args.mode == "archive": if not os.path.isdir(args.dirname): os.makedirs(args.dirname) if not os.path.exists(full_filename): shutil.copy(args.pathname, full_filename) else: shutil.copy(full_filename, args.pathname) patroni-4.0.4/features/backup_create.py000077500000000000000000000010561472010352700201610ustar00rootroot00000000000000#!/usr/bin/env python import argparse import subprocess import sys if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--datadir", required=True) parser.add_argument("--dbname", required=True) parser.add_argument("--walmethod", required=True, choices=("fetch", "stream", "none")) args, _ = parser.parse_known_args() walmethod = ["-X", args.walmethod] if args.walmethod != "none" else [] sys.exit(subprocess.call(["pg_basebackup", "-D", args.datadir, "-c", "fast", "-d", args.dbname] + walmethod)) patroni-4.0.4/features/backup_restore.py000077500000000000000000000005661472010352700204060ustar00rootroot00000000000000#!/usr/bin/env python import argparse import shutil if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--datadir", required=True) parser.add_argument("--sourcedir", required=True) parser.add_argument("--test-argument", required=True) args, _ = parser.parse_known_args() shutil.copytree(args.sourcedir, args.datadir) patroni-4.0.4/features/basic_replication.feature000066400000000000000000000112531472010352700220430ustar00rootroot00000000000000Feature: basic replication We should check that the basic bootstrapping, replication and failover works. Scenario: check replication of a single table Given I start postgres-0 Then postgres-0 is a leader after 10 seconds And there is a non empty initialize key in DCS after 15 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": true} Then I receive a response code 200 When I start postgres-1 And I configure and start postgres-2 with a tag replicatefrom postgres-0 And "sync" key in DCS has leader=postgres-0 after 20 seconds And I add the table foo to postgres-0 Then table foo is present on postgres-1 after 20 seconds Then table foo is present on postgres-2 after 20 seconds Scenario: check restart of sync replica Given I shut down postgres-2 Then "sync" key in DCS has sync_standby=postgres-1 after 5 seconds When I start postgres-2 And I shut down postgres-1 Then "sync" key in DCS has sync_standby=postgres-2 after 10 seconds When I start postgres-1 Then "members/postgres-1" key in DCS has state=running after 10 seconds And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds And Status code on GET http://127.0.0.1:8009/async is 200 after 3 seconds Scenario: check stuck sync replica Given I issue a PATCH request to http://127.0.0.1:8008/config with {"pause": true, "maximum_lag_on_syncnode": 15000000, "postgresql": {"parameters": {"synchronous_commit": "remote_apply"}}} Then I receive a response code 200 And I create table on postgres-0 And table mytest is present on postgres-1 after 2 seconds And table mytest is present on postgres-2 after 2 seconds When I pause wal replay on postgres-2 And I load data on postgres-0 Then "sync" key in DCS has sync_standby=postgres-1 after 15 seconds And I resume wal replay on postgres-2 And Status code on GET http://127.0.0.1:8009/sync is 200 after 3 seconds And Status code on GET http://127.0.0.1:8010/async is 200 after 3 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"pause": null, "maximum_lag_on_syncnode": -1, "postgresql": {"parameters": {"synchronous_commit": "on"}}} Then I receive a response code 200 And I drop table on postgres-0 Scenario: check multi sync replication Given I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_node_count": 2} Then I receive a response code 200 Then "sync" key in DCS has sync_standby=postgres-1,postgres-2 after 10 seconds And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds And Status code on GET http://127.0.0.1:8009/sync is 200 after 3 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_node_count": 1} Then I receive a response code 200 And I shut down postgres-1 Then "sync" key in DCS has sync_standby=postgres-2 after 10 seconds When I start postgres-1 Then "members/postgres-1" key in DCS has state=running after 10 seconds And Status code on GET http://127.0.0.1:8010/sync is 200 after 3 seconds And Status code on GET http://127.0.0.1:8009/async is 200 after 3 seconds Scenario: check the basic failover in synchronous mode Given I run patronictl.py pause batman Then I receive a response returncode 0 When I sleep for 2 seconds And I shut down postgres-0 And I run patronictl.py resume batman Then I receive a response returncode 0 And postgres-2 role is the primary after 24 seconds And Response on GET http://127.0.0.1:8010/history contains recovery after 10 seconds And there is a postgres-2_cb.log with "on_role_change primary batman" in postgres-2 data directory When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0} Then I receive a response code 200 When I add the table bar to postgres-2 Then table bar is present on postgres-1 after 20 seconds And Response on GET http://127.0.0.1:8010/config contains master_start_timeout after 10 seconds Scenario: check rejoin of the former primary with pg_rewind Given I add the table splitbrain to postgres-0 And I start postgres-0 Then postgres-0 role is the secondary after 20 seconds When I add the table buz to postgres-2 Then table buz is present on postgres-0 after 20 seconds @reject-duplicate-name Scenario: check graceful rejection when two nodes have the same name Given I start duplicate postgres-0 on port 8011 Then there is one of ["Can't start; there is already a node named 'postgres-0' running"] CRITICAL in the dup-postgres-0 patroni log after 5 seconds patroni-4.0.4/features/callback2.py000077500000000000000000000002241472010352700172030ustar00rootroot00000000000000#!/usr/bin/env python import sys with open("data/{0}/{0}_cb.log".format(sys.argv[1]), "a+") as log: log.write(" ".join(sys.argv[-3:]) + "\n") patroni-4.0.4/features/cascading_replication.feature000066400000000000000000000014501472010352700226740ustar00rootroot00000000000000Feature: cascading replication We should check that patroni can do base backup and streaming from the replica Scenario: check a base backup and streaming replication from a replica Given I start postgres-0 And postgres-0 is a leader after 10 seconds And I configure and start postgres-1 with a tag clonefrom true And replication works from postgres-0 to postgres-1 after 20 seconds And I create label with "postgres-0" in postgres-0 data directory And I create label with "postgres-1" in postgres-1 data directory And "members/postgres-1" key in DCS has state=running after 12 seconds And I configure and start postgres-2 with a tag replicatefrom postgres-1 Then replication works from postgres-0 to postgres-2 after 30 seconds And there is a label with "postgres-1" in postgres-2 data directory patroni-4.0.4/features/citus.feature000066400000000000000000000120731472010352700175210ustar00rootroot00000000000000Feature: citus We should check that coordinator discovers and registers workers and clients don't have errors when worker cluster switches over Scenario: check that worker cluster is registered in the coordinator Given I start postgres-0 in citus group 0 And I start postgres-2 in citus group 1 Then postgres-0 is a leader in a group 0 after 10 seconds And postgres-2 is a leader in a group 1 after 10 seconds When I start postgres-1 in citus group 0 And I start postgres-3 in citus group 1 Then replication works from postgres-0 to postgres-1 after 15 seconds Then replication works from postgres-2 to postgres-3 after 15 seconds And postgres-0 is registered in the postgres-0 as the primary in group 0 after 5 seconds And postgres-1 is registered in the postgres-0 as the secondary in group 0 after 5 seconds And postgres-2 is registered in the postgres-0 as the primary in group 1 after 5 seconds And postgres-3 is registered in the postgres-0 as the secondary in group 1 after 5 seconds Scenario: coordinator failover updates pg_dist_node Given I run patronictl.py failover batman --group 0 --candidate postgres-1 --force Then postgres-1 role is the primary after 10 seconds And "members/postgres-0" key in a group 0 in DCS has state=running after 15 seconds And replication works from postgres-1 to postgres-0 after 15 seconds And postgres-1 is registered in the postgres-2 as the primary in group 0 after 5 seconds And postgres-0 is registered in the postgres-2 as the secondary in group 0 after 15 seconds And "sync" key in a group 0 in DCS has sync_standby=postgres-0 after 15 seconds When I run patronictl.py switchover batman --group 0 --candidate postgres-0 --force Then postgres-0 role is the primary after 10 seconds And replication works from postgres-0 to postgres-1 after 15 seconds And postgres-0 is registered in the postgres-2 as the primary in group 0 after 5 seconds And postgres-1 is registered in the postgres-2 as the secondary in group 0 after 15 seconds And "sync" key in a group 0 in DCS has sync_standby=postgres-1 after 15 seconds Scenario: worker switchover doesn't break client queries on the coordinator Given I create a distributed table on postgres-0 And I start a thread inserting data on postgres-0 When I run patronictl.py switchover batman --group 1 --force Then I receive a response returncode 0 And postgres-3 role is the primary after 10 seconds And "members/postgres-2" key in a group 1 in DCS has state=running after 15 seconds And replication works from postgres-3 to postgres-2 after 15 seconds And postgres-3 is registered in the postgres-0 as the primary in group 1 after 5 seconds And postgres-2 is registered in the postgres-0 as the secondary in group 1 after 15 seconds And "sync" key in a group 1 in DCS has sync_standby=postgres-2 after 15 seconds And a thread is still alive When I run patronictl.py switchover batman --group 1 --force Then I receive a response returncode 0 And postgres-2 role is the primary after 10 seconds And replication works from postgres-2 to postgres-3 after 15 seconds And postgres-2 is registered in the postgres-0 as the primary in group 1 after 5 seconds And postgres-3 is registered in the postgres-0 as the secondary in group 1 after 15 seconds And "sync" key in a group 1 in DCS has sync_standby=postgres-3 after 15 seconds And a thread is still alive When I stop a thread Then a distributed table on postgres-0 has expected rows Scenario: worker primary restart doesn't break client queries on the coordinator Given I cleanup a distributed table on postgres-0 And I start a thread inserting data on postgres-0 When I run patronictl.py restart batman postgres-2 --group 1 --force Then I receive a response returncode 0 And postgres-2 role is the primary after 10 seconds And replication works from postgres-2 to postgres-3 after 15 seconds And postgres-2 is registered in the postgres-0 as the primary in group 1 after 5 seconds And postgres-3 is registered in the postgres-0 as the secondary in group 1 after 15 seconds And a thread is still alive When I stop a thread Then a distributed table on postgres-0 has expected rows Scenario: check that in-flight transaction is rolled back after timeout when other workers need to change pg_dist_node Given I start postgres-4 in citus group 2 Then postgres-4 is a leader in a group 2 after 10 seconds And "members/postgres-4" key in a group 2 in DCS has role=primary after 3 seconds When I run patronictl.py edit-config batman --group 2 -s ttl=20 --force Then I receive a response returncode 0 And I receive a response output "+ttl: 20" Then postgres-4 is registered in the postgres-2 as the primary in group 2 after 5 seconds When I shut down postgres-4 Then there is a transaction in progress on postgres-0 changing pg_dist_node after 5 seconds When I run patronictl.py restart batman postgres-2 --group 1 --force Then a transaction finishes in 20 seconds patroni-4.0.4/features/custom_bootstrap.feature000066400000000000000000000014301472010352700217740ustar00rootroot00000000000000Feature: custom bootstrap We should check that patroni can bootstrap a new cluster from a backup Scenario: clone existing cluster using pg_basebackup Given I start postgres-0 Then postgres-0 is a leader after 10 seconds When I add the table foo to postgres-0 And I start postgres-1 in a cluster batman1 as a clone of postgres-0 Then postgres-1 is a leader of batman1 after 10 seconds Then table foo is present on postgres-1 after 10 seconds Scenario: make a backup and do a restore into a new cluster Given I add the table bar to postgres-1 And I do a backup of postgres-1 When I start postgres-2 in a cluster batman2 from backup Then postgres-2 is a leader of batman2 after 30 seconds And table bar is present on postgres-2 after 10 seconds patroni-4.0.4/features/dcs_failsafe_mode.feature000066400000000000000000000151621472010352700220030ustar00rootroot00000000000000Feature: dcs failsafe mode We should check the basic dcs failsafe mode functioning Scenario: check failsafe mode can be successfully enabled Given I start postgres-0 And postgres-0 is a leader after 10 seconds Then "config" key in DCS has ttl=30 after 10 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 3, "failsafe_mode": true} Then I receive a response code 200 And Response on GET http://127.0.0.1:8008/failsafe contains postgres-0 after 10 seconds When I issue a GET request to http://127.0.0.1:8008/failsafe Then I receive a response code 200 And I receive a response postgres-0 http://127.0.0.1:8008/patroni When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}},"slots":{"dcs_slot_1": null,"postgres_0":null}} Then I receive a response code 200 When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots": {"dcs_slot_0": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}} Then I receive a response code 200 @dcs-failsafe Scenario: check one-node cluster is functioning while DCS is down Given DCS is down Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds And postgres-0 role is the primary after 10 seconds @dcs-failsafe Scenario: check new replica isn't promoted when leader is down and DCS is up Given DCS is up When I do a backup of postgres-0 And I shut down postgres-0 When I start postgres-1 in a cluster batman from backup with no_leader Then postgres-1 role is the replica after 12 seconds Scenario: check leader and replica are both in /failsafe key after leader is back Given I start postgres-0 And I start postgres-1 Then "members/postgres-0" key in DCS has state=running after 10 seconds And "members/postgres-1" key in DCS has state=running after 2 seconds And Response on GET http://127.0.0.1:8009/failsafe contains postgres-1 after 10 seconds When I issue a GET request to http://127.0.0.1:8009/failsafe Then I receive a response code 200 And I receive a response postgres-0 http://127.0.0.1:8008/patroni And I receive a response postgres-1 http://127.0.0.1:8009/patroni @dcs-failsafe @slot-advance Scenario: check leader and replica are functioning while DCS is down Given I get all changes from physical slot dcs_slot_1 on postgres-0 Then physical slot dcs_slot_1 is in sync between postgres-0 and postgres-1 after 10 seconds And logical slot dcs_slot_0 is in sync between postgres-0 and postgres-1 after 10 seconds And DCS is down Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds Then postgres-0 role is the primary after 10 seconds And postgres-1 role is the replica after 2 seconds And replication works from postgres-0 to postgres-1 after 10 seconds When I get all changes from logical slot dcs_slot_0 on postgres-0 And I get all changes from physical slot dcs_slot_1 on postgres-0 Then logical slot dcs_slot_0 is in sync between postgres-0 and postgres-1 after 20 seconds And physical slot dcs_slot_1 is in sync between postgres-0 and postgres-1 after 10 seconds @dcs-failsafe Scenario: check primary is demoted when one replica is shut down and DCS is down Given DCS is down And I kill postgres-1 And I kill postmaster on postgres-1 Then postgres-0 role is the replica after 12 seconds @dcs-failsafe Scenario: check known replica is promoted when leader is down and DCS is up Given I kill postgres-0 And I shut down postmaster on postgres-0 And DCS is up When I start postgres-1 Then "members/postgres-1" key in DCS has state=running after 10 seconds And postgres-1 role is the primary after 25 seconds @dcs-failsafe Scenario: scale to three-node cluster Given I start postgres-0 And I configure and start postgres-2 with a tag replicatefrom postgres-0 Then "members/postgres-2" key in DCS has state=running after 10 seconds And "members/postgres-0" key in DCS has state=running after 20 seconds And Response on GET http://127.0.0.1:8008/failsafe contains postgres-2 after 10 seconds And replication works from postgres-1 to postgres-0 after 10 seconds And replication works from postgres-1 to postgres-2 after 10 seconds @dcs-failsafe @slot-advance Scenario: make sure permanent slots exist on replicas Given I issue a PATCH request to http://127.0.0.1:8009/config with {"slots":{"dcs_slot_0":null,"dcs_slot_2":{"type":"logical","database":"postgres","plugin":"test_decoding"}}} Then logical slot dcs_slot_2 is in sync between postgres-1 and postgres-0 after 20 seconds And logical slot dcs_slot_2 is in sync between postgres-1 and postgres-2 after 20 seconds When I get all changes from physical slot dcs_slot_1 on postgres-1 Then physical slot dcs_slot_1 is in sync between postgres-1 and postgres-0 after 10 seconds And physical slot dcs_slot_1 is in sync between postgres-1 and postgres-2 after 10 seconds And physical slot postgres_0 is in sync between postgres-1 and postgres-2 after 10 seconds And physical slot postgres_2 is in sync between postgres-0 and postgres-1 after 10 seconds @dcs-failsafe Scenario: check three-node cluster is functioning while DCS is down Given DCS is down Then Response on GET http://127.0.0.1:8009/primary contains failsafe_mode_is_active after 12 seconds Then postgres-1 role is the primary after 10 seconds And postgres-0 role is the replica after 2 seconds And postgres-2 role is the replica after 2 seconds @dcs-failsafe @slot-advance Scenario: check that permanent slots are in sync between nodes while DCS is down Given replication works from postgres-1 to postgres-0 after 10 seconds And replication works from postgres-1 to postgres-2 after 10 seconds When I get all changes from logical slot dcs_slot_2 on postgres-1 And I get all changes from physical slot dcs_slot_1 on postgres-1 Then logical slot dcs_slot_2 is in sync between postgres-1 and postgres-0 after 20 seconds And logical slot dcs_slot_2 is in sync between postgres-1 and postgres-2 after 20 seconds And physical slot dcs_slot_1 is in sync between postgres-1 and postgres-0 after 10 seconds And physical slot dcs_slot_1 is in sync between postgres-1 and postgres-2 after 10 seconds And physical slot postgres_0 is in sync between postgres-1 and postgres-2 after 10 seconds And physical slot postgres_2 is in sync between postgres-0 and postgres-1 after 10 seconds patroni-4.0.4/features/environment.py000066400000000000000000001327111472010352700177350ustar00rootroot00000000000000import abc import datetime import glob import json import os import re import shutil import signal import stat import subprocess import sys import tempfile import threading import time from http.server import BaseHTTPRequestHandler, HTTPServer import psutil import yaml import patroni.psycopg as psycopg from patroni.request import PatroniRequest class AbstractController(abc.ABC): def __init__(self, context, name, work_directory, output_dir): self._context = context self._name = name self._work_directory = work_directory self._output_dir = output_dir self._handle = None self._log = None def _has_started(self): return self._handle and self._handle.pid and self._handle.poll() is None def _is_running(self): return self._has_started() @abc.abstractmethod def _is_accessible(self): """process is accessible for queries""" @abc.abstractmethod def _start(self): """start process""" def start(self, max_wait_limit=5): if self._is_running(): return True self._log = open(os.path.join(self._output_dir, self._name + '.log'), 'a') self._handle = self._start() max_wait_limit *= self._context.timeout_multiplier for _ in range(max_wait_limit): assert self._has_started(), "Process {0} is not running after being started".format(self._name) if self._is_accessible(): break time.sleep(1) else: assert False, \ "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit) def stop(self, kill=False, timeout=15, _=False): term = False start_time = time.time() timeout *= self._context.timeout_multiplier while self._handle and self._is_running(): if kill: self._handle.kill() elif not term: self._handle.terminate() term = True time.sleep(1) if not kill and time.time() - start_time > timeout: kill = True if self._log: self._log.close() def cancel_background(self): pass class PatroniController(AbstractController): __PORT = 5360 PATRONI_CONFIG = '{}.yml' """ starts and stops individual patronis""" def __init__(self, context, name, work_directory, output_dir, custom_config=None): super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir) PatroniController.__PORT += 1 self._data_dir = os.path.join(work_directory, 'data', name) self._connstring = None if custom_config and 'watchdog' in custom_config: self.watchdog = WatchdogMonitor(name, work_directory, output_dir) custom_config['watchdog'] = {'driver': 'testing', 'device': self.watchdog.fifo_path, 'mode': 'required'} else: self.watchdog = None self._scope = (custom_config or {}).get('scope', 'batman') self._citus_group = (custom_config or {}).get('citus', {}).get('group') self._config = self._make_patroni_test_config(name, custom_config) self._closables = [] self._conn = None self._curs = None def write_label(self, content): with open(os.path.join(self._data_dir, 'label'), 'w') as f: f.write(content) def read_label(self, label): try: with open(os.path.join(self._data_dir, label), 'r') as f: return f.read().strip() except IOError: return None @staticmethod def recursive_update(dst, src): for k, v in src.items(): if k in dst and isinstance(dst[k], dict): PatroniController.recursive_update(dst[k], v) else: dst[k] = v def update_config(self, custom_config): with open(self._config) as r: config = yaml.safe_load(r) self.recursive_update(config, custom_config) with open(self._config, 'w') as w: yaml.safe_dump(config, w, default_flow_style=False) self._scope = config.get('scope', 'batman') def add_tag_to_config(self, tag, value): self.update_config({'tags': {tag: value}}) def _start(self): if self.watchdog: self.watchdog.start() env = os.environ.copy() if isinstance(self._context.dcs_ctl, KubernetesController): self._context.dcs_ctl.create_pod(self._name[8:], self._scope, self._citus_group) env['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1] if os.name == 'nt': env['BEHAVE_DEBUG'] = 'true' patroni = subprocess.Popen([sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config], env=env, stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory) if os.name == 'nt': patroni.terminate = self.terminate return patroni def terminate(self): try: self._context.request_executor.request('POST', self._restapi_url + '/sigterm') except Exception: pass def stop(self, kill=False, timeout=15, postgres=False): if postgres: mode = 'i' if kill else 'f' return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-m' + mode, '-w']) super(PatroniController, self).stop(kill, timeout) if isinstance(self._context.dcs_ctl, KubernetesController) and not kill: self._context.dcs_ctl.delete_pod(self._name[8:]) if self.watchdog: self.watchdog.stop() def _is_accessible(self): cursor = self.query("SELECT 1", fail_ok=True) if cursor is not None: cursor.execute("SET synchronous_commit TO 'local'") return True def _make_patroni_test_config(self, name, custom_config): patroni_config_name = self.PATRONI_CONFIG.format(name) patroni_config_path = os.path.join(self._output_dir, patroni_config_name) with open('postgres0.yml') as f: config = yaml.safe_load(f) config.pop('etcd', None) raft_port = os.environ.get('RAFT_PORT') # If patroni_raft_controller is suspended two Patroni members is enough to get a quorum, # therefore we don't want Patroni to join as a voting member when testing dcs_failsafe_mode. if raft_port and not self._output_dir.endswith('dcs_failsafe_mode'): os.environ['RAFT_PORT'] = str(int(raft_port) + 1) config['raft'] = {'data_dir': self._output_dir, 'self_addr': 'localhost:' + os.environ['RAFT_PORT']} host = config['restapi']['listen'].rsplit(':', 1)[0] config['restapi']['listen'] = config['restapi']['connect_address'] = '{}:{}'.format(host, 8008 + int(name[-1])) host = config['postgresql']['listen'].rsplit(':', 1)[0] config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT) config['name'] = name config['postgresql']['data_dir'] = self._data_dir.replace('\\', '/') config['postgresql']['basebackup'] = [{'checkpoint': 'fast'}] config['postgresql']['callbacks'] = { 'on_role_change': '{0} features/callback2.py {1}'.format(self._context.pctl.PYTHON, name)} config['postgresql']['use_unix_socket'] = os.name != 'nt' # windows doesn't yet support unix-domain sockets config['postgresql']['use_unix_socket_repl'] = os.name != 'nt' config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name).replace('\\', '/') config['postgresql']['parameters'].update({ 'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir.replace('\\', '/'), 'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1', 'shared_buffers': '1MB', 'unix_socket_directories': tempfile.gettempdir().replace('\\', '/')}) config['postgresql']['pg_hba'] = [ 'local all all trust', 'local replication all trust', 'host replication replicator all md5', 'host all all all md5' ] if self._context.postgres_supports_ssl and self._context.certfile: config['postgresql']['parameters'].update({ 'ssl': 'on', 'ssl_ca_file': self._context.certfile.replace('\\', '/'), 'ssl_cert_file': self._context.certfile.replace('\\', '/'), 'ssl_key_file': self._context.keyfile.replace('\\', '/') }) for user in config['postgresql'].get('authentication').keys(): config['postgresql'].get('authentication', {}).get(user, {}).update({ 'sslmode': 'verify-ca', 'sslrootcert': self._context.certfile, 'sslcert': self._context.certfile, 'sslkey': self._context.keyfile }) for i, line in enumerate(list(config['postgresql']['pg_hba'])): if line.endswith('md5'): # we want to verify client cert first and than password config['postgresql']['pg_hba'][i] = 'hostssl' + line[4:] + ' clientcert=verify-ca' if 'bootstrap' in config: config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"' if 'initdb' in config['bootstrap']: config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}]) if custom_config is not None: self.recursive_update(config, custom_config) self.recursive_update(config, { 'log': { 'format': '%(asctime)s %(levelname)s [%(pathname)s:%(lineno)d - %(funcName)s]: %(message)s', 'loggers': {'patroni.postgresql.callback_executor': 'DEBUG'} }, 'bootstrap': { 'dcs': { 'loop_wait': 2, 'postgresql': { 'parameters': { 'wal_keep_segments': 100, 'archive_mode': 'on', 'archive_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT + ' --mode archive ' + '--dirname {} --filename %f --pathname %p').format( os.path.join(self._work_directory, 'data', f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/'), 'restore_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' + '--dirname {} --filename %f --pathname %p').format( os.path.join(self._work_directory, 'data', f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/') } } } } }) if config['postgresql'].get('callbacks', {}).get('on_role_change'): config['postgresql']['callbacks']['on_role_change'] += ' ' + str(self.__PORT) with open(patroni_config_path, 'w') as f: yaml.safe_dump(config, f, default_flow_style=False) self._connkwargs = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {}) self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres', 'user': self._connkwargs.pop('username', None)}) self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {}) self._replication.update({'host': host, 'port': self.__PORT, 'user': self._replication.pop('username', None)}) self._restapi_url = 'http://{0}'.format(config['restapi']['connect_address']) if self._context.certfile: self._restapi_url = self._restapi_url.replace('http://', 'https://') return patroni_config_path def _connection(self): if not self._conn or self._conn.closed != 0: self._conn = psycopg.connect(**self._connkwargs) return self._conn def _cursor(self): if not self._curs or self._curs.closed or self._curs.connection.closed != 0: self._curs = self._connection().cursor() return self._curs def query(self, query, fail_ok=False): try: cursor = self._cursor() cursor.execute(query) return cursor except psycopg.Error: if not fail_ok: raise def check_role_has_changed_to(self, new_role, timeout=10): bound_time = time.time() + timeout recovery_status = new_role != 'primary' while time.time() < bound_time: cur = self.query("SELECT pg_is_in_recovery()", fail_ok=True) if cur: row = cur.fetchone() if row and row[0] == recovery_status: return True time.sleep(1) return False def get_watchdog(self): return self.watchdog def _get_pid(self): try: pidfile = os.path.join(self._data_dir, 'postmaster.pid') if not os.path.exists(pidfile): return None return int(open(pidfile).readline().strip()) except Exception: return None def patroni_hang(self, timeout): hang = ProcessHang(self._handle.pid, timeout) self._closables.append(hang) hang.start() def cancel_background(self): for obj in self._closables: obj.close() self._closables = [] @property def backup_source(self): def escape(value): return re.sub(r'([\'\\ ])', r'\\\1', str(value)) return ' '.join('{0}={1}'.format(k, escape(v)) for k, v in self._replication.items()) def backup(self, dest=os.path.join('data', 'basebackup')): subprocess.call(PatroniPoolController.BACKUP_SCRIPT + ['--walmethod=none', '--datadir=' + os.path.join(self._work_directory, dest), '--dbname=' + self.backup_source]) def read_patroni_log(self, level): try: with open(str(os.path.join(self._output_dir or '', self._name + ".log"))) as f: return [line for line in f.readlines() if line[24:24 + len(level)] == level] except IOError: return [] class ProcessHang(object): """A background thread implementing a cancelable process hang via SIGSTOP.""" def __init__(self, pid, timeout): self._cancelled = threading.Event() self._thread = threading.Thread(target=self.run) self.pid = pid self.timeout = timeout def start(self): self._thread.start() def run(self): os.kill(self.pid, signal.SIGSTOP) try: self._cancelled.wait(self.timeout) finally: os.kill(self.pid, signal.SIGCONT) def close(self): self._cancelled.set() self._thread.join() class AbstractDcsController(AbstractController): _CLUSTER_NODE = '/service/{0}' def __init__(self, context, mktemp=True): work_directory = mktemp and tempfile.mkdtemp() or None self._paused = False super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir) def _is_accessible(self): return self._is_running() def stop(self, kill=False, timeout=15): """ terminate process and wipe out the temp work directory, but only if we actually started it""" super(AbstractDcsController, self).stop(kill=kill, timeout=timeout) if self._work_directory: shutil.rmtree(self._work_directory) def path(self, key=None, scope='batman', group=None): citus_group = '/{0}'.format(group) if group is not None else '' return self._CLUSTER_NODE.format(scope) + citus_group + (key and '/' + key or '') def start_outage(self): if not self._paused and self._handle: self._handle.suspend() self._paused = True def stop_outage(self): if self._paused and self._handle: self._handle.resume() self._paused = False @abc.abstractmethod def query(self, key, scope='batman', group=None): """ query for a value of a given key """ @abc.abstractmethod def cleanup_service_tree(self): """ clean all contents stored in the tree used for the tests """ @classmethod def get_subclasses(cls): for subclass in cls.__subclasses__(): for subsubclass in subclass.get_subclasses(): yield subsubclass yield subclass @classmethod def name(cls): return cls.__name__[:-10].lower() class ConsulController(AbstractDcsController): def __init__(self, context): super(ConsulController, self).__init__(context) os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500' os.environ['PATRONI_CONSUL_REGISTER_SERVICE'] = 'on' self._config_file = None import consul self._client = consul.Consul() def _start(self): self._config_file = self._work_directory + '.json' with open(self._config_file, 'wb') as f: f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}') return psutil.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT) def stop(self, kill=False, timeout=15): super(ConsulController, self).stop(kill=kill, timeout=timeout) if self._config_file: os.unlink(self._config_file) def _is_running(self): try: return bool(self._client.status.leader()) except Exception: return False def path(self, key=None, scope='batman', group=None): return super(ConsulController, self).path(key, scope, group)[1:] def query(self, key, scope='batman', group=None): _, value = self._client.kv.get(self.path(key, scope, group)) return value and value['Value'].decode('utf-8') def cleanup_service_tree(self): self._client.kv.delete(self.path(scope=''), recurse=True) def start(self, max_wait_limit=15): super(ConsulController, self).start(max_wait_limit) class AbstractEtcdController(AbstractDcsController): """ handles all etcd related tasks, used for the tests setup and cleanup """ def __init__(self, context, client_cls): super(AbstractEtcdController, self).__init__(context) self._client_cls = client_cls def _start(self): return psutil.Popen(["etcd", "--enable-v2=true", "--data-dir", self._work_directory], stdout=self._log, stderr=subprocess.STDOUT) def _is_running(self): from patroni.dcs.etcd import DnsCachingResolver # if etcd is running, but we didn't start it try: self._client = self._client_cls({'host': 'localhost', 'port': 2379, 'retry_timeout': 30, 'patronictl': 1}, DnsCachingResolver()) return True except Exception: return False class EtcdController(AbstractEtcdController): def __init__(self, context): from patroni.dcs.etcd import EtcdClient super(EtcdController, self).__init__(context, EtcdClient) os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379' def query(self, key, scope='batman', group=None): import etcd try: return self._client.get(self.path(key, scope, group)).value except etcd.EtcdKeyNotFound: return None def cleanup_service_tree(self): import etcd try: self._client.delete(self.path(scope=''), recursive=True) except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed): return except Exception as e: assert False, "exception when cleaning up etcd contents: {0}".format(e) class Etcd3Controller(AbstractEtcdController): def __init__(self, context): from patroni.dcs.etcd3 import Etcd3Client super(Etcd3Controller, self).__init__(context, Etcd3Client) os.environ['PATRONI_ETCD3_HOST'] = 'localhost:2379' def query(self, key, scope='batman', group=None): import base64 response = self._client.range(self.path(key, scope, group)) for k in response.get('kvs', []): return base64.b64decode(k['value']).decode('utf-8') if 'value' in k else None def cleanup_service_tree(self): try: self._client.deleteprefix(self.path(scope='')) except Exception as e: assert False, "exception when cleaning up etcd contents: {0}".format(e) class AbstractExternalDcsController(AbstractDcsController): def __init__(self, context, mktemp=True): super(AbstractExternalDcsController, self).__init__(context, mktemp) self._wrapper = ['sudo'] def _start(self): return self._external_pid def start_outage(self): if not self._paused: subprocess.call(self._wrapper + ['kill', '-SIGSTOP', self._external_pid]) self._paused = True def stop_outage(self): if self._paused: subprocess.call(self._wrapper + ['kill', '-SIGCONT', self._external_pid]) self._paused = False def _has_started(self): return True @abc.abstractmethod def process_name(): """process name to search with pgrep""" def _is_running(self): if not self._handle: self._external_pid = subprocess.check_output(['pgrep', '-nf', self.process_name()]).decode('utf-8').strip() return False return True def stop(self): pass class KubernetesController(AbstractExternalDcsController): def __init__(self, context): super(KubernetesController, self).__init__(context) self._namespace = 'default' self._labels = {"application": "patroni"} self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items()) os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels) os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true' os.environ.setdefault('PATRONI_KUBERNETES_BYPASS_API_SERVICE', 'true') from patroni.dcs.kubernetes import k8s_client, k8s_config k8s_config.load_kube_config(context=os.environ.setdefault('PATRONI_KUBERNETES_CONTEXT', 'kind-kind')) self._client = k8s_client self._api = self._client.CoreV1Api() def process_name(self): return "localkube" def _is_running(self): if not self._handle: context = os.environ.get('PATRONI_KUBERNETES_CONTEXT') if context.startswith('kind-'): container = '{0}-control-plane'.format(context[5:]) api_process = 'kube-apiserver' elif context.startswith('k3d-'): container = '{0}-server-0'.format(context) api_process = 'k3s server' else: return super(KubernetesController, self)._is_running() try: docker = 'docker' with open(os.devnull, 'w') as null: if subprocess.call([docker, 'info'], stdout=null, stderr=null) != 0: raise Exception except Exception: docker = 'podman' with open(os.devnull, 'w') as null: if subprocess.call([docker, 'info'], stdout=null, stderr=null) != 0: raise Exception self._wrapper = [docker, 'exec', container] self._external_pid = subprocess.check_output(self._wrapper + ['pidof', api_process]).decode('utf-8').strip() return False return True def create_pod(self, name, scope, group=None): self.delete_pod(name) labels = self._labels.copy() labels['cluster-name'] = scope if group is not None: labels['citus-group'] = str(group) metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels) spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')]) body = self._client.V1Pod(metadata=metadata, spec=spec) self._api.create_namespaced_pod(self._namespace, body) def delete_pod(self, name): try: self._api.delete_namespaced_pod(name, self._namespace, body=self._client.V1DeleteOptions()) except Exception: pass while True: try: self._api.read_namespaced_pod(name, self._namespace) except Exception: break def query(self, key, scope='batman', group=None): if key.startswith('members/'): pod = self._api.read_namespaced_pod(key[8:], self._namespace) return (pod.metadata.annotations or {}).get('status', '') else: try: if group is not None: scope = '{0}-{1}'.format(scope, group) rkey = 'leader' if key in ('status', 'failsafe') else key ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(rkey, '-' + rkey) e = self._api.read_namespaced_endpoints(ep, self._namespace) if key not in ('sync', 'status', 'failsafe'): return e.metadata.annotations[key] else: return json.dumps(e.metadata.annotations) except Exception: return None def cleanup_service_tree(self): try: self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector) except Exception: pass try: self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector) except Exception: pass while True: result = self._api.list_namespaced_pod(self._namespace, label_selector=self._label_selector) if len(result.items) < 1: break class ZooKeeperController(AbstractExternalDcsController): """ handles all zookeeper related tasks, used for the tests setup and cleanup """ def __init__(self, context, export_env=True): super(ZooKeeperController, self).__init__(context, False) if export_env: os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'" import kazoo.client self._client = kazoo.client.KazooClient() def process_name(self): return "java .*zookeeper" def query(self, key, scope='batman', group=None): import kazoo.exceptions try: return self._client.get(self.path(key, scope, group))[0].decode('utf-8') except kazoo.exceptions.NoNodeError: return None def cleanup_service_tree(self): import kazoo.exceptions try: self._client.delete(self.path(scope=''), recursive=True) except (kazoo.exceptions.NoNodeError): return except Exception as e: assert False, "exception when cleaning up zookeeper contents: {0}".format(e) def _is_running(self): if not super(ZooKeeperController, self)._is_running(): return False # if zookeeper is running, but we didn't start it if self._client.connected: return True try: return self._client.start(1) or True except Exception: return False class MockExhibitor(BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.end_headers() self.wfile.write(b'{"servers":["127.0.0.1"],"port":2181}') def log_message(self, fmt, *args): pass class ExhibitorController(ZooKeeperController): def __init__(self, context): super(ExhibitorController, self).__init__(context, False) port = 8181 exhibitor = HTTPServer(('', port), MockExhibitor) exhibitor.daemon_thread = True exhibitor_thread = threading.Thread(target=exhibitor.serve_forever) exhibitor_thread.daemon = True exhibitor_thread.start() os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': str(port)}) class RaftController(AbstractDcsController): CONTROLLER_ADDR = 'localhost:1234' PASSWORD = '12345' def __init__(self, context): super(RaftController, self).__init__(context) os.environ.update(PATRONI_RAFT_PARTNER_ADDRS="'" + self.CONTROLLER_ADDR + "'", PATRONI_RAFT_PASSWORD=self.PASSWORD, RAFT_PORT='1234') self._raft = None def _start(self): env = os.environ.copy() del env['PATRONI_RAFT_PARTNER_ADDRS'] env['PATRONI_RAFT_SELF_ADDR'] = self.CONTROLLER_ADDR env['PATRONI_RAFT_DATA_DIR'] = self._work_directory return psutil.Popen([sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p', 'patroni_raft_controller.py'], stdout=self._log, stderr=subprocess.STDOUT, env=env) def query(self, key, scope='batman', group=None): ret = self._raft.get(self.path(key, scope, group)) return ret and ret['value'] def set(self, key, value): self._raft.set(self.path(key), value) def cleanup_service_tree(self): from patroni.dcs.raft import KVStoreTTL if self._raft: self._raft.destroy() self.stop() os.makedirs(self._work_directory) self.start() ready_event = threading.Event() self._raft = KVStoreTTL(ready_event.set, None, None, partner_addrs=[self.CONTROLLER_ADDR], password=self.PASSWORD) self._raft.startAutoTick() ready_event.wait() class PatroniPoolController(object): PYTHON = sys.executable.replace('\\', '/') BACKUP_SCRIPT = [PYTHON, 'features/backup_create.py'] BACKUP_RESTORE_SCRIPT = ' '.join((PYTHON, os.path.abspath('features/backup_restore.py'))).replace('\\', '/') ARCHIVE_RESTORE_SCRIPT = ' '.join((PYTHON, os.path.abspath('features/archive-restore.py'))) def __init__(self, context): self._context = context self._dcs = None self._output_dir = None self._patroni_path = None self._processes = {} self.create_and_set_output_directory('') self._check_postgres_ssl() self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()} def _check_postgres_ssl(self): try: subprocess.check_output(['postgres', '-D', os.devnull, '-c', 'ssl=on'], stderr=subprocess.STDOUT) raise Exception # this one should never happen because the previous line will always raise and exception except Exception as e: self._context.postgres_supports_ssl = isinstance(e, subprocess.CalledProcessError)\ and 'SSL is not supported by this build' not in e.output.decode() @property def patroni_path(self): if self._patroni_path is None: cwd = os.path.realpath(__file__) while True: cwd, entry = os.path.split(cwd) if entry == 'features' or cwd == '/': break self._patroni_path = cwd return self._patroni_path @property def output_dir(self): return self._output_dir def start(self, name, max_wait_limit=40, custom_config=None): if name not in self._processes: self._processes[name] = PatroniController(self._context, name, self.patroni_path, self._output_dir, custom_config) self._processes[name].start(max_wait_limit) def __getattr__(self, func): if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config', 'get_watchdog', 'patroni_hang', 'backup', 'read_patroni_log']: raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func)) def wrapper(name, *args, **kwargs): return getattr(self._processes[name], func)(*args, **kwargs) return wrapper def stop_all(self): for ctl in self._processes.values(): ctl.cancel_background() ctl.stop() self._processes.clear() def create_and_set_output_directory(self, feature_name): feature_dir = os.path.join(self.patroni_path, 'features', 'output', feature_name.replace(' ', '_')) if os.path.exists(feature_dir): shutil.rmtree(feature_dir) os.makedirs(feature_dir) self._output_dir = feature_dir def clone(self, from_name, cluster_name, to_name): f = self._processes[from_name] custom_config = { 'scope': cluster_name, 'bootstrap': { 'method': 'pg_basebackup', 'pg_basebackup': { 'command': " ".join(self.BACKUP_SCRIPT + ['--walmethod=stream', '--dbname="{0}"'.format(f.backup_source)]) }, 'dcs': { 'postgresql': { 'parameters': { 'max_connections': 101 } } } }, 'postgresql': { 'parameters': { 'archive_mode': 'on', 'archive_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode archive ' + '--dirname {} --filename %f --pathname %p') .format(os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/')) }, 'authentication': { 'superuser': {'password': 'patroni1'}, 'replication': {'password': 'rep-pass1'} } } } self.start(to_name, custom_config=custom_config) def backup_restore_config(self, params=None): return { 'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' + os.path.join(self.patroni_path, 'data', 'basebackup')).replace('\\', '/'), 'test-argument': 'test-value', # test config mapping approach on custom bootstrap/replica creation **(params or {}), } def bootstrap_from_backup(self, name, cluster_name): custom_config = { 'scope': cluster_name, 'bootstrap': { 'method': 'backup_restore', 'backup_restore': self.backup_restore_config({ 'recovery_conf': { 'recovery_target_action': 'promote', 'recovery_target_timeline': 'latest', 'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' + '--dirname {} --filename %f --pathname %p').format( os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/')) }, }) }, 'postgresql': { 'authentication': { 'superuser': {'password': 'patroni2'}, 'replication': {'password': 'rep-pass2'} } } } self.start(name, custom_config=custom_config) def bootstrap_from_backup_no_leader(self, name, cluster_name): custom_config = { 'scope': cluster_name, 'postgresql': { 'create_replica_methods': ['no_leader_bootstrap'], 'no_leader_bootstrap': self.backup_restore_config({'no_leader': '1'}) } } self.start(name, custom_config=custom_config) @property def dcs(self): if self._dcs is None: self._dcs = os.environ.pop('DCS', 'etcd') assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs return self._dcs class WatchdogMonitor(object): """Testing harness for emulating a watchdog device as a named pipe. Because we can't easily emulate ioctl's we require a custom driver on Patroni side. The device takes no action, only notes if it was pinged and/or triggered. """ def __init__(self, name, work_directory, output_dir): self.fifo_path = os.path.join(work_directory, 'data', 'watchdog.{0}.fifo'.format(name)) self.fifo_file = None self._stop_requested = False # Relying on bool setting being atomic self._thread = None self.last_ping = None self.was_pinged = False self.was_closed = False self._was_triggered = False self.timeout = 60 self._log_file = open(os.path.join(output_dir, 'watchdog.{0}.log'.format(name)), 'w') self._log("watchdog {0} initialized".format(name)) def _log(self, msg): tstamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f") self._log_file.write("{0}: {1}\n".format(tstamp, msg)) def start(self): assert self._thread is None self._stop_requested = False self._log("starting fifo {0}".format(self.fifo_path)) fifo_dir = os.path.dirname(self.fifo_path) if os.path.exists(self.fifo_path): os.unlink(self.fifo_path) elif not os.path.exists(fifo_dir): os.mkdir(fifo_dir) os.mkfifo(self.fifo_path) self.last_ping = time.time() self._thread = threading.Thread(target=self.run) self._thread.start() def run(self): try: while not self._stop_requested: self._log("opening") self.fifo_file = os.open(self.fifo_path, os.O_RDONLY) try: self._log("Fifo {0} connected".format(self.fifo_path)) self.was_closed = False while not self._stop_requested: c = os.read(self.fifo_file, 1) if c == b'X': self._log("Stop requested") return elif c == b'': self._log("Pipe closed") break elif c == b'C': command = b'' c = os.read(self.fifo_file, 1) while c != b'\n' and c != b'': command += c c = os.read(self.fifo_file, 1) command = command.decode('utf8') if command.startswith('timeout='): self.timeout = int(command.split('=')[1]) self._log("timeout={0}".format(self.timeout)) elif c in [b'V', b'1']: cur_time = time.time() if cur_time - self.last_ping > self.timeout: self._log("Triggered") self._was_triggered = True if c == b'V': self._log("magic close") self.was_closed = True elif c == b'1': self.was_pinged = True self._log("ping after {0} seconds".format(cur_time - (self.last_ping or cur_time))) self.last_ping = cur_time else: self._log('Unknown command {0} received from fifo'.format(c)) finally: self.was_closed = True self._log("closing") os.close(self.fifo_file) except Exception as e: self._log("Error {0}".format(e)) finally: self._log("stopping") self._log_file.flush() if os.path.exists(self.fifo_path): os.unlink(self.fifo_path) def stop(self): self._log("Monitor stop") self._stop_requested = True try: if os.path.exists(self.fifo_path): fd = os.open(self.fifo_path, os.O_WRONLY) os.write(fd, b'X') os.close(fd) except Exception as e: self._log("err while closing: {0}".format(str(e))) if self._thread: self._thread.join() self._thread = None def reset(self): self._log("reset") self.was_pinged = self.was_closed = self._was_triggered = False @property def was_triggered(self): delta = time.time() - self.last_ping triggered = self._was_triggered or not self.was_closed and delta > self.timeout self._log("triggered={0}, {1}s left".format(triggered, self.timeout - delta)) return triggered # actions to execute on start/stop of the tests and before running individual features def before_all(context): context.ci = os.name == 'nt' or\ any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS')) context.timeout_multiplier = 5 if context.ci else 1 # MacOS sometimes is VERY slow context.pctl = PatroniPoolController(context) context.keyfile = os.path.join(context.pctl.output_dir, 'patroni.key') context.certfile = os.path.join(context.pctl.output_dir, 'patroni.crt') try: with open(os.devnull, 'w') as null: ret = subprocess.call(['openssl', 'req', '-nodes', '-new', '-x509', '-subj', '/CN=batman.patroni', '-addext', 'subjectAltName=IP:127.0.0.1', '-keyout', context.keyfile, '-out', context.certfile], stdout=null, stderr=null) if ret != 0: raise Exception os.chmod(context.keyfile, stat.S_IWRITE | stat.S_IREAD) except Exception: context.keyfile = context.certfile = None os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'}) ctl = {'auth': os.environ['PATRONI_RESTAPI_USERNAME'] + ':' + os.environ['PATRONI_RESTAPI_PASSWORD']} if context.certfile: os.environ.update({'PATRONI_RESTAPI_CAFILE': context.certfile, 'PATRONI_RESTAPI_CERTFILE': context.certfile, 'PATRONI_RESTAPI_KEYFILE': context.keyfile, 'PATRONI_RESTAPI_VERIFY_CLIENT': 'required', 'PATRONI_CTL_INSECURE': 'on', 'PATRONI_CTL_CERTFILE': context.certfile, 'PATRONI_CTL_KEYFILE': context.keyfile}) ctl.update({'cacert': context.certfile, 'certfile': context.certfile, 'keyfile': context.keyfile}) context.request_executor = PatroniRequest({'ctl': ctl}, True) context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context) context.dcs_ctl.start() try: context.dcs_ctl.cleanup_service_tree() except AssertionError: # after_all handlers won't be executed in before_all context.dcs_ctl.stop() raise def after_all(context): context.dcs_ctl.stop() subprocess.call([sys.executable, '-m', 'coverage', 'combine']) subprocess.call([sys.executable, '-m', 'coverage', 'report']) def before_feature(context, feature): """ create per-feature output directory to collect Patroni and PostgreSQL logs """ if feature.name == 'watchdog' and os.name == 'nt': return feature.skip("Watchdog isn't supported on Windows") elif feature.name == 'citus': lib = subprocess.check_output(['pg_config', '--pkglibdir']).decode('utf-8').strip() if not os.path.exists(os.path.join(lib, 'citus.so')): return feature.skip("Citus extension isn't available") context.pctl.create_and_set_output_directory(feature.name) def after_feature(context, feature): """ send SIGCONT to a dcs if necessary, stop all Patronis remove their data directory and cleanup the keys in etcd """ context.dcs_ctl.stop_outage() context.pctl.stop_all() data = os.path.join(context.pctl.patroni_path, 'data') if os.path.exists(data): shutil.rmtree(data) context.dcs_ctl.cleanup_service_tree() found = False logs = glob.glob(context.pctl.output_dir + '/patroni_*.log') for log in logs: with open(log) as f: for line in f: if 'please report it as a BUG' in line: print(':'.join([log, line.rstrip()])) found = True if feature.status == 'failed' or found: shutil.copytree(context.pctl.output_dir, context.pctl.output_dir + '_failed') if found: raise Exception('Unexpected errors in Patroni log files') def before_scenario(context, scenario): if 'slot-advance' in scenario.effective_tags: for p in context.pctl._processes.values(): if p._conn and p._conn.server_version < 110000: scenario.skip('pg_replication_slot_advance() is not supported on {0}'.format(p._conn.server_version)) break if 'dcs-failsafe' in scenario.effective_tags and not context.dcs_ctl._handle: scenario.skip('it is not possible to control state of {0} from tests'.format(context.dcs_ctl.name())) if 'reject-duplicate-name' in scenario.effective_tags and context.dcs_ctl.name() == 'raft': scenario.skip('Flaky test with Raft') patroni-4.0.4/features/ignored_slots.feature000066400000000000000000000111061472010352700212410ustar00rootroot00000000000000Feature: ignored slots Scenario: check ignored slots aren't removed on failover/switchover Given I start postgres-1 Then postgres-1 is a leader after 10 seconds And there is a non empty initialize key in DCS after 15 seconds When I issue a PATCH request to http://127.0.0.1:8009/config with {"ignore_slots": [{"name": "unmanaged_slot_0", "database": "postgres", "plugin": "test_decoding", "type": "logical"}, {"name": "unmanaged_slot_1", "database": "postgres", "plugin": "test_decoding"}, {"name": "unmanaged_slot_2", "database": "postgres"}, {"name": "unmanaged_slot_3"}], "postgresql": {"parameters": {"wal_level": "logical"}}} Then I receive a response code 200 And Response on GET http://127.0.0.1:8009/config contains ignore_slots after 10 seconds # Make sure the wal_level has been changed. When I shut down postgres-1 And I start postgres-1 Then postgres-1 is a leader after 10 seconds And "members/postgres-1" key in DCS has role=primary after 10 seconds # Make sure Patroni has finished telling Postgres it should be accepting writes. And postgres-1 role is the primary after 20 seconds # 1. Create our test logical replication slot. # Test that ny subset of attributes in the ignore slots matcher is enough to match a slot # by using 3 different slots. When I create a logical replication slot unmanaged_slot_0 on postgres-1 with the test_decoding plugin And I create a logical replication slot unmanaged_slot_1 on postgres-1 with the test_decoding plugin And I create a logical replication slot unmanaged_slot_2 on postgres-1 with the test_decoding plugin And I create a logical replication slot unmanaged_slot_3 on postgres-1 with the test_decoding plugin And I create a logical replication slot dummy_slot on postgres-1 with the test_decoding plugin # It seems like it'd be obvious that these slots exist since we just created them, # but Patroni can actually end up dropping them almost immediately, so it's helpful # to verify they exist before we begin testing whether they persist through failover # cycles. Then postgres-1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds When I start postgres-0 Then "members/postgres-0" key in DCS has role=replica after 10 seconds And postgres-0 role is the secondary after 20 seconds # Verify that the replica has advanced beyond the point in the WAL # where we created the replication slot so that on the next failover # cycle we don't accidentally rewind to before the slot creation. And replication works from postgres-1 to postgres-0 after 20 seconds When I shut down postgres-1 Then "members/postgres-0" key in DCS has role=primary after 10 seconds # 2. After a failover the server (now a replica) still has the slot. When I start postgres-1 Then postgres-1 role is the secondary after 20 seconds And "members/postgres-1" key in DCS has role=replica after 10 seconds # give Patroni time to sync replication slots And I sleep for 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds And postgres-1 does not have a replication slot named dummy_slot # 3. After a failover the server (now a primary) still has the slot. When I shut down postgres-0 Then "members/postgres-1" key in DCS has role=primary after 10 seconds And postgres-1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds And postgres-1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds patroni-4.0.4/features/nostream_node.feature000066400000000000000000000027701472010352700212320ustar00rootroot00000000000000Feature: nostream node Scenario: check nostream node is recovering from archive When I start postgres-0 And I configure and start postgres-1 with a tag nostream true Then "members/postgres-1" key in DCS has replication_state=in archive recovery after 10 seconds And replication works from postgres-0 to postgres-1 after 30 seconds @slot-advance Scenario: check permanent logical replication slots are not copied When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}, "slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}} Then I receive a response code 200 When I run patronictl.py restart batman postgres-0 --force Then postgres-0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds When I configure and start postgres-2 with a tag replicatefrom postgres-1 Then "members/postgres-2" key in DCS has replication_state=streaming after 10 seconds And postgres-1 does not have a replication slot named test_logical And postgres-2 does not have a replication slot named test_logical @slot-advance Scenario: check that slots are written to the /status key Given "status" key in DCS has postgres_0 in slots And "status" key in DCS has postgres_2 in slots And "status" key in DCS has test_logical in slots And "status" key in DCS has test_logical in slots And "status" key in DCS does not have postgres_1 in slots patroni-4.0.4/features/patroni_api.feature000066400000000000000000000151411472010352700206760ustar00rootroot00000000000000Feature: patroni api We should check that patroni correctly responds to valid and not-valid API requests. Scenario: check API requests on a stand-alone server Given I start postgres-0 And postgres-0 is a leader after 10 seconds When I issue a GET request to http://127.0.0.1:8008/ Then I receive a response code 200 And I receive a response state running And I receive a response role primary When I issue a GET request to http://127.0.0.1:8008/standby_leader Then I receive a response code 503 When I issue a GET request to http://127.0.0.1:8008/health Then I receive a response code 200 When I issue a GET request to http://127.0.0.1:8008/replica Then I receive a response code 503 When I issue a POST request to http://127.0.0.1:8008/reinitialize with {"force": true} Then I receive a response code 503 And I receive a response text I am the leader, can not reinitialize When I run patronictl.py switchover batman --primary postgres-0 --force Then I receive a response returncode 1 And I receive a response output "Error: No candidates found to switchover to" When I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres-0"} Then I receive a response code 412 And I receive a response text switchover is not possible: cluster does not have members except leader When I issue an empty POST request to http://127.0.0.1:8008/failover Then I receive a response code 400 When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"} Then I receive a response code 400 And I receive a response text "Failover could be performed only to a specific candidate" Scenario: check local configuration reload Given I add tag new_tag new_value to postgres-0 config And I issue an empty POST request to http://127.0.0.1:8008/reload Then I receive a response code 202 Scenario: check dynamic configuration change via DCS Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "postgresql": {"parameters": {"max_connections": "101"}}} Then I receive a response code 200 And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds When I issue a GET request to http://127.0.0.1:8008/config Then I receive a response code 200 And I receive a response ttl 20 When I issue a GET request to http://127.0.0.1:8008/patroni Then I receive a response code 200 And I receive a response tags {'new_tag': 'new_value'} And I sleep for 4 seconds Scenario: check the scheduled restart Given I run patronictl.py edit-config -p 'superuser_reserved_connections=6' --force batman Then I receive a response returncode 0 And I receive a response output "+ superuser_reserved_connections: 6" And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds Given I issue a scheduled restart at http://127.0.0.1:8008 in 5 seconds with {"role": "replica"} Then I receive a response code 202 And I sleep for 8 seconds And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds Given I issue a scheduled restart at http://127.0.0.1:8008 in 5 seconds with {"restart_pending": "True"} Then I receive a response code 202 And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds And postgres-0 role is the primary after 10 seconds Scenario: check API requests for the primary-replica pair in the pause mode Given I start postgres-1 Then replication works from postgres-0 to postgres-1 after 20 seconds When I run patronictl.py pause batman Then I receive a response returncode 0 When I kill postmaster on postgres-1 And I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 503 And "members/postgres-1" key in DCS has state=stopped after 10 seconds When I run patronictl.py restart batman postgres-1 --force Then I receive a response returncode 0 Then replication works from postgres-0 to postgres-1 after 20 seconds And I sleep for 2 seconds When I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 200 And I receive a response state running And I receive a response role replica When I run patronictl.py reinit batman postgres-1 --force --wait Then I receive a response returncode 0 And I receive a response output "Success: reinitialize for member postgres-1" And postgres-1 role is the secondary after 30 seconds And replication works from postgres-0 to postgres-1 after 20 seconds When I run patronictl.py restart batman postgres-0 --force Then I receive a response returncode 0 And I receive a response output "Success: restart on member postgres-0" And postgres-0 role is the primary after 5 seconds Scenario: check the switchover via the API in the pause mode Given I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres-0", "candidate": "postgres-1"} Then I receive a response code 200 And postgres-1 is a leader after 5 seconds And postgres-1 role is the primary after 10 seconds And postgres-0 role is the secondary after 10 seconds And replication works from postgres-1 to postgres-0 after 20 seconds And "members/postgres-0" key in DCS has state=running after 10 seconds When I issue a GET request to http://127.0.0.1:8008/primary Then I receive a response code 503 When I issue a GET request to http://127.0.0.1:8008/replica Then I receive a response code 200 When I issue a GET request to http://127.0.0.1:8009/primary Then I receive a response code 200 When I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 503 Scenario: check the scheduled switchover Given I issue a scheduled switchover from postgres-1 to postgres-0 in 10 seconds Then I receive a response returncode 1 And I receive a response output "Can't schedule switchover in the paused state" When I run patronictl.py resume batman Then I receive a response returncode 0 Given I issue a scheduled switchover from postgres-1 to postgres-0 in 10 seconds Then I receive a response returncode 0 And postgres-0 is a leader after 20 seconds And postgres-0 role is the primary after 10 seconds And postgres-1 role is the secondary after 10 seconds And replication works from postgres-0 to postgres-1 after 25 seconds And "members/postgres-1" key in DCS has state=running after 10 seconds When I issue a GET request to http://127.0.0.1:8008/primary Then I receive a response code 200 When I issue a GET request to http://127.0.0.1:8008/replica Then I receive a response code 503 When I issue a GET request to http://127.0.0.1:8009/primary Then I receive a response code 503 When I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 200 patroni-4.0.4/features/permanent_slots.feature000066400000000000000000000125311472010352700216060ustar00rootroot00000000000000Feature: permanent slots Scenario: check that physical permanent slots are created Given I start postgres-0 Then postgres-0 is a leader after 10 seconds And there is a non empty initialize key in DCS after 15 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots":{"test_physical":0,"postgres_3":0},"postgresql":{"parameters":{"wal_level":"logical"}}} Then I receive a response code 200 And Response on GET http://127.0.0.1:8008/config contains slots after 10 seconds When I start postgres-1 And I configure and start postgres-2 with a tag nofailover true And I configure and start postgres-3 with a tag replicatefrom postgres-2 Then postgres-0 has a physical replication slot named test_physical after 10 seconds And postgres-0 has a physical replication slot named postgres_1 after 10 seconds And postgres-0 has a physical replication slot named postgres_2 after 10 seconds And postgres-2 has a physical replication slot named postgres_3 after 10 seconds And postgres-2 does not have a replication slot named test_physical @slot-advance Scenario: check that logical permanent slots are created Given I run patronictl.py restart batman postgres-0 --force And I issue a PATCH request to http://127.0.0.1:8008/config with {"slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}} Then postgres-0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds @slot-advance Scenario: check that permanent slots are created on replicas Given postgres-1 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds Then Logical slot test_logical is in sync between postgres-0 and postgres-1 after 10 seconds And Logical slot test_logical is in sync between postgres-0 and postgres-3 after 10 seconds And postgres-1 has a physical replication slot named test_physical after 2 seconds And postgres-2 does not have a replication slot named test_logical And postgres-3 has a physical replication slot named test_physical after 2 seconds @slot-advance Scenario: check permanent physical slots that match with member names Given postgres-0 has a physical replication slot named postgres_3 after 2 seconds And postgres-1 has a physical replication slot named postgres_0 after 2 seconds And postgres-1 has a physical replication slot named postgres_2 after 2 seconds And postgres-1 has a physical replication slot named postgres_3 after 2 seconds And postgres-2 does not have a replication slot named postgres_0 And postgres-2 does not have a replication slot named postgres_1 And postgres-2 has a physical replication slot named postgres_3 after 2 seconds And postgres-3 has a physical replication slot named postgres_0 after 2 seconds And postgres-3 has a physical replication slot named postgres_1 after 2 seconds And postgres-3 has a physical replication slot named postgres_2 after 2 seconds @slot-advance Scenario: check that permanent slots are advanced on replicas Given I add the table replicate_me to postgres-0 When I get all changes from logical slot test_logical on postgres-0 And I get all changes from physical slot test_physical on postgres-0 Then Logical slot test_logical is in sync between postgres-0 and postgres-1 after 10 seconds And Physical slot test_physical is in sync between postgres-0 and postgres-1 after 10 seconds And Logical slot test_logical is in sync between postgres-0 and postgres-3 after 10 seconds And Physical slot test_physical is in sync between postgres-0 and postgres-3 after 10 seconds And Physical slot postgres_1 is in sync between postgres-0 and postgres-3 after 10 seconds And Physical slot postgres_3 is in sync between postgres-2 and postgres-0 after 20 seconds And Physical slot postgres_3 is in sync between postgres-2 and postgres-1 after 10 seconds @slot-advance Scenario: check that permanent slots and member slots are written to the /status key Given "status" key in DCS has test_physical in slots And "status" key in DCS has postgres_0 in slots And "status" key in DCS has postgres_1 in slots And "status" key in DCS has postgres_2 in slots And "status" key in DCS has postgres_3 in slots @slot-advance Scenario: check that only non-permanent member slots are written to the retain_slots in /status key Given "status" key in DCS has postgres_0 in retain_slots And "status" key in DCS has postgres_1 in retain_slots And "status" key in DCS has postgres_2 in retain_slots And "status" key in DCS does not have postgres_3 in retain_slots Scenario: check permanent physical replication slot after failover Given I shut down postgres-3 And I shut down postgres-2 And I shut down postgres-0 Then postgres-1 has a physical replication slot named test_physical after 10 seconds And postgres-1 has a physical replication slot named postgres_0 after 10 seconds And postgres-1 has a physical replication slot named postgres_3 after 10 seconds When I start postgres-0 Then postgres-0 role is the replica after 20 seconds And physical replication slot named postgres_1 on postgres-0 has no xmin value after 10 seconds And physical replication slot named postgres_2 on postgres-0 has no xmin value after 10 seconds patroni-4.0.4/features/priority_failover.feature000066400000000000000000000053301472010352700221400ustar00rootroot00000000000000Feature: priority replication We should check that we can give nodes priority during failover Scenario: check failover priority 0 prevents leaderships Given I configure and start postgres-0 with a tag failover_priority 1 And I configure and start postgres-1 with a tag failover_priority 0 Then replication works from postgres-0 to postgres-1 after 20 seconds When I shut down postgres-0 And there is one of ["following a different leader because I am not allowed to promote"] INFO in the postgres-1 patroni log after 5 seconds Then postgres-1 role is the secondary after 10 seconds When I start postgres-0 Then postgres-0 role is the primary after 10 seconds Scenario: check higher failover priority is respected Given I configure and start postgres-2 with a tag failover_priority 1 And I configure and start postgres-3 with a tag failover_priority 2 Then replication works from postgres-0 to postgres-2 after 20 seconds And replication works from postgres-0 to postgres-3 after 20 seconds When I shut down postgres-0 Then postgres-3 role is the primary after 10 seconds And there is one of ["postgres-3 has equally tolerable WAL position and priority 2, while this node has priority 1","Wal position of postgres-3 is ahead of my wal position"] INFO in the postgres-2 patroni log after 5 seconds Scenario: check conflicting configuration handling When I set nofailover tag in postgres-2 config And I issue an empty POST request to http://127.0.0.1:8010/reload Then I receive a response code 202 And there is one of ["Conflicting configuration between nofailover: True and failover_priority: 1. Defaulting to nofailover: True"] WARNING in the postgres-2 patroni log after 5 seconds And "members/postgres-2" key in DCS has tags={'failover_priority': '1', 'nofailover': True} after 10 seconds When I issue a POST request to http://127.0.0.1:8010/failover with {"candidate": "postgres-2"} Then I receive a response code 412 And I receive a response text "failover is not possible: no good candidates have been found" When I reset nofailover tag in postgres-1 config And I issue an empty POST request to http://127.0.0.1:8009/reload Then I receive a response code 202 And there is one of ["Conflicting configuration between nofailover: False and failover_priority: 0. Defaulting to nofailover: False"] WARNING in the postgres-1 patroni log after 5 seconds And "members/postgres-1" key in DCS has tags={'failover_priority': '0', 'nofailover': False} after 10 seconds And I issue a POST request to http://127.0.0.1:8009/failover with {"candidate": "postgres-1"} Then I receive a response code 200 And postgres-1 role is the primary after 10 seconds patroni-4.0.4/features/quorum_commit.feature000066400000000000000000000077671472010352700213100ustar00rootroot00000000000000Feature: quorum commit Check basic workfrlows when quorum commit is enabled Scenario: check enable quorum commit and that the only leader promotes after restart Given I start postgres-0 Then postgres-0 is a leader after 10 seconds And there is a non empty initialize key in DCS after 15 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": "quorum"} Then I receive a response code 200 And sync key in DCS has leader=postgres-0 after 20 seconds And sync key in DCS has quorum=0 after 2 seconds And synchronous_standby_names on postgres-0 is set to '_empty_str_' after 2 seconds When I shut down postgres-0 And sync key in DCS has leader=postgres-0 after 2 seconds When I start postgres-0 Then postgres-0 role is the primary after 10 seconds When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_mode_strict": true} Then synchronous_standby_names on postgres-0 is set to 'ANY 1 (*)' after 10 seconds Scenario: check failover with one quorum standby Given I start postgres-1 Then sync key in DCS has sync_standby=postgres-1 after 10 seconds And synchronous_standby_names on postgres-0 is set to 'ANY 1 ("postgres-1")' after 2 seconds When I shut down postgres-0 Then postgres-1 role is the primary after 10 seconds And sync key in DCS has quorum=0 after 10 seconds Then synchronous_standby_names on postgres-1 is set to 'ANY 1 (*)' after 10 seconds When I start postgres-0 Then sync key in DCS has leader=postgres-1 after 10 seconds Then sync key in DCS has sync_standby=postgres-0 after 10 seconds And synchronous_standby_names on postgres-1 is set to 'ANY 1 ("postgres-0")' after 2 seconds Scenario: check behavior with three nodes and different replication factor Given I start postgres-2 Then sync key in DCS has sync_standby=postgres-0,postgres-2 after 10 seconds And sync key in DCS has quorum=1 after 2 seconds And synchronous_standby_names on postgres-1 is set to 'ANY 1 ("postgres-0","postgres-2")' after 2 seconds When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_node_count": 2} Then sync key in DCS has quorum=0 after 10 seconds And synchronous_standby_names on postgres-1 is set to 'ANY 2 ("postgres-0","postgres-2")' after 2 seconds Scenario: switch from quorum replication to good old multisync and back Given I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": true, "synchronous_node_count": 1} And I shut down postgres-0 Then synchronous_standby_names on postgres-1 is set to '"postgres-2"' after 10 seconds And sync key in DCS has sync_standby=postgres-2 after 10 seconds Then sync key in DCS has quorum=0 after 2 seconds When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": "quorum"} And I start postgres-0 Then synchronous_standby_names on postgres-1 is set to 'ANY 1 ("postgres-0","postgres-2")' after 10 seconds And sync key in DCS has sync_standby=postgres-0,postgres-2 after 10 seconds Then sync key in DCS has quorum=1 after 2 seconds Scenario: REST API and patronictl Given I run patronictl.py list batman Then I receive a response returncode 0 And I receive a response output "Quorum Standby" And Status code on GET http://127.0.0.1:8008/quorum is 200 after 3 seconds And Status code on GET http://127.0.0.1:8010/quorum is 200 after 3 seconds Scenario: nosync node is removed from voters and synchronous_standby_names Given I add tag nosync true to postgres-2 config When I issue an empty POST request to http://127.0.0.1:8010/reload Then I receive a response code 202 And sync key in DCS has quorum=0 after 10 seconds And sync key in DCS has sync_standby=postgres-0 after 10 seconds And synchronous_standby_names on postgres-1 is set to 'ANY 1 ("postgres-0")' after 2 seconds And Status code on GET http://127.0.0.1:8010/quorum is 503 after 10 seconds patroni-4.0.4/features/recovery.feature000066400000000000000000000034551472010352700202340ustar00rootroot00000000000000Feature: recovery We want to check that crashed postgres is started back Scenario: check that timeline is not incremented when primary is started after crash Given I start postgres-0 Then postgres-0 is a leader after 10 seconds And there is a non empty initialize key in DCS after 15 seconds When I start postgres-1 And I add the table foo to postgres-0 Then table foo is present on postgres-1 after 20 seconds When I kill postmaster on postgres-0 Then postgres-0 role is the primary after 10 seconds When I issue a GET request to http://127.0.0.1:8008/ Then I receive a response code 200 And I receive a response role primary And I receive a response timeline 1 And "members/postgres-0" key in DCS has state=running after 12 seconds And replication works from postgres-0 to postgres-1 after 15 seconds Scenario: check immediate failover when master_start_timeout=0 Given I issue a PATCH request to http://127.0.0.1:8008/config with {"master_start_timeout": 0} Then I receive a response code 200 And Response on GET http://127.0.0.1:8008/config contains master_start_timeout after 10 seconds When I kill postmaster on postgres-0 Then postgres-1 is a leader after 10 seconds And postgres-1 role is the primary after 10 seconds Scenario: check crashed primary demotes after failed attempt to start Given I issue a PATCH request to http://127.0.0.1:8009/config with {"master_start_timeout": null} Then I receive a response code 200 And postgres-0 role is the replica after 10 seconds When I ensure postgres-1 fails to start after a failure When I kill postmaster on postgres-1 Then postgres-0 is a leader after 10 seconds And there is a postgres-1_cb.log with "on_role_change demoted batman" in postgres-1 data directory patroni-4.0.4/features/standby_cluster.feature000066400000000000000000000100101472010352700215640ustar00rootroot00000000000000Feature: standby cluster Scenario: prepare the cluster with logical slots Given I start postgres-1 Then postgres-1 is a leader after 10 seconds And there is a non empty initialize key in DCS after 15 seconds When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}} Then I receive a response code 200 And Response on GET http://127.0.0.1:8009/config contains slots after 10 seconds And I sleep for 3 seconds When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"test_logical": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}} Then I receive a response code 200 And I do a backup of postgres-1 When I start postgres-0 Then "members/postgres-0" key in DCS has state=running after 10 seconds And replication works from postgres-1 to postgres-0 after 15 seconds When I issue a GET request to http://127.0.0.1:8008/patroni Then I receive a response code 200 And I receive a response replication_state streaming And "members/postgres-0" key in DCS has replication_state=streaming after 10 seconds @slot-advance Scenario: check permanent logical slots are synced to the replica Given I run patronictl.py restart batman postgres-1 --force Then Logical slot test_logical is in sync between postgres-0 and postgres-1 after 10 seconds Scenario: Detach exiting node from the cluster When I shut down postgres-1 Then postgres-0 is a leader after 10 seconds And "members/postgres-0" key in DCS has role=primary after 5 seconds When I issue a GET request to http://127.0.0.1:8008/ Then I receive a response code 200 Scenario: check replication of a single table in a standby cluster Given I start postgres-1 in a standby cluster batman1 as a clone of postgres-0 Then postgres-1 is a leader of batman1 after 10 seconds When I add the table foo to postgres-0 Then table foo is present on postgres-1 after 20 seconds When I issue a GET request to http://127.0.0.1:8009/patroni Then I receive a response code 200 And I receive a response replication_state streaming And I sleep for 3 seconds When I issue a GET request to http://127.0.0.1:8009/primary Then I receive a response code 503 When I issue a GET request to http://127.0.0.1:8009/standby_leader Then I receive a response code 200 And I receive a response role standby_leader And there is a postgres-1_cb.log with "on_role_change standby_leader batman1" in postgres-1 data directory When I start postgres-2 in a cluster batman1 Then postgres-2 role is the replica after 24 seconds And postgres-2 is replicating from postgres-1 after 10 seconds And table foo is present on postgres-2 after 20 seconds When I issue a GET request to http://127.0.0.1:8010/patroni Then I receive a response code 200 And I receive a response replication_state streaming And postgres-1 does not have a replication slot named test_logical Scenario: check switchover Given I run patronictl.py switchover batman1 --force Then Status code on GET http://127.0.0.1:8010/standby_leader is 200 after 10 seconds And postgres-1 is replicating from postgres-2 after 32 seconds And there is a postgres-2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres-2 data directory Scenario: check failover When I kill postgres-2 And I kill postmaster on postgres-2 Then postgres-1 is replicating from postgres-0 after 32 seconds And Status code on GET http://127.0.0.1:8009/standby_leader is 200 after 10 seconds When I issue a GET request to http://127.0.0.1:8009/primary Then I receive a response code 503 And I receive a response role standby_leader And replication works from postgres-0 to postgres-1 after 15 seconds And there is a postgres-1_cb.log with "on_role_change replica batman1\non_role_change standby_leader batman1" in postgres-1 data directory patroni-4.0.4/features/steps/000077500000000000000000000000001472010352700161505ustar00rootroot00000000000000patroni-4.0.4/features/steps/basic_replication.py000066400000000000000000000122631472010352700222000ustar00rootroot00000000000000import json from time import sleep, time import parse from behave import register_type, step, then import patroni.psycopg as pg @parse.with_pattern(r'[a-z][a-z0-9_\-]*[a-z0-9]') def parse_name(text): return text register_type(name=parse_name) @step('I start {name:name}') def start_patroni(context, name): return context.pctl.start(name) @step('I start duplicate {name:name} on port {port:d}') def start_duplicate_patroni(context, name, port): config = { "name": name, "restapi": { "listen": "127.0.0.1:{0}".format(port) } } try: context.pctl.start('dup-' + name, custom_config=config) assert False, "Process was expected to fail" except AssertionError as e: assert 'is not running after being started' in str(e), \ "No error was raised by duplicate start of {0} ".format(name) @step('I shut down {name:name}') def stop_patroni(context, name): return context.pctl.stop(name, timeout=60) @step('I kill {name:name}') def kill_patroni(context, name): return context.pctl.stop(name, kill=True) @step('I shut down postmaster on {name:name}') def stop_postgres(context, name): return context.pctl.stop(name, postgres=True) @step('I kill postmaster on {name:name}') def kill_postgres(context, name): return context.pctl.stop(name, kill=True, postgres=True) def get_wal_name(context, pg_name): version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0] return 'xlog' if int(version) / 10000 < 10 else 'wal' @step('I add the table {table_name:w} to {pg_name:name}') def add_table(context, table_name, pg_name): # parse the configuration file and get the port try: context.pctl.query(pg_name, "CREATE TABLE public.{0}()".format(table_name)) context.pctl.query(pg_name, "SELECT pg_switch_{0}()".format(get_wal_name(context, pg_name))) except pg.Error as e: assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) @step('I {action:w} wal replay on {pg_name:name}') def toggle_wal_replay(context, action, pg_name): # pause or resume the wal replay process try: context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(get_wal_name(context, pg_name), action)) except pg.Error as e: assert False, "Error during {0} wal recovery on {1}: {2}".format(action, pg_name, e) @step('I {action:w} table on {pg_name:name}') def crdr_mytest(context, action, pg_name): try: if (action == "create"): context.pctl.query(pg_name, "create table if not exists public.mytest(id numeric)") else: context.pctl.query(pg_name, "drop table if exists public.mytest") except pg.Error as e: assert False, "Error {0} table mytest on {1}: {2}".format(action, pg_name, e) @step('I load data on {pg_name:name}') def initiate_load(context, pg_name): # perform dummy load try: context.pctl.query(pg_name, "insert into public.mytest select r::numeric from generate_series(1, 350000) r") except pg.Error as e: assert False, "Error loading test data on {0}: {1}".format(pg_name, e) @then('Table {table_name:w} is present on {pg_name:name} after {max_replication_delay:d} seconds') def table_is_present_on(context, table_name, pg_name, max_replication_delay): max_replication_delay *= context.timeout_multiplier for _ in range(int(max_replication_delay)): if context.pctl.query(pg_name, "SELECT 1 FROM public.{0}".format(table_name), fail_ok=True) is not None: break sleep(1) else: assert False, \ "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) @then('{pg_name:name} role is the {pg_role:w} after {max_promotion_timeout:d} seconds') def check_role(context, pg_name, pg_role, max_promotion_timeout): max_promotion_timeout *= context.timeout_multiplier assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)), \ "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) @step('replication works from {primary:name} to {replica:name} after {time_limit:d} seconds') @then('replication works from {primary:name} to {replica:name} after {time_limit:d} seconds') def replication_works(context, primary, replica, time_limit): context.execute_steps(u""" When I add the table test_{0} to {1} Then table test_{0} is present on {2} after {3} seconds """.format(str(time()).replace('.', '_').replace(',', '_'), primary, replica, time_limit)) @step('there is one of {message_list} {level:w} in the {node} patroni log after {timeout:d} seconds') def check_patroni_log(context, message_list, level, node, timeout): timeout *= context.timeout_multiplier message_list = json.loads(message_list) for _ in range(int(timeout)): messages_of_level = context.pctl.read_patroni_log(node, level) if any(any(message in line for line in messages_of_level) for message in message_list): break sleep(1) else: assert False, f"There were none of {message_list} {level} in the {node} patroni log after {timeout} seconds" patroni-4.0.4/features/steps/cascading_replication.py000066400000000000000000000036771472010352700230440ustar00rootroot00000000000000import json import time from behave import step, then @step('I configure and start {name:name} with a tag {tag_name:w} {tag_value}') def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value): return context.pctl.start(name, custom_config={'tags': {tag_name: tag_value}}) @then('There is a {label} with "{content}" in {name:name} data directory') def check_label(context, label, content, name): value = (context.pctl.read_label(name, label) or '').replace('\n', '\\n') assert content in value, "\"{0}\" in {1} doesn't contain {2}".format(value, label, content) @step('I create label with "{content}" in {name:name} data directory') def write_label(context, content, name): context.pctl.write_label(name, content) @step('"{name}" key in DCS has {key:w}={value} after {time_limit:d} seconds') def check_member(context, name, key, value, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) dcs_value = None while time.time() < max_time: try: response = json.loads(context.dcs_ctl.query(name)) dcs_value = str(response.get(key)) if dcs_value == value: return except Exception: pass time.sleep(1) assert False, "{0} does not have {1}={2} (found {3}) in dcs after {4} seconds".format(name, key, value, dcs_value, time_limit) @step('there is a non empty {key:w} key in DCS after {time_limit:d} seconds') def check_initialize(context, key, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while time.time() < max_time: try: if context.dcs_ctl.query(key): return except Exception: pass time.sleep(1) assert False, "There is no {0} in dcs after {1} seconds".format(key, time_limit) patroni-4.0.4/features/steps/citus.py000066400000000000000000000122171472010352700176540ustar00rootroot00000000000000import json import time from datetime import datetime from functools import partial from threading import Event, Thread from behave import step, then from dateutil import tz tzutc = tz.tzutc() @step('{name:name} is a leader in a group {group:d} after {time_limit:d} seconds') @then('{name:name} is a leader in a group {group:d} after {time_limit:d} seconds') def is_a_group_leader(context, name, group, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while (context.dcs_ctl.query("leader", group=group) != name): time.sleep(1) assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit) @step('"{name}" key in a group {group:d} in DCS has {key:w}={value} after {time_limit:d} seconds') def check_group_member(context, name, group, key, value, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) dcs_value = None response = None while time.time() < max_time: try: response = json.loads(context.dcs_ctl.query(name, group=group)) dcs_value = response.get(key) if dcs_value == value: return except Exception: pass time.sleep(1) assert False, ("{0} in a group {1} does not have {2}={3} (found {4}) in dcs" " after {5} seconds").format(name, group, key, value, response, time_limit) @step('I start {name:name} in citus group {group:d}') def start_citus(context, name, group): return context.pctl.start(name, custom_config={"citus": {"database": "postgres", "group": int(group)}}) @step('{name1:name} is registered in the {name2:name} as the {role:w} in group {group:d} after {time_limit:d} seconds') def check_registration(context, name1, name2, role, group, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) worker_port = int(context.pctl.query(name1, "SHOW port").fetchone()[0]) while time.time() < max_time: try: cur = context.pctl.query(name2, "SELECT nodeport, noderole" " FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group)) mapping = {r[0]: r[1] for r in cur} if mapping.get(worker_port) == role: return except Exception: pass time.sleep(1) assert False, "Node {0} is not registered in pg_dist_node on the node {1}".format(name1, name2) @step('I create a distributed table on {name:name}') def create_distributed_table(context, name): context.pctl.query(name, 'CREATE TABLE public.d(id int not null)') context.pctl.query(name, "SELECT create_distributed_table('public.d', 'id')") @step('I cleanup a distributed table on {name:name}') def cleanup_distributed_table(context, name): context.pctl.query(name, 'TRUNCATE public.d') def insert_thread(query_func, context): while True: if context.thread_stop_event.is_set(): break context.insert_counter += 1 query_func('INSERT INTO public.d VALUES({0})'.format(context.insert_counter)) context.thread_stop_event.wait(0.01) @step('I start a thread inserting data on {name:name}') def start_insert_thread(context, name): context.thread_stop_event = Event() context.insert_counter = 0 query_func = partial(context.pctl.query, name) thread_func = partial(insert_thread, query_func, context) context.thread = Thread(target=thread_func) context.thread.daemon = True context.thread.start() @then('a thread is still alive') def thread_is_alive(context): assert context.thread.is_alive(), "Thread is not alive" @step("I stop a thread") def stop_insert_thread(context): context.thread_stop_event.set() context.thread.join(1 * context.timeout_multiplier) assert not context.thread.is_alive(), "Thread is still alive" @step("a distributed table on {name:name} has expected rows") def count_rows(context, name): rows = context.pctl.query(name, "SELECT COUNT(*) FROM public.d").fetchone()[0] assert rows == context.insert_counter, "Distributed table doesn't have expected amount of rows" @step("there is a transaction in progress on {name:name} changing pg_dist_node after {time_limit:d} seconds") def check_transaction(context, name, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while time.time() < max_time: cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()" " AND state = 'idle in transaction' AND query ~ 'citus_update_node'") if cur.rowcount == 1: context.xact_start = cur.fetchone()[0] return time.sleep(1) assert False, f"There is no idle in transaction on {name} updating pg_dist_node after {time_limit} seconds" @step("a transaction finishes in {timeout:d} seconds") def check_transaction_timeout(context, timeout): assert (datetime.now(tzutc) - context.xact_start).seconds >= timeout, \ "a transaction finished earlier than in {0} seconds".format(timeout) patroni-4.0.4/features/steps/custom_bootstrap.py000066400000000000000000000017141472010352700221340ustar00rootroot00000000000000import time from behave import step, then @step('I start {name:name} in a cluster {cluster_name:w} as a clone of {name2:name}') def start_cluster_clone(context, name, cluster_name, name2): context.pctl.clone(name2, cluster_name, name) @step('I start {name:name} in a cluster {cluster_name:w} from backup') def start_cluster_from_backup(context, name, cluster_name): context.pctl.bootstrap_from_backup(name, cluster_name) @then('{name:name} is a leader of {cluster_name:w} after {time_limit:d} seconds') def is_a_leader(context, name, cluster_name, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while (context.dcs_ctl.query("leader", scope=cluster_name) != name): time.sleep(1) assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit) @step('I do a backup of {name:name}') def do_backup(context, name): context.pctl.backup(name) patroni-4.0.4/features/steps/dcs_failsafe_mode.py000066400000000000000000000006511472010352700221330ustar00rootroot00000000000000from behave import step @step('DCS is down') def start_dcs_outage(context): context.dcs_ctl.start_outage() @step('DCS is up') def stop_dcs_outage(context): context.dcs_ctl.stop_outage() @step('I start {name:name} in a cluster {cluster_name:w} from backup with no_leader') def start_cluster_from_backup_no_leader(context, name, cluster_name): context.pctl.bootstrap_from_backup_no_leader(name, cluster_name) patroni-4.0.4/features/steps/patroni_api.py000066400000000000000000000150211472010352700210260ustar00rootroot00000000000000import json import shlex import subprocess import sys import time from datetime import datetime, timedelta import parse import yaml from behave import register_type, step, then from dateutil import tz tzutc = tz.tzutc() @parse.with_pattern(r'https?://(?:\w|\.|:|/)+') def parse_url(text): return text register_type(url=parse_url) # there is no way we can find out if the node has already # started as a leader without checking the DCS. We cannot # just rely on the database availability, since there is # a short gap between the time PostgreSQL becomes available # and Patroni assuming the leader role. @step('{name:name} is a leader after {time_limit:d} seconds') @then('{name:name} is a leader after {time_limit:d} seconds') def is_a_leader(context, name, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while (context.dcs_ctl.query("leader") != name): time.sleep(1) assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit) @step('I sleep for {value:d} seconds') def sleep_for_n_seconds(context, value): time.sleep(int(value)) def _set_response(context, response): context.status_code = response.status data = response.data.decode('utf-8') ct = response.getheader('content-type', '') if ct.startswith('application/json') or\ ct.startswith('text/yaml') or\ ct.startswith('text/x-yaml') or\ ct.startswith('application/yaml') or\ ct.startswith('application/x-yaml'): try: context.response = yaml.safe_load(data) except ValueError: context.response = data else: context.response = data @step('I issue a GET request to {url:url}') def do_get(context, url): do_request(context, 'GET', url, None) @step('I issue an empty POST request to {url:url}') def do_post_empty(context, url): do_request(context, 'POST', url, None) @step('I issue a {request_method:w} request to {url:url} with {data}') def do_request(context, request_method, url, data): if context.certfile: url = url.replace('http://', 'https://') data = data and json.loads(data) try: r = context.request_executor.request(request_method, url, data) if request_method == 'PATCH' and r.status == 409: r = context.request_executor.request(request_method, url, data) except Exception: context.status_code = context.response = None else: _set_response(context, r) @step('I run {cmd}') def do_run(context, cmd): cmd = [sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd) try: response = subprocess.check_output(cmd, stderr=subprocess.STDOUT) context.status_code = 0 except subprocess.CalledProcessError as e: response = e.output context.status_code = e.returncode context.response = response.decode('utf-8').strip() @then('I receive a response {component:name} {data}') def check_response(context, component, data): if component == 'code': assert context.status_code == int(data), \ "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response) elif component == 'returncode': assert context.status_code == int(data), "return code {0} != {1}, {2}".format(context.status_code, data, context.response) elif component == 'text': assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data) elif component == 'output': assert data.strip('"') in context.response, "response {0} does not contain {1}".format(context.response, data) else: assert component in context.response, "{0} is not part of the response".format(component) if context.certfile: data = data.replace('http://', 'https://') assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data) @step('I issue a scheduled switchover from {from_host:name} to {to_host:name} in {in_seconds:d} seconds') def scheduled_switchover(context, from_host, to_host, in_seconds): context.execute_steps(u""" Given I run patronictl.py switchover batman --primary {0} --candidate {1} --scheduled "{2}" --force """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds)))) @step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}') def scheduled_restart(context, url, in_seconds, data): data = data and json.loads(data) or {} data.update(schedule='{0}'.format((datetime.now(tzutc) + timedelta(seconds=int(in_seconds))).isoformat())) context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data))) @step('I {action:w} {tag:w} tag in {pg_name:name} config') def add_bool_tag_to_config(context, action, tag, pg_name): value = action == 'set' context.pctl.add_tag_to_config(pg_name, tag, value) @step('I add tag {tag:w} {value:w} to {pg_name:name} config') def add_tag_to_config(context, tag, value, pg_name): context.pctl.add_tag_to_config(pg_name, tag, value) @then('Status code on GET {url:url} is {code:d} after {timeout:d} seconds') def check_http_code(context, url, code, timeout): if context.certfile: url = url.replace('http://', 'https://') timeout *= context.timeout_multiplier for _ in range(int(timeout)): r = context.request_executor.request('GET', url) if int(code) == int(r.status): break time.sleep(1) else: assert False, "HTTP Status Code is not {0} after {1} seconds".format(code, timeout) @then('Response on GET {url:url} contains {value} after {timeout:d} seconds') def check_http_response(context, url, value, timeout, negate=False): if context.certfile: url = url.replace('http://', 'https://') timeout *= context.timeout_multiplier for _ in range(int(timeout)): r = context.request_executor.request('GET', url) if (value in r.data.decode('utf-8')) != negate: break time.sleep(1) else: assert False, \ "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout) @then('Response on GET {url} does not contain {value} after {timeout:d} seconds') def check_not_in_http_response(context, url, value, timeout): check_http_response(context, url, value, timeout, negate=True) patroni-4.0.4/features/steps/quorum_commit.py000066400000000000000000000043051472010352700214240ustar00rootroot00000000000000import json import re import time from behave import step, then @step('sync key in DCS has {key:w}={value} after {time_limit:d} seconds') def check_sync(context, key, value, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) dcs_value = None while time.time() < max_time: try: response = json.loads(context.dcs_ctl.query('sync')) dcs_value = response.get(key) if key == 'sync_standby' and set((dcs_value or '').split(',')) == set(value.split(',')): return elif str(dcs_value) == value: return except Exception: pass time.sleep(1) assert False, "sync does not have {0}={1} (found {2}) in dcs after {3} seconds".format(key, value, dcs_value, time_limit) def _parse_synchronous_standby_names(value): if '(' in value: m = re.match(r'.*(\d+) \(([^)]+)\)', value) expected_value = set(m.group(2).split()) expected_num = m.group(1) else: expected_value = set([value]) expected_num = '1' return expected_num, expected_value @then("synchronous_standby_names on {name:2} is set to '{value}' after {time_limit:d} seconds") def check_synchronous_standby_names(context, name, value, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) if value == '_empty_str_': value = '' expected_num, expected_value = _parse_synchronous_standby_names(value) ssn = None while time.time() < max_time: try: ssn = context.pctl.query(name, "SHOW synchronous_standby_names").fetchone()[0] db_num, db_value = _parse_synchronous_standby_names(ssn) if expected_value == db_value and expected_num == db_num: return except Exception: pass time.sleep(1) assert False, "synchronous_standby_names is not set to '{0}' (found '{1}') after {2} seconds".format(value, ssn, time_limit) patroni-4.0.4/features/steps/recovery.py000066400000000000000000000004101472010352700203530ustar00rootroot00000000000000import os from behave import step @step('I ensure {name:name} fails to start after a failure') def spoil_autoconf(context, name): with open(os.path.join(context.pctl._processes[name]._data_dir, 'postgresql.auto.conf'), 'w') as f: f.write('foo=bar') patroni-4.0.4/features/steps/slots.py000066400000000000000000000134711472010352700176740ustar00rootroot00000000000000import json import time from behave import step, then import patroni.psycopg as pg @step('I create a logical replication slot {slot_name} on {pg_name:name} with the {plugin:w} plugin') def create_logical_replication_slot(context, slot_name, pg_name, plugin): try: output = context.pctl.query(pg_name, ("SELECT pg_create_logical_replication_slot('{0}', '{1}')," " current_database()").format(slot_name, plugin)) print(output.fetchone()) except pg.Error as e: print(e) assert False, "Error creating slot {0} on {1} with plugin {2}".format(slot_name, pg_name, plugin) @step('{pg_name:name} has a logical replication slot named {slot_name}' ' with the {plugin:w} plugin after {time_limit:d} seconds') @then('{pg_name:name} has a logical replication slot named {slot_name}' ' with the {plugin:w} plugin after {time_limit:d} seconds') def has_logical_replication_slot(context, pg_name, slot_name, plugin, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while time.time() < max_time: try: row = context.pctl.query(pg_name, ("SELECT slot_type, plugin FROM pg_replication_slots" f" WHERE slot_name = '{slot_name}'")).fetchone() if row: assert row[0] == "logical", f"Replication slot {slot_name} isn't a logical but {row[0]}" assert row[1] == plugin, f"Replication slot {slot_name} using plugin {row[1]} rather than {plugin}" return except Exception: pass time.sleep(1) assert False, f"Error looking for slot {slot_name} on {pg_name} with plugin {plugin}" @step('{pg_name:name} does not have a replication slot named {slot_name:w}') @then('{pg_name:name} does not have a replication slot named {slot_name:w}') def does_not_have_replication_slot(context, pg_name, slot_name): try: row = context.pctl.query(pg_name, ("SELECT 1 FROM pg_replication_slots" " WHERE slot_name = '{0}'").format(slot_name)).fetchone() assert not row, "Found unexpected replication slot named {0}".format(slot_name) except pg.Error: assert False, "Error looking for slot {0} on {1}".format(slot_name, pg_name) @step('{slot_type:w} slot {slot_name:w} is in sync between ' '{pg_name1:name} and {pg_name2:name} after {time_limit:d} seconds') def slots_in_sync(context, slot_type, slot_name, pg_name1, pg_name2, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) column = 'confirmed_flush_lsn' if slot_type.lower() == 'logical' else 'restart_lsn' query = f"SELECT {column} FROM pg_replication_slots WHERE slot_name = '{slot_name}'" while time.time() < max_time: try: slot1 = context.pctl.query(pg_name1, query).fetchone() slot2 = context.pctl.query(pg_name2, query).fetchone() if slot1[0] == slot2[0]: return except Exception: pass time.sleep(1) assert False, \ f"{slot_type} slot {slot_name} is not in sync between {pg_name1} and {pg_name2} after {time_limit} seconds" @step('I get all changes from logical slot {slot_name:w} on {pg_name:name}') def logical_slot_get_changes(context, slot_name, pg_name): context.pctl.query(pg_name, "SELECT * FROM pg_logical_slot_get_changes('{0}', NULL, NULL)".format(slot_name)) @step('I get all changes from physical slot {slot_name:w} on {pg_name:name}') def physical_slot_get_changes(context, slot_name, pg_name): context.pctl.query(pg_name, f"SELECT * FROM pg_replication_slot_advance('{slot_name}', pg_current_wal_lsn())") @step('{pg_name:name} has a physical replication slot named {slot_name} after {time_limit:d} seconds') def has_physical_replication_slot(context, pg_name, slot_name, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) query = f"SELECT * FROM pg_catalog.pg_replication_slots WHERE slot_type = 'physical' AND slot_name = '{slot_name}'" while time.time() < max_time: try: row = context.pctl.query(pg_name, query).fetchone() if row: return except Exception: pass time.sleep(1) assert False, f"Physical slot {slot_name} doesn't exist after {time_limit} seconds" @step('physical replication slot named {slot_name} on {pg_name:name} has no xmin value after {time_limit:d} seconds') def physical_slot_no_xmin(context, pg_name, slot_name, time_limit): time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) query = "SELECT xmin FROM pg_catalog.pg_replication_slots WHERE slot_type = 'physical'" f" AND slot_name = '{slot_name}'" exists = False while time.time() < max_time: try: row = context.pctl.query(pg_name, query).fetchone() exists = bool(row) if exists and row[0] is None: return except Exception: pass time.sleep(1) assert False, f"Physical slot {slot_name} doesn't exist after {time_limit} seconds" if not exists \ else f"Physical slot {slot_name} has xmin value after {time_limit} seconds" @step('"{name}" key in DCS has {subkey} in {key:w}') def dcs_key_contains(context, name, subkey, key): response = json.loads(context.dcs_ctl.query(name)) assert key in response and subkey in response[key], f"{name} key in DCS doesn't have {subkey} in {key}" @step('"{name}" key in DCS does not have {subkey} in {key:w}') def dcs_key_does_not_contain(context, name, subkey, key): response = json.loads(context.dcs_ctl.query(name)) assert key not in response or subkey not in response[key], f"{name} key in DCS has {subkey} in {key}" patroni-4.0.4/features/steps/standby_cluster.py000066400000000000000000000046411472010352700217340ustar00rootroot00000000000000import os import time from behave import step def callbacks(context, name): return {c: '{0} features/callback2.py {1}'.format(context.pctl.PYTHON, name) for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')} @step('I start {name:name} in a cluster {cluster_name:w}') def start_patroni(context, name, cluster_name): return context.pctl.start(name, custom_config={ "scope": cluster_name, "postgresql": { "callbacks": callbacks(context, name), "backup_restore": context.pctl.backup_restore_config() } }) @step('I start {name:name} in a standby cluster {cluster_name:w} as a clone of {name2:name}') def start_patroni_standby_cluster(context, name, cluster_name, name2): # we need to remove patroni.dynamic.json in order to "bootstrap" standby cluster with existing PGDATA os.unlink(os.path.join(context.pctl._processes[name]._data_dir, 'patroni.dynamic.json')) port = context.pctl._processes[name2]._connkwargs.get('port') context.pctl._processes[name].update_config({ "scope": cluster_name, "bootstrap": { "dcs": { "ttl": 20, "loop_wait": 2, "retry_timeout": 5, "synchronous_mode": True, # should be completely ignored "standby_cluster": { "host": "localhost", "port": port, "primary_slot_name": "pm_1", "create_replica_methods": ["backup_restore", "basebackup"] }, "postgresql": {"parameters": {"wal_level": "logical"}} } }, "postgresql": { "callbacks": callbacks(context, name) } }) return context.pctl.start(name) @step('{pg_name1:name} is replicating from {pg_name2:name} after {timeout:d} seconds') def check_replication_status(context, pg_name1, pg_name2, timeout): bound_time = time.time() + timeout * context.timeout_multiplier while time.time() < bound_time: cur = context.pctl.query( pg_name2, "SELECT * FROM pg_catalog.pg_stat_replication WHERE application_name = '{0}'".format(pg_name1), fail_ok=True ) if cur and len(cur.fetchall()) != 0: break time.sleep(1) else: assert False, "{0} is not replicating from {1} after {2} seconds".format(pg_name1, pg_name2, timeout) patroni-4.0.4/features/steps/watchdog.py000066400000000000000000000032531472010352700203250ustar00rootroot00000000000000import time from behave import step, then def polling_loop(timeout, interval=1): """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration.""" start_time = time.time() iteration = 0 end_time = start_time + timeout while time.time() < end_time: yield iteration iteration += 1 time.sleep(interval) @step('I start {name:name} with watchdog') def start_patroni_with_watchdog(context, name): return context.pctl.start(name, custom_config={'watchdog': True, 'bootstrap': {'dcs': {'ttl': 20}}}) @step('{name:name} watchdog has been pinged after {timeout:d} seconds') def watchdog_was_pinged(context, name, timeout): for _ in polling_loop(timeout): if context.pctl.get_watchdog(name).was_pinged: return True return False @then('{name:name} watchdog has been closed') def watchdog_was_closed(context, name): assert context.pctl.get_watchdog(name).was_closed @step('{name:name} watchdog has a {timeout:d} second timeout') def watchdog_has_timeout(context, name, timeout): assert context.pctl.get_watchdog(name).timeout == timeout @step('I reset {name:name} watchdog state') def watchdog_reset_pinged(context, name): context.pctl.get_watchdog(name).reset() @then('{name:name} watchdog is triggered after {timeout:d} seconds') def watchdog_was_triggered(context, name, timeout): for _ in polling_loop(timeout): if context.pctl.get_watchdog(name).was_triggered: return True assert False @step('{name:name} hangs for {timeout:d} seconds') def patroni_hang(context, name, timeout): return context.pctl.patroni_hang(name, timeout) patroni-4.0.4/features/watchdog.feature000066400000000000000000000030771472010352700201760ustar00rootroot00000000000000Feature: watchdog Verify that watchdog gets pinged and triggered under appropriate circumstances. Scenario: watchdog is opened and pinged Given I start postgres-0 with watchdog Then postgres-0 is a leader after 10 seconds And postgres-0 role is the primary after 10 seconds And postgres-0 watchdog has been pinged after 10 seconds And postgres-0 watchdog has a 15 second timeout Scenario: watchdog is reconfigured after global ttl changed Given I run patronictl.py edit-config batman -s ttl=30 --force Then I receive a response returncode 0 And I receive a response output "+ttl: 30" When I sleep for 4 seconds Then postgres-0 watchdog has a 25 second timeout Scenario: watchdog is disabled during pause Given I run patronictl.py pause batman Then I receive a response returncode 0 When I sleep for 2 seconds Then postgres-0 watchdog has been closed Scenario: watchdog is opened and pinged after resume Given I reset postgres-0 watchdog state And I run patronictl.py resume batman Then I receive a response returncode 0 And postgres-0 watchdog has been pinged after 10 seconds Scenario: watchdog is disabled when shutting down Given I shut down postgres-0 Then postgres-0 watchdog has been closed Scenario: watchdog is triggered if patroni stops responding Given I reset postgres-0 watchdog state And I start postgres-0 with watchdog Then postgres-0 role is the primary after 10 seconds When postgres-0 hangs for 30 seconds Then postgres-0 watchdog is triggered after 30 seconds patroni-4.0.4/haproxy.cfg000066400000000000000000000010601472010352700153440ustar00rootroot00000000000000global maxconn 100 defaults log global mode tcp retries 2 timeout client 30m timeout connect 4s timeout server 30m timeout check 5s listen stats mode http bind *:7000 stats enable stats uri / listen batman bind *:5000 option httpchk http-check expect status 200 default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions server postgresql_127.0.0.1_5432 127.0.0.1:5432 maxconn 100 check port 8008 server postgresql_127.0.0.1_5433 127.0.0.1:5433 maxconn 100 check port 8009 patroni-4.0.4/kubernetes/000077500000000000000000000000001472010352700153435ustar00rootroot00000000000000patroni-4.0.4/kubernetes/Dockerfile000066400000000000000000000026421472010352700173410ustar00rootroot00000000000000FROM postgres:16 LABEL maintainer="Alexander Kukushkin " RUN export DEBIAN_FRONTEND=noninteractive \ && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \ && apt-get update -y \ && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ | xargs apt-get install -y vim-tiny curl jq locales git python3-pip python3-wheel \ ## Make sure we have a en_US.UTF-8 locale available && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ && pip3 install --break-system-packages setuptools \ && pip3 install --break-system-packages 'git+https://github.com/patroni/patroni.git#egg=patroni[kubernetes]' \ && PGHOME=/home/postgres \ && mkdir -p $PGHOME \ && chown postgres $PGHOME \ && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \ # Set permissions for OpenShift && chmod 775 $PGHOME \ && chmod 664 /etc/passwd \ # Clean up && apt-get remove -y git python3-pip python3-wheel \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /root/.cache COPY entrypoint.sh / EXPOSE 5432 8008 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor USER postgres WORKDIR /home/postgres CMD ["/bin/bash", "/entrypoint.sh"] patroni-4.0.4/kubernetes/Dockerfile.citus000066400000000000000000000074611472010352700204730ustar00rootroot00000000000000FROM postgres:16 LABEL maintainer="Alexander Kukushkin " RUN export DEBIAN_FRONTEND=noninteractive \ && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \ && apt-get update -y \ && apt-get upgrade -y \ && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ | xargs apt-get install -y busybox vim-tiny curl jq less locales git python3-pip python3-wheel lsb-release \ ## Make sure we have a en_US.UTF-8 locale available && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ apt-get install -y postgresql-server-dev-16 \ gcc make autoconf \ libc6-dev flex libcurl4-gnutls-dev \ libicu-dev libkrb5-dev liblz4-dev \ libpam0g-dev libreadline-dev libselinux1-dev\ libssl-dev libxslt1-dev libzstd-dev uuid-dev \ && git clone -b "main" https://github.com/citusdata/citus.git \ && MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \ && cd citus && ./configure && make install && cd ../ && rm -rf /citus; \ else \ echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \ && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \ && apt-get update -y \ && apt-get -y install postgresql-16-citus-12.1; \ fi \ && pip3 install --break-system-packages setuptools \ && pip3 install --break-system-packages 'git+https://github.com/patroni/patroni.git#egg=patroni[kubernetes]' \ && PGHOME=/home/postgres \ && mkdir -p $PGHOME \ && chown postgres $PGHOME \ && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \ && /bin/busybox --install -s \ # Set permissions for OpenShift && chmod 775 $PGHOME \ && chmod 664 /etc/passwd \ # Clean up && apt-get remove -y git python3-pip python3-wheel \ postgresql-server-dev-16 gcc make autoconf \ libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \ libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /root/.cache ADD entrypoint.sh / ENV PGSSLMODE=verify-ca PGSSLKEY=/etc/ssl/private/ssl-cert-snakeoil.key PGSSLCERT=/etc/ssl/certs/ssl-cert-snakeoil.pem PGSSLROOTCERT=/etc/ssl/certs/ssl-cert-snakeoil.pem RUN sed -i 's/^postgresql:/&\n basebackup:\n checkpoint: fast/' /entrypoint.sh \ && sed -i "s|^ postgresql:|&\n parameters:\n max_connections: 100\n shared_buffers: 16MB\n ssl: 'on'\n ssl_ca_file: $PGSSLROOTCERT\n ssl_cert_file: $PGSSLCERT\n ssl_key_file: $PGSSLKEY\n citus.node_conninfo: 'sslrootcert=$PGSSLROOTCERT sslkey=$PGSSLKEY sslcert=$PGSSLCERT sslmode=$PGSSLMODE'|" /entrypoint.sh \ && sed -i 's/^ pg_hba:/&\n - local all all trust/' /entrypoint.sh \ && sed -i "s/^\(.*\) \(.*\) \(.*\) \(.*\) \(.*\) md5.*$/\1 hostssl \3 \4 all md5 clientcert=$PGSSLMODE/" /entrypoint.sh \ && sed -i "s#^ \(superuser\|replication\):#&\n sslmode: $PGSSLMODE\n sslkey: $PGSSLKEY\n sslcert: $PGSSLCERT\n sslrootcert: $PGSSLROOTCERT#" /entrypoint.sh EXPOSE 5432 8008 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor USER postgres WORKDIR /home/postgres CMD ["/bin/bash", "/entrypoint.sh"] patroni-4.0.4/kubernetes/README.md000066400000000000000000000167011472010352700166270ustar00rootroot00000000000000# Kubernetes deployment examples Below you will find examples of Patroni deployments using [kind](https://kind.sigs.k8s.io/). # Patroni on K8s The Patroni cluster deployment with a StatefulSet consisting of three Pods. Example session: $ kind create cluster Creating cluster "kind" ... ✓ Ensuring node image (kindest/node:v1.25.3) 🖼 ✓ Preparing nodes 📦 ✓ Writing configuration 📜 ✓ Starting control-plane ðŸ•¹ï¸ âœ“ Installing CNI 🔌 ✓ Installing StorageClass 💾 Set kubectl context to "kind-kind" You can now use your cluster with: kubectl cluster-info --context kind-kind Thanks for using kind! 😊 $ docker build -t patroni . Sending build context to Docker daemon 138.8kB Step 1/9 : FROM postgres:16 ... Successfully built e9bfe69c5d2b Successfully tagged patroni:latest $ kind load docker-image patroni Image: "" with ID "sha256:e9bfe69c5d2b319dec0cf564fb895484537664775e18f37f9b707914cc5537e6" not yet present on node "kind-control-plane", loading... $ kubectl apply -f patroni_k8s.yaml service/patronidemo-config created statefulset.apps/patronidemo created endpoints/patronidemo created service/patronidemo created service/patronidemo-repl created secret/patronidemo created serviceaccount/patronidemo created role.rbac.authorization.k8s.io/patronidemo created rolebinding.rbac.authorization.k8s.io/patronidemo created clusterrole.rbac.authorization.k8s.io/patroni-k8s-ep-access created clusterrolebinding.rbac.authorization.k8s.io/patroni-k8s-ep-access created $ kubectl get pods -L role NAME READY STATUS RESTARTS AGE ROLE patronidemo-0 1/1 Running 0 34s primary patronidemo-1 1/1 Running 0 30s replica patronidemo-2 1/1 Running 0 26s replica $ kubectl exec -ti patronidemo-0 -- bash postgres@patronidemo-0:~$ patronictl list + Cluster: patronidemo (7186662553319358497) ----+----+-----------+ | Member | Host | Role | State | TL | Lag in MB | +---------------+------------+---------+---------+----+-----------+ | patronidemo-0 | 10.244.0.5 | Leader | running | 1 | | | patronidemo-1 | 10.244.0.6 | Replica | running | 1 | 0 | | patronidemo-2 | 10.244.0.7 | Replica | running | 1 | 0 | +---------------+------------+---------+---------+----+-----------+ # Citus on K8s The Citus cluster with the StatefulSets, one coordinator with three Pods and two workers with two pods each. Example session: $ kind create cluster Creating cluster "kind" ... ✓ Ensuring node image (kindest/node:v1.25.3) 🖼 ✓ Preparing nodes 📦 ✓ Writing configuration 📜 ✓ Starting control-plane ðŸ•¹ï¸ âœ“ Installing CNI 🔌 ✓ Installing StorageClass 💾 Set kubectl context to "kind-kind" You can now use your cluster with: kubectl cluster-info --context kind-kind Thanks for using kind! 😊 demo@localhost:~/git/patroni/kubernetes$ docker build -f Dockerfile.citus -t patroni-citus-k8s . Sending build context to Docker daemon 138.8kB Step 1/11 : FROM postgres:16 ... Successfully built 8cd73e325028 Successfully tagged patroni-citus-k8s:latest $ kind load docker-image patroni-citus-k8s Image: "" with ID "sha256:8cd73e325028d7147672494965e53453f5540400928caac0305015eb2c7027c7" not yet present on node "kind-control-plane", loading... $ kubectl apply -f citus_k8s.yaml service/citusdemo-0-config created service/citusdemo-1-config created service/citusdemo-2-config created statefulset.apps/citusdemo-0 created statefulset.apps/citusdemo-1 created statefulset.apps/citusdemo-2 created endpoints/citusdemo-0 created service/citusdemo-0 created endpoints/citusdemo-1 created service/citusdemo-1 created endpoints/citusdemo-2 created service/citusdemo-2 created service/citusdemo-workers created secret/citusdemo created serviceaccount/citusdemo created role.rbac.authorization.k8s.io/citusdemo created rolebinding.rbac.authorization.k8s.io/citusdemo created clusterrole.rbac.authorization.k8s.io/patroni-k8s-ep-access created clusterrolebinding.rbac.authorization.k8s.io/patroni-k8s-ep-access created $ kubectl get sts NAME READY AGE citusdemo-0 1/3 6s # coodinator (group=0) citusdemo-1 1/2 6s # worker (group=1) citusdemo-2 1/2 6s # worker (group=2) $ kubectl get pods -l cluster-name=citusdemo -L role NAME READY STATUS RESTARTS AGE ROLE citusdemo-0-0 1/1 Running 0 105s primary citusdemo-0-1 1/1 Running 0 101s replica citusdemo-0-2 1/1 Running 0 96s replica citusdemo-1-0 1/1 Running 0 105s primary citusdemo-1-1 1/1 Running 0 101s replica citusdemo-2-0 1/1 Running 0 105s primary citusdemo-2-1 1/1 Running 0 101s replica $ kubectl exec -ti citusdemo-0-0 -- bash postgres@citusdemo-0-0:~$ patronictl list + Citus cluster: citusdemo -----------+----------------+---------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------------+-------------+----------------+---------+----+-----------+ | 0 | citusdemo-0-0 | 10.244.0.10 | Leader | running | 1 | | | 0 | citusdemo-0-1 | 10.244.0.12 | Replica | running | 1 | 0 | | 0 | citusdemo-0-2 | 10.244.0.14 | Quorum Standby | running | 1 | 0 | | 1 | citusdemo-1-0 | 10.244.0.8 | Leader | running | 1 | | | 1 | citusdemo-1-1 | 10.244.0.11 | Quorum Standby | running | 1 | 0 | | 2 | citusdemo-2-0 | 10.244.0.9 | Leader | running | 1 | | | 2 | citusdemo-2-1 | 10.244.0.13 | Quorum Standby | running | 1 | 0 | +-------+---------------+-------------+----------------+---------+----+-----------+ postgres@citusdemo-0-0:~$ psql citus psql (16.4 (Debian 16.4-1.pgdg120+1)) Type "help" for help. citus=# table pg_dist_node; nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+-------------+----------+----------+-------------+----------+-----------+-------------+----------------+------------------ 1 | 0 | 10.244.0.10 | 5432 | default | t | t | primary | default | t | f 2 | 1 | 10.244.0.8 | 5432 | default | t | t | primary | default | t | t 3 | 2 | 10.244.0.9 | 5432 | default | t | t | primary | default | t | t 4 | 0 | 10.244.0.14 | 5432 | default | t | t | secondary | default | t | f 5 | 0 | 10.244.0.12 | 5432 | default | t | t | secondary | default | t | f 6 | 1 | 10.244.0.11 | 5432 | default | t | t | secondary | default | t | t 7 | 2 | 10.244.0.13 | 5432 | default | t | t | secondary | default | t | t (7 rows) patroni-4.0.4/kubernetes/citus_k8s.yaml000066400000000000000000000327261472010352700201550ustar00rootroot00000000000000# headless services to avoid deletion of citusdemo-*-config endpoints apiVersion: v1 kind: Service metadata: name: citusdemo-0-config labels: application: patroni cluster-name: citusdemo citus-group: '0' spec: clusterIP: None --- apiVersion: v1 kind: Service metadata: name: citusdemo-1-config labels: application: patroni cluster-name: citusdemo citus-group: '1' spec: clusterIP: None --- apiVersion: v1 kind: Service metadata: name: citusdemo-2-config labels: application: patroni cluster-name: citusdemo citus-group: '2' spec: clusterIP: None --- apiVersion: apps/v1 kind: StatefulSet metadata: name: &cluster_name citusdemo-0 labels: &labels application: patroni cluster-name: citusdemo citus-group: '0' citus-type: coordinator spec: replicas: 3 serviceName: *cluster_name selector: matchLabels: <<: *labels template: metadata: labels: <<: *labels spec: serviceAccountName: citusdemo containers: - name: *cluster_name image: patroni-citus-k8s # docker build -f Dockerfile.citus -t patroni-citus-k8s . imagePullPolicy: IfNotPresent readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ports: - containerPort: 8008 protocol: TCP - containerPort: 5432 protocol: TCP volumeMounts: - mountPath: /home/postgres/pgdata name: pgdata env: - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE value: 'true' - name: PATRONI_KUBERNETES_USE_ENDPOINTS value: 'true' - name: PATRONI_KUBERNETES_LABELS value: '{application: patroni, cluster-name: citusdemo}' - name: PATRONI_CITUS_DATABASE value: citus - name: PATRONI_CITUS_GROUP value: '0' - name: PATRONI_SUPERUSER_USERNAME value: postgres - name: PATRONI_SUPERUSER_PASSWORD valueFrom: secretKeyRef: name: citusdemo key: superuser-password - name: PATRONI_REPLICATION_USERNAME value: standby - name: PATRONI_REPLICATION_PASSWORD valueFrom: secretKeyRef: name: citusdemo key: replication-password - name: PATRONI_SCOPE value: citusdemo - name: PATRONI_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: PATRONI_POSTGRESQL_DATA_DIR value: /home/postgres/pgdata/pgroot/data - name: PATRONI_POSTGRESQL_PGPASS value: /tmp/pgpass - name: PATRONI_POSTGRESQL_LISTEN value: '0.0.0.0:5432' - name: PATRONI_RESTAPI_LISTEN value: '0.0.0.0:8008' terminationGracePeriodSeconds: 0 volumes: - name: pgdata emptyDir: {} # volumeClaimTemplates: # - metadata: # labels: # application: spilo # spilo-cluster: *cluster_name # annotations: # volume.alpha.kubernetes.io/storage-class: anything # name: pgdata # spec: # accessModes: # - ReadWriteOnce # resources: # requests: # storage: 5Gi --- apiVersion: apps/v1 kind: StatefulSet metadata: name: &cluster_name citusdemo-1 labels: &labels application: patroni cluster-name: citusdemo citus-group: '1' citus-type: worker spec: replicas: 2 serviceName: *cluster_name selector: matchLabels: <<: *labels template: metadata: labels: <<: *labels spec: serviceAccountName: citusdemo containers: - name: *cluster_name image: patroni-citus-k8s # docker build -f Dockerfile.citus -t patroni-citus-k8s . imagePullPolicy: IfNotPresent readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ports: - containerPort: 8008 protocol: TCP - containerPort: 5432 protocol: TCP volumeMounts: - mountPath: /home/postgres/pgdata name: pgdata env: - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE value: 'true' - name: PATRONI_KUBERNETES_USE_ENDPOINTS value: 'true' - name: PATRONI_KUBERNETES_LABELS value: '{application: patroni, cluster-name: citusdemo}' - name: PATRONI_CITUS_DATABASE value: citus - name: PATRONI_CITUS_GROUP value: '1' - name: PATRONI_SUPERUSER_USERNAME value: postgres - name: PATRONI_SUPERUSER_PASSWORD valueFrom: secretKeyRef: name: citusdemo key: superuser-password - name: PATRONI_REPLICATION_USERNAME value: standby - name: PATRONI_REPLICATION_PASSWORD valueFrom: secretKeyRef: name: citusdemo key: replication-password - name: PATRONI_SCOPE value: citusdemo - name: PATRONI_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: PATRONI_POSTGRESQL_DATA_DIR value: /home/postgres/pgdata/pgroot/data - name: PATRONI_POSTGRESQL_PGPASS value: /tmp/pgpass - name: PATRONI_POSTGRESQL_LISTEN value: '0.0.0.0:5432' - name: PATRONI_RESTAPI_LISTEN value: '0.0.0.0:8008' terminationGracePeriodSeconds: 0 volumes: - name: pgdata emptyDir: {} # volumeClaimTemplates: # - metadata: # labels: # application: spilo # spilo-cluster: *cluster_name # annotations: # volume.alpha.kubernetes.io/storage-class: anything # name: pgdata # spec: # accessModes: # - ReadWriteOnce # resources: # requests: # storage: 5Gi --- apiVersion: apps/v1 kind: StatefulSet metadata: name: &cluster_name citusdemo-2 labels: &labels application: patroni cluster-name: citusdemo citus-group: '2' citus-type: worker spec: replicas: 2 serviceName: *cluster_name selector: matchLabels: <<: *labels template: metadata: labels: <<: *labels spec: serviceAccountName: citusdemo containers: - name: *cluster_name image: patroni-citus-k8s # docker build -f Dockerfile.citus -t patroni-citus-k8s . imagePullPolicy: IfNotPresent readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ports: - containerPort: 8008 protocol: TCP - containerPort: 5432 protocol: TCP volumeMounts: - mountPath: /home/postgres/pgdata name: pgdata env: - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE value: 'true' - name: PATRONI_KUBERNETES_USE_ENDPOINTS value: 'true' - name: PATRONI_KUBERNETES_LABELS value: '{application: patroni, cluster-name: citusdemo}' - name: PATRONI_CITUS_DATABASE value: citus - name: PATRONI_CITUS_GROUP value: '2' - name: PATRONI_SUPERUSER_USERNAME value: postgres - name: PATRONI_SUPERUSER_PASSWORD valueFrom: secretKeyRef: name: citusdemo key: superuser-password - name: PATRONI_REPLICATION_USERNAME value: standby - name: PATRONI_REPLICATION_PASSWORD valueFrom: secretKeyRef: name: citusdemo key: replication-password - name: PATRONI_SCOPE value: citusdemo - name: PATRONI_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: PATRONI_POSTGRESQL_DATA_DIR value: /home/postgres/pgdata/pgroot/data - name: PATRONI_POSTGRESQL_PGPASS value: /tmp/pgpass - name: PATRONI_POSTGRESQL_LISTEN value: '0.0.0.0:5432' - name: PATRONI_RESTAPI_LISTEN value: '0.0.0.0:8008' terminationGracePeriodSeconds: 0 volumes: - name: pgdata emptyDir: {} # volumeClaimTemplates: # - metadata: # labels: # application: spilo # spilo-cluster: *cluster_name # annotations: # volume.alpha.kubernetes.io/storage-class: anything # name: pgdata # spec: # accessModes: # - ReadWriteOnce # resources: # requests: # storage: 5Gi --- apiVersion: v1 kind: Endpoints metadata: name: citusdemo-0 labels: application: patroni cluster-name: citusdemo citus-group: '0' citus-type: coordinator subsets: [] --- apiVersion: v1 kind: Service metadata: name: citusdemo-0 labels: application: patroni cluster-name: citusdemo citus-group: '0' citus-type: coordinator spec: type: ClusterIP ports: - port: 5432 targetPort: 5432 --- apiVersion: v1 kind: Endpoints metadata: name: citusdemo-1 labels: application: patroni cluster-name: citusdemo citus-group: '1' citus-type: worker subsets: [] --- apiVersion: v1 kind: Service metadata: name: citusdemo-1 labels: application: patroni cluster-name: citusdemo citus-group: '1' citus-type: worker spec: type: ClusterIP ports: - port: 5432 targetPort: 5432 --- apiVersion: v1 kind: Endpoints metadata: name: citusdemo-2 labels: application: patroni cluster-name: citusdemo citus-group: '2' citus-type: worker subsets: [] --- apiVersion: v1 kind: Service metadata: name: citusdemo-2 labels: application: patroni cluster-name: citusdemo citus-group: '2' citus-type: worker spec: type: ClusterIP ports: - port: 5432 targetPort: 5432 --- apiVersion: v1 kind: Service metadata: name: citusdemo-workers labels: &labels application: patroni cluster-name: citusdemo citus-type: worker role: primary spec: type: ClusterIP selector: <<: *labels ports: - port: 5432 targetPort: 5432 --- apiVersion: v1 kind: Secret metadata: name: &cluster_name citusdemo labels: application: patroni cluster-name: *cluster_name type: Opaque data: superuser-password: emFsYW5kbw== replication-password: cmVwLXBhc3M= --- apiVersion: v1 kind: ServiceAccount metadata: name: citusdemo --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: citusdemo rules: - apiGroups: - "" resources: - configmaps verbs: - create - get - list - patch - update - watch # delete and deletecollection are required only for 'patronictl remove' - delete - deletecollection - apiGroups: - "" resources: - endpoints verbs: - get - patch - update # the following three privileges are necessary only when using endpoints - create - list - watch # delete and deletecollection are required only for for 'patronictl remove' - delete - deletecollection - apiGroups: - "" resources: - pods verbs: - get - list - patch - update - watch # The following privilege is only necessary for creation of headless service # for citusdemo-config endpoint, in order to prevent cleaning it up by the # k8s master. You can avoid giving this privilege by explicitly creating the # service like it is done in this manifest (lines 2..10) - apiGroups: - "" resources: - services verbs: - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: citusdemo roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: citusdemo subjects: - kind: ServiceAccount name: citusdemo # Following privileges are only required if deployed not in the "default" # namespace and you want Patroni to bypass kubernetes service # (PATRONI_KUBERNETES_BYPASS_API_SERVICE=true) --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: patroni-k8s-ep-access rules: - apiGroups: - "" resources: - endpoints resourceNames: - kubernetes verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: patroni-k8s-ep-access roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: patroni-k8s-ep-access subjects: - kind: ServiceAccount name: citusdemo # The namespace must be specified explicitly. # If deploying to the different namespace you have to change it. namespace: default patroni-4.0.4/kubernetes/entrypoint.sh000077500000000000000000000020451472010352700201160ustar00rootroot00000000000000#!/bin/bash if [[ $UID -ge 10000 ]]; then GID=$(id -g) sed -e "s/^postgres:x:[^:]*:[^:]*:/postgres:x:$UID:$GID:/" /etc/passwd > /tmp/passwd cat /tmp/passwd > /etc/passwd rm /tmp/passwd fi cat > /home/postgres/patroni.yml <<__EOF__ bootstrap: dcs: postgresql: use_pg_rewind: true pg_hba: - host all all 0.0.0.0/0 md5 - host replication ${PATRONI_REPLICATION_USERNAME} ${PATRONI_KUBERNETES_POD_IP}/16 md5 - host replication ${PATRONI_REPLICATION_USERNAME} 127.0.0.1/32 md5 initdb: - auth-host: md5 - auth-local: trust - encoding: UTF8 - locale: en_US.UTF-8 - data-checksums restapi: connect_address: '${PATRONI_KUBERNETES_POD_IP}:8008' postgresql: connect_address: '${PATRONI_KUBERNETES_POD_IP}:5432' authentication: superuser: password: '${PATRONI_SUPERUSER_PASSWORD}' replication: password: '${PATRONI_REPLICATION_PASSWORD}' __EOF__ unset PATRONI_SUPERUSER_PASSWORD PATRONI_REPLICATION_PASSWORD exec /usr/bin/python3 /usr/local/bin/patroni /home/postgres/patroni.yml patroni-4.0.4/kubernetes/openshift-example/000077500000000000000000000000001472010352700207735ustar00rootroot00000000000000patroni-4.0.4/kubernetes/openshift-example/README.md000066400000000000000000000025551472010352700222610ustar00rootroot00000000000000# Patroni OpenShift Configuration Patroni can be run in OpenShift. Based on the kubernetes configuration, the Dockerfile and Entrypoint has been modified to support the dynamic UID/GID configuration that is applied in OpenShift. This can be run under the standard `restricted` SCC. # Examples ## Create test project ``` oc new-project patroni-test ``` ## Build the image Note: Update the references when merged upstream. Note: If deploying as a template for multiple users, the following commands should be performed in a shared namespace like `openshift`. ``` oc import-image postgres:10 --confirm -n openshift oc new-build https://github.com/patroni/patroni --context-dir=kubernetes -n openshift ``` ## Deploy the Image Two configuration templates exist in [templates](templates) directory: - Patroni Ephemeral - Patroni Persistent The only difference is whether or not the statefulset requests persistent storage. ## Create the Template Install the template into the `openshift` namespace if this should be shared across projects: ``` oc create -f templates/template_patroni_ephemeral.yml -n openshift ``` Then, from your own project: ``` oc new-app patroni-pgsql-ephemeral ``` Once the pods are running, two configmaps should be available: ``` $ oc get configmap NAME DATA AGE patroniocp-config 0 1m patroniocp-leader 0 1m ``` patroni-4.0.4/kubernetes/openshift-example/templates/000077500000000000000000000000001472010352700227715ustar00rootroot00000000000000patroni-4.0.4/kubernetes/openshift-example/templates/template_patroni_ephemeral.yml000066400000000000000000000224121472010352700311060ustar00rootroot00000000000000apiVersion: v1 kind: Template metadata: name: patroni-pgsql-ephemeral annotations: description: |- Patroni Postgresql database cluster, without persistent storage. WARNING: Any data stored will be lost upon pod destruction. Only use this template for testing. iconClass: icon-postgresql openshift.io/display-name: Patroni Postgresql (Ephemeral) openshift.io/long-description: This template deploys a a patroni postgresql HA cluster without persistent storage. tags: postgresql objects: - apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${PATRONI_CLUSTER_NAME} spec: ports: - port: 5432 protocol: TCP targetPort: 5432 sessionAffinity: None type: ClusterIP status: loadBalancer: {} - apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${PATRONI_PRIMARY_SERVICE_NAME} spec: ports: - port: 5432 protocol: TCP targetPort: 5432 selector: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} role: primary sessionAffinity: None type: ClusterIP status: loadBalancer: {} - apiVersion: v1 kind: Secret metadata: name: ${PATRONI_CLUSTER_NAME} labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} stringData: superuser-password: ${PATRONI_SUPERUSER_PASSWORD} replication-password: ${PATRONI_REPLICATION_PASSWORD} - apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${PATRONI_REPLICA_SERVICE_NAME} spec: ports: - port: 5432 protocol: TCP targetPort: 5432 selector: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} role: replica sessionAffinity: None type: ClusterIP status: loadBalancer: {} - apiVersion: apps/v1 kind: StatefulSet metadata: creationTimestamp: null generation: 3 labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${APPLICATION_NAME} spec: podManagementPolicy: OrderedReady replicas: 3 revisionHistoryLimit: 10 selector: matchLabels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} serviceName: ${APPLICATION_NAME} template: metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} spec: containers: - env: - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE value: 'true' - name: PATRONI_KUBERNETES_LABELS value: '{application: ${APPLICATION_NAME}, cluster-name: ${PATRONI_CLUSTER_NAME}}' - name: PATRONI_SUPERUSER_USERNAME value: ${PATRONI_SUPERUSER_USERNAME} - name: PATRONI_SUPERUSER_PASSWORD valueFrom: secretKeyRef: key: superuser-password name: ${PATRONI_CLUSTER_NAME} - name: PATRONI_REPLICATION_USERNAME value: ${PATRONI_REPLICATION_USERNAME} - name: PATRONI_REPLICATION_PASSWORD valueFrom: secretKeyRef: key: replication-password name: ${PATRONI_CLUSTER_NAME} - name: PATRONI_SCOPE value: ${PATRONI_CLUSTER_NAME} - name: PATRONI_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: PATRONI_POSTGRESQL_DATA_DIR value: /home/postgres/pgdata/pgroot/data - name: PATRONI_POSTGRESQL_PGPASS value: /tmp/pgpass - name: PATRONI_POSTGRESQL_LISTEN value: 0.0.0.0:5432 - name: PATRONI_RESTAPI_LISTEN value: 0.0.0.0:8008 image: docker-registry.default.svc:5000/${NAMESPACE}/patroni:latest imagePullPolicy: IfNotPresent name: ${APPLICATION_NAME} readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ports: - containerPort: 8008 protocol: TCP - containerPort: 5432 protocol: TCP resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /home/postgres/pgdata name: pgdata dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: ${SERVICE_ACCOUNT} serviceAccountName: ${SERVICE_ACCOUNT} terminationGracePeriodSeconds: 0 volumes: - name: pgdata emptyDir: {} updateStrategy: type: OnDelete - apiVersion: v1 kind: Endpoints metadata: name: ${APPLICATION_NAME} labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} subsets: [] - apiVersion: v1 kind: ServiceAccount metadata: name: ${SERVICE_ACCOUNT} - apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: ${SERVICE_ACCOUNT} rules: - apiGroups: - "" resources: - configmaps verbs: - create - get - list - patch - update - watch # delete is required only for 'patronictl remove' - delete - apiGroups: - "" resources: - endpoints verbs: - get - patch - update # the following three privileges are necessary only when using endpoints - create - list - watch # delete is required only for for 'patronictl remove' - delete - apiGroups: - "" resources: - pods verbs: - get - list - patch - update - watch - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ${SERVICE_ACCOUNT} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: ${SERVICE_ACCOUNT} subjects: - kind: ServiceAccount name: ${SERVICE_ACCOUNT} # Following privileges are only required if deployed not in the "default" # namespace and you want Patroni to bypass kubernetes service # (PATRONI_KUBERNETES_BYPASS_API_SERVICE=true) - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: patroni-k8s-ep-access rules: - apiGroups: - "" resources: - endpoints resourceNames: - kubernetes verbs: - get - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: ${NAMESPACE}-${SERVICE_ACCOUNT}-k8s-ep-access roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: patroni-k8s-ep-access subjects: - kind: ServiceAccount name: ${SERVICE_ACCOUNT} namespace: ${NAMESPACE} parameters: - description: The name of the application for labelling all artifacts. displayName: Application Name name: APPLICATION_NAME value: patroni-ephemeral - description: The name of the patroni-pgsql cluster. displayName: Cluster Name name: PATRONI_CLUSTER_NAME value: patroni-ephemeral - description: The name of the OpenShift Service exposed for the patroni-ephemeral-primary container. displayName: Primary service name. name: PATRONI_PRIMARY_SERVICE_NAME value: patroni-ephemeral-primary - description: The name of the OpenShift Service exposed for the patroni-ephemeral-replica containers. displayName: Replica service name. name: PATRONI_REPLICA_SERVICE_NAME value: patroni-ephemeral-replica - description: Maximum amount of memory the container can use. displayName: Memory Limit name: MEMORY_LIMIT value: 512Mi - description: The OpenShift Namespace where the patroni and postgresql ImageStream resides. displayName: ImageStream Namespace name: NAMESPACE value: openshift - description: Username of the superuser account for initialization. displayName: Superuser Username name: PATRONI_SUPERUSER_USERNAME value: postgres - description: Password of the superuser account for initialization. displayName: Superuser Password name: PATRONI_SUPERUSER_PASSWORD value: postgres - description: Username of the replication account for initialization. displayName: Replication Username name: PATRONI_REPLICATION_USERNAME value: postgres - description: Password of the replication account for initialization. displayName: Repication Password name: PATRONI_REPLICATION_PASSWORD value: postgres - description: Service account name used for pods and rolebindings to form a cluster in the project. displayName: Service Account name: SERVICE_ACCOUNT value: patroniocp patroni-4.0.4/kubernetes/openshift-example/templates/template_patroni_persistent.yaml000066400000000000000000000242511472010352700315100ustar00rootroot00000000000000apiVersion: v1 kind: Template metadata: name: patroni-pgsql-persistent annotations: description: |- Patroni Postgresql database cluster, with persistent storage. iconClass: icon-postgresql openshift.io/display-name: Patroni Postgresql (Persistent) openshift.io/long-description: This template deploys a a patroni postgresql HA cluster with persistent storage. tags: postgresql objects: - apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${PATRONI_CLUSTER_NAME} spec: ports: - port: 5432 protocol: TCP targetPort: 5432 sessionAffinity: None type: ClusterIP status: loadBalancer: {} - apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${PATRONI_PRIMARY_SERVICE_NAME} spec: ports: - port: 5432 protocol: TCP targetPort: 5432 selector: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} role: primary sessionAffinity: None type: ClusterIP status: loadBalancer: {} - apiVersion: v1 kind: Secret metadata: name: ${PATRONI_CLUSTER_NAME} labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} stringData: superuser-password: ${PATRONI_SUPERUSER_PASSWORD} replication-password: ${PATRONI_REPLICATION_PASSWORD} - apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${PATRONI_REPLICA_SERVICE_NAME} spec: ports: - port: 5432 protocol: TCP targetPort: 5432 selector: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} role: replica sessionAffinity: None type: ClusterIP status: loadBalancer: {} - apiVersion: apps/v1 kind: StatefulSet metadata: creationTimestamp: null generation: 3 labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} name: ${APPLICATION_NAME} spec: podManagementPolicy: OrderedReady replicas: 3 revisionHistoryLimit: 10 selector: matchLabels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} serviceName: ${APPLICATION_NAME} template: metadata: creationTimestamp: null labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} spec: initContainers: - command: - sh - -c - "mkdir -p /home/postgres/pgdata/pgroot/data && chmod 0700 /home/postgres/pgdata/pgroot/data" image: docker-registry.default.svc:5000/${NAMESPACE}/patroni:latest imagePullPolicy: IfNotPresent name: fix-perms resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /home/postgres/pgdata name: ${APPLICATION_NAME} containers: - env: - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE value: 'true' - name: PATRONI_KUBERNETES_LABELS value: '{application: ${APPLICATION_NAME}, cluster-name: ${PATRONI_CLUSTER_NAME}}' - name: PATRONI_SUPERUSER_USERNAME value: ${PATRONI_SUPERUSER_USERNAME} - name: PATRONI_SUPERUSER_PASSWORD valueFrom: secretKeyRef: key: superuser-password name: ${PATRONI_CLUSTER_NAME} - name: PATRONI_REPLICATION_USERNAME value: ${PATRONI_REPLICATION_USERNAME} - name: PATRONI_REPLICATION_PASSWORD valueFrom: secretKeyRef: key: replication-password name: ${PATRONI_CLUSTER_NAME} - name: PATRONI_SCOPE value: ${PATRONI_CLUSTER_NAME} - name: PATRONI_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: PATRONI_POSTGRESQL_DATA_DIR value: /home/postgres/pgdata/pgroot/data - name: PATRONI_POSTGRESQL_PGPASS value: /tmp/pgpass - name: PATRONI_POSTGRESQL_LISTEN value: 0.0.0.0:5432 - name: PATRONI_RESTAPI_LISTEN value: 0.0.0.0:8008 image: docker-registry.default.svc:5000/${NAMESPACE}/patroni:latest imagePullPolicy: IfNotPresent name: ${APPLICATION_NAME} readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ports: - containerPort: 8008 protocol: TCP - containerPort: 5432 protocol: TCP resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /home/postgres/pgdata name: ${APPLICATION_NAME} dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: ${SERVICE_ACCOUNT} serviceAccountName: ${SERVICE_ACCOUNT} terminationGracePeriodSeconds: 0 volumes: - name: ${APPLICATION_NAME} persistentVolumeClaim: claimName: ${APPLICATION_NAME} volumeClaimTemplates: - metadata: labels: application: ${APPLICATION_NAME} name: ${APPLICATION_NAME} spec: accessModes: - ReadWriteOnce resources: requests: storage: ${PVC_SIZE} updateStrategy: type: OnDelete - apiVersion: v1 kind: Endpoints metadata: name: ${APPLICATION_NAME} labels: application: ${APPLICATION_NAME} cluster-name: ${PATRONI_CLUSTER_NAME} subsets: [] - apiVersion: v1 kind: ServiceAccount metadata: name: ${SERVICE_ACCOUNT} - apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: ${SERVICE_ACCOUNT} rules: - apiGroups: - "" resources: - configmaps verbs: - create - get - list - patch - update - watch # delete is required only for 'patronictl remove' - delete - apiGroups: - "" resources: - endpoints verbs: - get - patch - update # the following three privileges are necessary only when using endpoints - create - list - watch # delete is required only for for 'patronictl remove' - delete - apiGroups: - "" resources: - pods verbs: - get - list - patch - update - watch - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ${SERVICE_ACCOUNT} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: ${SERVICE_ACCOUNT} subjects: - kind: ServiceAccount name: ${SERVICE_ACCOUNT} # Following privileges are only required if deployed not in the "default" # namespace and you want Patroni to bypass kubernetes service # (PATRONI_KUBERNETES_BYPASS_API_SERVICE=true) - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: patroni-k8s-ep-access rules: - apiGroups: - "" resources: - endpoints resourceNames: - kubernetes verbs: - get - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: ${NAMESPACE}-${SERVICE_ACCOUNT}-k8s-ep-access roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: patroni-k8s-ep-access subjects: - kind: ServiceAccount name: ${SERVICE_ACCOUNT} namespace: ${NAMESPACE} parameters: - description: The name of the application for labelling all artifacts. displayName: Application Name name: APPLICATION_NAME value: patroni-persistent - description: The name of the patroni-pgsql cluster. displayName: Cluster Name name: PATRONI_CLUSTER_NAME value: patroni-persistent - description: The name of the OpenShift Service exposed for the patroni-persistent-primary container. displayName: Primary service name. name: PATRONI_PRIMARY_SERVICE_NAME value: patroni-persistent-primary - description: The name of the OpenShift Service exposed for the patroni-persistent-replica containers. displayName: Replica service name. name: PATRONI_REPLICA_SERVICE_NAME value: patroni-persistent-replica - description: Maximum amount of memory the container can use. displayName: Memory Limit name: MEMORY_LIMIT value: 512Mi - description: The OpenShift Namespace where the patroni and postgresql ImageStream resides. displayName: ImageStream Namespace name: NAMESPACE value: openshift - description: Username of the superuser account for initialization. displayName: Superuser Username name: PATRONI_SUPERUSER_USERNAME value: postgres - description: Password of the superuser account for initialization. displayName: Superuser Password name: PATRONI_SUPERUSER_PASSWORD value: postgres - description: Username of the replication account for initialization. displayName: Replication Username name: PATRONI_REPLICATION_USERNAME value: postgres - description: Password of the replication account for initialization. displayName: Repication Password name: PATRONI_REPLICATION_PASSWORD value: postgres - description: Service account name used for pods and rolebindings to form a cluster in the project. displayName: Service Account name: SERVICE_ACCOUNT value: patroni-persistent - description: The size of the persistent volume to create. displayName: Persistent Volume Size name: PVC_SIZE value: 5Gi patroni-4.0.4/kubernetes/openshift-example/test/000077500000000000000000000000001472010352700217525ustar00rootroot00000000000000patroni-4.0.4/kubernetes/openshift-example/test/Jenkinsfile000066400000000000000000000023601472010352700241370ustar00rootroot00000000000000pipeline { agent any stages { stage ('Deploy test pod'){ when { expression { openshift.withCluster() { openshift.withProject() { return !openshift.selector( "dc", "pgbench" ).exists() } } } } steps { script { openshift.withCluster() { openshift.withProject() { def pgbench = openshift.newApp( "https://github.com/stewartshea/docker-pgbench/", "--name=pgbench", "-e PGPASSWORD=postgres", "-e PGUSER=postgres", "-e PGHOST=patroni-persistent-primary", "-e PGDATABASE=postgres", "-e TEST_CLIENT_COUNT=20", "-e TEST_DURATION=120" ) def pgbenchdc = openshift.selector( "dc", "pgbench" ) timeout(5) { pgbenchdc.rollout().status() } } } } } } stage ('Run benchmark Test'){ steps { sh ''' oc exec $(oc get pods -l app=pgbench | grep Running | awk '{print $1}') ./test.sh ''' } } stage ('Clean up pgtest pod'){ steps { sh ''' oc delete all -l app=pgbench ''' } } } } patroni-4.0.4/kubernetes/openshift-example/test/README.md000066400000000000000000000002701472010352700232300ustar00rootroot00000000000000# Jenkins Test This pipeline test will create a separate deployment config for a pgbench pod and execute a test against the patroni cluster. This is a sample and should be customized. patroni-4.0.4/kubernetes/patroni_k8s.yaml000066400000000000000000000142061472010352700204730ustar00rootroot00000000000000# headless service to avoid deletion of patronidemo-config endpoint apiVersion: v1 kind: Service metadata: name: patronidemo-config labels: application: patroni cluster-name: patronidemo spec: clusterIP: None --- apiVersion: apps/v1 kind: StatefulSet metadata: name: &cluster_name patronidemo labels: application: patroni cluster-name: *cluster_name spec: replicas: 3 serviceName: *cluster_name selector: matchLabels: application: patroni cluster-name: *cluster_name template: metadata: labels: application: patroni cluster-name: *cluster_name spec: serviceAccountName: patronidemo containers: - name: *cluster_name image: patroni # docker build -t patroni . imagePullPolicy: IfNotPresent readinessProbe: httpGet: scheme: HTTP path: /readiness port: 8008 initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 ports: - containerPort: 8008 protocol: TCP - containerPort: 5432 protocol: TCP volumeMounts: - mountPath: /home/postgres/pgdata name: pgdata env: - name: PATRONI_KUBERNETES_POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: PATRONI_KUBERNETES_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: PATRONI_KUBERNETES_BYPASS_API_SERVICE value: 'true' - name: PATRONI_KUBERNETES_USE_ENDPOINTS value: 'true' - name: PATRONI_KUBERNETES_LABELS value: '{application: patroni, cluster-name: patronidemo}' - name: PATRONI_SUPERUSER_USERNAME value: postgres - name: PATRONI_SUPERUSER_PASSWORD valueFrom: secretKeyRef: name: *cluster_name key: superuser-password - name: PATRONI_REPLICATION_USERNAME value: standby - name: PATRONI_REPLICATION_PASSWORD valueFrom: secretKeyRef: name: *cluster_name key: replication-password - name: PATRONI_SCOPE value: *cluster_name - name: PATRONI_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: PATRONI_POSTGRESQL_DATA_DIR value: /home/postgres/pgdata/pgroot/data - name: PATRONI_POSTGRESQL_PGPASS value: /tmp/pgpass - name: PATRONI_POSTGRESQL_LISTEN value: '0.0.0.0:5432' - name: PATRONI_RESTAPI_LISTEN value: '0.0.0.0:8008' terminationGracePeriodSeconds: 0 volumes: - name: pgdata emptyDir: {} # volumeClaimTemplates: # - metadata: # labels: # application: spilo # spilo-cluster: *cluster_name # annotations: # volume.alpha.kubernetes.io/storage-class: anything # name: pgdata # spec: # accessModes: # - ReadWriteOnce # resources: # requests: # storage: 5Gi --- apiVersion: v1 kind: Endpoints metadata: name: &cluster_name patronidemo labels: application: patroni cluster-name: *cluster_name subsets: [] --- apiVersion: v1 kind: Service metadata: name: &cluster_name patronidemo labels: application: patroni cluster-name: *cluster_name spec: type: ClusterIP ports: - port: 5432 targetPort: 5432 --- apiVersion: v1 kind: Service metadata: name: patronidemo-repl labels: application: patroni cluster-name: &cluster_name patronidemo role: replica spec: type: ClusterIP selector: application: patroni cluster-name: *cluster_name role: replica ports: - port: 5432 targetPort: 5432 --- apiVersion: v1 kind: Secret metadata: name: &cluster_name patronidemo labels: application: patroni cluster-name: *cluster_name type: Opaque data: superuser-password: emFsYW5kbw== replication-password: cmVwLXBhc3M= --- apiVersion: v1 kind: ServiceAccount metadata: name: patronidemo --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: patronidemo rules: - apiGroups: - "" resources: - configmaps verbs: - create - get - list - patch - update - watch # delete and deletecollection are required only for 'patronictl remove' - delete - deletecollection - apiGroups: - "" resources: - endpoints verbs: - get - patch - update # the following three privileges are necessary only when using endpoints - create - list - watch # delete and deletecollection are required only for for 'patronictl remove' - delete - deletecollection - apiGroups: - "" resources: - pods verbs: - get - list - patch - update - watch # The following privilege is only necessary for creation of headless service # for patronidemo-config endpoint, in order to prevent cleaning it up by the # k8s master. You can avoid giving this privilege by explicitly creating the # service like it is done in this manifest (lines 2..10) - apiGroups: - "" resources: - services verbs: - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: patronidemo roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: patronidemo subjects: - kind: ServiceAccount name: patronidemo # Following privileges are only required if deployed not in the "default" # namespace and you want Patroni to bypass kubernetes service # (PATRONI_KUBERNETES_BYPASS_API_SERVICE=true) --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: patroni-k8s-ep-access rules: - apiGroups: - "" resources: - endpoints resourceNames: - kubernetes verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: patroni-k8s-ep-access roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: patroni-k8s-ep-access subjects: - kind: ServiceAccount name: patronidemo # The namespace must be specified explicitly. # If deploying to the different namespace you have to change it. namespace: default patroni-4.0.4/mkbinary.sh000077500000000000000000000001361472010352700153470ustar00rootroot00000000000000#!/bin/sh set -e pip install --ignore-installed pyinstaller pyinstaller --clean patroni.spec patroni-4.0.4/patroni.py000077500000000000000000000001401472010352700152200ustar00rootroot00000000000000#!/usr/bin/env python from patroni.__main__ import main if __name__ == '__main__': main() patroni-4.0.4/patroni.spec000066400000000000000000000020611472010352700155230ustar00rootroot00000000000000# -*- mode: python -*- block_cipher = None def hiddenimports(): import sys sys.path.insert(0, '.') try: import patroni.dcs return patroni.dcs.dcs_modules() + ['http.server'] finally: sys.path.pop(0) def resources(): import os res_dir = 'patroni/postgresql/available_parameters/' exts = set(f.split('.')[-1] for f in os.listdir(res_dir)) return [(res_dir + '*.' + e, res_dir) for e in exts if e.lower() in {'yml', 'yaml'}] a = Analysis(['patroni/__main__.py'], pathex=[], binaries=None, datas=resources(), hiddenimports=hiddenimports(), hookspath=[], runtime_hooks=[], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=False, cipher=block_cipher) pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, a.scripts, a.binaries, a.zipfiles, a.datas, name='patroni', debug=False, strip=False, upx=True, console=True) patroni-4.0.4/patroni/000077500000000000000000000000001472010352700146505ustar00rootroot00000000000000patroni-4.0.4/patroni/__init__.py000066400000000000000000000031201472010352700167550ustar00rootroot00000000000000"""Define general variables and functions for :mod:`patroni`. :var PATRONI_ENV_PREFIX: prefix for Patroni related configuration environment variables. :var KUBERNETES_ENV_PREFIX: prefix for Kubernetes related configuration environment variables. :var MIN_PSYCOPG2: minimum version of :mod:`psycopg2` required by Patroni to work. :var MIN_PSYCOPG3: minimum version of :mod:`psycopg` required by Patroni to work. """ from typing import Iterator, Tuple PATRONI_ENV_PREFIX = 'PATRONI_' KUBERNETES_ENV_PREFIX = 'KUBERNETES_' MIN_PSYCOPG2 = (2, 5, 4) MIN_PSYCOPG3 = (3, 0, 0) def parse_version(version: str) -> Tuple[int, ...]: """Convert *version* from human-readable format to tuple of integers. .. note:: Designed for easy comparison of software versions in Python. :param version: human-readable software version, e.g. ``2.5.4.dev1 (dt dec pq3 ext lo64)``. :returns: tuple of *version* parts, each part as an integer. :Example: >>> parse_version('2.5.4.dev1 (dt dec pq3 ext lo64)') (2, 5, 4) """ def _parse_version(version: str) -> Iterator[int]: """Yield each part of a human-readable version string as an integer. :param version: human-readable software version, e.g. ``2.5.4.dev1``. :yields: each part of *version* as an integer. :Example: >>> tuple(_parse_version('2.5.4.dev1')) (2, 5, 4) """ for e in version.split('.'): try: yield int(e) except ValueError: break return tuple(_parse_version(version.split(' ')[0])) patroni-4.0.4/patroni/__main__.py000066400000000000000000000372611472010352700167530ustar00rootroot00000000000000"""Patroni main entry point. Implement ``patroni`` main daemon and expose its entry point. """ import logging import os import signal import sys import time from argparse import Namespace from typing import Any, Dict, List, Optional, TYPE_CHECKING from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3, parse_version from patroni.daemon import abstract_main, AbstractPatroniDaemon, get_base_arg_parser from patroni.tags import Tags if TYPE_CHECKING: # pragma: no cover from .config import Config from .dcs import Cluster logger = logging.getLogger(__name__) class Patroni(AbstractPatroniDaemon, Tags): """Implement ``patroni`` command daemon. :ivar version: Patroni version. :ivar dcs: DCS object. :ivar watchdog: watchdog handler, if configured to use watchdog. :ivar postgresql: managed Postgres instance. :ivar api: REST API server instance of this node. :ivar request: wrapper for performing HTTP requests. :ivar ha: HA handler. :ivar next_run: time when to run the next HA loop cycle. :ivar scheduled_restart: when a restart has been scheduled to occur, if any. In that case, should contain two keys: * ``schedule``: timestamp when restart should occur; * ``postmaster_start_time``: timestamp when Postgres was last started. """ def __init__(self, config: 'Config') -> None: """Create a :class:`Patroni` instance with the given *config*. Get a connection to the DCS, configure watchdog (if required), set up Patroni interface with Postgres, configure the HA loop and bring the REST API up. .. note:: Expected to be instantiated and run through :func:`~patroni.daemon.abstract_main`. :param config: Patroni configuration. """ from patroni.api import RestApiServer from patroni.dcs import get_dcs from patroni.ha import Ha from patroni.postgresql import Postgresql from patroni.request import PatroniRequest from patroni.version import __version__ from patroni.watchdog import Watchdog super(Patroni, self).__init__(config) self.version = __version__ self.dcs = get_dcs(self.config) self.request = PatroniRequest(self.config, True) cluster = self.ensure_dcs_access() self.ensure_unique_name(cluster) self.watchdog = Watchdog(self.config) self.apply_dynamic_configuration(cluster) self.postgresql = Postgresql(self.config['postgresql'], self.dcs.mpp) self.api = RestApiServer(self, self.config['restapi']) self.ha = Ha(self) self._tags = self._get_tags() self.next_run = time.time() self.scheduled_restart: Dict[str, Any] = {} def ensure_dcs_access(self, sleep_time: int = 5) -> 'Cluster': """Continuously attempt to retrieve cluster from DCS with delay. :param sleep_time: seconds to wait between retry attempts after dcs connection raise :exc:`DCSError`. :returns: a PostgreSQL or MPP implementation of :class:`Cluster`. """ from patroni.exceptions import DCSError while True: try: return self.dcs.get_cluster() except DCSError: logger.warning('Can not get cluster from dcs') time.sleep(sleep_time) def apply_dynamic_configuration(self, cluster: 'Cluster') -> None: """Apply Patroni dynamic configuration. Apply dynamic configuration from the DCS, if `/config` key is available in the DCS, otherwise fall back to ``bootstrap.dcs`` section from the configuration file. .. note:: This method is called only once, at the time when Patroni is started. :param cluster: a PostgreSQL or MPP implementation of :class:`Cluster`. """ if cluster and cluster.config and cluster.config.data: if self.config.set_dynamic_configuration(cluster.config): self.dcs.reload_config(self.config) self.watchdog.reload_config(self.config) elif not self.config.dynamic_configuration and 'bootstrap' in self.config: if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']): self.dcs.reload_config(self.config) self.watchdog.reload_config(self.config) def ensure_unique_name(self, cluster: 'Cluster') -> None: """A helper method to prevent splitbrain from operator naming error. :param cluster: a PostgreSQL or MPP implementation of :class:`Cluster`. """ from patroni.dcs import Member if not cluster: return member = cluster.get_member(self.config['name'], False) if not isinstance(member, Member): return try: # Silence annoying WARNING: Retrying (...) messages when Patroni is quickly restarted. # At this moment we don't have custom log levels configured and hence shouldn't lose anything useful. self.logger.update_loggers({'urllib3.connectionpool': 'ERROR'}) _ = self.request(member, endpoint="/liveness", timeout=3) logger.fatal("Can't start; there is already a node named '%s' running", self.config['name']) sys.exit(1) except Exception: self.logger.update_loggers({}) def _get_tags(self) -> Dict[str, Any]: """Get tags configured for this node, if any. :returns: a dictionary of tags set for this node. """ return self._filter_tags(self.config.get('tags', {})) def reload_config(self, sighup: bool = False, local: Optional[bool] = False) -> None: """Apply new configuration values for ``patroni`` daemon. Reload: * Cached tags; * Request wrapper configuration; * REST API configuration; * Watchdog configuration; * Postgres configuration; * DCS configuration. :param sighup: if it is related to a SIGHUP signal. :param local: if there has been changes to the local configuration file. """ try: super(Patroni, self).reload_config(sighup, local) if local: self._tags = self._get_tags() self.request.reload_config(self.config) if local or sighup and self.api.reload_local_certificate(): self.api.reload_config(self.config['restapi']) self.watchdog.reload_config(self.config) self.postgresql.reload_config(self.config['postgresql'], sighup) self.dcs.reload_config(self.config) except Exception: logger.exception('Failed to reload config_file=%s', self.config.config_file) @property def tags(self) -> Dict[str, Any]: """Tags configured for this node, if any.""" return self._tags def schedule_next_run(self) -> None: """Schedule the next run of the ``patroni`` daemon main loop. Next run is scheduled based on previous run plus value of ``loop_wait`` configuration from DCS. If that has already been exceeded, run the next cycle immediately. """ self.next_run += self.dcs.loop_wait current_time = time.time() nap_time = self.next_run - current_time if nap_time <= 0: self.next_run = current_time # Release the GIL so we don't starve anyone waiting on async_executor lock time.sleep(0.001) # Warn user that Patroni is not keeping up logger.warning("Loop time exceeded, rescheduling immediately.") elif self.ha.watch(nap_time): self.next_run = time.time() def run(self) -> None: """Run ``patroni`` daemon process main loop. Start the REST API and keep running HA cycles every ``loop_wait`` seconds. """ self.api.start() self.next_run = time.time() super(Patroni, self).run() def _run_cycle(self) -> None: """Run a cycle of the ``patroni`` daemon main loop. Run an HA cycle and schedule the next cycle run. If any dynamic configuration change request is detected, apply the change and cache the new dynamic configuration values in ``patroni.dynamic.json`` file under Postgres data directory. """ logger.info(self.ha.run_cycle()) if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \ and self.config.set_dynamic_configuration(self.dcs.cluster.config): self.reload_config() if self.postgresql.role != 'uninitialized': self.config.save_cache() self.schedule_next_run() def _shutdown(self) -> None: """Perform shutdown of ``patroni`` daemon process. Shut down the REST API and the HA handler. """ try: self.api.shutdown() except Exception: logger.exception('Exception during RestApi.shutdown') try: self.ha.shutdown() except Exception: logger.exception('Exception during Ha.shutdown') def patroni_main(configfile: str) -> None: """Configure and start ``patroni`` main daemon process. :param configfile: path to Patroni configuration file. """ abstract_main(Patroni, configfile) def process_arguments() -> Namespace: """Process command-line arguments. Create a basic command-line parser through :func:`~patroni.daemon.get_base_arg_parser`, extend its capabilities by adding these flags and parse command-line arguments.: * ``--validate-config`` -- used to validate the Patroni configuration file * ``--generate-config`` -- used to generate Patroni configuration from a running PostgreSQL instance * ``--generate-sample-config`` -- used to generate a sample Patroni configuration * ``--ignore-listen-port`` | ``-i`` -- used to ignore ``listen`` ports already in use. Can be used only with ``--validate-config`` .. note:: If running with ``--generate-config``, ``--generate-sample-config`` or ``--validate-flag`` will exit after generating or validating configuration. :returns: parsed arguments, if not running with ``--validate-config`` flag. """ from patroni.config_generator import generate_config parser = get_base_arg_parser() group = parser.add_mutually_exclusive_group() group.add_argument('--validate-config', action='store_true', help='Run config validator and exit') group.add_argument('--generate-sample-config', action='store_true', help='Generate a sample Patroni yaml configuration file') group.add_argument('--generate-config', action='store_true', help='Generate a Patroni yaml configuration file for a running instance') parser.add_argument('--dsn', help='Optional DSN string of the instance to be used as a source \ for config generation. Superuser connection is required.') parser.add_argument('--ignore-listen-port', '-i', action='store_true', help='Ignore `listen` ports already in use.\ Can only be used with --validate-config') args = parser.parse_args() if args.generate_sample_config: generate_config(args.configfile, True, None) sys.exit(0) elif args.generate_config: generate_config(args.configfile, False, args.dsn) sys.exit(0) elif args.validate_config: from patroni.config import Config, ConfigParseError from patroni.validator import populate_validate_params, schema populate_validate_params(ignore_listen_port=args.ignore_listen_port) try: Config(args.configfile, validator=schema) sys.exit() except ConfigParseError as e: sys.exit(e.value) return args def check_psycopg() -> None: """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment. .. note:: Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible. If nothing meeting the requirements is found, then exit with a fatal message. """ min_psycopg2_str = '.'.join(map(str, MIN_PSYCOPG2)) min_psycopg3_str = '.'.join(map(str, MIN_PSYCOPG3)) available_versions: List[str] = [] # try psycopg2 try: from psycopg2 import __version__ if parse_version(__version__) >= MIN_PSYCOPG2: return available_versions.append('psycopg2=={0}'.format(__version__.split(' ')[0])) except ImportError: logger.debug('psycopg2 module is not available') # try psycopg3 try: from psycopg import __version__ if parse_version(__version__) >= MIN_PSYCOPG3: return available_versions.append('psycopg=={0}'.format(__version__.split(' ')[0])) except ImportError: logger.debug('psycopg module is not available') error = f'FATAL: Patroni requires psycopg2>={min_psycopg2_str}, psycopg2-binary, or psycopg>={min_psycopg3_str}' if available_versions: error += ', but only {0} {1} available'.format( ' and '.join(available_versions), 'is' if len(available_versions) == 1 else 'are') sys.exit(error) def main() -> None: """Main entrypoint of :mod:`patroni.__main__`. Process command-line arguments, ensure :mod:`psycopg2` (or :mod:`psycopg`) attendee the pre-requisites and start ``patroni`` daemon process. .. note:: If running through a Docker container, make the main process take care of init process duties and run ``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded to ``patroni`` daemon process. """ from multiprocessing import freeze_support # Executables created by PyInstaller are frozen, thus we need to enable frozen support for # :mod:`multiprocessing` to avoid :class:`RuntimeError` exceptions. freeze_support() check_psycopg() args = process_arguments() if os.getpid() != 1: return patroni_main(args.configfile) # Patroni started with PID=1, it looks like we are in the container from types import FrameType pid = 0 # Looks like we are in a docker, so we will act like init def sigchld_handler(signo: int, stack_frame: Optional[FrameType]) -> None: """Handle ``SIGCHLD`` received by main process from ``patroni`` daemon when the daemon terminates. :param signo: signal number. :param stack_frame: current stack frame. """ try: # log exit code of all children processes, and break loop when there is none left while True: ret = os.waitpid(-1, os.WNOHANG) if ret == (0, 0): break elif ret[0] != pid: logger.info('Reaped pid=%s, exit status=%s', *ret) except OSError: pass def passtochild(signo: int, stack_frame: Optional[FrameType]) -> None: """Forward a signal *signo* from main process to child process. :param signo: signal number. :param stack_frame: current stack frame. """ if pid: os.kill(pid, signo) if os.name != 'nt': signal.signal(signal.SIGCHLD, sigchld_handler) signal.signal(signal.SIGHUP, passtochild) signal.signal(signal.SIGQUIT, passtochild) signal.signal(signal.SIGUSR1, passtochild) signal.signal(signal.SIGUSR2, passtochild) signal.signal(signal.SIGINT, passtochild) signal.signal(signal.SIGABRT, passtochild) signal.signal(signal.SIGTERM, passtochild) import multiprocessing patroni = multiprocessing.Process(target=patroni_main, args=(args.configfile,)) patroni.start() pid = patroni.pid patroni.join() if __name__ == '__main__': main() patroni-4.0.4/patroni/api.py000066400000000000000000002514351472010352700160050ustar00rootroot00000000000000"""Implement Patroni's REST API. Exposes a REST API of patroni operations functions, such as status, performance and management to web clients. Much of what can be achieved with the command line tool patronictl can be done via the API. Patroni CLI and daemon utilises the API to perform these functions. """ import base64 import datetime import hmac import json import logging import os import socket import sys import time import traceback from http.server import BaseHTTPRequestHandler, HTTPServer from ipaddress import ip_address, ip_network, IPv4Network, IPv6Network from socketserver import ThreadingMixIn from threading import Thread from typing import Any, Callable, cast, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union from urllib.parse import parse_qs, urlparse import dateutil.parser from . import global_config, psycopg from .__main__ import Patroni from .dcs import Cluster from .exceptions import PostgresConnectionException, PostgresException from .postgresql.misc import postgres_version_to_int from .utils import cluster_as_json, deep_compare, enable_keepalive, parse_bool, \ parse_int, patch_config, Retry, RetryFailedError, split_host_port, tzutc, uri logger = logging.getLogger(__name__) def check_access(*args: Any, **kwargs: Any) -> Callable[..., Any]: """Check the source ip, authorization header, or client certificates. .. note:: The actual logic to check access is implemented through :func:`RestApiServer.check_access`. Optionally it is possible to skip source ip check by specifying ``allowlist_check_members=False``. :returns: a decorator that executes *func* only if :func:`RestApiServer.check_access` returns ``True``. :Example: >>> class FooServer: ... def check_access(self, *args, **kwargs): ... print(f'In FooServer: {args[0].__class__.__name__}') ... return True ... >>> class Foo: ... server = FooServer() ... @check_access ... def do_PUT_foo(self): ... print('In do_PUT_foo') ... @check_access(allowlist_check_members=False) ... def do_POST_bar(self): ... print('In do_POST_bar') >>> f = Foo() >>> f.do_PUT_foo() In FooServer: Foo In do_PUT_foo """ allowlist_check_members = kwargs.get('allowlist_check_members', True) def inner_decorator(func: Callable[..., Any]) -> Callable[..., Any]: def wrapper(self: 'RestApiHandler', *args: Any, **kwargs: Any) -> Any: if self.server.check_access(self, allowlist_check_members=allowlist_check_members): return func(self, *args, **kwargs) return wrapper # A hacky way to have decorators that work with and without parameters. if len(args) == 1 and callable(args[0]): # The first parameter is a function, it means decorator is used as "@check_access" return inner_decorator(args[0]) else: # @check_access(allowlist_check_members=False) case return inner_decorator class RestApiHandler(BaseHTTPRequestHandler): """Define how to handle each of the requests that are made against the REST API server.""" # Comment from pyi stub file. These unions can cause typing errors with IDEs, e.g. PyCharm # # Those are technically of types, respectively: # * _RequestType = Union[socket.socket, Tuple[bytes, socket.socket]] # * _AddressType = Tuple[str, int] # But there are some concerns that having unions here would cause # too much inconvenience to people using it (see # https://github.com/python/typeshed/pull/384#issuecomment-234649696) def __init__(self, request: Any, client_address: Any, server: Union['RestApiServer', HTTPServer]) -> None: """Create a :class:`RestApiHandler` instance. .. note:: Currently not different from its superclass :func:`__init__`, and only used so ``pyright`` can understand the type of ``server`` attribute. :param request: client request to be processed. :param client_address: address of the client connection. :param server: HTTP server that received the request. """ if TYPE_CHECKING: # pragma: no cover assert isinstance(server, RestApiServer) super(RestApiHandler, self).__init__(request, client_address, server) self.server: 'RestApiServer' = server # pyright: ignore [reportIncompatibleVariableOverride] self.__start_time: float = 0.0 self.path_query: Dict[str, List[str]] = {} def _write_status_code_only(self, status_code: int) -> None: """Write a response that is composed only of the HTTP status. The response is written with these values separated by space: * HTTP protocol version; * *status_code*; * description of *status_code*. .. note:: This is usually useful for replying to requests from software like HAProxy. :param status_code: HTTP status code. :Example: * ``_write_status_code_only(200)`` would write a response like ``HTTP/1.0 200 OK``. """ message = self.responses[status_code][0] self.wfile.write('{0} {1} {2}\r\n\r\n'.format(self.protocol_version, status_code, message).encode('utf-8')) self.log_request(status_code) def write_response(self, status_code: int, body: str, content_type: str = 'text/html', headers: Optional[Dict[str, str]] = None) -> None: """Write an HTTP response. .. note:: Besides ``Content-Type`` header, and the HTTP headers passed through *headers*, this function will also write the HTTP headers defined through ``restapi.http_extra_headers`` and ``restapi.https_extra_headers`` from Patroni configuration. :param status_code: response HTTP status code. :param body: response body. :param content_type: value for ``Content-Type`` HTTP header. :param headers: dictionary of additional HTTP headers to set for the response. Each key is the header name, and the corresponding value is the value for the header in the response. """ # TODO: try-catch ConnectionResetError: [Errno 104] Connection reset by peer and log it in DEBUG level self.send_response(status_code) headers = headers or {} if content_type: headers['Content-Type'] = content_type for name, value in headers.items(): self.send_header(name, value) for name, value in (self.server.http_extra_headers or {}).items(): self.send_header(name, value) self.end_headers() self.wfile.write(body.encode('utf-8')) def _write_json_response(self, status_code: int, response: Any) -> None: """Write an HTTP response with a JSON content type. Call :func:`write_response` with ``content_type`` as ``application/json``. :param status_code: response HTTP status code. :param response: value to be dumped as a JSON string and to be used as the response body. """ self.write_response(status_code, json.dumps(response, default=str), content_type='application/json') def _write_status_response(self, status_code: int, response: Dict[str, Any]) -> None: """Write an HTTP response with Patroni/Postgres status in JSON format. Modifies *response* before sending it to the client. Defines the ``patroni`` key, which is a dictionary that contains the mandatory keys: * ``version``: Patroni version, e.g. ``3.0.2``; * ``scope``: value of ``scope`` setting from Patroni configuration. May also add the following optional keys, depending on the status of this Patroni/PostgreSQL node: * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags; * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output; * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted; * ``pending_restart_reason``: dictionary where each key is the parameter that caused "pending restart" flag to be set and the value is a dictionary with the old and the new value. * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the scheduled restart; * ``watchdog_failed``: ``True`` if watchdog device is unhealthy; * ``logger_queue_size``: log queue length if it is longer than expected; * ``logger_records_lost``: number of log records that have been lost while the log queue was full. :param status_code: response HTTP status code. :param response: represents the status of the PostgreSQL node, and is used as a basis for the HTTP response. This dictionary is built through :func:`get_postgresql_status`. """ patroni = self.server.patroni tags = patroni.ha.get_effective_tags() if tags: response['tags'] = tags if patroni.postgresql.sysid: response['database_system_identifier'] = patroni.postgresql.sysid if patroni.postgresql.pending_restart_reason: response['pending_restart'] = True response['pending_restart_reason'] = dict(patroni.postgresql.pending_restart_reason) response['patroni'] = { 'version': patroni.version, 'scope': patroni.postgresql.scope, 'name': patroni.postgresql.name } if patroni.scheduled_restart: response['scheduled_restart'] = patroni.scheduled_restart.copy() del response['scheduled_restart']['postmaster_start_time'] response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat() if not patroni.ha.watchdog.is_healthy: response['watchdog_failed'] = True qsize = patroni.logger.queue_size if qsize > patroni.logger.NORMAL_LOG_QUEUE_SIZE: response['logger_queue_size'] = qsize lost = patroni.logger.records_lost if lost: response['logger_records_lost'] = lost self._write_json_response(status_code, response) def do_GET(self, write_status_code_only: bool = False) -> None: """Process all GET requests which can not be routed to other methods. Is used for handling all health-checks requests. E.g. "GET /(primary|replica|sync|async|etc...)". The (optional) query parameters and the HTTP response status depend on the requested path: * ``/``, ``primary``, or ``read-write``: * HTTP status ``200``: if a primary with the leader lock. * ``/standby-leader``: * HTTP status ``200``: if holds the leader lock in a standby cluster. * ``/leader``: * HTTP status ``200``: if holds the leader lock. * ``/replica``: * Query parameters: * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g. ``10MB``). * Any custom parameter: will attempt to match them against node tags. * HTTP status ``200``: if up and running as a standby and without ``noloadbalance`` tag. * ``/read-only``: * HTTP status ``200``: if up and running and without ``noloadbalance`` tag. * ``/quorum``: * HTTP status ``200``: if up and running as a quorum synchronous standby. * ``/read-only-quorum``: * HTTP status ``200``: if up and running as a quorum synchronous standby or primary. * ``/synchronous`` or ``/sync``: * HTTP status ``200``: if up and running as a synchronous standby. * ``/read-only-sync``: * HTTP status ``200``: if up and running as a synchronous standby or primary. * ``/asynchronous``: * Query parameters: * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g. ``10MB``). * HTTP status ``200``: if up and running as an asynchronous standby. * ``/health``: * HTTP status ``200``: if up and running. .. note:: If not able to honor the query parameter, or not able to match the condition described for HTTP status ``200`` in each path above, then HTTP status will be ``503``. .. note:: Independently of the requested path, if *write_status_code_only* is ``False``, then it always write an HTTP response through :func:`_write_status_response`, with the node status. :param write_status_code_only: indicates that instead of a normal HTTP response we should send only the HTTP Status Code and close the connection. Useful when health-checks are executed by HAProxy. """ path = '/primary' if self.path == '/' else self.path response = self.get_postgresql_status() patroni = self.server.patroni cluster = patroni.dcs.cluster config = global_config.from_cluster(cluster) leader_optime = cluster and cluster.status.last_lsn replayed_location = response.get('xlog', {}).get('replayed_location', 0) max_replica_lag = parse_int(self.path_query.get('lag', [sys.maxsize])[0], 'B') if max_replica_lag is None: max_replica_lag = sys.maxsize is_lagging = leader_optime and leader_optime > replayed_location + max_replica_lag replica_status_code = 200 if not patroni.noloadbalance and not is_lagging and \ response.get('role') == 'replica' and response.get('state') == 'running' else 503 if not cluster and response.get('pause'): leader_status_code = 200 if response.get('role') in ('primary', 'standby_leader') else 503 primary_status_code = 200 if response.get('role') == 'primary' else 503 standby_leader_status_code = 200 if response.get('role') == 'standby_leader' else 503 elif patroni.ha.is_leader(): leader_status_code = 200 if config.is_standby_cluster: primary_status_code = replica_status_code = 503 standby_leader_status_code = 200 if response.get('role') in ('replica', 'standby_leader') else 503 else: primary_status_code = 200 standby_leader_status_code = 503 else: leader_status_code = primary_status_code = standby_leader_status_code = 503 status_code = 503 ignore_tags = False if 'standby_leader' in path or 'standby-leader' in path: status_code = standby_leader_status_code ignore_tags = True elif 'leader' in path: status_code = leader_status_code ignore_tags = True elif 'master' in path or 'primary' in path or 'read-write' in path: status_code = primary_status_code ignore_tags = True elif 'replica' in path: status_code = replica_status_code elif 'read-only' in path and 'sync' not in path and 'quorum' not in path: status_code = 200 if 200 in (primary_status_code, standby_leader_status_code) else replica_status_code elif 'health' in path: status_code = 200 if response.get('state') == 'running' else 503 elif cluster: # dcs is available is_quorum = response.get('quorum_standby') is_synchronous = response.get('sync_standby') if path in ('/sync', '/synchronous') and is_synchronous: status_code = replica_status_code elif path == '/quorum' and is_quorum: status_code = replica_status_code elif path in ('/async', '/asynchronous') and not is_synchronous and not is_quorum: status_code = replica_status_code elif path == '/read-only-quorum': if 200 in (primary_status_code, standby_leader_status_code): status_code = 200 elif is_quorum: status_code = replica_status_code elif path in ('/read-only-sync', '/read-only-synchronous'): if 200 in (primary_status_code, standby_leader_status_code): status_code = 200 elif is_synchronous: status_code = replica_status_code # check for user defined tags in query params if not ignore_tags and status_code == 200: qs_tag_prefix = "tag_" for qs_key, qs_value in self.path_query.items(): if not qs_key.startswith(qs_tag_prefix): continue qs_key = qs_key[len(qs_tag_prefix):] qs_value = qs_value[0] instance_tag_value = patroni.tags.get(qs_key) # tag not registered for instance if instance_tag_value is None: status_code = 503 break if not isinstance(instance_tag_value, str): instance_tag_value = str(instance_tag_value).lower() if instance_tag_value != qs_value: status_code = 503 break if write_status_code_only: # when haproxy sends OPTIONS request it reads only status code and nothing more self._write_status_code_only(status_code) else: self._write_status_response(status_code, response) def do_OPTIONS(self) -> None: """Handle an ``OPTIONS`` request. Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or ``503 Service Unavailable`` as a response and nothing more, particularly no headers. """ self.do_GET(write_status_code_only=True) def do_HEAD(self) -> None: """Handle a ``HEAD`` request. Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or ``503 Service Unavailable`` as a response and nothing more, particularly no headers. """ self.do_GET(write_status_code_only=True) def do_GET_liveness(self) -> None: """Handle a ``GET`` request to ``/liveness`` path. Write a simple HTTP response with HTTP status: * ``200``: * If the cluster is in maintenance mode; or * If Patroni heartbeat loop is properly running; * ``503``: * if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the value of ``ttl`` on a replica). """ patroni: Patroni = self.server.patroni is_primary = patroni.postgresql.role == 'primary' and patroni.postgresql.is_running() # We can tolerate Patroni problems longer on the replica. # On the primary the liveness probe most likely will start failing only after the leader key expired. # It should not be a big problem because replicas will see that the primary is still alive via REST API call. liveness_threshold = patroni.dcs.ttl * (1 if is_primary else 2) # In maintenance mode (pause) we are fine if heartbeat loop stuck. status_code = 200 if patroni.ha.is_paused() or patroni.next_run + liveness_threshold > time.time() else 503 self._write_status_code_only(status_code) def do_GET_readiness(self) -> None: """Handle a ``GET`` request to ``/readiness`` path. Write a simple HTTP response which HTTP status can be: * ``200``: * If this Patroni node holds the DCS leader lock; or * If this PostgreSQL instance is up and running; * ``503``: if none of the previous conditions apply. """ patroni = self.server.patroni if patroni.ha.is_leader(): status_code = 200 elif patroni.postgresql.state == 'running': status_code = 200 if patroni.dcs.cluster else 503 else: status_code = 503 self._write_status_code_only(status_code) def do_GET_patroni(self) -> None: """Handle a ``GET`` request to ``/patroni`` path. Write an HTTP response through :func:`_write_status_response`, with HTTP status ``200`` and the status of Postgres. """ response = self.get_postgresql_status(True) self._write_status_response(200, response) def do_GET_cluster(self) -> None: """Handle a ``GET`` request to ``/cluster`` path. Write an HTTP response with JSON content based on the output of :func:`~patroni.utils.cluster_as_json`, with HTTP status ``200`` and the JSON representation of the cluster topology. """ cluster = self.server.patroni.dcs.get_cluster() response = cluster_as_json(cluster) response['scope'] = self.server.patroni.postgresql.scope self._write_json_response(200, response) def do_GET_history(self) -> None: """Handle a ``GET`` request to ``/history`` path. Write an HTTP response with a JSON content representing the history of events in the cluster, with HTTP status ``200``. The response contains a :class:`list` of failover/switchover events. Each item is a :class:`list` with the following items: * Timeline when the event occurred (class:`int`); * LSN at which the event occurred (class:`int`); * The reason for the event (class:`str`); * Timestamp when the new timeline was created (class:`str`); * Name of the involved Patroni node (class:`str`). """ cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster() self._write_json_response(200, cluster.history and cluster.history.lines or []) def do_GET_config(self) -> None: """Handle a ``GET`` request to ``/config`` path. Write an HTTP response with a JSON content representing the Patroni configuration that is stored in the DCS, with HTTP status ``200``. If the cluster information is not available in the DCS, then it will respond with no body and HTTP status ``502`` instead. """ cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster() if cluster.config: self._write_json_response(200, cluster.config.data) else: self.send_error(502) def do_GET_metrics(self) -> None: """Handle a ``GET`` request to ``/metrics`` path. Write an HTTP response with plain text content in the format used by Prometheus, with HTTP status ``200``. The response contains the following items: * ``patroni_version``: Patroni version without periods, e.g. ``030002`` for Patroni ``3.0.2``; * ``patroni_postgres_running``: ``1`` if PostgreSQL is running, else ``0``; * ``patroni_postmaster_start_time``: epoch timestamp since Postmaster was started; * ``patroni_primary``: ``1`` if this node holds the leader lock, else ``0``; * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_flush_lsn(), '0/0')`` if leader, else ``0``; * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``; * ``patroni_replica``: ``1`` if a replica, else ``0``; * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``; * ``patroni_quorum_standby``: ``1`` if a quorum sync replica, else ``0``; * ``patroni_xlog_received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``; * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``; * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``; * ``patroni_xlog_paused``: ``pg_is_wal_replay_paused()``; * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``; * ``patroni_cluster_unlocked``: ``1`` if no one holds the leader lock, else ``0``; * ``patroni_failsafe_mode_is_active``: ``1`` if ``failsafe_mode`` is currently active, else ``0``; * ``patroni_postgres_timeline``: PostgreSQL timeline based on current WAL file name; * ``patroni_dcs_last_seen``: epoch timestamp when DCS was last contacted successfully; * ``patroni_pending_restart``: ``1`` if this PostgreSQL node is pending a restart, else ``0``; * ``patroni_is_paused``: ``1`` if Patroni is in maintenance node, else ``0``. For PostgreSQL v9.6+ the response will also have the following: * ``patroni_postgres_streaming``: 1 if Postgres is streaming from another node, else ``0``; * ``patroni_postgres_in_archive_recovery``: ``1`` if Postgres isn't streaming and there is ``restore_command`` available, else ``0``. """ postgres = self.get_postgresql_status(True) patroni = self.server.patroni epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc) metrics: List[str] = [] labels = f'{{scope="{patroni.postgresql.scope}",name="{patroni.postgresql.name}"}}' metrics.append("# HELP patroni_version Patroni semver without periods.") metrics.append("# TYPE patroni_version gauge") padded_semver = ''.join([x.zfill(2) for x in patroni.version.split('.')]) # 2.0.2 => 020002 metrics.append("patroni_version{0} {1}".format(labels, padded_semver)) metrics.append("# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.") metrics.append("# TYPE patroni_postgres_running gauge") metrics.append("patroni_postgres_running{0} {1}".format(labels, int(postgres['state'] == 'running'))) metrics.append("# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.") metrics.append("# TYPE patroni_postmaster_start_time gauge") postmaster_start_time = postgres.get('postmaster_start_time') postmaster_start_time = (postmaster_start_time - epoch).total_seconds() if postmaster_start_time else 0 metrics.append("patroni_postmaster_start_time{0} {1}".format(labels, postmaster_start_time)) metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.") metrics.append("# TYPE patroni_primary gauge") metrics.append("patroni_primary{0} {1}".format(labels, int(postgres['role'] == 'primary'))) metrics.append("# HELP patroni_xlog_location Current location of the Postgres" " transaction log, 0 if this node is not the leader.") metrics.append("# TYPE patroni_xlog_location counter") metrics.append("patroni_xlog_location{0} {1}".format(labels, postgres.get('xlog', {}).get('location', 0))) metrics.append("# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.") metrics.append("# TYPE patroni_standby_leader gauge") metrics.append("patroni_standby_leader{0} {1}".format(labels, int(postgres['role'] == 'standby_leader'))) metrics.append("# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.") metrics.append("# TYPE patroni_replica gauge") metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica'))) metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby, 0 otherwise.") metrics.append("# TYPE patroni_sync_standby gauge") metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False)))) metrics.append("# HELP patroni_quorum_standby Value is 1 if this node is a quorum standby, 0 otherwise.") metrics.append("# TYPE patroni_quorum_standby gauge") metrics.append("patroni_quorum_standby{0} {1}".format(labels, int(postgres.get('quorum_standby', False)))) metrics.append("# HELP patroni_xlog_received_location Current location of the received" " Postgres transaction log, 0 if this node is not a replica.") metrics.append("# TYPE patroni_xlog_received_location counter") metrics.append("patroni_xlog_received_location{0} {1}" .format(labels, postgres.get('xlog', {}).get('received_location', 0))) metrics.append("# HELP patroni_xlog_replayed_location Current location of the replayed" " Postgres transaction log, 0 if this node is not a replica.") metrics.append("# TYPE patroni_xlog_replayed_location counter") metrics.append("patroni_xlog_replayed_location{0} {1}" .format(labels, postgres.get('xlog', {}).get('replayed_location', 0))) metrics.append("# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed" " Postgres transaction log, 0 if null.") metrics.append("# TYPE patroni_xlog_replayed_timestamp gauge") replayed_timestamp = postgres.get('xlog', {}).get('replayed_timestamp') replayed_timestamp = (replayed_timestamp - epoch).total_seconds() if replayed_timestamp else 0 metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(labels, replayed_timestamp)) metrics.append("# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.") metrics.append("# TYPE patroni_xlog_paused gauge") metrics.append("patroni_xlog_paused{0} {1}" .format(labels, int(postgres.get('xlog', {}).get('paused', False) is True))) if postgres.get('server_version', 0) >= 90600: metrics.append("# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.") metrics.append("# TYPE patroni_postgres_streaming gauge") metrics.append("patroni_postgres_streaming{0} {1}" .format(labels, int(postgres.get('replication_state') == 'streaming'))) metrics.append("# HELP patroni_postgres_in_archive_recovery Value is 1" " if Postgres is replicating from archive, 0 otherwise.") metrics.append("# TYPE patroni_postgres_in_archive_recovery gauge") metrics.append("patroni_postgres_in_archive_recovery{0} {1}" .format(labels, int(postgres.get('replication_state') == 'in archive recovery'))) metrics.append("# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.") metrics.append("# TYPE patroni_postgres_server_version gauge") metrics.append("patroni_postgres_server_version {0} {1}".format(labels, postgres.get('server_version', 0))) metrics.append("# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.") metrics.append("# TYPE patroni_cluster_unlocked gauge") metrics.append("patroni_cluster_unlocked{0} {1}".format(labels, int(postgres.get('cluster_unlocked', 0)))) metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if failsafe mode is active, 0 if inactive.") metrics.append("# TYPE patroni_failsafe_mode_is_active gauge") metrics.append("patroni_failsafe_mode_is_active{0} {1}" .format(labels, int(postgres.get('failsafe_mode_is_active', 0)))) metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.") metrics.append("# TYPE patroni_postgres_timeline counter") metrics.append("patroni_postgres_timeline{0} {1}".format(labels, postgres.get('timeline') or 0)) metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully" " by Patroni.") metrics.append("# TYPE patroni_dcs_last_seen gauge") metrics.append("patroni_dcs_last_seen{0} {1}".format(labels, postgres.get('dcs_last_seen', 0))) metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.") metrics.append("# TYPE patroni_pending_restart gauge") metrics.append("patroni_pending_restart{0} {1}" .format(labels, int(bool(patroni.postgresql.pending_restart_reason)))) metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.") metrics.append("# TYPE patroni_is_paused gauge") metrics.append("patroni_is_paused{0} {1}".format(labels, int(postgres.get('pause', 0)))) self.write_response(200, '\n'.join(metrics) + '\n', content_type='text/plain') def _read_json_content(self, body_is_optional: bool = False) -> Optional[Dict[Any, Any]]: """Read JSON from HTTP request body. .. note:: Retrieves the request body based on `content-length` HTTP header. The body is expected to be a JSON string with that length. If request body is expected but `content-length` HTTP header is absent, then write an HTTP response with HTTP status ``411``. If request body is expected but contains nothing, or if an exception is faced, then write an HTTP response with HTTP status ``400``. :param body_is_optional: if ``False`` then the request must contain a body. If ``True``, then the request may or may not contain a body. :returns: deserialized JSON string from request body, if present. If body is absent, but *body_is_optional* is ``True``, then return an empty dictionary. Returns ``None`` otherwise. """ if 'content-length' not in self.headers: return self.send_error(411) if not body_is_optional else {} try: content_length = int(self.headers.get('content-length') or 0) if content_length == 0 and body_is_optional: return {} request = json.loads(self.rfile.read(content_length).decode('utf-8')) if isinstance(request, dict) and (request or body_is_optional): return cast(Dict[str, Any], request) except Exception: logger.exception('Bad request') self.send_error(400) @check_access def do_PATCH_config(self) -> None: """Handle a ``PATCH`` request to ``/config`` path. Updates the Patroni configuration based on the JSON request body, then writes a response with the new configuration, with HTTP status ``200``. .. note:: If the configuration has been previously wiped out from DCS, then write a response with HTTP status ``503``. If applying a configuration value fails, then write a response with HTTP status ``409``. """ request = self._read_json_content() if request: cluster = self.server.patroni.dcs.get_cluster() if not (cluster.config and cluster.config.modify_version): return self.send_error(503) data = cluster.config.data.copy() if patch_config(data, request): value = json.dumps(data, separators=(',', ':')) if not self.server.patroni.dcs.set_config_value(value, cluster.config.version): return self.send_error(409) self.server.patroni.ha.wakeup() self._write_json_response(200, data) @check_access def do_PUT_config(self) -> None: """Handle a ``PUT`` request to ``/config`` path. Overwrites the Patroni configuration based on the JSON request body, then writes a response with the new configuration, with HTTP status ``200``. .. note:: If applying the new configuration fails, then write a response with HTTP status ``502``. """ request = self._read_json_content() if request: cluster = self.server.patroni.dcs.get_cluster() if not (cluster.config and deep_compare(request, cluster.config.data)): value = json.dumps(request, separators=(',', ':')) if not self.server.patroni.dcs.set_config_value(value): return self.send_error(502) self._write_json_response(200, request) @check_access def do_POST_reload(self) -> None: """Handle a ``POST`` request to ``/reload`` path. Schedules a reload to Patroni and writes a response with HTTP status ``202``. """ self.server.patroni.sighup_handler() self.write_response(202, 'reload scheduled') def do_GET_failsafe(self) -> None: """Handle a ``GET`` request to ``/failsafe`` path. Writes a response with a JSON string body containing all nodes that are known to Patroni at a given point in time, with HTTP status ``200``. The JSON contains a dictionary, each key is the name of the Patroni node, and the corresponding value is the URI to access `/patroni` path of its REST API. .. note:: If ``failsafe_mode`` is not enabled, then write a response with HTTP status ``502``. """ failsafe = self.server.patroni.dcs.failsafe if isinstance(failsafe, dict): self._write_json_response(200, failsafe) else: self.send_error(502) @check_access(allowlist_check_members=False) def do_POST_failsafe(self) -> None: """Handle a ``POST`` request to ``/failsafe`` path. Writes a response with HTTP status ``200`` if this node is a Standby, or with HTTP status ``500`` if this is the primary. In addition to that it returns absolute value of received/replayed LSN in the ``lsn`` header. .. note:: If ``failsafe_mode`` is not enabled, then write a response with HTTP status ``502``. """ if self.server.patroni.ha.is_failsafe_mode(): request = self._read_json_content() if request: ret = self.server.patroni.ha.update_failsafe(request) headers = {'lsn': str(ret)} if isinstance(ret, int) else {} message = ret if isinstance(ret, str) else 'Accepted' code = 200 if message == 'Accepted' else 500 self.write_response(code, message, headers=headers) else: self.send_error(502) @check_access def do_POST_sigterm(self) -> None: """Handle a ``POST`` request to ``/sigterm`` path. Schedule a shutdown and write a response with HTTP status ``202``. .. note:: Only for behave testing on Windows. """ if os.name == 'nt' and os.getenv('BEHAVE_DEBUG'): self.server.patroni.api_sigterm() self.write_response(202, 'shutdown scheduled') @staticmethod def parse_schedule(schedule: str, action: str) -> Tuple[Union[int, None], Union[str, None], Union[datetime.datetime, None]]: """Parse the given *schedule* and validate it. :param schedule: a string representing a timestamp, e.g. ``2023-04-14T20:27:00+00:00``. :param action: the action to be scheduled (``restart``, ``switchover``, or ``failover``). :returns: a tuple composed of 3 items: * Suggested HTTP status code for a response: * ``None``: if no issue was faced while parsing, leaving it up to the caller to decide the status; or * ``400``: if no timezone information could be found in *schedule*; or * ``422``: if *schedule* is invalid -- in the past or not parsable. * An error message, if any error is faced, otherwise ``None``; * Parsed *schedule*, if able to parse, otherwise ``None``. """ error = None scheduled_at = None try: scheduled_at = dateutil.parser.parse(schedule) if scheduled_at.tzinfo is None: error = 'Timezone information is mandatory for the scheduled {0}'.format(action) status_code = 400 elif scheduled_at < datetime.datetime.now(tzutc): error = 'Cannot schedule {0} in the past'.format(action) status_code = 422 else: status_code = None except (ValueError, TypeError): logger.exception('Invalid scheduled %s time: %s', action, schedule) error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' status_code = 422 return status_code, error, scheduled_at @check_access def do_POST_restart(self) -> None: """Handle a ``POST`` request to ``/restart`` path. Used to restart postgres (or schedule a restart), mainly by ``patronictl restart``. The request body should be a JSON dictionary, and it can contain the following keys: * ``schedule``: timestamp at which the restart should occur; * ``role``: restart only nodes which role is ``role``. Can be either: * ``primary`; or * ``replica``. * ``postgres_version``: restart only nodes which PostgreSQL version is less than ``postgres_version``, e.g. ``15.2``; * ``timeout``: if restart takes longer than ``timeout`` return an error and fail over to a replica; * ``restart_pending``: if we should restart only when have ``pending restart`` flag; Response HTTP status codes: * ``200``: if successfully performed an immediate restart; or * ``202``: if successfully scheduled a restart for later; or * ``500``: if the cluster is in maintenance mode; or * ``400``: if * ``role`` value is invalid; or * ``postgres_version`` value is invalid; or * ``timeout`` is not a number, or lesser than ``0``; or * request contains an unknown key; or * exception is faced while performing an immediate restart. * ``409``: if another restart was already previously scheduled; or * ``503``: if any issue was found while performing an immediate restart; or * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule. .. note:: If it's not able to parse the request body, then the request is silently discarded. """ status_code = 500 data = 'restart failed' request = self._read_json_content(body_is_optional=True) cluster = self.server.patroni.dcs.get_cluster() if request is None: # failed to parse the json return if request: logger.debug("received restart request: {0}".format(request)) if global_config.from_cluster(cluster).is_paused and 'schedule' in request: self.write_response(status_code, "Can't schedule restart in the paused state") return for k in request: if k == 'schedule': (_, data, request[k]) = self.parse_schedule(request[k], "restart") if _: status_code = _ break elif k == 'role': if request[k] not in ('primary', 'standby_leader', 'replica'): status_code = 400 data = "PostgreSQL role should be either primary, standby_leader, or replica" break elif k == 'postgres_version': try: postgres_version_to_int(request[k]) except PostgresException as e: status_code = 400 data = e.value break elif k == 'timeout': request[k] = parse_int(request[k], 's') if request[k] is None or request[k] <= 0: status_code = 400 data = "Timeout should be a positive number of seconds" break elif k != 'restart_pending': status_code = 400 data = "Unknown filter for the scheduled restart: {0}".format(k) break else: if 'schedule' not in request: try: status, data = self.server.patroni.ha.restart(request) status_code = 200 if status else 503 except Exception: logger.exception('Exception during restart') status_code = 400 else: if self.server.patroni.ha.schedule_future_restart(request): data = "Restart scheduled" status_code = 202 else: data = "Another restart is already scheduled" status_code = 409 # pyright thinks ``data`` can be ``None`` because ``parse_schedule`` call may return ``None``. However, if # that's the case, ``data`` will be overwritten when the ``for`` loop ends if TYPE_CHECKING: # pragma: no cover assert isinstance(data, str) self.write_response(status_code, data) @check_access def do_DELETE_restart(self) -> None: """Handle a ``DELETE`` request to ``/restart`` path. Used to remove a scheduled restart of PostgreSQL. Response HTTP status codes: * ``200``: if a scheduled restart was removed; or * ``404``: if no scheduled restart could be found. """ if self.server.patroni.ha.delete_future_restart(): data = "scheduled restart deleted" code = 200 else: data = "no restarts are scheduled" code = 404 self.write_response(code, data) @check_access def do_DELETE_switchover(self) -> None: """Handle a ``DELETE`` request to ``/switchover`` path. Used to remove a scheduled switchover in the cluster. It writes a response, and the HTTP status code can be: * ``200``: if a scheduled switchover was removed; or * ``404``: if no scheduled switchover could be found; or * ``409``: if not able to update the switchover info in the DCS. """ failover = self.server.patroni.dcs.get_cluster().failover if failover and failover.scheduled_at: if not self.server.patroni.dcs.manual_failover('', '', version=failover.version): return self.send_error(409) else: data = "scheduled switchover deleted" code = 200 else: data = "no switchover is scheduled" code = 404 self.write_response(code, data) @check_access def do_POST_reinitialize(self) -> None: """Handle a ``POST`` request to ``/reinitialize`` path. The request body may contain a JSON dictionary with the following key: * ``force``: ``True`` if we want to cancel an already running task in order to reinit a replica. Response HTTP status codes: * ``200``: if the reinit operation has started; or * ``503``: if any error is returned by :func:`~patroni.ha.Ha.reinitialize`. """ request = self._read_json_content(body_is_optional=True) if request: logger.debug('received reinitialize request: %s', request) force = isinstance(request, dict) and parse_bool(request.get('force')) or False data = self.server.patroni.ha.reinitialize(force) if data is None: status_code = 200 data = 'reinitialize started' else: status_code = 503 self.write_response(status_code, data) def poll_failover_result(self, leader: Optional[str], candidate: Optional[str], action: str) -> Tuple[int, str]: """Poll failover/switchover operation until it finishes or times out. :param leader: name of the current Patroni leader. :param candidate: name of the Patroni node to be promoted. :param action: the action that is ongoing (``switchover`` or ``failover``). :returns: a tuple composed of 2 items: * Response HTTP status codes: * ``200``: if the operation succeeded; or * ``503``: if the operation failed or timed out. * A status message about the operation. """ timeout = max(10, self.server.patroni.dcs.loop_wait) for _ in range(0, timeout * 2): time.sleep(1) try: cluster = self.server.patroni.dcs.get_cluster() if not cluster.is_unlocked() and cluster.leader and cluster.leader.name != leader: if not candidate or candidate == cluster.leader.name: return 200, 'Successfully {0}ed over to "{1}"'.format(action[:-4], cluster.leader.name) else: return 200, '{0}ed over to "{1}" instead of "{2}"'.format(action[:-4].title(), cluster.leader.name, candidate) if not cluster.failover: return 503, action.title() + ' failed' except Exception as e: logger.debug('Exception occurred during polling %s result: %s', action, e) return 503, action.title() + ' status unknown' def is_failover_possible(self, cluster: Cluster, leader: Optional[str], candidate: Optional[str], action: str) -> Optional[str]: """Checks whether there are nodes that could take over after demoting the primary. :param cluster: the Patroni cluster. :param leader: name of the current Patroni leader. :param candidate: name of the Patroni node to be promoted. :param action: the action to be performed (``switchover`` or ``failover``). :returns: a string with the error message or ``None`` if good nodes are found. """ config = global_config.from_cluster(cluster) if leader and (not cluster.leader or cluster.leader.name != leader): return 'leader name does not match' if candidate: if action == 'switchover' and config.is_synchronous_mode\ and not config.is_quorum_commit_mode and not cluster.sync.matches(candidate): return 'candidate name does not match with sync_standby' members = [m for m in cluster.members if m.name == candidate] if not members: return 'candidate does not exists' elif config.is_synchronous_mode and not config.is_quorum_commit_mode: members = [m for m in cluster.members if cluster.sync.matches(m.name)] if not members: return action + ' is not possible: can not find sync_standby' else: members = [m for m in cluster.members if not cluster.leader or m.name != cluster.leader.name and m.api_url] if not members: return action + ' is not possible: cluster does not have members except leader' for st in self.server.patroni.ha.fetch_nodes_statuses(members): if st.failover_limitation() is None: return None return action + ' is not possible: no good candidates have been found' @check_access def do_POST_failover(self, action: str = 'failover') -> None: """Handle a ``POST`` request to ``/failover`` path. Handles manual failovers/switchovers, mainly from ``patronictl``. The request body should be a JSON dictionary, and it can contain the following keys: * ``leader``: name of the current leader in the cluster; * ``candidate``: name of the Patroni node to be promoted; * ``scheduled_at``: a string representing the timestamp when to execute the switchover/failover, e.g. ``2023-04-14T20:27:00+00:00``. Response HTTP status codes: * ``202``: if operation has been scheduled; * ``412``: if operation is not possible; * ``503``: if unable to register the operation to the DCS; * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule; * HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately; * ``400``: if none of the above applies. .. note:: If unable to parse the request body, then the request is silently discarded. :param action: the action to be performed (``switchover`` or ``failover``). """ request = self._read_json_content() (status_code, data) = (400, '') if not request: return leader = request.get('leader') candidate = request.get('candidate') or request.get('member') scheduled_at = request.get('scheduled_at') cluster = self.server.patroni.dcs.get_cluster() config = global_config.from_cluster(cluster) logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s", action, leader, candidate, scheduled_at) if action == 'failover' and not candidate: data = 'Failover could be performed only to a specific candidate' elif action == 'switchover' and not leader: data = 'Switchover could be performed only from a specific leader' if not data and scheduled_at: if action == 'failover': data = "Failover can't be scheduled" elif config.is_paused: data = "Can't schedule switchover in the paused state" else: (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action) if not data and config.is_paused and not candidate: data = 'Switchover is possible only to a specific candidate in a paused state' if action == 'failover' and leader: logger.warning('received failover request with leader specified - performing switchover instead') action = 'switchover' if not data and leader == candidate: data = 'Switchover target and source are the same' if not data and not scheduled_at: data = self.is_failover_possible(cluster, leader, candidate, action) if data: status_code = 412 if not data: if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at): self.server.patroni.ha.wakeup() if scheduled_at: data = action.title() + ' scheduled' status_code = 202 else: status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate, action) else: data = 'failed to write failover key into DCS' status_code = 503 # pyright thinks ``status_code`` can be ``None`` because ``parse_schedule`` call may return ``None``. However, # if that's the case, ``status_code`` will be overwritten somewhere between ``parse_schedule`` and # ``write_response`` calls. if TYPE_CHECKING: # pragma: no cover assert isinstance(status_code, int) self.write_response(status_code, data) def do_POST_switchover(self) -> None: """Handle a ``POST`` request to ``/switchover`` path. Calls :func:`do_POST_failover` with ``switchover`` option. """ self.do_POST_failover(action='switchover') @check_access def do_POST_citus(self) -> None: """Handle a ``POST`` request to ``/citus`` path. .. note:: We keep this entrypoint for backward compatibility and simply dispatch the request to :meth:`do_POST_mpp`. """ self.do_POST_mpp() def do_POST_mpp(self) -> None: """Handle a ``POST`` request to ``/mpp`` path. Call :func:`~patroni.postgresql.mpp.AbstractMPPHandler.handle_event` to handle the request, then write a response with HTTP status code ``200``. .. note:: If unable to parse the request body, then the request is silently discarded. """ request = self._read_json_content() if not request: return patroni = self.server.patroni if patroni.postgresql.mpp_handler.is_coordinator() and patroni.ha.is_leader(): cluster = patroni.dcs.get_cluster() patroni.postgresql.mpp_handler.handle_event(cluster, request) self.write_response(200, 'OK') def parse_request(self) -> bool: """Override :func:`parse_request` to enrich basic functionality of :class:`~http.server.BaseHTTPRequestHandler`. Original class can only invoke :func:`do_GET`, :func:`do_POST`, :func:`do_PUT`, etc method implementations if they are defined. But we would like to have at least some simple routing mechanism, i.e.: * ``GET /uri1/part2`` request should invoke :func:`do_GET_uri1()` * ``POST /other`` should invoke :func:`do_POST_other()` If the :func:`do__` method does not exist we'll fall back to original behavior. :returns: ``True`` for success, ``False`` for failure; on failure, any relevant error response has already been sent back. """ ret = BaseHTTPRequestHandler.parse_request(self) if ret: urlpath = urlparse(self.path) self.path = urlpath.path self.path_query = parse_qs(urlpath.query) or {} mname = self.path.lstrip('/').split('/')[0] mname = self.command + ('_' + mname if mname else '') if hasattr(self, 'do_' + mname): self.command = mname return ret def query(self, sql: str, *params: Any, retry: bool = False) -> List[Tuple[Any, ...]]: """Execute *sql* query with *params* and optionally return results. :param sql: the SQL statement to be run. :param params: positional arguments to call :func:`RestApiServer.query` with. :param retry: whether the query should be retried upon failure or given up immediately. :returns: a list of rows that were fetched from the database. """ if not retry: return self.server.query(sql, *params) return Retry(delay=1, retry_exceptions=PostgresConnectionException)(self.server.query, sql, *params) def get_postgresql_status(self, retry: bool = False) -> Dict[str, Any]: """Builds an object representing a status of "postgres". Some of the values are collected by executing a query and other are taken from the state stored in memory. :param retry: whether the query should be retried if failed or give up immediately :returns: a dict with the status of Postgres/Patroni. The keys are: * ``state``: Postgres state among ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``, ``creating replica``, or ``unknown``; * ``postmaster_start_time``: ``pg_postmaster_start_time()``; * ``role``: ``replica`` or ``primary`` based on ``pg_is_in_recovery()`` output; * ``server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``; * ``xlog``: dictionary. Its structure depends on ``role``: * If ``primary``: * ``location``: ``pg_current_wal_flush_lsn()`` * If ``replica``: * ``received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``; * ``replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``; * ``replayed_timestamp``: ``pg_last_xact_replay_timestamp``; * ``paused``: ``pg_is_wal_replay_paused()``; * ``sync_standby``: ``True`` if replication mode is synchronous and this is a sync standby; * ``quorum_standby``: ``True`` if replication mode is quorum and this is a quorum standby; * ``timeline``: PostgreSQL primary node timeline; * ``replication``: :class:`list` of :class:`dict` entries, one for each replication connection. Each entry contains the following keys: * ``application_name``: ``pg_stat_activity.application_name``; * ``client_addr``: ``pg_stat_activity.client_addr``; * ``state``: ``pg_stat_replication.state``; * ``sync_priority``: ``pg_stat_replication.sync_priority``; * ``sync_state``: ``pg_stat_replication.sync_state``; * ``usename``: ``pg_stat_activity.usename``. * ``pause``: ``True`` if cluster is in maintenance mode; * ``cluster_unlocked``: ``True`` if cluster has no node holding the leader lock; * ``failsafe_mode_is_active``: ``True`` if DCS failsafe mode is currently active; * ``dcs_last_seen``: epoch timestamp DCS was last reached by Patroni. """ postgresql = self.server.patroni.postgresql cluster = self.server.patroni.dcs.cluster config = global_config.from_cluster(cluster) try: if postgresql.state not in ('running', 'restarting', 'starting'): raise RetryFailedError('') replication_state = ('(pg_catalog.pg_stat_get_wal_receiver()).status' if postgresql.major_version >= 90600 else 'NULL') + ", " +\ ("pg_catalog.current_setting('restore_command')" if postgresql.major_version >= 120000 else "NULL") stmt = ("SELECT " + postgresql.POSTMASTER_START_TIME + ", " + postgresql.TL_LSN + "," " pg_catalog.pg_last_xact_replay_timestamp(), " + replication_state + "," " pg_catalog.array_to_json(pg_catalog.array_agg(pg_catalog.row_to_json(ri))) " "FROM (SELECT (SELECT rolname FROM pg_catalog.pg_authid WHERE oid = usesysid) AS usename," " application_name, client_addr, w.state, sync_state, sync_priority" " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri") row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name, postgresql.wal_flush), retry=retry)[0] result = { 'state': postgresql.state, 'postmaster_start_time': row[0], 'role': 'replica' if row[1] == 0 else 'primary', 'server_version': postgresql.server_version, 'xlog': ({ 'received_location': row[4] or row[3], 'replayed_location': row[3], 'replayed_timestamp': row[6], 'paused': row[5]} if row[1] == 0 else { 'location': row[2] }) } if result['role'] == 'replica' and config.is_standby_cluster: result['role'] = postgresql.role if result['role'] == 'replica' and config.is_synchronous_mode\ and cluster and cluster.sync.matches(postgresql.name): result['quorum_standby' if global_config.is_quorum_commit_mode else 'sync_standby'] = True if row[1] > 0: result['timeline'] = row[1] else: leader_timeline = None\ if not cluster or cluster.is_unlocked() or not cluster.leader else cluster.leader.timeline result['timeline'] = postgresql.replica_cached_timeline(leader_timeline) replication_state = postgresql.replication_state_from_parameters(row[1] > 0, row[7], row[8]) if replication_state: result['replication_state'] = replication_state if row[9]: result['replication'] = row[9] except (psycopg.Error, RetryFailedError, PostgresConnectionException): state = postgresql.state if state == 'running': logger.exception('get_postgresql_status') state = 'unknown' result: Dict[str, Any] = {'state': state, 'role': postgresql.role} if config.is_paused: result['pause'] = True if not cluster or cluster.is_unlocked(): result['cluster_unlocked'] = True if self.server.patroni.ha.failsafe_is_active(): result['failsafe_mode_is_active'] = True result['dcs_last_seen'] = self.server.patroni.dcs.last_seen return result def handle_one_request(self) -> None: """Parse and dispatch a request to the appropriate ``do_*`` method. .. note:: This is only used to keep track of latency when logging messages through :func:`log_message`. """ self.__start_time = time.time() BaseHTTPRequestHandler.handle_one_request(self) def log_message(self, format: str, *args: Any) -> None: """Log a custom ``debug`` message. Additionally, to *format*, the log entry contains the client IP address and the current latency of the request. :param format: printf-style format string message to be logged. :param args: arguments to be applied as inputs to *format*. """ latency = 1000.0 * (time.time() - self.__start_time) logger.debug("API thread: %s - - %s latency: %0.3f ms", self.client_address[0], format % args, latency) class RestApiServer(ThreadingMixIn, HTTPServer, Thread): """Patroni REST API server. An asynchronous thread-based HTTP server. """ # On 3.7+ the `ThreadingMixIn` gathers all non-daemon worker threads in order to join on them at server close. daemon_threads = True # Make worker threads "fire and forget" to prevent a memory leak. def __init__(self, patroni: Patroni, config: Dict[str, Any]) -> None: """Establish patroni configuration for the REST API daemon. Create a :class:`RestApiServer` instance. :param patroni: Patroni daemon process. :param config: ``restapi`` section of Patroni configuration. """ self.connection_string: str self.__auth_key = None self.__allowlist_include_members: Optional[bool] = None self.__allowlist: Tuple[Union[IPv4Network, IPv6Network], ...] = () self.http_extra_headers: Dict[str, str] = {} self.patroni = patroni self.__listen = None self.request_queue_size = int(config.get('request_queue_size', 5)) self.__ssl_options: Dict[str, Any] = {} self.__ssl_serial_number = None self._received_new_cert = False self.reload_config(config) self.daemon = True def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]: """Execute *sql* query with *params* and optionally return results. .. note:: Prefer to use own connection to postgres and fallback to ``heartbeat`` when own isn't available. :param sql: the SQL statement to be run. :param params: positional arguments to be used as parameters for *sql*. :returns: a list of rows that were fetched from the database. :raises: :class:`psycopg.Error`: if had issues while executing *sql*. :class:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database. """ # We first try to get a heartbeat connection because it is always required for the main thread. try: heartbeat_connection = self.patroni.postgresql.connection_pool.get('heartbeat') heartbeat_connection.get() # try to open psycopg connection to postgres except psycopg.Error as exc: raise PostgresConnectionException('connection problems') from exc try: connection = self.patroni.postgresql.connection_pool.get('restapi') connection.get() # try to open psycopg connection to postgres except psycopg.Error: logger.debug('restapi connection to postgres is not available') connection = heartbeat_connection return connection.query(sql, *params) @staticmethod def _set_fd_cloexec(fd: socket.socket) -> None: """Set ``FD_CLOEXEC`` for *fd*. It is used to avoid inheriting the REST API port when forking its process. .. note:: Only takes effect on non-Windows environments. :param fd: socket file descriptor. """ if os.name != 'nt': import fcntl flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) def check_basic_auth_key(self, key: str) -> bool: """Check if *key* matches the password configured for the REST API. :param key: the password received through the Basic authorization header of an HTTP request. :returns: ``True`` if *key* matches the password configured for the REST API. """ # pyright -- ``__auth_key`` was already checked through the caller method (:func:`check_auth_header`). if TYPE_CHECKING: # pragma: no cover assert self.__auth_key is not None return hmac.compare_digest(self.__auth_key, key.encode('utf-8')) def check_auth_header(self, auth_header: Optional[str]) -> Optional[str]: """Validate HTTP Basic authorization header, if present. :param auth_header: value of ``Authorization`` HTTP header, if present, else ``None``. :returns: an error message if any issue is found, ``None`` otherwise. """ if self.__auth_key: if auth_header is None: return 'no auth header received' if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]): return 'not authenticated' @staticmethod def __resolve_ips(host: str, port: int) -> Iterator[Union[IPv4Network, IPv6Network]]: """Resolve *host* + *port* to one or more IP networks. :param host: hostname to be checked. :param port: port to be checked. :yields: *host* + *port* resolved to IP networks. """ try: for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP): yield ip_network(sa[0], False) except Exception as e: logger.error('Failed to resolve %s: %r', host, e) def __members_ips(self) -> Iterator[Union[IPv4Network, IPv6Network]]: """Resolve each Patroni node ``restapi.connect_address`` to IP networks. .. note:: Only yields object if ``restapi.allowlist_include_members`` setting is enabled. :yields: each node ``restapi.connect_address`` resolved to an IP network. """ cluster = self.patroni.dcs.cluster if self.__allowlist_include_members and cluster: for cluster in [cluster] + list(cluster.workers.values()): for member in cluster.members: if member.api_url: try: r = urlparse(member.api_url) if r.hostname: port = r.port or (443 if r.scheme == 'https' else 80) for ip in self.__resolve_ips(r.hostname, port): yield ip except Exception as e: logger.debug('Failed to parse url %s: %r', member.api_url, e) def check_access(self, rh: RestApiHandler, allowlist_check_members: bool = True) -> Optional[bool]: """Ensure client has enough privileges to perform a given request. Write a response back to the client if any issue is observed, and the HTTP status may be: * ``401``: if ``Authorization`` header is missing or contain an invalid password; * ``403``: if: * ``restapi.allowlist`` was configured, but client IP is not in the allowed list; or * ``restapi.allowlist_include_members`` is enabled, but client IP is not in the members list; or * a client certificate is expected by the server, but is missing in the request. :param rh: the request which access should be checked. :param allowlist_check_members: whether we should check the source ip against existing cluster members. :returns: ``True`` if client access verification succeeded, otherwise ``None``. """ allowlist_check_members = allowlist_check_members and bool(self.__allowlist_include_members) if self.__allowlist or allowlist_check_members: incoming_ip = ip_address(rh.client_address[0]) members_ips = tuple(self.__members_ips()) if allowlist_check_members else () if not any(incoming_ip in net for net in self.__allowlist + members_ips): return rh.write_response(403, 'Access is denied') if not hasattr(rh.request, 'getpeercert') or not rh.request.getpeercert(): # valid client cert isn't present if self.__protocol == 'https' and self.__ssl_options.get('verify_client') in ('required', 'optional'): return rh.write_response(403, 'client certificate required') reason = self.check_auth_header(rh.headers.get('Authorization')) if reason: headers = {'WWW-Authenticate': 'Basic realm="' + self.patroni.__class__.__name__ + '"'} return rh.write_response(401, reason, headers=headers) return True @staticmethod def __has_dual_stack() -> bool: """Check if the system has support for dual stack sockets. :returns: ``True`` if it has support for dual stack sockets. """ if hasattr(socket, 'AF_INET6') and hasattr(socket, 'IPPROTO_IPV6') and hasattr(socket, 'IPV6_V6ONLY'): sock = None try: sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False) return True except socket.error as e: logger.debug('Error when working with ipv6 socket: %s', e) finally: if sock: sock.close() return False def __httpserver_init(self, host: str, port: int) -> None: """Start REST API HTTP server. .. note:: If system has no support for dual stack sockets, then IPv4 is preferred over IPv6. :param host: host to bind REST API to. :param port: port to bind REST API to. """ dual_stack = self.__has_dual_stack() hostname = host if hostname in ('', '*'): hostname = None info = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE) # in case dual stack is not supported we want IPv4 to be preferred over IPv6 info.sort(key=lambda x: x[0] == socket.AF_INET, reverse=not dual_stack) self.address_family = info[0][0] try: HTTPServer.__init__(self, info[0][-1][:2], RestApiHandler) except socket.error: logger.error( "Couldn't start a service on '%s:%s', please check your `restapi.listen` configuration", hostname, port) raise def __initialize(self, listen: str, ssl_options: Dict[str, Any]) -> None: """Configure and start REST API HTTP server. .. note:: This method can be called upon first initialization, and also when reloading Patroni. When reloading Patroni, it restarts the HTTP server thread. :param listen: IP and port to bind REST API to. It should be a string in the format ``host:port``, where ``host`` can be a hostname or IP address. It is the value of ``restapi.listen`` setting. :param ssl_options: dictionary that may contain the following keys, depending on what has been configured in ``restapi`` section: * ``certfile``: path to PEM certificate. If given, will start in HTTPS mode; * ``keyfile``: path to key of ``certfile``; * ``keyfile_password``: password for decrypting ``keyfile``; * ``cafile``: path to CA file to validate client certificates; * ``ciphers``: permitted cipher suites; * ``verify_client``: value can be one among: * ``none``: do not check client certificates; * ``optional``: check client certificate only for unsafe REST API endpoints; * ``required``: check client certificate for all REST API endpoints. :raises: :class:`ValueError`: if any issue is faced while parsing *listen*. """ try: host, port = split_host_port(listen, None) except Exception: raise ValueError('Invalid "restapi" config: expected : for "listen", but got "{0}"' .format(listen)) reloading_config = self.__listen is not None # changing config in runtime if reloading_config: self.shutdown() # Rely on ThreadingMixIn.server_close() to have all requests terminate before we continue self.server_close() self.__listen = listen self.__ssl_options = ssl_options self._received_new_cert = False # reset to False after reload_config() self.__httpserver_init(host, port) Thread.__init__(self, target=self.serve_forever) self._set_fd_cloexec(self.socket) # wrap socket with ssl if 'certfile' is defined in a config.yaml # Sometime it's also needed to pass reference to a 'keyfile'. self.__protocol = 'https' if ssl_options.get('certfile') else 'http' if self.__protocol == 'https': import ssl ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=ssl_options.get('cafile')) if ssl_options.get('ciphers'): ctx.set_ciphers(ssl_options['ciphers']) ctx.load_cert_chain(certfile=ssl_options['certfile'], keyfile=ssl_options.get('keyfile'), password=ssl_options.get('keyfile_password')) verify_client = ssl_options.get('verify_client') if verify_client: modes = {'none': ssl.CERT_NONE, 'optional': ssl.CERT_OPTIONAL, 'required': ssl.CERT_REQUIRED} if verify_client in modes: ctx.verify_mode = modes[verify_client] else: logger.error('Bad value in the "restapi.verify_client": %s', verify_client) self.__ssl_serial_number = self.get_certificate_serial_number() self.socket = ctx.wrap_socket(self.socket, server_side=True, do_handshake_on_connect=False) if reloading_config: self.start() def process_request_thread(self, request: Union[socket.socket, Tuple[bytes, socket.socket]], client_address: Tuple[str, int]) -> None: """Process a request to the REST API. Wrapper for :func:`~socketserver.ThreadingMixIn.process_request_thread` that additionally: * Enable TCP keepalive * Perform SSL handshake (if an SSL socket). :param request: socket to handle the client request. :param client_address: tuple containing the client IP and port. """ if isinstance(request, socket.socket): enable_keepalive(request, 10, 3) if hasattr(request, 'context'): # SSLSocket from ssl import SSLSocket if isinstance(request, SSLSocket): # pyright request.do_handshake() super(RestApiServer, self).process_request_thread(request, client_address) def shutdown_request(self, request: Union[socket.socket, Tuple[bytes, socket.socket]]) -> None: """Shut down a request to the REST API. Wrapper for :func:`http.server.HTTPServer.shutdown_request` that additionally: * Perform SSL shutdown handshake (if a SSL socket). :param request: socket to handle the client request. """ if hasattr(request, 'context'): # SSLSocket try: from ssl import SSLSocket if isinstance(request, SSLSocket): # pyright request.unwrap() except Exception as e: logger.debug('Failed to shutdown SSL connection: %r', e) super(RestApiServer, self).shutdown_request(request) def get_certificate_serial_number(self) -> Optional[str]: """Get serial number of the certificate used by the REST API. :returns: serial number of the certificate configured through ``restapi.certfile`` setting. """ certfile: Optional[str] = self.__ssl_options.get('certfile') if certfile: import ssl try: crt = cast(Dict[str, Any], ssl._ssl._test_decode_cert(certfile)) # pyright: ignore return crt.get('serialNumber') except ssl.SSLError as e: logger.error('Failed to get serial number from certificate %s: %r', self.__ssl_options['certfile'], e) def reload_local_certificate(self) -> Optional[bool]: """Reload the SSL certificate used by the REST API. :return: ``True`` if a different certificate has been configured through ``restapi.certfile` setting, ``None`` otherwise. """ if self.__protocol == 'https': on_disk_cert_serial_number = self.get_certificate_serial_number() if on_disk_cert_serial_number != self.__ssl_serial_number: self._received_new_cert = True self.__ssl_serial_number = on_disk_cert_serial_number return True def _build_allowlist(self, value: Optional[List[str]]) -> Iterator[Union[IPv4Network, IPv6Network]]: """Resolve each entry in *value* to an IP network object. :param value: list of IPs and/or networks contained in ``restapi.allowlist`` setting. Each item can be a host, an IP, or a network in CIDR format. :yields: *host* + *port* resolved to IP networks. """ if isinstance(value, list): for v in value: if '/' in v: # netmask try: yield ip_network(v, False) except Exception as e: logger.error('Invalid value "%s" in the allowlist: %r', v, e) else: # ip or hostname, try to resolve it for ip in self.__resolve_ips(v, 8080): yield ip def reload_config(self, config: Dict[str, Any]) -> None: """Reload REST API configuration. :param config: dictionary representing values under the ``restapi`` configuration section. :raises: :class:`ValueError`: if ``listen`` key is not present in *config*. """ if 'listen' not in config: # changing config in runtime raise ValueError('Can not find "restapi.listen" config') self.__allowlist = tuple(self._build_allowlist(config.get('allowlist'))) self.__allowlist_include_members = config.get('allowlist_include_members') ssl_options = {n: config[n] for n in ('certfile', 'keyfile', 'keyfile_password', 'cafile', 'ciphers') if n in config} self.http_extra_headers = config.get('http_extra_headers') or {} self.http_extra_headers.update((config.get('https_extra_headers') or {}) if ssl_options.get('certfile') else {}) if isinstance(config.get('verify_client'), str): ssl_options['verify_client'] = config['verify_client'].lower() if self.__listen != config['listen'] or self.__ssl_options != ssl_options or self._received_new_cert: self.__initialize(config['listen'], ssl_options) self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')) if 'auth' in config else None # pyright -- ``__listen`` is initially created as ``None``, but right after that it is replaced with a string # through :func:`__initialize`. if TYPE_CHECKING: # pragma: no cover assert isinstance(self.__listen, str) self.connection_string = uri(self.__protocol, config.get('connect_address') or self.__listen, 'patroni') def handle_error(self, request: Union[socket.socket, Tuple[bytes, socket.socket]], client_address: Tuple[str, int]) -> None: """Handle any exception that is thrown while handling a request to the REST API. Logs ``WARNING`` messages with the client information, and the stack trace of the faced exception. :param request: the request that faced an exception. :param client_address: a tuple composed of the IP and port of the client connection. """ logger.warning('Exception happened during processing of request from %s:%s', client_address[0], client_address[1]) logger.warning(traceback.format_exc()) patroni-4.0.4/patroni/async_executor.py000066400000000000000000000215431472010352700202620ustar00rootroot00000000000000"""Implement facilities for executing asynchronous tasks.""" import logging from threading import Event, Lock, RLock, Thread from types import TracebackType from typing import Any, Callable, Optional, Tuple, Type from .postgresql.cancellable import CancellableSubprocess logger = logging.getLogger(__name__) class CriticalTask(object): """Represents a critical task in a background process that we either need to cancel or get the result of. Fields of this object may be accessed only when holding a lock on it. To perform the critical task the background thread must, while holding lock on this object, check ``is_cancelled`` flag, run the task and mark the task as complete using :func:`complete`. The main thread must hold async lock to prevent the task from completing, hold lock on critical task object, call :func:`cancel`. If the task has completed :func:`cancel` will return ``False`` and ``result`` field will contain the result of the task. When :func:`cancel` returns ``True`` it is guaranteed that the background task will notice the ``is_cancelled`` flag. :ivar is_cancelled: if the critical task has been cancelled. :ivar result: contains the result of the task, if it has already been completed. """ def __init__(self) -> None: """Create a new instance of :class:`CriticalTask`. Instantiate the lock and the task control attributes. """ self._lock = Lock() self.is_cancelled = False self.result = None def reset(self) -> None: """Must be called every time the background task is finished. .. note:: Must be called from async thread. Caller must hold lock on async executor when calling. """ self.is_cancelled = False self.result = None def cancel(self) -> bool: """Tries to cancel the task. .. note:: Caller must hold lock on async executor and the task when calling. :returns: ``False`` if the task has already run, or ``True`` it has been cancelled. """ if self.result is not None: return False self.is_cancelled = True return True def complete(self, result: Any) -> None: """Mark task as completed along with a *result*. .. note:: Must be called from async thread. Caller must hold lock on task when calling. """ self.result = result def __enter__(self) -> 'CriticalTask': """Acquire the object lock when entering the context manager.""" self._lock.acquire() return self def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: """Release the object lock when exiting the context manager.""" self._lock.release() class AsyncExecutor(object): """Asynchronous executor of (long) tasks. :ivar critical_task: a :class:`CriticalTask` instance to handle execution of critical background tasks. """ def __init__(self, cancellable: CancellableSubprocess, ha_wakeup: Callable[..., None]) -> None: """Create a new instance of :class:`AsyncExecutor`. Configure the given *cancellable* and *ha_wakeup*, initializes the control attributes, and instantiate the lock and event objects that are used to access attributes and manage communication between threads. :param cancellable: a subprocess that supports being cancelled. :param ha_wakeup: function to wake up the HA loop. """ self._cancellable = cancellable self._ha_wakeup = ha_wakeup self._thread_lock = RLock() self._scheduled_action: Optional[str] = None self._scheduled_action_lock = RLock() self._is_cancelled = False self._finish_event = Event() self.critical_task = CriticalTask() @property def busy(self) -> bool: """``True`` if there is an action scheduled to occur, else ``False``.""" return self.scheduled_action is not None def schedule(self, action: str) -> Optional[str]: """Schedule *action* to be executed. .. note:: Must be called before executing a task. .. note:: *action* can only be scheduled if there is no other action currently scheduled. :param action: action to be executed. :returns: ``None`` if *action* has been successfully scheduled, or the previously scheduled action, if any. """ with self._scheduled_action_lock: if self._scheduled_action is not None: return self._scheduled_action self._scheduled_action = action self._is_cancelled = False self._finish_event.set() return None @property def scheduled_action(self) -> Optional[str]: """The currently scheduled action, if any, else ``None``.""" with self._scheduled_action_lock: return self._scheduled_action def reset_scheduled_action(self) -> None: """Unschedule a previously scheduled action, if any. .. note:: Must be called once the scheduled task finishes or is cancelled. """ with self._scheduled_action_lock: self._scheduled_action = None def run(self, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> Optional[Any]: """Run *func* with *args*. .. note:: Expected to be executed through a thread. :param func: function to be run. If it returns anything other than ``None``, HA loop will be woken up at the end of :func:`run` execution. :param args: arguments to be passed to *func*. :returns: ``None`` if *func* execution has been cancelled or faced any exception, otherwise the result of *func*. """ wakeup = False try: with self: if self._is_cancelled: return self._finish_event.clear() self._cancellable.reset_is_cancelled() # if the func returned something (not None) - wake up main HA loop wakeup = func(*args) if args else func() return wakeup except Exception: logger.exception('Exception during execution of long running task %s', self.scheduled_action) finally: with self: self.reset_scheduled_action() self._finish_event.set() with self.critical_task: self.critical_task.reset() if wakeup is not None: self._ha_wakeup() def run_async(self, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> None: """Start an async thread that runs *func* with *args*. :param func: function to be run. Will be passed through args to :class:`~threading.Thread` with a target of :func:`run`. :param args: arguments to be passed along to :class:`~threading.Thread` with *func*. """ Thread(target=self.run, args=(func, args)).start() def try_run_async(self, action: str, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> Optional[str]: """Try to run an async task, if none is currently being executed. :param action: name of the task to be executed. :param func: actual function that performs the task *action*. :param args: arguments to be passed to *func*. :returns: ``None`` if *func* was scheduled successfully, otherwise an error message informing of an already ongoing task. """ prev = self.schedule(action) if prev is None: return self.run_async(func, args) return 'Failed to run {0}, {1} is already in progress'.format(action, prev) def cancel(self) -> None: """Request cancellation of a scheduled async task, if any. .. note:: Wait until task is cancelled before returning control to caller. """ with self: with self._scheduled_action_lock: if self._scheduled_action is None: return logger.warning('Cancelling long running task %s', self._scheduled_action) self._is_cancelled = True self._cancellable.cancel() self._finish_event.wait() with self: self.reset_scheduled_action() def __enter__(self) -> 'AsyncExecutor': """Acquire the thread lock when entering the context manager.""" self._thread_lock.acquire() return self def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: """Release the thread lock when exiting the context manager. .. note:: The arguments are not used, but we need them to match the expected method signature. """ self._thread_lock.release() patroni-4.0.4/patroni/collections.py000066400000000000000000000202761472010352700175470ustar00rootroot00000000000000"""Patroni custom object types somewhat like :mod:`collections` module. Provides a case insensitive :class:`dict` and :class:`set` object types, and `EMPTY_DICT` frozen dictionary object. """ from collections import OrderedDict from copy import deepcopy from typing import Any, Collection, Dict, Iterator, KeysView, Mapping, MutableMapping, MutableSet, Optional class CaseInsensitiveSet(MutableSet[str]): """A case-insensitive :class:`set`-like object. Implements all methods and operations of :class:`~typing.MutableSet`. All values are expected to be strings. The structure remembers the case of the last value set, however, contains testing is case insensitive. """ def __init__(self, values: Optional[Collection[str]] = None) -> None: """Create a new instance of :class:`CaseInsensitiveSet` with the given *values*. :param values: values to be added to the set. """ self._values: Dict[str, str] = {} for v in values or (): self.add(v) def __repr__(self) -> str: """Get a string representation of the set. Provide a helpful way of recreating the set. :returns: representation of the set, showing its values. :Example: >>> repr(CaseInsensitiveSet(('1', 'test', 'Test', 'TESt', 'test2'))) # doctest: +ELLIPSIS "'.format(type(self).__name__, tuple(self._values.values()), id(self)) def __str__(self) -> str: """Get set values for printing. :returns: set of values in string format. :Example: >>> str(CaseInsensitiveSet(('1', 'test', 'Test', 'TESt', 'test2'))) # doctest: +SKIP "{'TESt', 'test2', '1'}" """ return str(set(self._values.values())) def __contains__(self, value: object) -> bool: """Check if set contains *value*. The check is performed case-insensitively. :param value: value to be checked. :returns: ``True`` if *value* is already in the set, ``False`` otherwise. """ return isinstance(value, str) and value.lower() in self._values def __iter__(self) -> Iterator[str]: """Iterate over the values in this set. :yields: values from set. """ return iter(self._values.values()) def __len__(self) -> int: """Get the length of this set. :returns: number of values in the set. :Example: >>> len(CaseInsensitiveSet(('1', 'test', 'Test', 'TESt', 'test2'))) 3 """ return len(self._values) def add(self, value: str) -> None: """Add *value* to this set. Search is performed case-insensitively. If *value* is already in the set, overwrite it with *value*, so we "remember" the last case of *value*. :param value: value to be added to the set. """ self._values[value.lower()] = value def discard(self, value: str) -> None: """Remove *value* from this set. Search is performed case-insensitively. If *value* is not present in the set, no exception is raised. :param value: value to be removed from the set. """ self._values.pop(value.lower(), None) def issubset(self, other: 'CaseInsensitiveSet') -> bool: """Check if this set is a subset of *other*. :param other: another set to be compared with this set. :returns: ``True`` if this set is a subset of *other*, else ``False``. """ return self <= other class CaseInsensitiveDict(MutableMapping[str, Any]): """A case-insensitive :class:`dict`-like object. Implements all methods and operations of :class:`~typing.MutableMapping` as well as :class:`dict`'s :func:`~dict.copy`. All keys are expected to be strings. The structure remembers the case of the last key to be set, and :func:`iter`, :func:`dict.keys`, :func:`dict.items`, :func:`dict.iterkeys`, and :func:`dict.iteritems` will contain case-sensitive keys. However, querying and contains testing is case insensitive. """ def __init__(self, data: Optional[Dict[str, Any]] = None) -> None: """Create a new instance of :class:`CaseInsensitiveDict` with the given *data*. :param data: initial dictionary to create a :class:`CaseInsensitiveDict` from. """ self._values: OrderedDict[str, Any] = OrderedDict() self.update(data or {}) def __setitem__(self, key: str, value: Any) -> None: """Assign *value* to *key* in this dict. *key* is searched/stored case-insensitively in the dict. The corresponding value in the dict is a tuple of: * original *key*; * *value*. :param key: key to be created or updated in the dict. :param value: value for *key*. """ self._values[key.lower()] = (key, value) def __getitem__(self, key: str) -> Any: """Get the value corresponding to *key*. *key* is searched case-insensitively in the dict. .. note: If *key* is not present in the dict, :class:`KeyError` will be triggered. :param key: key to be searched in the dict. :returns: value corresponding to *key*. """ return self._values[key.lower()][1] def __delitem__(self, key: str) -> None: """Remove *key* from this dict. *key* is searched case-insensitively in the dict. .. note: If *key* is not present in the dict, :class:`KeyError` will be triggered. :param key: key to be removed from the dict. """ del self._values[key.lower()] def __iter__(self) -> Iterator[str]: """Iterate over keys of this dict. :yields: each key present in the dict. Yields each key with its last case that has been stored. """ return iter(key for key, _ in self._values.values()) def __len__(self) -> int: """Get the length of this dict. :returns: number of keys in the dict. :Example: >>> len(CaseInsensitiveDict({'a': 'b', 'A': 'B', 'c': 'd'})) 2 """ return len(self._values) def copy(self) -> 'CaseInsensitiveDict': """Create a copy of this dict. :return: a new dict object with the same keys and values of this dict. """ return CaseInsensitiveDict({v[0]: v[1] for v in self._values.values()}) def keys(self) -> KeysView[str]: """Return a new view of the dict's keys. :returns: a set-like object providing a view on the dict's keys """ return self._values.keys() def __repr__(self) -> str: """Get a string representation of the dict. Provide a helpful way of recreating the dict. :returns: representation of the dict, showing its keys and values. :Example: >>> repr(CaseInsensitiveDict({'a': 'b', 'A': 'B', 'c': 'd'})) # doctest: +ELLIPSIS "'.format(type(self).__name__, dict(self.items()), id(self)) class _FrozenDict(Mapping[str, Any]): """Frozen dictionary object.""" def __init__(self, *args: Any, **kwargs: Any) -> None: """Create a new instance of :class:`_FrozenDict` with given data.""" self.__values: Dict[str, Any] = dict(*args, **kwargs) def __iter__(self) -> Iterator[str]: """Iterate over keys of this dict. :yields: each key present in the dict. Yields each key with its last case that has been stored. """ return iter(self.__values) def __len__(self) -> int: """Get the length of this dict. :returns: number of keys in the dict. :Example: >>> len(_FrozenDict()) 0 """ return len(self.__values) def __getitem__(self, key: str) -> Any: """Get the value corresponding to *key*. :returns: value corresponding to *key*. """ return self.__values[key] def copy(self) -> Dict[str, Any]: """Create a copy of this dict. :return: a new dict object with the same keys and values of this dict. """ return deepcopy(self.__values) EMPTY_DICT = _FrozenDict() patroni-4.0.4/patroni/config.py000066400000000000000000001133611472010352700164740ustar00rootroot00000000000000"""Facilities related to Patroni configuration.""" import json import logging import os import re import shutil import tempfile from collections import defaultdict from copy import deepcopy from typing import Any, Callable, cast, Collection, Dict, List, Optional, TYPE_CHECKING, Union import yaml from . import PATRONI_ENV_PREFIX from .collections import CaseInsensitiveDict, EMPTY_DICT from .dcs import ClusterConfig from .exceptions import ConfigParseError from .file_perm import pg_perm from .postgresql.config import ConfigHandler from .utils import deep_compare, parse_bool, parse_int, patch_config from .validator import IntValidator logger = logging.getLogger(__name__) _AUTH_ALLOWED_PARAMETERS = ( 'username', 'password', 'sslmode', 'sslcert', 'sslkey', 'sslpassword', 'sslrootcert', 'sslcrl', 'sslcrldir', 'gssencmode', 'channel_binding', 'sslnegotiation' ) def default_validator(conf: Dict[str, Any]) -> List[str]: """Ensure *conf* is not empty. Designed to be used as default validator for :class:`Config` objects, if no specific validator is provided. :param conf: configuration to be validated. :returns: an empty list -- :class:`Config` expects the validator to return a list of 0 or more issues found while validating the configuration. :raises: :class:`ConfigParseError`: if *conf* is empty. """ if not conf: raise ConfigParseError("Config is empty.") return [] class Config(object): """Handle Patroni configuration. This class is responsible for: 1) Building and giving access to ``effective_configuration`` from: * ``Config.__DEFAULT_CONFIG`` -- some sane default values; * ``dynamic_configuration`` -- configuration stored in DCS; * ``local_configuration`` -- configuration from `config.yml` or environment. 2) Saving and loading ``dynamic_configuration`` into 'patroni.dynamic.json' file located in local_configuration['postgresql']['data_dir'] directory. This is necessary to be able to restore ``dynamic_configuration`` if DCS was accidentally wiped. 3) Loading of configuration file in the old format and converting it into new format. 4) Mimicking some ``dict`` interfaces to make it possible to work with it as with the old ``config`` object. :cvar PATRONI_CONFIG_VARIABLE: name of the environment variable that can be used to load Patroni configuration from. :cvar __CACHE_FILENAME: name of the file used to cache dynamic configuration under Postgres data directory. :cvar __DEFAULT_CONFIG: default configuration values for some Patroni settings. """ PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION' __CACHE_FILENAME = 'patroni.dynamic.json' __DEFAULT_CONFIG: Dict[str, Any] = { 'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10, 'standby_cluster': { 'create_replica_methods': '', 'host': '', 'port': '', 'primary_slot_name': '', 'restore_command': '', 'archive_cleanup_command': '', 'recovery_min_apply_delay': '' }, 'postgresql': { 'use_slots': True, 'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items() if v[0] is not None and p not in ('wal_keep_segments', 'wal_keep_size')}) } } def __init__(self, configfile: str, validator: Optional[Callable[[Dict[str, Any]], List[str]]] = default_validator) -> None: """Create a new instance of :class:`Config` and validate the loaded configuration using *validator*. .. note:: Patroni will read configuration from these locations in this order: * file or directory path passed as command-line argument (*configfile*), if it exists and the file or files found in the directory can be parsed (see :meth:`~Config._load_config_path`), otherwise * YAML file passed via the environment variable (see :attr:`PATRONI_CONFIG_VARIABLE`), if the referenced file exists and can be parsed, otherwise * from configuration values defined as environment variables, see :meth:`~Config._build_environment_configuration`. :param configfile: path to Patroni configuration file. :param validator: function used to validate Patroni configuration. It should receive a dictionary which represents Patroni configuration, and return a list of zero or more error messages based on validation. :raises: :class:`ConfigParseError`: if any issue is reported by *validator*. """ self._modify_version = -1 self._dynamic_configuration = {} self.__environment_configuration = self._build_environment_configuration() self._config_file = configfile if configfile and os.path.exists(configfile) else None if self._config_file: self._local_configuration = self._load_config_file() else: config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None) self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration if validator: errors = validator(self._local_configuration) if errors: raise ConfigParseError("\n".join(errors)) self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration) self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "") self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME) if validator: # patronictl uses validator=None self._load_cache() # we don't want to load anything from local cache for ctl self._validate_failover_tags() # irrelevant for ctl self._cache_needs_saving = False @property def config_file(self) -> Optional[str]: """Path to Patroni configuration file, if any, else ``None``.""" return self._config_file @property def dynamic_configuration(self) -> Dict[str, Any]: """Deep copy of cached Patroni dynamic configuration.""" return deepcopy(self._dynamic_configuration) @property def local_configuration(self) -> Dict[str, Any]: """Deep copy of cached Patroni local configuration. :returns: copy of :attr:`~Config._local_configuration` """ return deepcopy(dict(self._local_configuration)) @classmethod def get_default_config(cls) -> Dict[str, Any]: """Deep copy default configuration. :returns: copy of :attr:`~Config.__DEFAULT_CONFIG` """ return deepcopy(cls.__DEFAULT_CONFIG) def _load_config_path(self, path: str) -> Dict[str, Any]: """Load Patroni configuration file(s) from *path*. If *path* is a file, load the yml file pointed to by *path*. If *path* is a directory, load all yml files in that directory in alphabetical order. :param path: path to either an YAML configuration file, or to a folder containing YAML configuration files. :returns: configuration after reading the configuration file(s) from *path*. :raises: :class:`ConfigParseError`: if *path* is invalid. """ if os.path.isfile(path): files = [path] elif os.path.isdir(path): files = [os.path.join(path, f) for f in sorted(os.listdir(path)) if (f.endswith('.yml') or f.endswith('.yaml')) and os.path.isfile(os.path.join(path, f))] else: logger.error('config path %s is neither directory nor file', path) raise ConfigParseError('invalid config path') overall_config: Dict[str, Any] = {} for fname in files: with open(fname) as f: config = yaml.safe_load(f) patch_config(overall_config, config) return overall_config def _load_config_file(self) -> Dict[str, Any]: """Load configuration file(s) from filesystem and apply values which were set via environment variables. :returns: final configuration after merging configuration file(s) and environment variables. """ if TYPE_CHECKING: # pragma: no cover assert self.config_file is not None config = self._load_config_path(self.config_file) patch_config(config, self.__environment_configuration) return config def _load_cache(self) -> None: """Load dynamic configuration from ``patroni.dynamic.json``.""" if os.path.isfile(self._cache_file): try: with open(self._cache_file) as f: self.set_dynamic_configuration(json.load(f)) except Exception: logger.exception('Exception when loading file: %s', self._cache_file) def save_cache(self) -> None: """Save dynamic configuration to ``patroni.dynamic.json`` under Postgres data directory. .. note:: ``patroni.dynamic.jsonXXXXXX`` is created as a temporary file and than renamed to ``patroni.dynamic.json``, where ``XXXXXX`` is a random suffix. """ if self._cache_needs_saving: tmpfile = fd = None try: pg_perm.set_permissions_from_data_directory(self._data_dir) (fd, tmpfile) = tempfile.mkstemp(prefix=self.__CACHE_FILENAME, dir=self._data_dir) with os.fdopen(fd, 'w') as f: fd = None json.dump(self.dynamic_configuration, f) tmpfile = shutil.move(tmpfile, self._cache_file) os.chmod(self._cache_file, pg_perm.file_create_mode) self._cache_needs_saving = False except Exception: logger.exception('Exception when saving file: %s', self._cache_file) if fd: try: os.close(fd) except Exception: logger.error('Can not close temporary file %s', tmpfile) if tmpfile and os.path.exists(tmpfile): try: os.remove(tmpfile) except Exception: logger.error('Can not remove temporary file %s', tmpfile) def __get_and_maybe_adjust_int_value(self, config: Dict[str, Any], param: str, min_value: int) -> int: """Get, validate and maybe adjust a *param* integer value from the *config* :class:`dict`. .. note: If the value is smaller than provided *min_value* we update the *config*. This method may raise an exception if value isn't :class:`int` or cannot be casted to :class:`int`. :param config: :class:`dict` object with new global configuration. :param param: name of the configuration parameter we want to read/validate/adjust. :param min_value: the minimum possible value that a given *param* could have. :returns: an integer value which corresponds to a provided *param*. """ value = int(config.get(param, self.__DEFAULT_CONFIG[param])) if value < min_value: logger.warning("%s=%d can't be smaller than %d, adjusting...", param, value, min_value) value = config[param] = min_value return value def _validate_and_adjust_timeouts(self, config: Dict[str, Any]) -> None: """Validate and adjust ``loop_wait``, ``retry_timeout``, and ``ttl`` values if necessary. Minimum values: * ``loop_wait``: 1 second; * ``retry_timeout``: 3 seconds. * ``ttl``: 20 seconds; Maximum values: In case if values don't fulfill the following rule, ``retry_timeout`` and ``loop_wait`` are reduced so that the rule is fulfilled: .. code-block:: python loop_wait + 2 * retry_timeout <= ttl .. note: We prefer to reduce ``loop_wait`` and will reduce ``retry_timeout`` only if ``loop_wait`` is already set to a minimal possible value. :param config: :class:`dict` object with new global configuration. """ min_loop_wait = 1 loop_wait = self. __get_and_maybe_adjust_int_value(config, 'loop_wait', min_loop_wait) retry_timeout = self. __get_and_maybe_adjust_int_value(config, 'retry_timeout', 3) ttl = self. __get_and_maybe_adjust_int_value(config, 'ttl', 20) if min_loop_wait + 2 * retry_timeout > ttl: config['loop_wait'] = min_loop_wait config['retry_timeout'] = (ttl - min_loop_wait) // 2 logger.warning('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. ' 'Adjusting loop_wait from %d to %d and retry_timeout from %d to %d', ttl, loop_wait, min_loop_wait, retry_timeout, config['retry_timeout']) elif loop_wait + 2 * retry_timeout > ttl: config['loop_wait'] = ttl - 2 * retry_timeout logger.warning('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d and retry_timeout=%d.' ' Adjusting loop_wait from %d to %d', ttl, retry_timeout, loop_wait, config['loop_wait']) # configuration could be either ClusterConfig or dict def set_dynamic_configuration(self, configuration: Union[ClusterConfig, Dict[str, Any]]) -> bool: """Set dynamic configuration values with given *configuration*. :param configuration: new dynamic configuration values. Supports :class:`dict` for backward compatibility. :returns: ``True`` if changes have been detected between current dynamic configuration and the new dynamic *configuration*, ``False`` otherwise. """ if isinstance(configuration, ClusterConfig): if self._modify_version == configuration.modify_version: return False # If the version didn't change there is nothing to do self._modify_version = configuration.modify_version configuration = configuration.data if not deep_compare(self._dynamic_configuration, configuration): try: self._validate_and_adjust_timeouts(configuration) self.__effective_configuration = self._build_effective_configuration(configuration, self._local_configuration) self._dynamic_configuration = configuration self._cache_needs_saving = True return True except Exception: logger.exception('Exception when setting dynamic_configuration') return False def reload_local_configuration(self) -> Optional[bool]: """Reload configuration values from the configuration file(s). .. note:: Designed to be used when user applies changes to configuration file(s), so Patroni can use the new values with a reload instead of a restart. :returns: ``True`` if changes have been detected between current local configuration """ if self.config_file: try: configuration = self._load_config_file() if not deep_compare(self._local_configuration, configuration): new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration) self._local_configuration = configuration self.__effective_configuration = new_configuration self._validate_failover_tags() return True else: logger.info('No local configuration items changed.') except Exception: logger.exception('Exception when reloading local configuration from %s', self.config_file) @staticmethod def _process_postgresql_parameters(parameters: Dict[str, Any], is_local: bool = False) -> Dict[str, Any]: """Process Postgres *parameters*. .. note:: If *is_local* configuration discard any setting from *parameters* that is listed under :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` as those are supposed to be set only through dynamic configuration. When setting parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` through dynamic configuration their value will be validated as per the validator defined in that very same attribute entry. If the given value cannot be validated, a warning will be logged and the default value of the GUC will be used instead. Some parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` cannot be set even if not *is_local* configuration: * ``listen_addresses``: inferred from ``postgresql.listen`` local configuration or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable; * ``port``: inferred from ``postgresql.listen`` local configuration or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable; * ``cluster_name``: set through ``scope`` local configuration or through ``PATRONI_SCOPE`` environment variable; * ``hot_standby``: always enabled; :param parameters: Postgres parameters to be processed. Should be the parsed YAML value of ``postgresql.parameters`` configuration, either from local or from dynamic configuration. :param is_local: should be ``True`` if *parameters* refers to local configuration, or ``False`` if *parameters* refers to dynamic configuration. :returns: new value for ``postgresql.parameters`` after processing and validating *parameters*. """ pg_params: Dict[str, Any] = {} for name, value in (parameters or {}).items(): if name not in ConfigHandler.CMDLINE_OPTIONS: pg_params[name] = value elif not is_local: validator = ConfigHandler.CMDLINE_OPTIONS[name][1] if validator(value): int_val = parse_int(value) if isinstance(validator, IntValidator) else None pg_params[name] = int_val if isinstance(int_val, int) else value else: logger.warning("postgresql parameter %s=%s failed validation, defaulting to %s", name, value, ConfigHandler.CMDLINE_OPTIONS[name][0]) return pg_params def _safe_copy_dynamic_configuration(self, dynamic_configuration: Dict[str, Any]) -> Dict[str, Any]: """Create a copy of *dynamic_configuration*. Merge *dynamic_configuration* with :attr:`__DEFAULT_CONFIG` (*dynamic_configuration* takes precedence), and process ``postgresql.parameters`` from *dynamic_configuration* through :func:`_process_postgresql_parameters`, if present. .. note:: The following settings are not allowed in ``postgresql`` section as they are intended to be local configuration, and are removed if present: * ``connect_address``; * ``proxy_address``; * ``listen``; * ``config_dir``; * ``data_dir``; * ``pgpass``; * ``authentication``; Besides that any setting present in *dynamic_configuration* but absent from :attr:`__DEFAULT_CONFIG` is discarded. :param dynamic_configuration: Patroni dynamic configuration. :returns: copy of *dynamic_configuration*, merged with default dynamic configuration and with some sanity checks performed over it. """ config = self.get_default_config() for name, value in dynamic_configuration.items(): if name == 'postgresql': for name, value in (value or EMPTY_DICT).items(): if name == 'parameters': config['postgresql'][name].update(self._process_postgresql_parameters(value)) elif name not in ('connect_address', 'proxy_address', 'listen', 'config_dir', 'data_dir', 'pgpass', 'authentication'): config['postgresql'][name] = deepcopy(value) elif name == 'standby_cluster': for name, value in (value or EMPTY_DICT).items(): if name in self.__DEFAULT_CONFIG['standby_cluster']: config['standby_cluster'][name] = deepcopy(value) elif name in config: # only variables present in __DEFAULT_CONFIG allowed to be overridden from DCS config[name] = int(value) return config @staticmethod def _build_environment_configuration() -> Dict[str, Any]: """Get local configuration settings that were specified through environment variables. :returns: dictionary containing the found environment variables and their values, respecting the expected structure of Patroni configuration. """ ret: Dict[str, Any] = defaultdict(dict) def _popenv(name: str) -> Optional[str]: """Get value of environment variable *name*. .. note:: *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment. Also, the corresponding environment variable is removed from the environment upon reading its value. :param name: name of the environment variable. :returns: value of *name*, if present in the environment, otherwise ``None``. """ return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None) for param in ('name', 'namespace', 'scope'): value = _popenv(param) if value: ret[param] = value def _fix_log_env(name: str, oldname: str) -> None: """Normalize a log related environment variable. .. note:: Patroni used to support different names for log related environment variables in the past. As the environment variables were renamed, this function takes care of mapping and normalizing the environment. *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` and ``LOG`` when searching in the environment. *oldname* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment. If both *name* and *oldname* are set in the environment, *name* takes precedence. :param name: new name of a log related environment variable. :param oldname: original name of a log related environment variable. :type oldname: str """ value = _popenv(oldname) name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper() if value and name not in os.environ: os.environ[name] = value for name, oldname in (('level', 'loglevel'), ('format', 'logformat'), ('dateformat', 'log_datefmt')): _fix_log_env(name, oldname) def _set_section_values(section: str, params: List[str]) -> None: """Get value of *params* environment variables that are related with *section*. .. note:: The values are retrieved from the environment and updated directly into the returning dictionary of :func:`_build_environment_configuration`. :param section: configuration section the *params* belong to. :param params: name of the Patroni settings. """ for param in params: value = _popenv(section + '_' + param) if value: ret[section][param] = value _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile', 'keyfile_password', 'cafile', 'ciphers', 'verify_client', 'http_extra_headers', 'https_extra_headers', 'allowlist', 'allowlist_include_members', 'request_queue_size']) _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile', 'keyfile_password']) _set_section_values('postgresql', ['listen', 'connect_address', 'proxy_address', 'config_dir', 'data_dir', 'pgpass', 'bin_dir']) _set_section_values('log', ['type', 'level', 'traceback_level', 'format', 'dateformat', 'static_fields', 'max_queue_size', 'dir', 'mode', 'file_size', 'file_num', 'loggers']) _set_section_values('raft', ['data_dir', 'self_addr', 'partner_addrs', 'password', 'bind_addr']) for binary in ('pg_ctl', 'initdb', 'pg_controldata', 'pg_basebackup', 'postgres', 'pg_isready', 'pg_rewind'): value = _popenv('POSTGRESQL_BIN_' + binary) if value: ret['postgresql'].setdefault('bin_name', {})[binary] = value # parse all values retrieved from the environment as Python objects, according to the expected type for first, second in (('restapi', 'allowlist_include_members'), ('ctl', 'insecure')): value = ret.get(first, {}).pop(second, None) if value: value = parse_bool(value) if value is not None: ret[first][second] = value for first, params in (('restapi', ('request_queue_size',)), ('log', ('max_queue_size', 'file_size', 'file_num', 'mode'))): for second in params: value = ret.get(first, {}).pop(second, None) if value: value = parse_int(value) if value is not None: ret[first][second] = value def _parse_list(value: str) -> Optional[List[str]]: """Parse an YAML list *value* as a :class:`list`. :param value: YAML list as a string. :returns: *value* as :class:`list`. """ if not (value.strip().startswith('-') or '[' in value): value = '[{0}]'.format(value) try: return yaml.safe_load(value) except Exception: logger.exception('Exception when parsing list %s', value) return None for first, second in (('raft', 'partner_addrs'), ('restapi', 'allowlist')): value = ret.get(first, {}).pop(second, None) if value: value = _parse_list(value) if value: ret[first][second] = value logformat = ret.get('log', {}).get('format') if logformat and not re.search(r'%\(\w+\)', logformat): logformat = _parse_list(logformat) if logformat: ret['log']['format'] = logformat def _parse_dict(value: str) -> Optional[Dict[str, Any]]: """Parse an YAML dictionary *value* as a :class:`dict`. :param value: YAML dictionary as a string. :returns: *value* as :class:`dict`. """ if not value.strip().startswith('{'): value = '{{{0}}}'.format(value) try: return yaml.safe_load(value) except Exception: logger.exception('Exception when parsing dict %s', value) return None dict_configs = ( ('restapi', ('http_extra_headers', 'https_extra_headers')), ('log', ('static_fields', 'loggers')) ) for first, params in dict_configs: for second in params: value = ret.get(first, {}).pop(second, None) if value: value = _parse_dict(value) if value: ret[first][second] = value def _get_auth(name: str, params: Collection[str] = _AUTH_ALLOWED_PARAMETERS[:2]) -> Dict[str, str]: """Get authorization related environment variables *params* from section *name*. :param name: name of a configuration section that may contain authorization *params*. :param params: the authorization settings that may be set under section *name*. :returns: dictionary containing environment values for authorization *params* of section *name*. """ ret: Dict[str, str] = {} for param in params: value = _popenv(name + '_' + param) if value: ret[param] = value return ret for section in ('ctl', 'restapi'): auth = _get_auth(section) if auth: ret[section]['authentication'] = auth authentication = {} for user_type in ('replication', 'superuser', 'rewind'): entry = _get_auth(user_type, _AUTH_ALLOWED_PARAMETERS) if entry: authentication[user_type] = entry if authentication: ret['postgresql']['authentication'] = authentication for param in list(os.environ.keys()): if param.startswith(PATRONI_ENV_PREFIX): # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..) name, suffix = (param[len(PATRONI_ENV_PREFIX):].split('_', 1) + [''])[:2] if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY', 'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME', 'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD', 'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE', 'LEADER_LABEL_VALUE', 'FOLLOWER_LABEL_VALUE', 'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL', 'AUTH_DATA') and name: value = os.environ.pop(param) if name == 'CITUS': if suffix == 'GROUP': value = parse_int(value) elif suffix != 'DATABASE': continue elif suffix == 'PORT': value = value and parse_int(value) elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS', 'RETRIABLE_HTTP_CODES'): value = value and _parse_list(value) elif suffix in ('LABELS', 'SET_ACLS', 'AUTH_DATA'): value = _parse_dict(value) elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE', 'USE_ENDPOINTS', 'BYPASS_API_SERVICE', 'VERIFY'): value = parse_bool(value) if value is not None: ret[name.lower()][suffix.lower()] = value for dcs in ('etcd', 'etcd3'): if dcs in ret: ret[dcs].update(_get_auth(dcs)) return ret def _build_effective_configuration(self, dynamic_configuration: Dict[str, Any], local_configuration: Dict[str, Union[Dict[str, Any], Any]]) -> Dict[str, Any]: """Build effective configuration by merging *dynamic_configuration* and *local_configuration*. .. note:: *local_configuration* takes precedence over *dynamic_configuration* if a setting is defined in both. :param dynamic_configuration: Patroni dynamic configuration. :param local_configuration: Patroni local configuration. :returns: _description_ """ config = self._safe_copy_dynamic_configuration(dynamic_configuration) for name, value in local_configuration.items(): if name == 'citus': # remove invalid citus configuration if isinstance(value, dict) and isinstance(cast(Dict[str, Any], value).get('group'), int) \ and isinstance(cast(Dict[str, Any], value).get('database'), str): config[name] = value elif name == 'postgresql': for name, value in (value or {}).items(): if name == 'parameters': config['postgresql'][name].update(self._process_postgresql_parameters(value, True)) elif name != 'use_slots': # replication slots must be enabled/disabled globally config['postgresql'][name] = deepcopy(value) elif name not in config or name in ['watchdog']: config[name] = deepcopy(value) if value else {} # restapi server expects to get restapi.auth = 'username:password' and similarly for `ctl` for section in ('ctl', 'restapi'): if section in config and 'authentication' in config[section]: config[section]['auth'] = '{username}:{password}'.format(**config[section]['authentication']) # special treatment for old config # 'exhibitor' inside 'zookeeper': if 'zookeeper' in config and 'exhibitor' in config['zookeeper']: config['exhibitor'] = config['zookeeper'].pop('exhibitor') config.pop('zookeeper') pg_config = config['postgresql'] # no 'authentication' in 'postgresql', but 'replication' and 'superuser' if 'authentication' not in pg_config: pg_config['use_pg_rewind'] = 'pg_rewind' in pg_config pg_config['authentication'] = {u: pg_config[u] for u in ('replication', 'superuser') if u in pg_config} # no 'superuser' in 'postgresql'.'authentication' if 'superuser' not in pg_config['authentication'] and 'pg_rewind' in pg_config: pg_config['authentication']['superuser'] = pg_config['pg_rewind'] # handle setting additional connection parameters that may be available # in the configuration file, such as SSL connection parameters for name, value in pg_config['authentication'].items(): pg_config['authentication'][name] = {n: v for n, v in value.items() if n in _AUTH_ALLOWED_PARAMETERS} # no 'name' in config if 'name' not in config and 'name' in pg_config: config['name'] = pg_config['name'] # when bootstrapping the new Citus cluster (coordinator/worker) enable sync replication in global configuration if 'citus' in config: bootstrap = config.setdefault('bootstrap', {}) dcs = bootstrap.setdefault('dcs', {}) dcs.setdefault('synchronous_mode', 'quorum') updated_fields = ( 'name', 'scope', 'retry_timeout', 'citus' ) pg_config.update({p: config[p] for p in updated_fields if p in config}) return config def get(self, key: str, default: Optional[Any] = None) -> Any: """Get effective value of ``key`` setting from Patroni configuration root. Designed to work the same way as :func:`dict.get`. :param key: name of the setting. :param default: default value if *key* is not present in the effective configuration. :returns: value of *key*, if present in the effective configuration, otherwise *default*. """ return self.__effective_configuration.get(key, default) def __contains__(self, key: str) -> bool: """Check if setting *key* is present in the effective configuration. Designed to work the same way as :func:`dict.__contains__`. :param key: name of the setting to be checked. :returns: ``True`` if setting *key* exists in effective configuration, else ``False``. """ return key in self.__effective_configuration def __getitem__(self, key: str) -> Any: """Get value of setting *key* from effective configuration. Designed to work the same way as :func:`dict.__getitem__`. :param key: name of the setting. :returns: value of setting *key*. :raises: :class:`KeyError`: if *key* is not present in effective configuration. """ return self.__effective_configuration[key] def copy(self) -> Dict[str, Any]: """Get a deep copy of effective Patroni configuration. :returns: a deep copy of the Patroni configuration. """ return deepcopy(self.__effective_configuration) def _validate_failover_tags(self) -> None: """Check ``nofailover``/``failover_priority`` config and warn user if it's contradictory. .. note:: To preserve sanity (and backwards compatibility) the ``nofailover`` tag will still exist. A contradictory configuration is one where ``nofailover`` is ``True`` but ``failover_priority > 0``, or where ``nofailover`` is ``False``, but ``failover_priority <= 0``. Essentially, ``nofailover`` and ``failover_priority`` are communicating different things. This checks for this edge case (which is a misconfiguration on the part of the user) and warns them. The behaviour is as if ``failover_priority`` were not provided (i.e ``nofailover`` is the bedrock source of truth) """ tags = self.get('tags', {}) if 'nofailover' not in tags: return nofailover_tag = tags.get('nofailover') failover_priority_tag = parse_int(tags.get('failover_priority')) if failover_priority_tag is not None \ and (bool(nofailover_tag) is True and failover_priority_tag > 0 or bool(nofailover_tag) is False and failover_priority_tag <= 0): logger.warning('Conflicting configuration between nofailover: %s and failover_priority: %s. ' 'Defaulting to nofailover: %s', nofailover_tag, failover_priority_tag, nofailover_tag) patroni-4.0.4/patroni/config_generator.py000066400000000000000000000567141472010352700205520ustar00rootroot00000000000000"""patroni ``--generate-config`` machinery.""" import abc import logging import os import socket import sys from contextlib import contextmanager from getpass import getpass, getuser from typing import Any, Dict, Iterator, List, Optional, TextIO, Tuple, TYPE_CHECKING, Union import psutil import yaml if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor from . import psycopg from .collections import EMPTY_DICT from .config import Config from .exceptions import PatroniException from .log import PatroniLogger from .postgresql.config import ConfigHandler, parse_dsn from .postgresql.misc import postgres_major_version_to_int from .utils import get_major_version, parse_bool, patch_config, read_stripped # Mapping between the libpq connection parameters and the environment variables. # This dict should be kept in sync with `patroni.utils._AUTH_ALLOWED_PARAMETERS` # (we use "username" in the Patroni config for some reason, other parameter names are the same). _AUTH_ALLOWED_PARAMETERS_MAPPING = { 'user': 'PGUSER', 'password': 'PGPASSWORD', 'sslmode': 'PGSSLMODE', 'sslcert': 'PGSSLCERT', 'sslkey': 'PGSSLKEY', 'sslpassword': '', 'sslrootcert': 'PGSSLROOTCERT', 'sslcrl': 'PGSSLCRL', 'sslcrldir': 'PGSSLCRLDIR', 'gssencmode': 'PGGSSENCMODE', 'channel_binding': 'PGCHANNELBINDING', 'sslnegotiation': 'PGSSLNEGOTIATION' } NO_VALUE_MSG = '#FIXME' def get_address() -> Tuple[str, str]: """Try to get hostname and the ip address for it returned by :func:`~socket.gethostname`. .. note:: Can also return local ip. :returns: tuple consisting of the hostname returned by :func:`~socket.gethostname` and the first element in the sorted list of the addresses returned by :func:`~socket.getaddrinfo`. Sorting guarantees it will prefer IPv4. If an exception occurred, hostname and ip values are equal to :data:`~patroni.config_generator.NO_VALUE_MSG`. """ hostname = None try: hostname = socket.gethostname() return hostname, sorted(socket.getaddrinfo(hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0), key=lambda x: x[0])[0][4][0] except Exception as err: logging.warning('Failed to obtain address: %r', err) return NO_VALUE_MSG, NO_VALUE_MSG class AbstractConfigGenerator(abc.ABC): """Object representing the generated Patroni config. :ivar output_file: full path to the output file to be used. :ivar pg_major: integer representation of the major PostgreSQL version. :ivar config: dictionary used for the generated configuration storage. """ def __init__(self, output_file: Optional[str]) -> None: """Set up the output file (if passed), helper vars and the minimal config structure. :param output_file: full path to the output file to be used. """ self.output_file = output_file self.pg_major = 0 self.config = self.get_template_config() self.generate() @classmethod def get_template_config(cls) -> Dict[str, Any]: """Generate a template config for further extension (e.g. in the inherited classes). :returns: dictionary with the values gathered from Patroni env, hopefully defined hostname and ip address (otherwise set to :data:`~patroni.config_generator.NO_VALUE_MSG`), and some sane defaults. """ _HOSTNAME, _IP = get_address() template_config: Dict[str, Any] = { 'scope': NO_VALUE_MSG, 'name': _HOSTNAME, 'restapi': { 'connect_address': _IP + ':8008', 'listen': _IP + ':8008' }, 'log': { 'type': PatroniLogger.DEFAULT_TYPE, 'level': PatroniLogger.DEFAULT_LEVEL, 'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL, 'format': PatroniLogger.DEFAULT_FORMAT, 'max_queue_size': PatroniLogger.DEFAULT_MAX_QUEUE_SIZE }, 'postgresql': { 'data_dir': NO_VALUE_MSG, 'connect_address': _IP + ':5432', 'listen': _IP + ':5432', 'bin_dir': '', 'authentication': { 'superuser': { 'username': 'postgres', 'password': NO_VALUE_MSG }, 'replication': { 'username': 'replicator', 'password': NO_VALUE_MSG } } }, 'tags': { 'failover_priority': 1, 'noloadbalance': False, 'clonefrom': True, 'nosync': False, 'nostream': False, } } dynamic_config = Config.get_default_config() # to properly dump CaseInsensitiveDict as YAML later dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters']) config = Config('', None).local_configuration # Get values from env config.setdefault('bootstrap', {})['dcs'] = dynamic_config config.setdefault('postgresql', {}) del config['bootstrap']['dcs']['standby_cluster'] patch_config(template_config, config) return template_config @abc.abstractmethod def generate(self) -> None: """Generate config and store in :attr:`~AbstractConfigGenerator.config`.""" @staticmethod def _format_block(block: Any, line_prefix: str = '') -> str: """Format a single YAML block. .. note:: Optionally the formatted block could be indented with the *line_prefix* :param block: the object that should be formatted to YAML. :param line_prefix: is used for indentation. :returns: a formatted and indented *block*. """ return line_prefix + yaml.safe_dump(block, default_flow_style=False, line_break='\n', allow_unicode=True, indent=2).strip().replace('\n', '\n' + line_prefix) def _format_config_section(self, section_name: str) -> Iterator[str]: """Format and yield as single section of the current :attr:`~AbstractConfigGenerator.config`. .. note:: If the section is a :class:`dict` object we put an empty line before it. :param section_name: a section name in the :attr:`~AbstractConfigGenerator.config`. :yields: a formatted section in case if it exists in the :attr:`~AbstractConfigGenerator.config`. """ if section_name in self.config: if isinstance(self.config[section_name], dict): yield '' yield self._format_block({section_name: self.config[section_name]}) def _format_config(self) -> Iterator[str]: """Format current :attr:`~AbstractConfigGenerator.config` and enrich it with some comments. :yields: formatted lines or blocks that represent a text output of the YAML document. """ for name in ('scope', 'namespace', 'name', 'log', 'restapi', 'ctl', 'citus', 'consul', 'etcd', 'etcd3', 'exhibitor', 'kubernetes', 'raft', 'zookeeper'): yield from self._format_config_section(name) if 'bootstrap' in self.config: yield '\n# The bootstrap configuration. Works only when the cluster is not yet initialized.' yield '# If the cluster is already initialized, all changes in the `bootstrap` section are ignored!' yield 'bootstrap:' if 'dcs' in self.config['bootstrap']: yield ' # This section will be written into :///config after initializing' yield ' # new cluster and all other cluster members will use it as a `global configuration`.' yield ' # WARNING! If you want to change any of the parameters that were set up' yield ' # via `bootstrap.dcs` section, please use `patronictl edit-config`!' yield ' dcs:' for name in ('loop_wait', 'retry_timeout', 'ttl'): if name in self.config['bootstrap']['dcs']: yield self._format_block({name: self.config['bootstrap']['dcs'].pop(name)}, ' ') for name, value in self.config['bootstrap']['dcs'].items(): yield self._format_block({name: value}, ' ') for name in ('postgresql', 'watchdog', 'tags'): yield from self._format_config_section(name) def _write_config_to_fd(self, fd: TextIO) -> None: """Format and write current :attr:`~AbstractConfigGenerator.config` to provided file descriptor. :param fd: where to write the config file. Could be ``sys.stdout`` or the real file. """ fd.write('\n'.join(self._format_config())) def write_config(self) -> None: """Write current :attr:`~AbstractConfigGenerator.config` to the output file if provided, to stdout otherwise.""" if self.output_file: dir_path = os.path.dirname(self.output_file) if dir_path and not os.path.isdir(dir_path): os.makedirs(dir_path) with open(self.output_file, 'w', encoding='UTF-8') as output_file: self._write_config_to_fd(output_file) else: self._write_config_to_fd(sys.stdout) class SampleConfigGenerator(AbstractConfigGenerator): """Object representing the generated sample Patroni config. Sane defaults are used based on the gathered PG version. """ @property def get_auth_method(self) -> str: """Return the preferred authentication method for a specific PG version if provided or the default ``md5``. :returns: :class:`str` value for the preferred authentication method. """ return 'scram-sha-256' if self.pg_major and self.pg_major >= 100000 else 'md5' def _get_int_major_version(self) -> int: """Get major PostgreSQL version from the binary as an integer. :returns: an integer PostgreSQL major version representation gathered from the PostgreSQL binary. See :func:`~patroni.postgresql.misc.postgres_major_version_to_int` and :func:`~patroni.utils.get_major_version`. """ postgres_bin = ((self.config.get('postgresql') or EMPTY_DICT).get('bin_name') or EMPTY_DICT).get('postgres', 'postgres') return postgres_major_version_to_int(get_major_version(self.config['postgresql'].get('bin_dir'), postgres_bin)) def generate(self) -> None: """Generate sample config using some sane defaults and update :attr:`~AbstractConfigGenerator.config`.""" self.pg_major = self._get_int_major_version() self.config['postgresql']['parameters'] = {'password_encryption': self.get_auth_method} username = self.config["postgresql"]["authentication"]["replication"]["username"] self.config['postgresql']['pg_hba'] = [ f'host all all all {self.get_auth_method}', f'host replication {username} all {self.get_auth_method}' ] # add version-specific configuration wal_keep_param = 'wal_keep_segments' if self.pg_major < 130000 else 'wal_keep_size' self.config['bootstrap']['dcs']['postgresql']['parameters'][wal_keep_param] = \ ConfigHandler.CMDLINE_OPTIONS[wal_keep_param][0] wal_level = 'hot_standby' if self.pg_major < 90600 else 'replica' self.config['bootstrap']['dcs']['postgresql']['parameters']['wal_level'] = wal_level self.config['bootstrap']['dcs']['postgresql']['use_pg_rewind'] = \ parse_bool(self.config['bootstrap']['dcs']['postgresql']['parameters']['wal_log_hints']) is True if self.pg_major >= 110000: self.config['postgresql']['authentication'].setdefault( 'rewind', {'username': 'rewind_user'}).setdefault('password', NO_VALUE_MSG) class RunningClusterConfigGenerator(AbstractConfigGenerator): """Object representing the Patroni config generated using information gathered from the running instance. :ivar dsn: DSN string for the local instance to get GUC values from (if provided). :ivar parsed_dsn: DSN string parsed into a dictionary (see :func:`~patroni.postgresql.config.parse_dsn`). """ def __init__(self, output_file: Optional[str] = None, dsn: Optional[str] = None) -> None: """Additionally store the passed dsn (if any) in both original and parsed version and run config generation. :param output_file: full path to the output file to be used. :param dsn: DSN string for the local instance to get GUC values from. :raises: :exc:`~patroni.exceptions.PatroniException`: if DSN parsing failed. """ self.dsn = dsn self.parsed_dsn = {} super().__init__(output_file) @property def _get_hba_conn_types(self) -> Tuple[str, ...]: """Return the connection types allowed. If :attr:`~RunningClusterConfigGenerator.pg_major` is defined, adds additional parameters for PostgreSQL version >=16. :returns: tuple of the connection methods allowed. """ allowed_types = ('local', 'host', 'hostssl', 'hostnossl', 'hostgssenc', 'hostnogssenc') if self.pg_major and self.pg_major >= 160000: allowed_types += ('include', 'include_if_exists', 'include_dir') return allowed_types @property def _required_pg_params(self) -> List[str]: """PG configuration parameters that have to be always present in the generated config. :returns: list of the parameter names. """ return ['hba_file', 'ident_file', 'config_file', 'data_directory'] + \ list(ConfigHandler.CMDLINE_OPTIONS.keys()) def _get_bin_dir_from_running_instance(self) -> str: """Define the directory postgres binaries reside using postmaster's pid executable. :returns: path to the PostgreSQL binaries directory. :raises: :exc:`~patroni.exceptions.PatroniException`: if: * pid could not be obtained from the ``postmaster.pid`` file; or * :exc:`OSError` occurred during ``postmaster.pid`` file handling; or * the obtained postmaster pid doesn't exist. """ postmaster_pid = None data_dir = self.config['postgresql']['data_dir'] try: with open(f"{data_dir}/postmaster.pid", 'r') as pid_file: postmaster_pid = pid_file.readline() if not postmaster_pid: raise PatroniException('Failed to obtain postmaster pid from postmaster.pid file') postmaster_pid = int(postmaster_pid.strip()) except OSError as err: raise PatroniException(f'Error while reading postmaster.pid file: {err}') try: return os.path.dirname(psutil.Process(postmaster_pid).exe()) except psutil.NoSuchProcess: raise PatroniException("Obtained postmaster pid doesn't exist.") @contextmanager def _get_connection_cursor(self) -> Iterator[Union['cursor', 'Cursor[Any]']]: """Get cursor for the PG connection established based on the stored information. :raises: :exc:`~patroni.exceptions.PatroniException`: if :exc:`psycopg.Error` occurred. """ try: conn = psycopg.connect(dsn=self.dsn, password=self.config['postgresql']['authentication']['superuser']['password']) with conn.cursor() as cur: yield cur conn.close() except psycopg.Error as e: raise PatroniException(f'Failed to establish PostgreSQL connection: {e}') def _set_pg_params(self, cur: Union['cursor', 'Cursor[Any]']) -> None: """Extend :attr:`~RunningClusterConfigGenerator.config` with the actual PG GUCs values. THe following GUC values are set: * Non-internal having configuration file, postmaster command line or environment variable as a source. * List of the always required parameters (see :meth:`~RunningClusterConfigGenerator._required_pg_params`). :param cur: connection cursor to use. """ cur.execute("SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings " "WHERE context <> 'internal' " "AND source IN ('configuration file', 'command line', 'environment variable') " "AND category <> 'Write-Ahead Log / Recovery Target' " "AND setting <> '(disabled)' " "OR name = ANY(%s)", (self._required_pg_params,)) helper_dict = dict.fromkeys(['port', 'listen_addresses']) self.config['postgresql'].setdefault('parameters', {}) for param, value in cur.fetchall(): if param == 'data_directory': self.config['postgresql']['data_dir'] = value elif param == 'cluster_name' and value: self.config['scope'] = value elif param in ('archive_command', 'restore_command', 'archive_cleanup_command', 'recovery_end_command', 'ssl_passphrase_command', 'hba_file', 'ident_file', 'config_file'): # write commands to the local config due to security implications # write hba/ident/config_file to local config to ensure they are not removed later self.config['postgresql']['parameters'][param] = value elif param in helper_dict: helper_dict[param] = value else: self.config['bootstrap']['dcs']['postgresql']['parameters'][param] = value connect_ip = self.config['postgresql']['connect_address'].rsplit(':')[0] connect_port = self.parsed_dsn.get('port', os.getenv('PGPORT', helper_dict['port'])) self.config['postgresql']['connect_address'] = f'{connect_ip}:{connect_port}' self.config['postgresql']['listen'] = f'{helper_dict["listen_addresses"]}:{helper_dict["port"]}' def _set_su_params(self) -> None: """Extend :attr:`~RunningClusterConfigGenerator.config` with the superuser auth information. Information set is based on the options used for connection. """ su_params: Dict[str, str] = {} for conn_param, env_var in _AUTH_ALLOWED_PARAMETERS_MAPPING.items(): val = self.parsed_dsn.get(conn_param, os.getenv(env_var)) if val: su_params[conn_param] = val patroni_env_su_username = ((self.config.get('authentication') or EMPTY_DICT).get('superuser') or EMPTY_DICT).get('username') patroni_env_su_pwd = ((self.config.get('authentication') or EMPTY_DICT).get('superuser') or EMPTY_DICT).get('password') # because we use "username" in the config for some reason su_params['username'] = su_params.pop('user', patroni_env_su_username) or getuser() su_params['password'] = su_params.get('password', patroni_env_su_pwd) or \ getpass('Please enter the user password:') self.config['postgresql']['authentication'] = { 'superuser': su_params, 'replication': {'username': NO_VALUE_MSG, 'password': NO_VALUE_MSG} } def _set_conf_files(self) -> None: """Extend :attr:`~RunningClusterConfigGenerator.config` with ``pg_hba.conf`` and ``pg_ident.conf`` content. .. note:: This function only defines ``postgresql.pg_hba`` and ``postgresql.pg_ident`` when ``hba_file`` and ``ident_file`` are set to the defaults. It may happen these files are located outside of ``PGDATA`` and Patroni doesn't have write permissions for them. :raises: :exc:`~patroni.exceptions.PatroniException`: if :exc:`OSError` occurred during the conf files handling. """ default_hba_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_hba.conf') if self.config['postgresql']['parameters']['hba_file'] == default_hba_path: try: self.config['postgresql']['pg_hba'] = list( filter(lambda i: i and i.split()[0] in self._get_hba_conn_types, read_stripped(default_hba_path))) except OSError as err: raise PatroniException(f'Failed to read pg_hba.conf: {err}') default_ident_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_ident.conf') if self.config['postgresql']['parameters']['ident_file'] == default_ident_path: try: self.config['postgresql']['pg_ident'] = [i for i in read_stripped(default_ident_path) if i and not i.startswith('#')] except OSError as err: raise PatroniException(f'Failed to read pg_ident.conf: {err}') if not self.config['postgresql']['pg_ident']: del self.config['postgresql']['pg_ident'] def _enrich_config_from_running_instance(self) -> None: """Extend :attr:`~RunningClusterConfigGenerator.config` with the values gathered from the running instance. Retrieve the following information from the running PostgreSQL instance: * superuser auth parameters (see :meth:`~RunningClusterConfigGenerator._set_su_params`); * some GUC values (see :meth:`~RunningClusterConfigGenerator._set_pg_params`); * ``postgresql.connect_address``, ``postgresql.listen``; * ``postgresql.pg_hba`` and ``postgresql.pg_ident`` (see :meth:`~RunningClusterConfigGenerator._set_conf_files`) And redefine ``scope`` with the ``cluster_name`` GUC value if set. :raises: :exc:`~patroni.exceptions.PatroniException`: if the provided user doesn't have superuser privileges. """ self._set_su_params() with self._get_connection_cursor() as cur: self.pg_major = getattr(cur.connection, 'server_version', 0) if not parse_bool(getattr(cur.connection, 'get_parameter_status')('is_superuser')): raise PatroniException('The provided user does not have superuser privilege') self._set_pg_params(cur) self._set_conf_files() def generate(self) -> None: """Generate config using the info gathered from the specified running PG instance. Result is written to :attr:`~RunningClusterConfigGenerator.config`. """ if self.dsn: self.parsed_dsn = parse_dsn(self.dsn) or {} if not self.parsed_dsn: raise PatroniException('Failed to parse DSN string') self._enrich_config_from_running_instance() self.config['postgresql']['bin_dir'] = self._get_bin_dir_from_running_instance() def generate_config(output_file: str, sample: bool, dsn: Optional[str]) -> None: """Generate Patroni configuration file. :param output_file: Full path to the configuration file to be used. If not provided, result is sent to ``stdout``. :param sample: Optional flag. If set, no source instance will be used - generate config with some sane defaults. :param dsn: Optional DSN string for the local instance to get GUC values from. """ try: if sample: config_generator = SampleConfigGenerator(output_file) else: config_generator = RunningClusterConfigGenerator(output_file, dsn) config_generator.write_config() except PatroniException as e: sys.exit(str(e)) except Exception as e: sys.exit(f'Unexpected exception: {e}') patroni-4.0.4/patroni/ctl.py000066400000000000000000003075731472010352700160230ustar00rootroot00000000000000"""Implement ``patronictl``: a command-line application which utilises the REST API to perform cluster operations. :var CONFIG_DIR_PATH: path to Patroni configuration directory as per :func:`click.get_app_dir` output. :var CONFIG_FILE_PATH: default path to ``patronictl.yaml`` configuration file. :var DCS_DEFAULTS: auxiliary dictionary to build the DCS section of the configuration file. Mainly used to help parsing ``--dcs-url`` command-line option of ``patronictl``. .. note:: Most of the ``patronictl`` commands (``restart``/``reinit``/``pause``/``resume``/``show-config``/``edit-config`` and similar) require the ``group`` argument and work only for that specific Citus ``group``. If not specified in the command line the ``group`` might be taken from the configuration file. If it is also missing in the configuration file we assume that this is just a normal Patroni cluster (not Citus). """ import codecs import copy import datetime import difflib import io import json import logging import os import random import shutil import subprocess import sys import tempfile import time from collections import defaultdict from contextlib import contextmanager from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union from urllib.parse import urlparse import click import dateutil.parser import dateutil.tz import urllib3 import yaml from prettytable import PrettyTable try: # pragma: no cover from prettytable import HRuleStyle hrule_all = HRuleStyle.ALL hrule_frame = HRuleStyle.FRAME except ImportError: # pragma: no cover from prettytable import ALL as hrule_all, FRAME as hrule_frame if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor try: # pragma: no cover from ydiff import markup_to_pager # pyright: ignore [reportMissingModuleSource] try: from ydiff import PatchStream # pyright: ignore [reportMissingModuleSource] except ImportError: PatchStream = iter except ImportError: # pragma: no cover from cdiff import markup_to_pager, PatchStream # pyright: ignore [reportMissingModuleSource] from . import global_config from .config import Config from .dcs import AbstractDCS, Cluster, get_dcs as _get_dcs, Member from .exceptions import PatroniException from .postgresql.misc import postgres_version_to_int from .postgresql.mpp import get_mpp from .request import PatroniRequest from .utils import cluster_as_json, patch_config, polling_loop from .version import __version__ CONFIG_DIR_PATH = click.get_app_dir('patroni') CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml') DCS_DEFAULTS: Dict[str, Dict[str, Any]] = { 'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"}, 'exhibitor': {'port': 8181, 'template': "exhibitor:\n hosts: [{host}]\n port: {port}"}, 'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"}, 'etcd': {'port': 2379, 'template': "etcd:\n host: '{host}:{port}'"}, 'etcd3': {'port': 2379, 'template': "etcd3:\n host: '{host}:{port}'"}} class PatroniCtlException(click.ClickException): """Raised upon issues faced by ``patronictl`` utility.""" pass class PatronictlPrettyTable(PrettyTable): """Utilitary class to print pretty tables. Extend :class:`~prettytable.PrettyTable` to make it print custom information in the header line. The idea is to print a header line like this: ``` + Cluster: batman --------+--------+---------+----+-----------+ ``` Instead of the default header line which would contain only dash and plus characters. """ def __init__(self, header: str, *args: Any, **kwargs: Any) -> None: """Create a :class:`PatronictlPrettyTable` instance with the given *header*. :param header: custom string to be put in the first header line of the table. :param args: positional arguments to be passed to :class:`~prettytable.PrettyTable` constructor. :param kwargs: keyword arguments to be passed to :class:`~prettytable.PrettyTable` constructor. """ super(PatronictlPrettyTable, self).__init__(*args, **kwargs) self.__table_header = header self.__hline_num = 0 self.__hline: str def __build_header(self, line: str) -> str: """Build the custom header line for the table. .. note:: Expected to be called only against the very first header line of the table. :param line: the original header line. :returns: the modified header line. """ header = self.__table_header[:len(line) - 2] return "".join([line[0], header, line[1 + len(header):]]) def _stringify_hrule(self, *args: Any, **kwargs: Any) -> str: """Get the string representation of a header line. Inject the custom header line, if processing the first header line. .. note:: New implementation for injecting a custom header line, which is used from :mod:`prettytable` 2.2.0 onwards. :returns: string representation of a header line. """ ret = super(PatronictlPrettyTable, self)._stringify_hrule(*args, **kwargs) where = args[1] if len(args) > 1 else kwargs.get('where') if where == 'top_' and self.__table_header: ret = self.__build_header(ret) self.__hline_num += 1 return ret def _is_first_hline(self) -> bool: """Check if the current line being processed is the very first line of the header. :returns: ``True`` if processing the first header line, ``False`` otherwise. """ return self.__hline_num == 0 def _set_hline(self, value: str) -> None: """Set header line string representation. :param value: string representing a header line. """ self.__hline = value def _get_hline(self) -> str: """Get string representation of a header line. Inject the custom header line, if processing the first header line. .. note:: Original implementation for injecting a custom header line, and is used up to :mod:`prettytable` 2.2.0. From :mod:`prettytable` 2.2.0 onwards :func:`_stringify_hrule` is used instead. :returns: string representing a header line. """ ret = self.__hline # Inject nice table header if self._is_first_hline() and self.__table_header: ret = self.__build_header(ret) self.__hline_num += 1 return ret _hrule = property(_get_hline, _set_hline) def parse_dcs(dcs: Optional[str]) -> Optional[Dict[str, Any]]: """Parse a DCS URL. :param dcs: the DCS URL in the format ``DCS://HOST:PORT/NAMESPACE``. ``DCS`` can be one among: * ``consul`` * ``etcd`` * ``etcd3`` * ``exhibitor`` * ``zookeeper`` If ``DCS`` is not specified, assume ``etcd`` by default. If ``HOST`` is not specified, assume ``localhost`` by default. If ``PORT`` is not specified, assume the default port of the given ``DCS``. If ``NAMESPACE`` is not specified, use whatever is in config. :returns: ``None`` if *dcs* is ``None``, otherwise a dictionary. The dictionary represents *dcs* as if it were parsed from the Patroni configuration file. Additionally, if a namespace is specified in *dcs*, return a ``namespace`` key with the parsed value. :raises: :class:`PatroniCtlException`: if the DCS name in *dcs* is not valid. :Example: >>> parse_dcs('') {'etcd': {'host': 'localhost:2379'}} >>> parse_dcs('etcd://:2399') {'etcd': {'host': 'localhost:2399'}} >>> parse_dcs('etcd://test') {'etcd': {'host': 'test:2379'}} >>> parse_dcs('etcd3://random.com:2399') {'etcd3': {'host': 'random.com:2399'}} >>> parse_dcs('etcd3://random.com:2399/customnamespace') {'etcd3': {'host': 'random.com:2399'}, 'namespace': '/customnamespace'} """ if dcs is None: return None elif '//' not in dcs: dcs = '//' + dcs parsed = urlparse(dcs) scheme = parsed.scheme port = int(parsed.port) if parsed.port else None if scheme == '': scheme = ([k for k, v in DCS_DEFAULTS.items() if v['port'] == port] or ['etcd'])[0] elif scheme not in DCS_DEFAULTS: raise PatroniCtlException('Unknown dcs scheme: {}'.format(scheme)) default = DCS_DEFAULTS[scheme] ret = yaml.safe_load(default['template'].format(host=parsed.hostname or 'localhost', port=port or default['port'])) if parsed.path and parsed.path.strip() != '/': ret['namespace'] = parsed.path.strip() return ret def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]: """Load configuration file from *path* and optionally override its DCS configuration with *dcs_url*. :param path: path to the configuration file. :param dcs_url: the DCS URL in the format ``DCS://HOST:PORT/NAMESPACE``, e.g. ``etcd3://random.com:2399/service``. If given, override whatever DCS and ``namespace`` that are set in the configuration file. See :func:`parse_dcs` for more information. :returns: a dictionary representing the configuration. :raises: :class:`PatroniCtlException`: if *path* does not exist or is not readable. """ if not (os.path.exists(path) and os.access(path, os.R_OK)): if path != CONFIG_FILE_PATH: # bail if non-default config location specified but file not found / readable raise PatroniCtlException('Provided config file {0} not existing or no read rights.' ' Check the -c/--config-file parameter'.format(path)) else: logging.debug('Ignoring configuration file "%s". It does not exists or is not readable.', path) else: logging.debug('Loading configuration from file %s', path) config = Config(path, validator=None).copy() dcs_kwargs = parse_dcs(dcs_url) or {} if dcs_kwargs: for d in DCS_DEFAULTS: config.pop(d, None) config.update(dcs_kwargs) return config def _get_configuration() -> Dict[str, Any]: """Get configuration object. :returns: configuration object from the current context. """ return click.get_current_context().obj['__config'] option_format = click.option('--format', '-f', 'fmt', help='Output format', default='pretty', type=click.Choice(['pretty', 'tsv', 'json', 'yaml', 'yml'])) option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds') option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds') option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point') arg_cluster_name = click.argument('cluster_name', required=False, default=lambda: _get_configuration().get('scope')) option_default_citus_group = click.option('--group', required=False, type=int, help='Citus group', default=lambda: _get_configuration().get('citus', {}).get('group')) option_citus_group = click.option('--group', required=False, type=int, help='Citus group') role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any']) @click.group(cls=click.Group) @click.option('--config-file', '-c', help='Configuration file', envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH) @click.option('--dcs-url', '--dcs', '-d', 'dcs_url', help='The DCS connect url', envvar='DCS_URL') @click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs') @click.pass_context def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure: bool) -> None: """Command-line interface for interacting with Patroni. \f Entry point of ``patronictl`` utility. Load the configuration file. .. note:: Besides *dcs_url* and *insecure*, which are used to override DCS configuration section and ``ctl.insecure`` setting, you can also override the value of ``log.level``, by default ``WARNING``, through either of these environment variables: * ``LOGLEVEL`` * ``PATRONI_LOGLEVEL`` * ``PATRONI_LOG_LEVEL`` :param ctx: click context to be passed to sub-commands. :param config_file: path to the configuration file. :param dcs_url: the DCS URL in the format ``DCS://HOST:PORT``, e.g. ``etcd3://random.com:2399``. If given override whatever DCS is set in the configuration file. :param insecure: if ``True`` allow SSL connections without client certificates. Override what is configured through ``ctl.insecure` in the configuration file. """ level = 'WARNING' for name in ('LOGLEVEL', 'PATRONI_LOGLEVEL', 'PATRONI_LOG_LEVEL'): level = os.environ.get(name, level) logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level) logging.captureWarnings(True) # Capture eventual SSL warning config = load_config(config_file, dcs_url) # backward compatibility for configuration file where ctl section is not defined config.setdefault('ctl', {})['insecure'] = config.get('ctl', {}).get('insecure') or insecure ctx.obj = {'__config': config, '__mpp': get_mpp(config)} def is_citus_cluster() -> bool: """Check if we are working with Citus cluster. :returns: ``True`` if configuration has ``citus`` section, otherwise ``False``. """ return click.get_current_context().obj['__mpp'].is_enabled() # Cache DCS instances for given scope and group __dcs_cache: Dict[Tuple[str, Optional[int]], AbstractDCS] = {} def get_dcs(scope: str, group: Optional[int]) -> AbstractDCS: """Get the DCS object. :param scope: cluster name. :param group: if *group* is defined, use it to select which alternative Citus group this DCS refers to. If *group* is ``None`` and a Citus configuration exists, assume this is the coordinator. Coordinator has the group ``0``. Refer to the module note for more details. :returns: a subclass of :class:`~patroni.dcs.AbstractDCS`, according to the DCS technology that is configured. :raises: :class:`PatroniCtlException`: if not suitable DCS configuration could be found. """ if (scope, group) in __dcs_cache: return __dcs_cache[(scope, group)] config = _get_configuration() config.update({'scope': scope, 'patronictl': True}) if group is not None: config['citus'] = {'group': group, 'database': 'postgres'} config.setdefault('name', scope) try: dcs = _get_dcs(config) if is_citus_cluster() and group is None: dcs.is_mpp_coordinator = lambda: True click.get_current_context().obj['__mpp'] = dcs.mpp __dcs_cache[(scope, group)] = dcs return dcs except PatroniException as e: raise PatroniCtlException(str(e)) def request_patroni(member: Member, method: str = 'GET', endpoint: Optional[str] = None, data: Optional[Any] = None) -> urllib3.response.HTTPResponse: """Perform a request to Patroni REST API. :param member: DCS member, used to get the base URL of its REST API server. :param method: HTTP method to be used, e.g. ``GET``. :param endpoint: URL path of the request, e.g. ``patroni``. :param data: anything to be used as the request body. :returns: the response for the request. """ ctx = click.get_current_context() # the current click context request_executor = ctx.obj.get('__request_patroni') if not request_executor: request_executor = ctx.obj['__request_patroni'] = PatroniRequest(_get_configuration()) return request_executor(member, method, endpoint, data) def print_output(columns: Optional[List[str]], rows: List[List[Any]], alignment: Optional[Dict[str, str]] = None, fmt: str = 'pretty', header: str = '', delimiter: str = '\t') -> None: """Print tabular information. :param columns: list of column names. :param rows: list of rows. Each item is a list of values for the columns. :param alignment: alignment to be applied to column values. Each key is the name of a column to be aligned, and the corresponding value can be one among: * ``l``: left-aligned * ``c``: center-aligned * ``r``: right-aligned A key in the dictionary is only required for a column that needs a specific alignment. Only apply when *fmt* is either ``pretty`` or ``topology``. :param fmt: the printing format. Can be one among: * ``json``: to print as a JSON string -- array of objects; * ``yaml`` or ``yml``: to print as a YAML string; * ``tsv``: to print a table of separated values, by default by tab; * ``pretty``: to print a pretty table; * ``topology``: similar to *pretty*, but with a topology view when printing cluster members. :param header: a string to be included in the first line of the table header, typically the cluster name. Only apply when *fmt* is either ``pretty`` or ``topology``. :param delimiter: the character to be used as delimiter when *fmt* is ``tsv``. """ if fmt in {'json', 'yaml', 'yml'}: elements = [{k: v for k, v in zip(columns or [], r) if not header or str(v)} for r in rows] func = json.dumps if fmt == 'json' else format_config_for_editing click.echo(func(elements)) elif fmt in {'pretty', 'tsv', 'topology'}: list_cluster = bool(header and columns and columns[0] == 'Cluster') if list_cluster and columns and 'Tags' in columns: # we want to format member tags as YAML i = columns.index('Tags') for row in rows: if row[i]: # Member tags are printed in YAML block format if *fmt* is ``pretty``. If *fmt* is either ``tsv`` # or ``topology``, then write in the YAML flow format, which is similar to JSON row[i] = format_config_for_editing(row[i], fmt != 'pretty').strip() if list_cluster and header and fmt != 'tsv': # skip cluster name and maybe Citus group if pretty-printing skip_cols = 2 if ' (group: ' in header else 1 columns = columns[skip_cols:] if columns else [] rows = [row[skip_cols:] for row in rows] # In ``tsv`` format print cluster name in every row as the first column if fmt == 'tsv': for r in ([columns] if columns else []) + rows: click.echo(delimiter.join(map(str, r))) # In ``pretty`` and ``topology`` formats print the cluster name only once, in the very first header line else: # If any value is multi-line, then add horizontal between all table rows while printing to get a clear # visual separation of rows. hrules = hrule_all if any(any(isinstance(c, str) and '\n' in c for c in r) for r in rows) else hrule_frame table = PatronictlPrettyTable(header, columns, hrules=hrules) table.align = 'l' for k, v in (alignment or {}).items(): table.align[k] = v for r in rows: table.add_row(r) click.echo(table) def watching(w: bool, watch: Optional[int], max_count: Optional[int] = None, clear: bool = True) -> Iterator[int]: """Yield a value every ``watch`` seconds. Used to run a command with a watch-based approach. :param w: if ``True`` and *watch* is ``None``, then *watch* assumes the value ``2``. :param watch: amount of seconds to wait before yielding another value. :param max_count: maximum number of yielded values. If ``None`` keep yielding values indefinitely. :param clear: if the screen should be cleared out at each iteration. :yields: ``0`` each time *watch* seconds have passed. :Example: >>> len(list(watching(True, 1, 0))) 1 >>> len(list(watching(True, 1, 1))) 2 >>> len(list(watching(True, None, 0))) 1 """ if w and not watch: watch = 2 if watch and clear: click.clear() yield 0 if max_count is not None and max_count < 1: return counter = 1 while watch and counter <= (max_count or counter): time.sleep(watch) counter += 1 if clear: click.clear() yield 0 def get_all_members(cluster: Cluster, group: Optional[int], role: str = 'leader') -> Iterator[Member]: """Get all cluster members that have the given *role*. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members from. If ``None`` get from all groups. :param role: role to filter members. Can be one among: * ``primary``: the primary PostgreSQL instance; * ``replica`` or ``standby``: a standby PostgreSQL instance; * ``leader``: the leader of a Patroni cluster. Can also be used to get the leader of a Patroni standby cluster; * ``standby-leader``: the leader of a Patroni standby cluster; * ``any``: matches any node independent of its role. :yields: members that have the given *role*. """ clusters = {0: cluster} if is_citus_cluster() and group is None: clusters.update(cluster.workers) if role in ('leader', 'primary', 'standby-leader'): # In the DCS the members' role can be one among: ``primary``, ``master``, ``replica`` or ``standby_leader``. # ``primary`` and ``master`` are the same thing. role = {'standby-leader': 'standby_leader'}.get(role, role) for cluster in clusters.values(): if cluster.leader is not None and cluster.leader.name and\ (role == 'leader' or cluster.leader.data.get('role') not in ('primary', 'master') and role == 'standby_leader' or cluster.leader.data.get('role') != 'standby_leader' and role == 'primary'): yield cluster.leader.member return for cluster in clusters.values(): leader_name = (cluster.leader.member.name if cluster.leader else None) for m in cluster.members: if role == 'any' or role in ('replica', 'standby') and m.name != leader_name: yield m def get_any_member(cluster: Cluster, group: Optional[int], role: Optional[str] = None, member: Optional[str] = None) -> Optional[Member]: """Get the first found cluster member that has the given *role*. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members from. If ``None`` get from all groups. :param role: role to filter members. See :func:`get_all_members` for available options. :param member: if specified, then besides having the given *role*, the cluster member's name should be *member*. :returns: the first found cluster member that has the given *role*. :raises: :class:`PatroniCtlException`: if both *role* and *member* are provided. """ if member is not None: if role is not None: raise PatroniCtlException('--role and --member are mutually exclusive options') role = 'any' elif role is None: role = 'leader' for m in get_all_members(cluster, group, role): if member is None or m.name == member: return m def get_all_members_leader_first(cluster: Cluster) -> Iterator[Member]: """Get all cluster members, with the cluster leader being yielded first. .. note:: Only yield members that have a ``restapi.connect_address`` configured. :yields: all cluster members, with the leader first. """ leader_name = cluster.leader.member.name if cluster.leader and cluster.leader.member.api_url else None if leader_name and cluster.leader: yield cluster.leader.member for member in cluster.members: if member.api_url and member.name != leader_name: yield member def get_cursor(cluster: Cluster, group: Optional[int], connect_parameters: Dict[str, Any], role: Optional[str] = None, member_name: Optional[str] = None) -> Union['cursor', 'Cursor[Any]', None]: """Get a cursor object to execute queries against a member that has the given *role* or *member_name*. .. note:: Besides what is passed through *connect_parameters*, this function also sets the following parameters: * ``fallback_application_name``: as ``Patroni ctl``; * ``connect_timeout``: as ``5``. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members to create a cursor against. If ``None`` consider members from all groups. :param connect_parameters: database connection parameters. :param role: role to filter members. See :func:`get_all_members` for available options. :param member_name: if specified, then besides having the given *role*, the cluster member's name should be *member_name*. :returns: a cursor object to execute queries against the database. Can be either: * A :class:`psycopg.Cursor` if using :mod:`psycopg`; or * A :class:`psycopg2.extensions.cursor` if using :mod:`psycopg2`; * ``None`` if not able to get a cursor that attendees *role* and *member_name*. """ member = get_any_member(cluster, group, role=role, member=member_name) if member is None: return None params = member.conn_kwargs(connect_parameters) params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'}) if 'dbname' in connect_parameters: params['dbname'] = connect_parameters['dbname'] else: params.pop('dbname') from . import psycopg conn = psycopg.connect(**params) cursor = conn.cursor() # If we want ``any`` node we are fine to return the cursor. ``None`` is similar to ``any`` at this point, as it's # been dealt with through :func:`get_any_member`. # If we want the Patroni leader node, :func:`get_any_member` already checks that for us if role in (None, 'any', 'leader'): return cursor # If we want something other than ``any`` or ``leader``, then we do not rely only on the DCS information about # members, but rather double check the underlying Postgres status. cursor.execute('SELECT pg_catalog.pg_is_in_recovery()') row = cursor.fetchone() in_recovery = not row or row[0] if in_recovery and role in ('replica', 'standby', 'standby-leader') or not in_recovery and role == 'primary': return cursor conn.close() return None def get_members(cluster: Cluster, cluster_name: str, member_names: List[str], role: str, force: bool, action: str, ask_confirmation: bool = True, group: Optional[int] = None) -> List[Member]: """Get the list of members based on the given filters. .. note:: Contain some filtering and checks processing that are common to several actions that are exposed by `patronictl`, like: * Get members of *cluster* that respect the given *member_names*, *role*, and *group*; * Bypass confirmations; * Prompt user for information that has not been passed through the command-line options; * etc. Designed to handle both attended and unattended ``patronictl`` commands execution that need to retrieve and validate the members before doing anything. In the very end may call :func:`confirm_members_action` to ask if the user would like to proceed with *action* over the retrieved members. That won't actually perform the action, but it works as the "last confirmation" before the *action* is processed by the caller method. Additional checks can also be implemented in the caller method, in which case you might want to pass ``ask_confirmation=False``, and later call :func:`confirm_members_action` manually in the caller method. That way the workflow won't look broken to the user that is interacting with ``patronictl``. :param cluster: Patroni cluster. :param cluster_name: name of the Patroni cluster. :param member_names: used to filter which members should take the *action* based on their names. Each item is the name of a Patroni member, as per ``name`` configuration. If *member_names* is an empty :class:`tuple` no filters are applied based on names. :param role: used to filter which members should take the *action* based on their role. See :func:`get_all_members` for available options. :param force: if ``True``, then it won't ask for confirmations at any point nor prompt the user to select values for options that were not specified through the command-line. :param action: the action that is being processed, one among: * ``reload``: reload PostgreSQL configuration; or * ``restart``: restart PostgreSQL; or * ``reinitialize``: reinitialize PostgreSQL data directory; or * ``flush``: discard scheduled actions. :param ask_confirmation: if ``False``, then it won't ask for the final confirmation regarding the *action* before returning the list of members. Usually useful as ``False`` if you want to perform additional checks in the caller method besides the checks that are performed through this generic method. :param group: filter which Citus group we should get members from. If ``None`` consider members from all groups. :returns: a list of members that respect the given filters. :raises: :class:`PatroniCtlException`: if * Cluster does not have members that match the given *role*; or * Cluster does not have members that match the given *member_names*; or * No member with given *role* is found among the specified *member_names*. """ members = list(get_all_members(cluster, group, role)) candidates = {m.name for m in members} if not force or role: if not member_names and not candidates: raise PatroniCtlException('{0} cluster doesn\'t have any members'.format(cluster_name)) output_members(cluster, cluster_name, group=group) if member_names: member_names = list(set(member_names) & candidates) if not member_names: raise PatroniCtlException('No {0} among provided members'.format(role)) elif action != 'reinitialize': member_names = list(candidates) if not member_names and not force: member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action, ', '.join(candidates)), type=str, default='')] for member_name in member_names: if member_name not in candidates: raise PatroniCtlException('{0} is not a member of cluster'.format(member_name)) members = [m for m in members if m.name in member_names] if ask_confirmation: confirm_members_action(members, force, action) return members def confirm_members_action(members: List[Member], force: bool, action: str, scheduled_at: Optional[datetime.datetime] = None) -> None: """Ask for confirmation if *action* should be taken by *members*. :param members: list of member which will take the *action*. :param force: if ``True`` skip the confirmation prompt and allow the *action* to proceed. :param action: the action that is being processed, one among: * ``reload``: reload PostgreSQL configuration; or * ``restart``: restart PostgreSQL; or * ``reinitialize``: reinitialize PostgreSQL data directory; or * ``flush``: discard scheduled actions. :param scheduled_at: timestamp at which the *action* should be scheduled to. If ``None`` *action* is taken immediately. :raises: :class:`PatroniCtlException`: if the user aborted the *action*. """ if scheduled_at: if not force: confirm = click.confirm('Are you sure you want to schedule {0} of members {1} at {2}?' .format(action, ', '.join([m.name for m in members]), scheduled_at)) if not confirm: raise PatroniCtlException('Aborted scheduled {0}'.format(action)) else: if not force: confirm = click.confirm('Are you sure you want to {0} members {1}?' .format(action, ', '.join([m.name for m in members]))) if not confirm: raise PatroniCtlException('Aborted {0}'.format(action)) @ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the leader') @click.option('--role', '-r', help='Give a dsn of any member with this role', type=role_choice, default=None) @click.option('--member', '-m', help='Generate a dsn for this member', type=str) @arg_cluster_name @option_citus_group def dsn(cluster_name: str, group: Optional[int], role: Optional[str], member: Optional[str]) -> None: """Process ``dsn`` command of ``patronictl`` utility. Get DSN to connect to *member*. .. note:: If no *role* nor *member* is given assume *role* as ``leader``. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get members to get DSN from. Refer to the module note for more details. :param role: filter which members to get DSN from based on their role. See :func:`get_all_members` for available options. :param member: filter which member to get DSN from based on its name. :raises: :class:`PatroniCtlException`: if * both *role* and *member* are provided; or * No member matches requested *member* or *role*. """ cluster = get_dcs(cluster_name, group).get_cluster() m = get_any_member(cluster, group, role=role, member=member) if m is None: raise PatroniCtlException('Can not find a suitable member') params = m.conn_kwargs() click.echo('host={host} port={port}'.format(**params)) @ctl.command('query', help='Query a Patroni PostgreSQL member') @arg_cluster_name @option_citus_group @click.option('--format', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='tsv') @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb')) @click.option('--password', help='force password prompt', is_flag=True) @click.option('-U', '--username', help='database user name', type=str) @option_watch @option_watchrefresh @click.option('--role', '-r', help='The role of the query', type=role_choice, default=None) @click.option('--member', '-m', help='Query a specific member', type=str) @click.option('--delimiter', help='The column delimiter', default='\t') @click.option('--command', '-c', help='The SQL commands to execute') @click.option('-d', '--dbname', help='database name to connect to', type=str) def query( cluster_name: str, group: Optional[int], role: Optional[str], member: Optional[str], w: bool, watch: Optional[int], delimiter: str, command: Optional[str], p_file: Optional[io.BufferedReader], password: Optional[bool], username: Optional[str], dbname: Optional[str], fmt: str = 'tsv' ) -> None: """Process ``query`` command of ``patronictl`` utility. Perform a Postgres query in a Patroni node. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for more details. :param role: filter which members to perform the query against based on their role. See :func:`get_all_members` for available options. :param member: filter which member to perform the query against based on its name. :param w: perform query with watch-based approach every 2 seconds. :param watch: perform query with watch-based approach every *watch* seconds. :param delimiter: column delimiter when *fmt* is ``tsv``. :param command: SQL query to execute. :param p_file: path to file containing SQL query to execute. :param password: if ``True`` then prompt for password. :param username: name of the database user. :param dbname: name of the database. :param fmt: the output table printing format. See :func:`print_output` for available options. :raises: :class:`PatroniCtlException`: if: * if * both *role* and *member* are provided; or * both *file* and *command* are provided; or * neither *file* nor *command* is provided. """ if p_file is not None: if command is not None: raise PatroniCtlException('--file and --command are mutually exclusive options') sql = p_file.read().decode('utf-8') else: if command is None: raise PatroniCtlException('You need to specify either --command or --file') sql = command connect_parameters: Dict[str, str] = {} if username: connect_parameters['username'] = username if password: connect_parameters['password'] = click.prompt('Password', hide_input=True, type=str) if dbname: connect_parameters['dbname'] = dbname dcs = get_dcs(cluster_name, group) cluster = cursor = None for _ in watching(w, watch, clear=False): if cluster is None: cluster = dcs.get_cluster() output, header = query_member(cluster, group, cursor, member, role, sql, connect_parameters) print_output(header, output, fmt=fmt, delimiter=delimiter) def query_member(cluster: Cluster, group: Optional[int], cursor: Union['cursor', 'Cursor[Any]', None], member: Optional[str], role: Optional[str], command: str, connect_parameters: Dict[str, Any]) -> Tuple[List[List[Any]], Optional[List[Any]]]: """Execute SQL *command* against a member. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for more details. :param cursor: cursor through which *command* is executed. If ``None`` a new cursor is instantiated through :func:`get_cursor`. :param member: filter which member to create a cursor against based on its name, if *cursor* is ``None``. :param role: filter which member to create a cursor against based on their role, if *cursor* is ``None``. See :func:`get_all_members` for available options. :param command: SQL command to be executed. :param connect_parameters: connection parameters to be passed down to :func:`get_cursor`, if *cursor* is ``None``. :returns: a tuple composed of two items: * List of rows returned by the executed *command*; * List of columns related to the rows returned by the executed *command*. If an error occurs while executing *command*, then returns the following values in the tuple: * List with 2 items: * Current timestamp; * Error message. * ``None``. """ from . import psycopg try: if cursor is None: cursor = get_cursor(cluster, group, connect_parameters, role=role, member_name=member) if cursor is None: if member is not None: message = f'No connection to member {member} is available' elif role is not None: message = f'No connection to role {role} is available' else: message = 'No connection is available' logging.debug(message) return [[timestamp(0), message]], None cursor.execute(command.encode('utf-8')) return [list(row) for row in cursor], cursor.description and [d.name for d in cursor.description] except psycopg.DatabaseError as de: logging.debug(de) if cursor is not None and not cursor.connection.closed: cursor.connection.close() message = de.diag.sqlstate or str(de) message = message.replace('\n', ' ') return [[timestamp(0), 'ERROR, SQLSTATE: {0}'.format(message)]], None @ctl.command('remove', help='Remove cluster from DCS') @click.argument('cluster_name') @option_citus_group @option_format def remove(cluster_name: str, group: Optional[int], fmt: str) -> None: """Process ``remove`` command of ``patronictl`` utility. Remove cluster *cluster_name* from the DCS. :param cluster_name: name of the cluster which information will be wiped out of the DCS. :param group: which Citus group should have its information wiped out of the DCS. Refer to the module note for more details. :param fmt: the output table printing format. See :func:`print_output` for available options. :raises: :class:`PatroniCtlException`: if: * Patroni is running on a Citus cluster, but no *group* was specified; or * *cluster_name* does not exist; or * user did not type the expected confirmation message when prompted for confirmation; or * use did not type the correct leader name when requesting removal of a healthy cluster. """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if is_citus_cluster() and group is None: raise PatroniCtlException('For Citus clusters the --group must me specified') output_members(cluster, cluster_name, fmt=fmt) confirm = click.prompt('Please confirm the cluster name to remove', type=str) if confirm != cluster_name: raise PatroniCtlException('Cluster names specified do not match') message = 'Yes I am aware' confirm = \ click.prompt('You are about to remove all information in DCS for {0}, please type: "{1}"'.format(cluster_name, message), type=str) if message != confirm: raise PatroniCtlException('You did not exactly type "{0}"'.format(message)) if cluster.leader and cluster.leader.name: confirm = click.prompt('This cluster currently is healthy. Please specify the leader name to continue') if confirm != cluster.leader.name: raise PatroniCtlException('You did not specify the current leader of the cluster') dcs.delete_cluster() def check_response(response: urllib3.response.HTTPResponse, member_name: str, action_name: str, silent_success: bool = False) -> bool: """Check an HTTP response and print a status message. :param response: the response to be checked. :param member_name: name of the member associated with the *response*. :param action_name: action associated with the *response*. :param silent_success: if a status message should be skipped upon a successful *response*. :returns: ``True`` if the response indicates a successful operation (HTTP status < ``400``), ``False`` otherwise. """ if response.status >= 400: click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format( action_name, member_name, response.status, response.data.decode('utf-8') )) return False elif not silent_success: click.echo('Success: {0} for member {1}'.format(action_name, member_name)) return True def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]: """Parse a string *scheduled* timestamp as a :class:`~datetime.datetime` object. :param scheduled: string representation of the timestamp. May also be ``now``. :returns: the corresponding :class:`~datetime.datetime` object, if *scheduled* is not ``now``, otherwise ``None``. :raises: :class:`PatroniCtlException`: if unable to parse *scheduled* from :class:`str` to :class:`~datetime.datetime`. :Example: >>> parse_scheduled(None) is None True >>> parse_scheduled('now') is None True >>> parse_scheduled('2023-05-29T04:32:31') datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzlocal()) >>> parse_scheduled('2023-05-29T04:32:31-3') datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzoffset(None, -10800)) """ if scheduled is not None and (scheduled or 'now') != 'now': try: scheduled_at = dateutil.parser.parse(scheduled) if scheduled_at.tzinfo is None: scheduled_at = scheduled_at.replace(tzinfo=dateutil.tz.tzlocal()) except (ValueError, TypeError): message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)' raise PatroniCtlException(message.format(scheduled)) return scheduled_at return None @ctl.command('reload', help='Reload cluster member configuration') @click.argument('cluster_name') @click.argument('member_names', nargs=-1) @option_citus_group @click.option('--role', '-r', help='Reload only members with this role', type=role_choice, default='any') @option_force def reload(cluster_name: str, member_names: List[str], group: Optional[int], force: bool, role: str) -> None: """Process ``reload`` command of ``patronictl`` utility. Reload configuration of cluster members based on given filters. :param cluster_name: name of the Patroni cluster. :param member_names: name of the members which configuration should be reloaded. :param group: filter which Citus group we should reload members. Refer to the module note for more details. :param force: perform the reload without asking for confirmations. :param role: role to filter members. See :func:`get_all_members` for available options. """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() members = get_members(cluster, cluster_name, member_names, role, force, 'reload', group=group) for member in members: r = request_patroni(member, 'post', 'reload') if r.status == 200: click.echo('No changes to apply on member {0}'.format(member.name)) elif r.status == 202: config = global_config.from_cluster(cluster) click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format( member.name, config.get('loop_wait') or dcs.loop_wait) ) else: click.echo('Failed: reload for member {0}, status code={1}, ({2})'.format( member.name, r.status, r.data.decode('utf-8')) ) @ctl.command('restart', help='Restart cluster member') @click.argument('cluster_name') @click.argument('member_names', nargs=-1) @option_citus_group @click.option('--role', '-r', help='Restart only members with this role', type=role_choice, default='any') @click.option('--any', 'p_any', help='Restart a single member only', is_flag=True) @click.option('--scheduled', help='Timestamp of a scheduled restart in unambiguous format (e.g. ISO 8601)', default=None) @click.option('--pg-version', 'version', help='Restart if the PostgreSQL version is less than provided (e.g. 9.5.2)', default=None) @click.option('--pending', help='Restart if pending', is_flag=True) @click.option('--timeout', help='Return error and fail over if necessary when restarting takes longer than this.') @option_force def restart(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, role: str, p_any: bool, scheduled: Optional[str], version: Optional[str], pending: bool, timeout: Optional[str]) -> None: """Process ``restart`` command of ``patronictl`` utility. Restart Postgres on cluster members based on given filters. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should restart members. Refer to the module note for more details. :param member_names: name of the members that should be restarted. :param force: perform the restart without asking for confirmations. :param role: role to filter members. See :func:`get_all_members` for available options. :param p_any: restart a single and random member among the ones that match the given filters. :param scheduled: timestamp when the restart should be scheduled to occur. If ``now`` restart immediately. :param version: restart only members which Postgres version is less than *version*. :param pending: restart only members that are flagged as ``pending restart``. :param timeout: timeout for the restart operation. If timeout is reached a failover may occur in the cluster. :raises: :class:`PatroniCtlException`: if: * *scheduled* could not be parsed; or * *version* could not be parsed; or * a restart is attempted against a cluster that is in maintenance mode. """ cluster = get_dcs(cluster_name, group).get_cluster() members = get_members(cluster, cluster_name, member_names, role, force, 'restart', False, group=group) if scheduled is None and not force: next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ', type=str, default='now') scheduled_at = parse_scheduled(scheduled) confirm_members_action(members, force, 'restart', scheduled_at) if p_any: random.shuffle(members) members = members[:1] if version is None and not force: version = click.prompt('Restart if the PostgreSQL version is less than provided (e.g. 9.5.2) ', type=str, default='') content: Dict[str, Any] = {} if pending: content['restart_pending'] = True if version: try: postgres_version_to_int(version) except PatroniException as e: raise PatroniCtlException(e.value) content['postgres_version'] = version if scheduled_at: if global_config.from_cluster(cluster).is_paused: raise PatroniCtlException("Can't schedule restart in the paused state") content['schedule'] = scheduled_at.isoformat() if timeout is not None: content['timeout'] = timeout for member in members: if 'schedule' in content: if force and member.data.get('scheduled_restart'): r = request_patroni(member, 'delete', 'restart') check_response(r, member.name, 'flush scheduled restart', True) r = request_patroni(member, 'post', 'restart', content) if r.status == 200: click.echo('Success: restart on member {0}'.format(member.name)) elif r.status == 202: click.echo('Success: restart scheduled on member {0}'.format(member.name)) elif r.status == 409: click.echo('Failed: another restart is already scheduled on member {0}'.format(member.name)) else: click.echo('Failed: restart for member {0}, status code={1}, ({2})'.format( member.name, r.status, r.data.decode('utf-8')) ) @ctl.command('reinit', help='Reinitialize cluster member') @click.argument('cluster_name') @option_citus_group @click.argument('member_names', nargs=-1) @option_force @click.option('--wait', help='Wait until reinitialization completes', is_flag=True) def reinit(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, wait: bool) -> None: """Process ``reinit`` command of ``patronictl`` utility. Reinitialize cluster members based on given filters. .. note:: Only reinitialize replica members, not a leader. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should reinit members. Refer to the module note for more details. :param member_names: name of the members that should be reinitialized. :param force: perform the restart without asking for confirmations. :param wait: wait for the operation to complete. """ cluster = get_dcs(cluster_name, group).get_cluster() members = get_members(cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group) wait_on_members: List[Member] = [] for member in members: body: Dict[str, bool] = {'force': force} while True: r = request_patroni(member, 'post', 'reinitialize', body) started = check_response(r, member.name, 'reinitialize') if not started and r.data.endswith(b' already in progress') \ and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'): body['force'] = True continue break if started and wait: wait_on_members.append(member) last_display = [] while wait_on_members: if wait_on_members != last_display: click.echo('Waiting for reinitialize to complete on: {0}'.format( ", ".join(member.name for member in wait_on_members)) ) last_display[:] = wait_on_members time.sleep(2) for member in wait_on_members: data = json.loads(request_patroni(member, 'get', 'patroni').data.decode('utf-8')) if data.get('state') != 'creating replica': click.echo('Reinitialize is completed on: {0}'.format(member.name)) wait_on_members.remove(member) def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[int], candidate: Optional[str], force: bool, switchover_leader: Optional[str] = None, switchover_scheduled: Optional[str] = None) -> None: """Perform a failover or a switchover operation in the cluster. Informational messages are printed in the console during the operation, as well as the list of members before and after the operation, so the user can follow the operation status. .. note:: If not able to perform the operation through the REST API, write directly to the DCS as a fall back. :param action: action to be taken -- ``failover`` or ``switchover``. :param cluster_name: name of the Patroni cluster. :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be prompted for filling it -- unless *force* is ``True``, in which case an exception is raised. :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used. :param force: perform the failover or switchover without asking for confirmations. :param switchover_leader: name of the leader passed to the switchover command if any. :param switchover_scheduled: timestamp when the switchover should be scheduled to occur. If ``now``, perform immediately. :raises: :class:`PatroniCtlException`: if: * Patroni is running on a Citus cluster, but no *group* was specified; or * a switchover was requested by the cluster has no leader; or * *switchover_leader* does not match the current leader of the cluster; or * cluster has no candidates available for the operation; or * no *candidate* is given for a failover operation; or * current leader and *candidate* are the same; or * *candidate* is tagged as nofailover; or * *candidate* is not a member of the cluster; or * trying to schedule a switchover in a cluster that is in maintenance mode; or * user aborts the operation. """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() click.echo('Current cluster topology') output_members(cluster, cluster_name, group=group) if is_citus_cluster() and group is None: if force: raise PatroniCtlException('For Citus clusters the --group must me specified') else: group = click.prompt('Citus group', type=int) dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() config = global_config.from_cluster(cluster) cluster_leader = cluster.leader and cluster.leader.name # leader has to be be defined for switchover only if action == 'switchover': if not cluster_leader: raise PatroniCtlException('This cluster has no leader') if switchover_leader is None: if force: switchover_leader = cluster_leader else: prompt = 'Standby Leader' if config.is_standby_cluster else 'Primary' switchover_leader = click.prompt(prompt, type=str, default=cluster_leader) if cluster_leader != switchover_leader: raise PatroniCtlException(f'Member {switchover_leader} is not the leader of cluster {cluster_name}') # excluding members with nofailover tag candidate_names = [str(m.name) for m in cluster.members if m.name != cluster_leader and not m.nofailover] # We sort the names for consistent output to the client candidate_names.sort() if not candidate_names: raise PatroniCtlException('No candidates found to {0} to'.format(action)) if candidate is None and not force: candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='') if action == 'failover' and not candidate: raise PatroniCtlException('Failover could be performed only to a specific candidate') if candidate and candidate not in candidate_names: if candidate == cluster_leader: raise PatroniCtlException( f'Member {candidate} is already the leader of cluster {cluster_name}') raise PatroniCtlException( f'Member {candidate} does not exist in cluster {cluster_name} or is tagged as nofailover') if all((not force, action == 'failover', config.is_synchronous_mode, not cluster.sync.is_empty, not cluster.sync.matches(candidate, True))): if not click.confirm(f'Are you sure you want to failover to the asynchronous node {candidate}?'): raise PatroniCtlException('Aborting ' + action) scheduled_at_str = None scheduled_at = None if action == 'switchover': if switchover_scheduled is None and not force: next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') switchover_scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ', type=str, default='now') scheduled_at = parse_scheduled(switchover_scheduled) if scheduled_at: if config.is_paused: raise PatroniCtlException("Can't schedule switchover in the paused state") scheduled_at_str = scheduled_at.isoformat() failover_value = {'candidate': candidate} if action == 'switchover': failover_value['leader'] = switchover_leader if scheduled_at_str: failover_value['scheduled_at'] = scheduled_at_str logging.debug(failover_value) # By now we have established that the leader exists and the candidate exists if not force: demote_msg = f', demoting current leader {cluster_leader}' if cluster_leader else '' if scheduled_at_str: # only switchover can be scheduled if not click.confirm(f'Are you sure you want to schedule switchover of cluster ' f'{cluster_name} at {scheduled_at_str}{demote_msg}?'): # action as a var to catch a regression in the tests raise PatroniCtlException('Aborting scheduled ' + action) else: if not click.confirm(f'Are you sure you want to {action} cluster {cluster_name}{demote_msg}?'): raise PatroniCtlException('Aborting ' + action) r = None try: member = cluster.leader.member if cluster.leader else candidate and cluster.get_member(candidate, False) if TYPE_CHECKING: # pragma: no cover assert isinstance(member, Member) r = request_patroni(member, 'post', action, failover_value) # probably old patroni, which doesn't support switchover yet if r.status == 501 and action == 'switchover' and b'Server does not support this operation' in r.data: r = request_patroni(member, 'post', 'failover', failover_value) if r.status in (200, 202): logging.debug(r) cluster = dcs.get_cluster() logging.debug(cluster) click.echo('{0} {1}'.format(timestamp(), r.data.decode('utf-8'))) else: click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status, r.data.decode('utf-8'))) return except Exception: logging.exception(r) logging.warning('Failing over to DCS') click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action)) dcs.manual_failover(switchover_leader, candidate, scheduled_at=scheduled_at) output_members(cluster, cluster_name, group=group) @ctl.command('failover', help='Failover to a replica') @arg_cluster_name @option_citus_group @click.option('--candidate', help='The name of the candidate', default=None) @option_force def failover(cluster_name: str, group: Optional[int], candidate: Optional[str], force: bool) -> None: """Process ``failover`` command of ``patronictl`` utility. Perform a failover operation immediately in the cluster. .. seealso:: Refer to :func:`_do_failover_or_switchover` for details. :param cluster_name: name of the Patroni cluster. :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be prompted for filling it -- unless *force* is ``True``, in which case an exception is raised by :func:`_do_failover_or_switchover`. :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used. :param force: perform the failover or switchover without asking for confirmations. """ _do_failover_or_switchover('failover', cluster_name, group, candidate, force) @ctl.command('switchover', help='Switchover to a replica') @arg_cluster_name @option_citus_group @click.option('--leader', '--primary', 'leader', help='The name of the current leader', default=None) @click.option('--candidate', help='The name of the candidate', default=None) @click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)', default=None) @option_force def switchover(cluster_name: str, group: Optional[int], leader: Optional[str], candidate: Optional[str], force: bool, scheduled: Optional[str]) -> None: """Process ``switchover`` command of ``patronictl`` utility. Perform a switchover operation in the cluster. .. seealso:: Refer to :func:`_do_failover_or_switchover` for details. :param cluster_name: name of the Patroni cluster. :param group: filter Citus group within we should perform a switchover. If ``None``, user will be prompted for filling it -- unless *force* is ``True``, in which case an exception is raised by :func:`_do_failover_or_switchover`. :param leader: name of the current leader member. :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used. :param force: perform the switchover without asking for confirmations. :param scheduled: timestamp when the switchover should be scheduled to occur. If ``now`` perform immediately. """ _do_failover_or_switchover('switchover', cluster_name, group, candidate, force, leader, scheduled) def generate_topology(level: int, member: Dict[str, Any], topology: Dict[Optional[str], List[Dict[str, Any]]]) -> Iterator[Dict[str, Any]]: """Recursively yield members with their names adjusted according to their *level* in the cluster topology. .. note:: The idea is to get a tree view of the members when printing their names. For example, suppose you have a cascading replication composed of 3 nodes, say ``postgresql0``, ``postgresql1``, and ``postgresql2``. This function would adjust their names to be like this: * ``'postgresql0'`` -> ``'postgresql0'`` * ``'postgresql1'`` -> ``'+ postgresql1'`` * ``'postgresql2'`` -> ``' + postgresql2'`` So, if you ever print their names line by line, you would see something like this: .. code-block:: postgresql0 + postgresql1 + postgresql2 :param level: the current level being inspected in the *topology*. :param member: information about the current member being inspected in *level* of *topology*. Should contain at least this key: * ``name``: name of the node, according to ``name`` configuration; But may contain others, which although ignored by this function, will be yielded as part of the resulting object. The value of key ``name`` is changed as explained in the note. :param topology: each key is the name of a node which has at least one replica attached to it. The corresponding value is a list of the attached replicas, each of them with the same structure described for *member*. :yields: the current member with its name changed. Besides that reyield values from recursive calls. """ members = topology.get(member['name'], []) if level > 0: member['name'] = '{0}+ {1}'.format((' ' * (level - 1) * 2), member['name']) if member['name']: yield member for member in members: yield from generate_topology(level + 1, member, topology) def topology_sort(members: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]: """Sort *members* according to their level in the replication topology tree. :param members: list of members in the cluster. Each item should contain at least these keys: * ``name``: name of the node, according to ``name`` configuration; * ``role``: ``leader``, ``standby_leader`` or ``replica``. Cascading replicas are identified through ``tags`` -> ``replicatefrom`` value -- if that is set, and they are in fact attached to another replica. Besides ``name``, ``role`` and ``tags`` keys, it may contain other keys, which although ignored by this function, will be yielded as part of the resulting object. The value of key ``name`` is changed through :func:`generate_topology`. :yields: *members* sorted by level in the topology, and with a new ``name`` value according to their level in the topology. """ topology: Dict[Optional[str], List[Dict[str, Any]]] = defaultdict(list) leader = next((m for m in members if m['role'].endswith('leader')), {'name': None}) replicas = set(member['name'] for member in members if not member['role'].endswith('leader')) for member in members: if not member['role'].endswith('leader'): parent = member.get('tags', {}).get('replicatefrom') parent = parent if parent and parent != member['name'] and parent in replicas else leader['name'] topology[parent].append(member) for member in generate_topology(0, leader, topology): yield member def get_cluster_service_info(cluster: Dict[str, Any]) -> List[str]: """Get complementary information about the cluster. :param cluster: a Patroni cluster represented as an object created through :func:`~patroni.utils.cluster_as_json`. :returns: a list of 0 or more informational messages. They can be about: * Cluster in maintenance mode; * Scheduled switchovers. """ service_info: List[str] = [] if cluster.get('pause'): service_info.append('Maintenance mode: on') if 'scheduled_switchover' in cluster: info = 'Switchover scheduled at: ' + cluster['scheduled_switchover']['at'] for name in ('from', 'to'): if name in cluster['scheduled_switchover']: info += '\n{0:>24}: {1}'.format(name, cluster['scheduled_switchover'][name]) service_info.append(info) return service_info def output_members(cluster: Cluster, name: str, extended: bool = False, fmt: str = 'pretty', group: Optional[int] = None) -> None: """Print information about the Patroni cluster and its members. Information is printed to console through :func:`print_output`, and contains: * ``Cluster``: name of the Patroni cluster, as per ``scope`` configuration; * ``Member``: name of the Patroni node, as per ``name`` configuration; * ``Host``: hostname (or IP) and port, as per ``postgresql.listen`` configuration; * ``Role``: ``Leader``, ``Standby Leader``, ``Sync Standby`` or ``Replica``; * ``State``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``, ``creating replica``, ``streaming``, ``in archive recovery``, and so on; * ``TL``: current timeline in Postgres; ``Lag in MB``: replication lag. Besides that it may also have: * ``Group``: Citus group ID -- showed only if Citus is enabled. * ``Pending restart``: if the node is pending a restart -- showed only if *extended*; * ``Scheduled restart``: timestamp for scheduled restart, if any -- showed only if *extended*; * ``Tags``: node tags, if any -- showed only if *extended*. The 3 extended columns are always included if *extended*, even if the member has no value for a given column. If not *extended*, these columns may still be shown if any of the members has any information for them. :param cluster: Patroni cluster. :param name: name of the Patroni cluster. :param extended: if extended information (pending restarts, scheduled restarts, node tags) should be printed, if available. :param fmt: the output table printing format. See :func:`print_output` for available options. If *fmt* is neither ``topology`` nor ``pretty``, then complementary information gathered through :func:`get_cluster_service_info` is not printed. :param group: filter which Citus group we should get members from. If ``None`` get from all groups. """ rows: List[List[Any]] = [] logging.debug(cluster) initialize = {None: 'uninitialized', '': 'initializing'}.get(cluster.initialize, cluster.initialize) columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB'] clusters = {group or 0: cluster_as_json(cluster)} if is_citus_cluster(): columns.insert(1, 'Group') if group is None: clusters.update({g: cluster_as_json(c) for g, c in cluster.workers.items()}) all_members = [m for c in clusters.values() for m in c['members'] if 'host' in m] for c in ('Pending restart', 'Pending restart reason', 'Scheduled restart', 'Tags'): if extended or any(m.get(c.lower().replace(' ', '_')) for m in all_members): columns.append(c) # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host append_port = any('port' in m and m['port'] != 5432 for m in all_members) or\ len(set(m['host'] for m in all_members)) < len(all_members) sort = topology_sort if fmt == 'topology' else iter for g, c in sorted(clusters.items()): for member in sort(c['members']): logging.debug(member) lag = member.get('lag', '') def format_diff(param: str, values: Dict[str, str], hide_long: bool): full_diff = param + ': ' + values['old_value'] + '->' + values['new_value'] return full_diff if not hide_long or len(full_diff) <= 50 else param + ': [hidden - too long]' restart_reason = '\n'.join([format_diff(k, v, fmt in ('pretty', 'topology')) for k, v in member.get('pending_restart_reason', {}).items()]) or '' member.update(cluster=name, member=member['name'], group=g, host=member.get('host', ''), tl=member.get('timeline', ''), role=member['role'].replace('_', ' ').title(), lag_in_mb=round(lag / 1024 / 1024) if isinstance(lag, int) else lag, pending_restart='*' if member.get('pending_restart') else '', pending_restart_reason=restart_reason) if append_port and member['host'] and member.get('port'): member['host'] = ':'.join([member['host'], str(member['port'])]) if 'scheduled_restart' in member: value = member['scheduled_restart']['schedule'] if 'postgres_version' in member['scheduled_restart']: value += ' if version < {0}'.format(member['scheduled_restart']['postgres_version']) member['scheduled_restart'] = value rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns]) if is_citus_cluster(): title = 'Citus cluster' title_details = '' if group is None else f' (group: {group}, {initialize})' else: title = 'Cluster' title_details = f' ({initialize})' title = f' {title}: {name}{title_details} ' print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title) if fmt not in ('pretty', 'topology'): # Omit service info when using machine-readable formats return for g, c in sorted(clusters.items()): service_info = get_cluster_service_info(c) if service_info: if is_citus_cluster() and group is None: click.echo('Citus group: {0}'.format(g)) click.echo(' ' + '\n '.join(service_info)) @ctl.command('list', help='List the Patroni members for a given Patroni') @click.argument('cluster_names', nargs=-1) @option_citus_group @click.option('--extended', '-e', help='Show some extra information', is_flag=True) @click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True) @option_format @option_watch @option_watchrefresh def members(cluster_names: List[str], group: Optional[int], fmt: str, watch: Optional[int], w: bool, extended: bool, ts: bool) -> None: """Process ``list`` command of ``patronictl`` utility. Print information about the Patroni cluster through :func:`output_members`. :param cluster_names: name of clusters that should be printed. If ``None`` consider only the cluster present in ``scope`` key of the configuration. :param group: filter which Citus group we should get members from. Refer to the module note for more details. :param fmt: the output table printing format. See :func:`print_output` for available options. :param watch: if given print output every *watch* seconds. :param w: if ``True`` print output every 2 seconds. :param extended: if extended information should be printed. See ``extended`` argument of :func:`output_members` for more details. :param ts: if timestamp should be included in the output. """ config = _get_configuration() if not cluster_names: if 'scope' in config: cluster_names = [config['scope']] if not cluster_names: return logging.warning('Listing members: No cluster names were provided') for _ in watching(w, watch): if ts: click.echo(timestamp(0)) for cluster_name in cluster_names: dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() output_members(cluster, cluster_name, extended, fmt, group) @ctl.command('topology', help='Prints ASCII topology for given cluster') @click.argument('cluster_names', nargs=-1) @option_citus_group @option_watch @option_watchrefresh @click.pass_context def topology(ctx: click.Context, cluster_names: List[str], group: Optional[int], watch: Optional[int], w: bool) -> None: """Process ``topology`` command of ``patronictl`` utility. Print information about the cluster in ``topology`` format through :func:`members`. :param ctx: click context to be passed to :func:`members`. :param cluster_names: name of clusters that should be printed. See ``cluster_names`` argument of :func:`output_members` for more details. :param group: filter which Citus group we should get members from. See ``group`` argument of :func:`output_members` for more details. :param watch: if given print output every *watch* seconds. :param w: if ``True`` print output every 2 seconds. """ ctx.forward(members, fmt='topology') def timestamp(precision: int = 6) -> str: """Get current timestamp with given *precision* as a string. :param precision: Amount of digits to be present in the precision. :returns: the current timestamp with given *precision*. """ return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:precision - 7] @ctl.command('flush', help='Discard scheduled events') @click.argument('cluster_name') @option_citus_group @click.argument('member_names', nargs=-1) @click.argument('target', type=click.Choice(['restart', 'switchover'])) @click.option('--role', '-r', help='Flush only members with this role', type=role_choice, default='any') @option_force def flush(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, role: str, target: str) -> None: """Process ``flush`` command of ``patronictl`` utility. Discard scheduled restart or switchover events. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should flush an event. Refer to the module note for more details. :param member_names: name of the members which events should be flushed. :param force: perform the operation without asking for confirmations. :param role: role to filter members. See :func:`get_all_members` for available options. :param target: the event that should be flushed -- ``restart`` or ``switchover``. """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if target == 'restart': for member in get_members(cluster, cluster_name, member_names, role, force, 'flush', group=group): if member.data.get('scheduled_restart'): r = request_patroni(member, 'delete', 'restart') check_response(r, member.name, 'flush scheduled restart') else: click.echo('No scheduled restart for member {0}'.format(member.name)) elif target == 'switchover': failover = cluster.failover if not failover or not failover.scheduled_at: return click.echo('No pending scheduled switchover') for member in get_all_members_leader_first(cluster): try: r = request_patroni(member, 'delete', 'switchover') if r.status in (200, 404): prefix = 'Success' if r.status == 200 else 'Failed' return click.echo('{0}: {1}'.format(prefix, r.data.decode('utf-8'))) click.echo('Failed: member={0}, status_code={1}, ({2})'.format( member.name, r.status, r.data.decode('utf-8'))) except Exception as err: logging.warning(str(err)) logging.warning('Member %s is not accessible', member.name) logging.warning('Failing over to DCS') click.echo('{0} Could not find any accessible member of cluster {1}'.format(timestamp(), cluster_name)) dcs.manual_failover('', '', version=failover.version) def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Cluster) -> None: """Wait for all members in the cluster to have ``pause`` state set to *paused*. :param dcs: DCS object from where to get fresh cluster information. :param paused: the desired state for ``pause`` in all nodes. :param old_cluster: original cluster information before pause or unpause has been requested. Used to report which nodes are still pending to have ``pause`` equal *paused* at a given point in time. """ config = global_config.from_cluster(old_cluster) click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume')) old = {m.name: m.version for m in old_cluster.members if m.api_url} loop_wait = config.get('loop_wait') or dcs.loop_wait cluster = None for _ in polling_loop(loop_wait + 1): cluster = dcs.get_cluster() if all(m.data.get('pause', False) == paused for m in cluster.members if m.name in old): break else: if TYPE_CHECKING: # pragma: no cover assert cluster is not None remaining = [m.name for m in cluster.members if m.data.get('pause', False) != paused and m.name in old and old[m.name] != m.version] if remaining: return click.echo("{0} members didn't recognized pause state after {1} seconds" .format(', '.join(remaining), loop_wait)) return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed')) def toggle_pause(cluster_name: str, group: Optional[int], paused: bool, wait: bool) -> None: """Toggle the ``pause`` state in the cluster members. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should toggle the pause state of. Refer to the module note for more details. :param paused: the desired state for ``pause`` in all nodes. :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. :raises: PatroniCtlException: if * ``pause`` state is already *paused*; or * cluster contains no accessible members. """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if global_config.from_cluster(cluster).is_paused == paused: raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not')) for member in get_all_members_leader_first(cluster): try: r = request_patroni(member, 'patch', 'config', {'pause': paused or None}) except Exception as err: logging.warning(str(err)) logging.warning('Member %s is not accessible', member.name) continue if r.status == 200: if wait: wait_until_pause_is_applied(dcs, paused, cluster) else: click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed')) else: click.echo('Failed: {0} cluster management status code={1}, ({2})'.format( paused and 'pause' or 'resume', r.status, r.data.decode('utf-8'))) break else: raise PatroniCtlException('Can not find accessible cluster member') @ctl.command('pause', help='Disable auto failover') @arg_cluster_name @option_default_citus_group @click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True) def pause(cluster_name: str, group: Optional[int], wait: bool) -> None: """Process ``pause`` command of ``patronictl`` utility. Put the cluster in maintenance mode. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should pause. Refer to the module note for more details. :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. """ return toggle_pause(cluster_name, group, True, wait) @ctl.command('resume', help='Resume auto failover') @arg_cluster_name @option_default_citus_group @click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True) def resume(cluster_name: str, group: Optional[int], wait: bool) -> None: """Process ``unpause`` command of ``patronictl`` utility. Put the cluster out of maintenance mode. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should unpause. Refer to the module note for more details. :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. """ return toggle_pause(cluster_name, group, False, wait) @contextmanager def temporary_file(contents: bytes, suffix: str = '', prefix: str = 'tmp') -> Iterator[str]: """Create a temporary file with specified contents that persists for the context. :param contents: binary string that will be written to the file. :param prefix: will be prefixed to the filename. :param suffix: will be appended to the filename. :yields: path of the created file. """ tmp = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, delete=False) with tmp: tmp.write(contents) try: yield tmp.name finally: os.unlink(tmp.name) def show_diff(before_editing: str, after_editing: str) -> None: """Show a diff between two strings. Inputs are expected to be unicode strings. If the output is to a tty the diff will be colored. .. note:: If tty it requires a pager program, and uses first found among: * Program given by ``PAGER`` environment variable; or * ``less``; or * ``more``. :param before_editing: string to be compared with *after_editing*. :param after_editing: string to be compared with *before_editing*. :raises: :class:`PatroniCtlException`: if no suitable pager can be found when printing diff output to a tty. """ def listify(string: str) -> List[str]: return [line + '\n' for line in string.rstrip('\n').split('\n')] unified_diff = difflib.unified_diff(listify(before_editing), listify(after_editing)) if sys.stdout.isatty(): buf = io.BytesIO() for line in unified_diff: buf.write(line.encode('utf-8')) buf.seek(0) class opts: theme = 'default' side_by_side = False width = 80 tab_width = 8 wrap = True pager = next( ( os.path.basename(p) for p in (os.environ.get('PAGER'), "less", "more") if p is not None and bool(shutil.which(p)) ), None, ) pager_options = None if opts.pager is None: raise PatroniCtlException( 'No pager could be found. Either set PAGER environment variable with ' 'your pager or install either "less" or "more" in the host.' ) # if we end up selecting "less" as "pager" then we set "pager" attribute # to "None". "less" is the default pager for "ydiff" module, and that # module adds some command-line options to "less" when "pager" is "None" if opts.pager == 'less': opts.pager = None markup_to_pager(PatchStream(buf), opts) else: for line in unified_diff: click.echo(line.rstrip('\n')) def format_config_for_editing(data: Any, default_flow_style: bool = False) -> str: """Format configuration as YAML for human consumption. :param data: configuration as nested dictionaries. :param default_flow_style: passed down as ``default_flow_style`` argument of :func:`yaml.safe_dump`. :returns: unicode YAML of the configuration. """ return yaml.safe_dump(data, default_flow_style=default_flow_style, encoding=None, allow_unicode=True, width=200) def apply_config_changes(before_editing: str, data: Dict[str, Any], kvpairs: List[str]) -> Tuple[str, Dict[str, Any]]: """Apply config changes specified as a list of key-value pairs. Keys are interpreted as dotted paths into the configuration data structure. Except for paths beginning with ``postgresql.parameters`` where rest of the path is used directly to allow for PostgreSQL GUCs containing dots. Values are interpreted as YAML values. :param before_editing: human representation before editing. :param data: configuration data structure. :param kvpairs: list of strings containing key value pairs separated by ``=``. :returns: tuple of human-readable, parsed data structure after changes. :raises: :class:`PatroniCtlException`: if any entry in *kvpairs* is ``None`` or not in the expected format. """ changed_data = copy.deepcopy(data) def set_path_value(config: Dict[str, Any], path: List[str], value: Any, prefix: Tuple[str, ...] = ()) -> None: """Recursively walk through *config* and update setting specified by *path* with *value*. :param config: configuration data structure with all settings found under *prefix* path. :param path: dotted path split by dot as delimiter into a list. Used to control the recursive calls and identify when a leaf node is reached. :param value: value for configuration described by *path*. If ``None`` the configuration key is removed from *config*. :param prefix: previous parts of *path* that have already been opened by parent recursive calls. Used to know if we are changing a Postgres related setting or not. *prefix* plus *path* compose the original *path* given on the root call. """ # Postgresql GUCs can't be nested, but can contain dots so we re-flatten the structure for this case if prefix == ('postgresql', 'parameters'): path = ['.'.join(path)] key = path[0] # When *path* contains a single item it means we reached a leaf node in the configuration, so we can remove or # update the configuration based on what has been requested by the user. if len(path) == 1: if value is None: config.pop(key, None) else: config[key] = value # Otherwise we need to keep navigating down in the configuration structure. else: if not isinstance(config.get(key), dict): config[key] = {} set_path_value(config[key], path[1:], value, prefix + (key,)) if config[key] == {}: del config[key] for pair in kvpairs: if not pair or "=" not in pair: raise PatroniCtlException("Invalid parameter setting {0}".format(pair)) key_path, value = pair.split("=", 1) set_path_value(changed_data, key_path.strip().split("."), yaml.safe_load(value)) return format_config_for_editing(changed_data), changed_data def apply_yaml_file(data: Dict[str, Any], filename: str) -> Tuple[str, Dict[str, Any]]: """Apply changes from a YAML file to configuration. :param data: configuration data structure. :param filename: name of the YAML file, ``-`` is taken to mean standard input. :returns: tuple of human-readable and parsed data structure after changes. """ changed_data = copy.deepcopy(data) if filename == '-': new_options = yaml.safe_load(sys.stdin) else: with open(filename) as fd: new_options = yaml.safe_load(fd) patch_config(changed_data, new_options) return format_config_for_editing(changed_data), changed_data def invoke_editor(before_editing: str, cluster_name: str) -> Tuple[str, Dict[str, Any]]: """Start editor command to edit configuration in human readable format. .. note:: Requires an editor program, and uses first found among: * Program given by ``EDITOR`` environment variable; or * ``editor``; or * ``vi``. :param before_editing: human representation before editing. :param cluster_name: name of the Patroni cluster. :returns: tuple of human-readable, parsed data structure after changes. :raises: :class:`PatroniCtlException`: if * No suitable editor can be found; or * Editor call exits with unexpected return code. """ editor_cmd = os.environ.get('EDITOR') if not editor_cmd: for editor in ('editor', 'vi'): editor_cmd = shutil.which(editor) if editor_cmd: logging.debug('Setting fallback editor_cmd=%s', editor) break if not editor_cmd: raise PatroniCtlException('EDITOR environment variable is not set. editor or vi are not available') with temporary_file(contents=before_editing.encode('utf-8'), suffix='.yaml', prefix='{0}-config-'.format(cluster_name)) as tmpfile: ret = subprocess.call([editor_cmd, tmpfile]) if ret: raise PatroniCtlException("Editor exited with return code {0}".format(ret)) with codecs.open(tmpfile, encoding='utf-8') as fd: after_editing = fd.read() return after_editing, yaml.safe_load(after_editing) @ctl.command('edit-config', help="Edit cluster configuration") @arg_cluster_name @option_default_citus_group @click.option('--quiet', '-q', is_flag=True, help='Do not show changes') @click.option('--set', '-s', 'kvpairs', multiple=True, help='Set specific configuration value. Can be specified multiple times') @click.option('--pg', '-p', 'pgkvpairs', multiple=True, help='Set specific PostgreSQL parameter value. Shorthand for -s postgresql.parameters. ' 'Can be specified multiple times') @click.option('--apply', 'apply_filename', help='Apply configuration from file. Use - for stdin.') @click.option('--replace', 'replace_filename', help='Apply configuration from file, replacing existing configuration.' ' Use - for stdin.') @option_force def edit_config(cluster_name: str, group: Optional[int], force: bool, quiet: bool, kvpairs: List[str], pgkvpairs: List[str], apply_filename: Optional[str], replace_filename: Optional[str]) -> None: """Process ``edit-config`` command of ``patronictl`` utility. Update or replace Patroni configuration in the DCS. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group configuration we should edit. Refer to the module note for more details. :param force: if ``True`` apply config changes without asking for confirmations. :param quiet: if ``True`` skip showing config diff in the console. :param kvpairs: list of key value general parameters to be changed. :param pgkvpairs: list of key value Postgres parameters to be changed. :param apply_filename: name of the file which contains with new configuration parameters to be applied. Pass ``-`` for using stdin instead. :param replace_filename: name of the file which contains the new configuration parameters to replace the existing configuration. Pass ``-`` for using stdin instead. :raises: :class:`PatroniCtlException`: if * Configuration is absent from DCS; or * Detected a concurrent modification of the configuration in the DCS. """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if not cluster.config: raise PatroniCtlException('The config key does not exist in the cluster {0}'.format(cluster_name)) before_editing = format_config_for_editing(cluster.config.data) after_editing = None # Serves as a flag if any changes were requested changed_data = cluster.config.data if replace_filename: after_editing, changed_data = apply_yaml_file({}, replace_filename) if apply_filename: after_editing, changed_data = apply_yaml_file(changed_data, apply_filename) if kvpairs or pgkvpairs: all_pairs = list(kvpairs) + ['postgresql.parameters.' + v.lstrip() for v in pgkvpairs] after_editing, changed_data = apply_config_changes(before_editing, changed_data, all_pairs) # If no changes were specified on the command line invoke editor if after_editing is None: after_editing, changed_data = invoke_editor(before_editing, cluster_name) if cluster.config.data == changed_data: if not quiet: click.echo("Not changed") return if not quiet: show_diff(before_editing, after_editing) if (apply_filename == '-' or replace_filename == '-') and not force: click.echo("Use --force option to apply changes") return if force or click.confirm('Apply these changes?'): if not dcs.set_config_value(json.dumps(changed_data, separators=(',', ':')), cluster.config.version): raise PatroniCtlException("Config modification aborted due to concurrent changes") click.echo("Configuration changed") @ctl.command('show-config', help="Show cluster configuration") @arg_cluster_name @option_default_citus_group def show_config(cluster_name: str, group: Optional[int]) -> None: """Process ``show-config`` command of ``patronictl`` utility. Show Patroni configuration stored in the DCS. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group configuration we should show. Refer to the module note for more details. """ cluster = get_dcs(cluster_name, group).get_cluster() if cluster.config: click.echo(format_config_for_editing(cluster.config.data)) @ctl.command('version', help='Output version of patronictl command or a running Patroni instance') @click.argument('cluster_name', required=False) @click.argument('member_names', nargs=-1) @option_citus_group def version(cluster_name: str, group: Optional[int], member_names: List[str]) -> None: """Process ``version`` command of ``patronictl`` utility. Show version of: * ``patronictl`` on invoker; * ``patroni`` on all members of the cluster; * ``PostgreSQL`` on all members of the cluster. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get members from. Refer to the module note for more details. :param member_names: filter which members we should get version information from. """ click.echo("patronictl version {0}".format(__version__)) if not cluster_name: return click.echo("") cluster = get_dcs(cluster_name, group).get_cluster() for m in get_all_members(cluster, group, 'any'): if m.api_url: if not member_names or m.name in member_names: try: response = request_patroni(m) data = json.loads(response.data.decode('utf-8')) version = data.get('patroni', {}).get('version') pg_version = data.get('server_version') pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else "" click.echo("{0}: Patroni {1}{2}".format(m.name, version, pg_version_str)) except Exception as e: click.echo("{0}: failed to get version: {1}".format(m.name, e)) @ctl.command('history', help="Show the history of failovers/switchovers") @arg_cluster_name @option_default_citus_group @option_format def history(cluster_name: str, group: Optional[int], fmt: str) -> None: """Process ``history`` command of ``patronictl`` utility. Show the history of failover/switchover events in the cluster. Information is printed to console through :func:`print_output`, and contains: * ``TL``: Postgres timeline when the event occurred; * ``LSN``: Postgres LSN, in bytes, when the event occurred; * ``Reason``: the reason that motivated the event, if any; * ``Timestamp``: timestamp when the event occurred; * ``New Leader``: the Postgres node that was promoted during the event. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get events from. Refer to the module note for more details. :param fmt: the output table printing format. See :func:`print_output` for available options. """ cluster = get_dcs(cluster_name, group).get_cluster() cluster_history = cluster.history.lines if cluster.history else [] history: List[List[Any]] = list(map(list, cluster_history)) table_header_row = ['TL', 'LSN', 'Reason', 'Timestamp', 'New Leader'] for line in history: if len(line) < len(table_header_row): add_column_num = len(table_header_row) - len(line) for _ in range(add_column_num): line.append('') print_output(table_header_row, history, {'TL': 'r', 'LSN': 'r'}, fmt) def format_pg_version(version: int) -> str: """Format Postgres version for human consumption. :param version: Postgres version represented as an integer. :returns: Postgres version represented as a human-readable string. :Example: >>> format_pg_version(90624) '9.6.24' >>> format_pg_version(100000) '10.0' >>> format_pg_version(140008) '14.8' """ if version < 100000: return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100) else: return "{0}.{1}".format(version // 10000, version % 100) patroni-4.0.4/patroni/daemon.py000066400000000000000000000132411472010352700164660ustar00rootroot00000000000000"""Daemon processes abstraction module. This module implements abstraction classes and functions for creating and managing daemon processes in Patroni. Currently it is only used for the main "Thread" of ``patroni`` and ``patroni_raft_controller`` commands. """ from __future__ import print_function import abc import argparse import os import signal import sys from threading import Lock from typing import Any, Optional, Type, TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover from .config import Config def get_base_arg_parser() -> argparse.ArgumentParser: """Create a basic argument parser with the arguments used for both patroni and raft controller daemon. :returns: 'argparse.ArgumentParser' object """ from .config import Config from .version import __version__ parser = argparse.ArgumentParser() parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__)) parser.add_argument('configfile', nargs='?', default='', help='Patroni may also read the configuration from the {0} environment variable' .format(Config.PATRONI_CONFIG_VARIABLE)) return parser class AbstractPatroniDaemon(abc.ABC): """A Patroni daemon process. .. note:: When inheriting from :class:`AbstractPatroniDaemon` you are expected to define the methods :func:`_run_cycle` to determine what it should do in each execution cycle, and :func:`_shutdown` to determine what it should do when shutting down. :ivar logger: log handler used by this daemon. :ivar config: configuration options for this daemon. """ def __init__(self, config: 'Config') -> None: """Set up signal handlers, logging handler and configuration. :param config: configuration options for this daemon. """ from patroni.log import PatroniLogger self.setup_signal_handlers() self.logger = PatroniLogger() self.config = config AbstractPatroniDaemon.reload_config(self, local=True) def sighup_handler(self, *_: Any) -> None: """Handle SIGHUP signals. Flag the daemon as "SIGHUP received". """ self._received_sighup = True def api_sigterm(self) -> bool: """Guarantee only a single SIGTERM is being processed. Flag the daemon as "SIGTERM received" with a lock-based approach. :returns: ``True`` if the daemon was flagged as "SIGTERM received". """ ret = False with self._sigterm_lock: if not self._received_sigterm: self._received_sigterm = True ret = True return ret def sigterm_handler(self, *_: Any) -> None: """Handle SIGTERM signals. Terminate the daemon process through :func:`api_sigterm`. """ if self.api_sigterm(): sys.exit() def setup_signal_handlers(self) -> None: """Set up daemon signal handlers. Set up SIGHUP and SIGTERM signal handlers. .. note:: SIGHUP is only handled in non-Windows environments. """ self._received_sighup = False self._sigterm_lock = Lock() self._received_sigterm = False if os.name != 'nt': signal.signal(signal.SIGHUP, self.sighup_handler) signal.signal(signal.SIGTERM, self.sigterm_handler) @property def received_sigterm(self) -> bool: """If daemon was signaled with SIGTERM.""" with self._sigterm_lock: return self._received_sigterm def reload_config(self, sighup: bool = False, local: Optional[bool] = False) -> None: """Reload configuration. :param sighup: if it is related to a SIGHUP signal. The sighup parameter could be used in the method overridden in a child class. :param local: will be ``True`` if there are changes in the local configuration file. """ if local: self.logger.reload_config(self.config.get('log', {})) @abc.abstractmethod def _run_cycle(self) -> None: """Define what the daemon should do in each execution cycle. Keep being called in the daemon's main loop until the daemon is eventually terminated. """ def run(self) -> None: """Run the daemon process. Start the logger thread and keep running execution cycles until a SIGTERM is eventually received. Also reload configuration upon receiving SIGHUP. """ self.logger.start() while not self.received_sigterm: if self._received_sighup: self._received_sighup = False self.reload_config(True, self.config.reload_local_configuration()) self._run_cycle() @abc.abstractmethod def _shutdown(self) -> None: """Define what the daemon should do when shutting down.""" def shutdown(self) -> None: """Shut the daemon down when a SIGTERM is received. Shut down the daemon process and the logger thread. """ with self._sigterm_lock: self._received_sigterm = True self._shutdown() self.logger.shutdown() def abstract_main(cls: Type[AbstractPatroniDaemon], configfile: str) -> None: """Create the main entry point of a given daemon process. :param cls: a class that should inherit from :class:`AbstractPatroniDaemon`. :param configfile: """ from .config import Config, ConfigParseError try: config = Config(configfile) except ConfigParseError as e: sys.exit(e.value) controller = cls(config) try: controller.run() except KeyboardInterrupt: pass finally: controller.shutdown() patroni-4.0.4/patroni/dcs/000077500000000000000000000000001472010352700154215ustar00rootroot00000000000000patroni-4.0.4/patroni/dcs/__init__.py000066400000000000000000002662401472010352700175440ustar00rootroot00000000000000"""Abstract classes for Distributed Configuration Store.""" import abc import datetime import json import logging import re import time from collections import defaultdict from copy import deepcopy from random import randint from threading import Event, Lock from typing import Any, Callable, cast, Collection, Dict, Iterator, \ List, NamedTuple, Optional, Set, Tuple, Type, TYPE_CHECKING, Union from urllib.parse import parse_qsl, urlparse, urlunparse import dateutil.parser from .. import global_config from ..dynamic_loader import iter_classes, iter_modules from ..exceptions import PatroniFatalException from ..tags import Tags from ..utils import deep_compare, parse_int, uri if TYPE_CHECKING: # pragma: no cover from ..config import Config from ..postgresql import Postgresql from ..postgresql.mpp import AbstractMPP slot_name_re = re.compile('^[a-z0-9_]{1,63}$') logger = logging.getLogger(__name__) def slot_name_from_member_name(member_name: str) -> str: """Translate member name to valid PostgreSQL slot name. .. note:: PostgreSQL's replication slot names must be valid PostgreSQL names. This function maps the wider space of member names to valid PostgreSQL names. Names have their case lowered, dashes and periods common in hostnames are replaced with underscores, other characters are encoded as their unicode codepoint. Name is truncated to 64 characters. Multiple different member names may map to a single slot name. :param member_name: The string to convert to a slot name. :returns: The string converted using the rules described above. """ def replace_char(match: Any) -> str: c = match.group(0) return '_' if c in '-.' else f"u{ord(c):04d}" slot_name = re.sub('[^a-z0-9_]', replace_char, member_name.lower()) return slot_name[0:63] def parse_connection_string(value: str) -> Tuple[str, Union[str, None]]: """Split and rejoin a URL string into a connection URL and an API URL. .. note:: Original Governor stores connection strings for each cluster members in a following format: postgres://{username}:{password}@{connect_address}/postgres Since each of our patroni instances provides their own REST API endpoint, it's good to store this information in DCS along with PostgreSQL connection string. In order to not introduce new keys and be compatible with original Governor we decided to extend original connection string in a following way: postgres://{username}:{password}@{connect_address}/postgres?application_name={api_url} This way original Governor could use such connection string as it is, because of feature of ``libpq`` library. :param value: The URL string to split. :returns: the connection string stored in DCS split into two parts, ``conn_url`` and ``api_url``. """ scheme, netloc, path, params, query, fragment = urlparse(value) conn_url = urlunparse((scheme, netloc, path, params, '', fragment)) api_url = ([v for n, v in parse_qsl(query) if n == 'application_name'] or [None])[0] return conn_url, api_url def dcs_modules() -> List[str]: """Get names of DCS modules, depending on execution environment. :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``. """ if TYPE_CHECKING: # pragma: no cover assert isinstance(__package__, str) return iter_modules(__package__) def iter_dcs_classes( config: Optional[Union['Config', Dict[str, Any]]] = None ) -> Iterator[Tuple[str, Type['AbstractDCS']]]: """Attempt to import DCS modules that are present in the given configuration. .. note:: If a module successfully imports we can assume that all its requirements are installed. :param config: configuration information with possible DCS names as keys. If given, only attempt to import DCS modules defined in the configuration. Else, if ``None``, attempt to import any supported DCS module. :returns: an iterator of tuples, each containing the module ``name`` and the imported DCS class object. """ if TYPE_CHECKING: # pragma: no cover assert isinstance(__package__, str) return iter_classes(__package__, AbstractDCS, config) def get_dcs(config: Union['Config', Dict[str, Any]]) -> 'AbstractDCS': """Attempt to load a Distributed Configuration Store from known available implementations. .. note:: Using the list of available DCS classes returned by :func:`iter_classes` attempt to dynamically instantiate the class that implements a DCS using the abstract class :class:`AbstractDCS`. Basic top-level configuration parameters retrieved from *config* are propagated to the DCS specific config before being passed to the module DCS class. If no module is found to satisfy configuration then report and log an error. This will cause Patroni to exit. :raises :exc:`PatroniFatalException`: if a load of all available DCS modules have been tried and none succeeded. :param config: object or dictionary with Patroni configuration. This is normally a representation of the main Patroni :returns: The first successfully loaded DCS module which is an implementation of :class:`AbstractDCS`. """ for name, dcs_class in iter_dcs_classes(config): # Propagate some parameters from top level of config if defined to the DCS specific config section. config[name].update({ p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait', 'patronictl', 'ttl', 'retry_timeout') if p in config}) from patroni.postgresql.mpp import get_mpp return dcs_class(config[name], get_mpp(config)) available_implementations = ', '.join(sorted([n for n, _ in iter_dcs_classes()])) raise PatroniFatalException("Can not find suitable configuration of distributed configuration store\n" f"Available implementations: {available_implementations}") _Version = Union[int, str] _Session = Union[int, float, str, None] class Member(Tags, NamedTuple('Member', [('version', _Version), ('name', str), ('session', _Session), ('data', Dict[str, Any])])): """Immutable object (namedtuple) which represents single member of PostgreSQL cluster. .. note:: We are using an old-style attribute declaration here because otherwise it is not possible to override ``__new__`` method in the :class:`RemoteMember` class. .. note:: These two keys in data are always written to the DCS, but care is taken to maintain consistency and resilience from data that is read: ``conn_url``: connection string containing host, user and password which could be used to access this member. ``api_url``: REST API url of patroni instance Consists of the following fields: :ivar version: modification version of a given member key in a Configuration Store. :ivar name: name of PostgreSQL cluster member. :ivar session: either session id or just ttl in seconds. :ivar data: dictionary containing arbitrary data i.e. ``conn_url``, ``api_url``, ``xlog_location``, ``state``, ``role``, ``tags``, etc... """ @staticmethod def from_node(version: _Version, name: str, session: _Session, value: str) -> 'Member': """Factory method for instantiating :class:`Member` from a JSON serialised string or object. :param version: modification version of a given member key in a Configuration Store. :param name: name of PostgreSQL cluster member. :param session: either session id or just ttl in seconds. :param value: JSON encoded string containing arbitrary data i.e. ``conn_url``, ``api_url``, ``xlog_location``, ``state``, ``role``, ``tags``, etc. OR a connection URL starting with ``postgres://``. :returns: an :class:`Member` instance built with the given arguments. :Example: >>> Member.from_node(-1, '', '', '{"conn_url": "postgres://foo@bar/postgres"}') is not None True >>> Member.from_node(-1, '', '', '{') Member(version=-1, name='', session='', data={}) """ if value.startswith('postgres'): conn_url, api_url = parse_connection_string(value) data = {'conn_url': conn_url, 'api_url': api_url} else: try: data = json.loads(value) assert isinstance(data, dict) except (AssertionError, TypeError, ValueError): data: Dict[str, Any] = {} return Member(version, name, session, data) @property def conn_url(self) -> Optional[str]: """The ``conn_url`` value from :attr:`~Member.data` if defined or constructed from ``conn_kwargs``.""" conn_url = self.data.get('conn_url') if conn_url: return conn_url conn_kwargs = self.data.get('conn_kwargs') if conn_kwargs: conn_url = uri('postgresql', (conn_kwargs.get('host'), conn_kwargs.get('port', 5432))) self.data['conn_url'] = conn_url return conn_url return None def conn_kwargs(self, auth: Optional[Any] = None) -> Dict[str, Any]: """Give keyword arguments used for PostgreSQL connection settings. :param auth: Authentication properties - can be defined as anything supported by the ``psycopg2`` or ``psycopg`` modules. Converts a key of ``username`` to ``user`` if supplied. :returns: A dictionary containing a merge of default parameter keys ``host``, ``port`` and ``dbname``, with the contents of :attr:`~Member.data` ``conn_kwargs`` key. If those are not defined will parse and reform connection parameters from :attr:`~Member.conn_url`. One of these two attributes needs to have data defined to construct the output dictionary. Finally, *auth* parameters are merged with the dictionary before returned. """ defaults = { "host": None, "port": None, "dbname": None } ret: Optional[Dict[str, Any]] = self.data.get('conn_kwargs') if ret: defaults.update(ret) ret = defaults else: conn_url = self.conn_url if not conn_url: return {} # due to the invalid conn_url we don't care about authentication parameters r = urlparse(conn_url) ret = { 'host': r.hostname, 'port': r.port or 5432, 'dbname': r.path[1:] } self.data['conn_kwargs'] = ret.copy() # apply any remaining authentication parameters if auth and isinstance(auth, dict): ret.update({k: v for k, v in cast(Dict[str, Any], auth).items() if v is not None}) if 'username' in auth: ret['user'] = ret.pop('username') return ret def get_endpoint_url(self, endpoint: Optional[str] = None) -> str: """Get URL from member :attr:`~Member.api_url` and endpoint. :param endpoint: URL path of REST API. :returns: full URL for this REST API. """ url = self.api_url or '' if endpoint: scheme, netloc, _, _, _, _ = urlparse(url) url = urlunparse((scheme, netloc, endpoint, '', '', '')) return url @property def api_url(self) -> Optional[str]: """The ``api_url`` value from :attr:`~Member.data` if defined.""" return self.data.get('api_url') @property def tags(self) -> Dict[str, Any]: """The ``tags`` value from :attr:`~Member.data` if defined, otherwise an empty dictionary.""" return self.data.get('tags', {}) @property def clonefrom(self) -> bool: """``True`` if both ``clonefrom`` tag is ``True`` and a connection URL is defined.""" return super().clonefrom and bool(self.conn_url) @property def state(self) -> str: """The ``state`` value of :attr:`~Member.data`.""" return self.data.get('state', 'unknown') @property def is_running(self) -> bool: """``True`` if the member :attr:`~Member.state` is ``running``.""" return self.state == 'running' @property def patroni_version(self) -> Optional[Tuple[int, ...]]: """The ``version`` string value from :attr:`~Member.data` converted to tuple. :Example: >>> Member.from_node(1, '', '', '{"version":"1.2.3"}').patroni_version (1, 2, 3) """ version = self.data.get('version') if version: try: return tuple(map(int, version.split('.'))) except Exception: logger.debug('Failed to parse Patroni version %s', version) return None @property def lsn(self) -> Optional[int]: """Current LSN (receive/flush/replay).""" return parse_int(self.data.get('xlog_location')) class RemoteMember(Member): """Represents a remote member (typically a primary) for a standby cluster. :cvar ALLOWED_KEYS: Controls access to relevant key names that could be in stored :attr:`~RemoteMember.data`. """ ALLOWED_KEYS: Tuple[str, ...] = ( 'primary_slot_name', 'create_replica_methods', 'restore_command', 'archive_cleanup_command', 'recovery_min_apply_delay', 'no_replication_slot' ) def __new__(cls, name: str, data: Dict[str, Any]) -> 'RemoteMember': """Factory method to construct instance from given *name* and *data*. :param name: name of the remote member. :param data: dictionary of member information, which can contain keys from :const:`~RemoteMember.ALLOWED_KEYS` but also member connection information ``api_url`` and ``conn_kwargs``, and slot information. :returns: constructed instance using supplied parameters. """ return super(RemoteMember, cls).__new__(cls, -1, name, None, data) def __getattr__(self, name: str) -> Any: """Dictionary style key lookup. :param name: key to lookup. :returns: value of *name* key in :attr:`~RemoteMember.data` if key *name* is in :cvar:`~RemoteMember.ALLOWED_KEYS`, else ``None``. """ return self.data.get(name) if name in RemoteMember.ALLOWED_KEYS else None class Leader(NamedTuple): """Immutable object (namedtuple) which represents leader key. Consists of the following fields: :ivar version: modification version of a leader key in a Configuration Store :ivar session: either session id or just ttl in seconds :ivar member: reference to a :class:`Member` object which represents current leader (see :attr:`Cluster.members`) """ version: _Version session: _Session member: Member @property def name(self) -> str: """The leader "member" name.""" return self.member.name def conn_kwargs(self, auth: Optional[Dict[str, str]] = None) -> Dict[str, str]: """Connection keyword arguments. :param auth: an optional dictionary containing authentication information. :returns: the result of the called :meth:`Member.conn_kwargs` method. """ return self.member.conn_kwargs(auth) @property def conn_url(self) -> Optional[str]: """Connection URL value of the :class:`Member` instance.""" return self.member.conn_url @property def data(self) -> Dict[str, Any]: """Data value of the :class:`Member` instance.""" return self.member.data @property def timeline(self) -> Optional[int]: """Timeline value of :attr:`~Member.data`.""" return self.data.get('timeline') @property def checkpoint_after_promote(self) -> Optional[bool]: """Determine whether a checkpoint has occurred for this leader after promotion. :returns: ``True`` if the role is ``master`` or ``primary`` and ``checkpoint_after_promote`` is not set, ``False`` if not a ``master`` or ``primary`` or if the checkpoint hasn't occurred. If the version of Patroni is older than 1.5.6, return ``None``. :Example: >>> Leader(1, '', Member.from_node(1, '', '', '{"version":"z"}')).checkpoint_after_promote """ version = self.member.patroni_version # 1.5.6 is the last version which doesn't expose checkpoint_after_promote: false if version and version > (1, 5, 6): return self.data.get('role') in ('master', 'primary') and 'checkpoint_after_promote' not in self.data return None class Failover(NamedTuple): """Immutable object (namedtuple) representing configuration information required for failover/switchover capability. :ivar version: version of the object. :ivar leader: name of the leader. If value isn't empty we treat it as a switchover from the specified node. :ivar candidate: the name of the member node to be considered as a failover candidate. :ivar scheduled_at: in the case of a switchover the :class:`~datetime.datetime` object to perform the scheduled switchover. :Example: >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}')) True >>> 'Failover' in str(Failover.from_node(1, {"leader": "cluster_leader"})) True >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}')) True >>> Failover.from_node(1, 'null') is None False >>> n = '''{"leader": "cluster_leader", "member": "cluster_candidate", ... "scheduled_at": "2016-01-14T10:09:57.1394Z"}''' >>> 'tzinfo=' in str(Failover.from_node(1, n)) True >>> Failover.from_node(1, None) is None False >>> Failover.from_node(1, '{}') is None False >>> 'abc' in Failover.from_node(1, 'abc:def') True """ version: _Version leader: Optional[str] candidate: Optional[str] scheduled_at: Optional[datetime.datetime] @staticmethod def from_node(version: _Version, value: Union[str, Dict[str, str]]) -> 'Failover': """Factory method to parse *value* as failover configuration. :param version: version number for the object. :param value: JSON serialized data or a dictionary of configuration. Can also be a colon ``:`` delimited list of leader, followed by candidate name (legacy format). If ``scheduled_at`` key is defined the value will be parsed by :func:`dateutil.parser.parse`. :returns: constructed :class:`Failover` information object """ if isinstance(value, dict): data: Dict[str, Any] = value elif value: try: data = json.loads(value) assert isinstance(data, dict) except AssertionError: data = {} except ValueError: t = [a.strip() for a in value.split(':')] leader = t[0] candidate = t[1] if len(t) > 1 else None return Failover(version, leader, candidate, None) else: data = {} if data.get('scheduled_at'): data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at']) return Failover(version, data.get('leader'), data.get('member'), data.get('scheduled_at')) def __len__(self) -> int: """Implement ``len`` function capability. .. note:: This magic method aids in the evaluation of "emptiness" of a :class:`Failover` instance. For example: >>> failover = Failover.from_node(1, None) >>> len(failover) 0 >>> assert bool(failover) is False >>> failover = Failover.from_node(1, {"leader": "cluster_leader"}) >>> len(failover) 1 >>> assert bool(failover) is True This makes it easier to write ``if cluster.failover`` rather than the longer statement. """ return int(bool(self.leader)) + int(bool(self.candidate)) class ClusterConfig(NamedTuple): """Immutable object (namedtuple) which represents cluster configuration. :ivar version: version number for the object. :ivar data: dictionary of configuration information. :ivar modify_version: modified version number. """ version: _Version data: Dict[str, Any] modify_version: _Version @staticmethod def from_node(version: _Version, value: str, modify_version: Optional[_Version] = None) -> 'ClusterConfig': """Factory method to parse *value* as configuration information. :param version: version number for object. :param value: raw JSON serialized data, if not parsable replaced with an empty dictionary. :param modify_version: optional modify version number, use *version* if not provided. :returns: constructed :class:`ClusterConfig` instance. :Example: >>> ClusterConfig.from_node(1, '{') is None False """ try: data = json.loads(value) assert isinstance(data, dict) except (AssertionError, TypeError, ValueError): data: Dict[str, Any] = {} modify_version = 0 return ClusterConfig(version, data, version if modify_version is None else modify_version) class SyncState(NamedTuple): """Immutable object (namedtuple) which represents last observed synchronous replication state. :ivar version: modification version of a synchronization key in a Configuration Store. :ivar leader: reference to member that was leader. :ivar sync_standby: synchronous standby list (comma delimited) which are last synchronized to leader. :ivar quorum: if the node from :attr:`~SyncState.sync_standby` list is doing a leader race it should see at least :attr:`~SyncState.quorum` other nodes from the :attr:`~SyncState.sync_standby` + :attr:`~SyncState.leader` list. """ version: Optional[_Version] leader: Optional[str] sync_standby: Optional[str] quorum: int @staticmethod def from_node(version: Optional[_Version], value: Union[str, Dict[str, Any], None]) -> 'SyncState': """Factory method to parse *value* as synchronisation state information. :param version: optional *version* number for the object. :param value: (optionally JSON serialised) synchronisation state information :returns: constructed :class:`SyncState` object. :Example: >>> SyncState.from_node(1, None).leader is None True >>> SyncState.from_node(1, '{}').leader is None True >>> SyncState.from_node(1, '{').leader is None True >>> SyncState.from_node(1, '[]').leader is None True >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader" True >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader" True """ try: if value and isinstance(value, str): value = json.loads(value) assert isinstance(value, dict) leader = value.get('leader') quorum = value.get('quorum') return SyncState(version, leader, value.get('sync_standby'), int(quorum) if leader and quorum else 0) except (AssertionError, TypeError, ValueError): return SyncState.empty(version) @staticmethod def empty(version: Optional[_Version] = None) -> 'SyncState': """Construct an empty :class:`SyncState` instance. :param version: optional version number. :returns: empty synchronisation state object. """ return SyncState(version, None, None, 0) @property def is_empty(self) -> bool: """``True`` if ``/sync`` key is not valid (doesn't have a leader).""" return not self.leader @staticmethod def _str_to_list(value: str) -> List[str]: """Splits a string by comma and returns list of strings. :param value: a comma separated string. :returns: list of non-empty strings after splitting an input value by comma. """ return list(filter(lambda a: a, [s.strip() for s in value.split(',')])) @property def voters(self) -> List[str]: """:attr:`~SyncState.sync_standby` as list or an empty list if undefined or object considered ``empty``.""" return self._str_to_list(self.sync_standby) if not self.is_empty and self.sync_standby else [] @property def members(self) -> List[str]: """:attr:`~SyncState.sync_standby` and :attr:`~SyncState.leader` as list or an empty list if object considered ``empty``. """ return [] if not self.leader else [self.leader] + self.voters def matches(self, name: Optional[str], check_leader: bool = False) -> bool: """Checks if node is presented in the /sync state. Since PostgreSQL does case-insensitive checks for synchronous_standby_name we do it also. :param name: name of the node. :param check_leader: by default the *name* is searched for only in members, a value of ``True`` will include the leader to list. :returns: ``True`` if the ``/sync`` key not :func:`is_empty` and the given *name* is among those presented in the sync state. :Example: >>> s = SyncState(1, 'foo', 'bar,zoo', 0) >>> s.matches('foo') False >>> s.matches('fOo', True) True >>> s.matches('Bar') True >>> s.matches('zoO') True >>> s.matches('baz') False >>> s.matches(None) False >>> SyncState.empty(1).matches('foo') False """ ret = False if name and not self.is_empty: search_str = (self.sync_standby or '') + (',' + (self.leader or '') if check_leader else '') ret = name.lower() in self._str_to_list(search_str.lower()) return ret def leader_matches(self, name: Optional[str]) -> bool: """Compare the given *name* to stored leader value. :returns: ``True`` if *name* is matching the :attr:`~SyncState.leader` value. """ return bool(name and not self.is_empty and name.lower() == (self.leader or '').lower()) _HistoryTuple = Union[Tuple[int, int, str], Tuple[int, int, str, str], Tuple[int, int, str, str, str]] class TimelineHistory(NamedTuple): """Object representing timeline history file. .. note:: The content held in *lines* deserialized from *value* are lines parsed from PostgreSQL timeline history files, consisting of the timeline number, the LSN where the timeline split and any other string held in the file. The files are parsed by :func:`~patroni.postgresql.misc.parse_history`. :ivar version: version number of the file. :ivar value: raw JSON serialised data consisting of parsed lines from history files. :ivar lines: ``List`` of ``Tuple`` parsed lines from history files. """ version: _Version value: Any lines: List[_HistoryTuple] @staticmethod def from_node(version: _Version, value: str) -> 'TimelineHistory': """Parse the given JSON serialized string as a list of timeline history lines. :param version: version number :param value: JSON serialized string, consisting of parsed lines of PostgreSQL timeline history files, see :class:`TimelineHistory`. :returns: composed timeline history object using parsed lines. :Example: If the passed *value* argument is not parsed an empty list of lines is returned: >>> h = TimelineHistory.from_node(1, 2) >>> h.lines [] """ try: lines = json.loads(value) assert isinstance(lines, list) except (AssertionError, TypeError, ValueError): lines: List[_HistoryTuple] = [] return TimelineHistory(version, value, lines) class Status(NamedTuple): """Immutable object (namedtuple) which represents `/status` key. Consists of the following fields: :ivar last_lsn: :class:`int` object containing position of last known leader LSN. :ivar slots: state of permanent replication slots on the primary in the format: ``{"slot_name": int}``. :ivar retain_slots: list physical replication slots for members that exist in the cluster. """ last_lsn: int slots: Optional[Dict[str, int]] retain_slots: List[str] @staticmethod def empty() -> 'Status': """Construct an empty :class:`Status` instance. :returns: empty :class:`Status` object. """ return Status(0, None, []) def is_empty(self): """Validate definition of all attributes of this :class:`Status` instance. :returns: ``True`` if all attributes of the current :class:`Status` are unpopulated. """ return self.last_lsn == 0 and self.slots is None and not self.retain_slots @staticmethod def from_node(value: Union[str, Dict[str, Any], None]) -> 'Status': """Factory method to parse *value* as :class:`Status` object. :param value: JSON serialized string or :class:`dict` object. :returns: constructed :class:`Status` object. """ try: if isinstance(value, str): value = json.loads(value) except Exception: return Status.empty() if isinstance(value, int): # legacy return Status(value, None, []) if not isinstance(value, dict): return Status.empty() try: last_lsn = int(value.get('optime', '')) except Exception: last_lsn = 0 slots: Union[str, Dict[str, int], None] = value.get('slots') if isinstance(slots, str): try: slots = json.loads(slots) except Exception: slots = None if not isinstance(slots, dict): slots = None retain_slots: Union[str, List[str], None] = value.get('retain_slots') if isinstance(retain_slots, str): try: retain_slots = json.loads(retain_slots) except Exception: retain_slots = [] if not isinstance(retain_slots, list): retain_slots = [] return Status(last_lsn, slots, retain_slots) class Cluster(NamedTuple('Cluster', [('initialize', Optional[str]), ('config', Optional[ClusterConfig]), ('leader', Optional[Leader]), ('status', Status), ('members', List[Member]), ('failover', Optional[Failover]), ('sync', SyncState), ('history', Optional[TimelineHistory]), ('failsafe', Optional[Dict[str, str]]), ('workers', Dict[int, 'Cluster'])])): """Immutable object (namedtuple) which represents PostgreSQL or MPP cluster. .. note:: We are using an old-style attribute declaration here because otherwise it is not possible to override `__new__` method. Without it the *workers* by default gets always the same :class:`dict` object that could be mutated. Consists of the following fields: :ivar initialize: shows whether this cluster has initialization key stored in DC or not. :ivar config: global dynamic configuration, reference to `ClusterConfig` object. :ivar leader: :class:`Leader` object which represents current leader of the cluster. :ivar status: :class:`Status` object which represents the `/status` key. :ivar members: list of:class:` Member` objects, all PostgreSQL cluster members including leader :ivar failover: reference to :class:`Failover` object. :ivar sync: reference to :class:`SyncState` object, last observed synchronous replication state. :ivar history: reference to `TimelineHistory` object. :ivar failsafe: failsafe topology. Node is allowed to become the leader only if its name is found in this list. :ivar workers: dictionary of workers of the MPP cluster, optional. Each key representing the group and the corresponding value is a :class:`Cluster` instance. """ def __new__(cls, *args: Any, **kwargs: Any): """Make workers argument optional and set it to an empty dict object.""" if len(args) < len(cls._fields) and 'workers' not in kwargs: kwargs['workers'] = {} return super(Cluster, cls).__new__(cls, *args, **kwargs) @property def slots(self) -> Dict[str, int]: """State of permanent replication slots on the primary in the format: ``{"slot_name": int}``. .. note:: We are trying to be foolproof here and for values that can't be parsed to :class:`int` will return ``0``. """ return {k: parse_int(v) or 0 for k, v in (self.status.slots or {}).items()} @staticmethod def empty() -> 'Cluster': """Produce an empty :class:`Cluster` instance.""" return Cluster(None, None, None, Status.empty(), [], None, SyncState.empty(), None, None, {}) def is_empty(self): """Validate definition of all attributes of this :class:`Cluster` instance. :returns: ``True`` if all attributes of the current :class:`Cluster` are unpopulated. """ return all((self.initialize is None, self.config is None, self.leader is None, self.status.is_empty(), self.members == [], self.failover is None, self.sync.version is None, self.history is None, self.failsafe is None, self.workers == {})) def __len__(self) -> int: """Implement ``len`` function capability. .. note:: This magic method aids in the evaluation of "emptiness" of a ``Cluster`` instance. For example: >>> cluster = Cluster.empty() >>> len(cluster) 0 >>> assert bool(cluster) is False >>> status = Status(0, None, []) >>> cluster = Cluster(None, None, None, status, [1, 2, 3], None, SyncState.empty(), None, None, {}) >>> len(cluster) 1 >>> assert bool(cluster) is True This makes it easier to write ``if cluster`` rather than the longer statement. """ return int(not self.is_empty()) @property def leader_name(self) -> Optional[str]: """The name of the leader if defined otherwise ``None``.""" return self.leader and self.leader.name def is_unlocked(self) -> bool: """Check if the cluster does not have the leader. :returns: ``True`` if a leader name is not defined. """ return not self.leader_name def has_member(self, member_name: str) -> bool: """Check if the given member name is present in the cluster. :param member_name: name to look up in the :attr:`~Cluster.members`. :returns: ``True`` if the member name is found. """ return any(m for m in self.members if m.name == member_name) def get_member(self, member_name: str, fallback_to_leader: bool = True) -> Union[Member, Leader, None]: """Get :class:`Member` object by name or the :class:`Leader`. :param member_name: name of the member to retrieve. :param fallback_to_leader: if ``True`` return the :class:`Leader` instead if the member cannot be found. :returns: the :class:`Member` if found or :class:`Leader` object. """ return next((m for m in self.members if m.name == member_name), self.leader if fallback_to_leader else None) def get_clone_member(self, exclude_name: str) -> Union[Member, Leader, None]: """Get member or leader object to use as clone source. :param exclude_name: name of a member name to exclude. :returns: a randomly selected candidate member from available running members that are configured to as viable sources for cloning (has tag ``clonefrom`` in configuration). If no member is appropriate the current leader is used. """ exclude = [exclude_name] + ([self.leader.name] if self.leader else []) candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude] return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader @staticmethod def is_physical_slot(value: Any) -> bool: """Check whether provided configuration is for permanent physical replication slot. :param value: configuration of the permanent replication slot. :returns: ``True`` if *value* is a physical replication slot, otherwise ``False``. """ return not value \ or (isinstance(value, dict) and not Cluster.is_logical_slot(cast(Dict[str, Any], value)) and cast(Dict[str, Any], value).get('type', 'physical') == 'physical') @staticmethod def is_logical_slot(value: Any) -> bool: """Check whether provided configuration is for permanent logical replication slot. :param value: configuration of the permanent replication slot. :returns: ``True`` if *value* is a logical replication slot, otherwise ``False``. """ return isinstance(value, dict) \ and cast(Dict[str, Any], value).get('type', 'logical') == 'logical' \ and bool(cast(Dict[str, Any], value).get('database') and cast(Dict[str, Any], value).get('plugin')) @property def __permanent_slots(self) -> Dict[str, Union[Dict[str, Any], Any]]: """Dictionary of permanent replication slots with their known LSN.""" ret: Dict[str, Union[Dict[str, Any], Any]] = global_config.permanent_slots members: Dict[str, int] = {slot_name_from_member_name(m.name): m.lsn or 0 for m in self.members if m.replicatefrom} slots: Dict[str, int] = self.slots for name, value in list(ret.items()): if not value: value = ret[name] = {} if isinstance(value, dict): # For permanent physical slots we want to get MAX LSN from the `Cluster.slots` and from the # member that does cascading replication with the matching name (see `replicatefrom` tag). # It is necessary because we may have the permanent replication slot on the primary for this node. lsn = max(members.get(name, 0) if self.is_physical_slot(value) else 0, slots.get(name, 0)) if lsn: value['lsn'] = lsn else: # Don't let anyone set 'lsn' in the global configuration :) value.pop('lsn', None) # pyright: ignore [reportUnknownMemberType] return ret @property def permanent_physical_slots(self) -> Dict[str, Any]: """Dictionary of permanent ``physical`` replication slots.""" return {name: value for name, value in self.__permanent_slots.items() if self.is_physical_slot(value)} @property def __permanent_logical_slots(self) -> Dict[str, Any]: """Dictionary of permanent ``logical`` replication slots.""" return {name: value for name, value in self.__permanent_slots.items() if self.is_logical_slot(value)} def get_replication_slots(self, postgresql: 'Postgresql', member: Tags, *, role: Optional[str] = None, show_error: bool = False) -> Dict[str, Dict[str, Any]]: """Lookup configured slot names in the DCS, report issues found and merge with permanent slots. Will log an error if: * Any logical slots are disabled, due to version compatibility, and *show_error* is ``True``. :param postgresql: reference to :class:`Postgresql` object. :param member: reference to an object implementing :class:`Tags` interface. :param role: role of the node, if not set will be taken from *postgresql*. :param show_error: if ``True`` report error if any disabled logical slots or conflicting slot names are found. :returns: final dictionary of slot names, after merging with permanent slots and performing sanity checks. """ name = member.name if isinstance(member, Member) else postgresql.name role = role or postgresql.role slots: Dict[str, Dict[str, Any]] = self._get_members_slots(name, role, member.nofailover, postgresql.can_advance_slots) permanent_slots: Dict[str, Any] = self._get_permanent_slots(postgresql, member, role) disabled_permanent_logical_slots: List[str] = self._merge_permanent_slots( slots, permanent_slots, name, role, postgresql.can_advance_slots) if disabled_permanent_logical_slots and show_error: logger.error("Permanent logical replication slots supported by Patroni only starting from PostgreSQL 11. " "Following slots will not be created: %s.", disabled_permanent_logical_slots) return slots def _merge_permanent_slots(self, slots: Dict[str, Dict[str, Any]], permanent_slots: Dict[str, Any], name: str, role: str, can_advance_slots: bool) -> List[str]: """Merge replication *slots* for members with *permanent_slots*. Perform validation of configured permanent slot name, skipping invalid names. Will update *slots* in-line based on ``type`` of slot, ``physical`` or ``logical``, and name of node. Type is assumed to be ``physical`` if there are no attributes stored as the slot value. :param slots: Slot names with existing attributes if known. :param name: name of this node. :param role: role of the node -- ``primary``, ``standby_leader`` or ``replica``. :param permanent_slots: dictionary containing slot name key and slot information values. :param can_advance_slots: ``True`` if ``pg_replication_slot_advance()`` function is available, ``False`` otherwise. :returns: List of disabled permanent, logical slot names, if postgresql version < 11. """ name = slot_name_from_member_name(name) topology = {slot_name_from_member_name(m.name): m.replicatefrom and slot_name_from_member_name(m.replicatefrom) for m in self.members} disabled_permanent_logical_slots: List[str] = [] for slot_name, value in permanent_slots.items(): if not slot_name_re.match(slot_name): logger.error("Invalid permanent replication slot name '%s'", slot_name) logger.error("Slot name may only contain lower case letters, numbers, and the underscore chars") continue tmp = deepcopy(value) if value else {'type': 'physical'} if isinstance(tmp, dict): value = cast(Dict[str, Any], tmp) if 'type' not in value: value['type'] = 'logical' if value.get('database') and value.get('plugin') else 'physical' if value['type'] == 'physical': # Don't try to create permanent physical replication slot for yourself if slot_name not in slots and slot_name != name: # On the leader we expected to have permanent slots active, except the case when it is a slot # for a cascading replica. Lets consider a configuration with C being a permanent slot. In this # case we should have the following: A(B: active, C: inactive) <- B (C: active) <- C # We don't consider the same situation on node B, because if node C doesn't exists, we will not # be able to know its `replicatefrom` tag value. expected_active = not topology.get(slot_name) and role in ('primary', 'standby_leader') slots[slot_name] = {**value, 'expected_active': expected_active} continue if self.is_logical_slot(value): if not can_advance_slots: disabled_permanent_logical_slots.append(slot_name) elif slot_name in slots: logger.error("Permanent logical replication slot {'%s': %s} is conflicting with" " physical replication slot for cluster member", slot_name, value) else: slots[slot_name] = value continue logger.error("Bad value for slot '%s' in permanent_slots: %s", slot_name, permanent_slots[slot_name]) return disabled_permanent_logical_slots def _get_permanent_slots(self, postgresql: 'Postgresql', tags: Tags, role: str) -> Dict[str, Any]: """Get configured permanent replication slots. .. note:: Permanent replication slots are only considered if ``use_slots`` configuration is enabled. A node that is not supposed to become a leader (*nofailover*) will not have permanent replication slots. Also node with disabled streaming (*nostream*) and its cascading followers must not have permanent logical slots due to lack of feedback from node to primary, which makes them unsafe to use. In a standby cluster we only support physical replication slots. The returned dictionary for a non-standby cluster always contains permanent logical replication slots in order to show a warning if they are not supported by PostgreSQL before v11. :param postgresql: reference to :class:`Postgresql` object. :param tags: reference to an object implementing :class:`Tags` interface. :param role: role of the node -- ``primary``, ``standby_leader`` or ``replica``. :returns: dictionary of permanent slot names mapped to attributes. """ if not global_config.use_slots or tags.nofailover: return {} if global_config.is_standby_cluster or self.get_slot_name_on_primary(postgresql.name, tags) is None: return self.permanent_physical_slots if postgresql.can_advance_slots or role == 'standby_leader' else {} return self.__permanent_slots if postgresql.can_advance_slots or role == 'primary' \ else self.__permanent_logical_slots def _get_members_slots(self, name: str, role: str, nofailover: bool, can_advance_slots: bool) -> Dict[str, Dict[str, Any]]: """Get physical replication slots configuration for a given member. There are following situations possible: * If the ``nostream`` tag is set on the member - we should not have the replication slot for it on the current primary or any other member even if ``replicatefrom`` is set, because ``nostream`` disables WAL streaming. * PostgreSQL is 11 and newer and configuration allows retention of member replication slots. In this case we want to have replication slots for every member except the case when we have ``nofailover`` tag set. * PostgreSQL is older than 11 or configuration doesn't allow member slots retention. In this case we want: * On primary have replication slots for all members that don't have ``replicatefrom`` tag pointing to the existing member. * On replica node have replication slots only for members which ``replicatefrom`` tag pointing to us. Will log an error if: * Conflicting slot names between members are found :param name: name of this node. :param role: role of this node, ``primary``, ``standby_leader``, or ``replica``. :param nofailover: ``True`` if this node is tagged to not be a failover candidate, ``False`` otherwise. :param can_advance_slots: ``True`` if ``pg_replication_slot_advance()`` function is available, ``False`` otherwise. :returns: dictionary of physical replication slots that should exist on a given node. """ if not global_config.use_slots: return {} # we always want to exclude the member with our name from the list, # also exclude members with disabled WAL streaming members = filter(lambda m: m.name != name and not m.nostream, self.members) def leader_filter(member: Member) -> bool: """Check whether provided *member* should replicate from the current node when it is running as a leader. :param member: a :class:`Member` object. :returns: ``True`` if provided member should replicate from the current node, ``False`` otherwise. """ return member.replicatefrom is None or\ member.replicatefrom == name or\ not self.has_member(member.replicatefrom) def replica_filter(member: Member) -> bool: """Check whether provided *member* should replicate from the current node when it is running as a replica. ..note:: We only consider members with ``replicatefrom`` tag that matches our name and always exclude the leader. :param member: a :class:`Member` object. :returns: ``True`` if provided member should replicate from the current node, ``False`` otherwise. """ return member.replicatefrom == name and member.name != self.leader_name # In case when retention of replication slots is possible the `expected_active` function # will be used to figure out whether the replication slot is expected to be active. # Otherwise it will be used to find replication slots that should exist on a current node. expected_active = leader_filter if role in ('primary', 'standby_leader') else replica_filter if can_advance_slots and global_config.member_slots_ttl > 0: # if the node does only cascading and can't become the leader, we # want only to have slots for members that could connect to it. members = [m for m in members if not nofailover or m.replicatefrom == name] else: members = [m for m in members if expected_active(m)] leader_patroni_version = self.leader and self.leader.member.patroni_version slots: Dict[str, int] = self.slots ret: Dict[str, Dict[str, Any]] = {} for member in members: slot_name = slot_name_from_member_name(member.name) lsn = slots.get(slot_name, 0) if member.replicatefrom or leader_patroni_version and leader_patroni_version < (4, 0, 0): # `/status` key is maintained by the leader, but `member` may be connected to some other node. # In that case, the slot in the leader is inactive and doesn't advance, so we use the LSN # reported by the member to advance replication slot LSN. # `max` is only a fallback so we take the LSN from the slot when there is no feedback from the member. lsn = max(member.lsn or 0, lsn) ret[slot_name] = {'type': 'physical', 'lsn': lsn, 'expected_active': expected_active(member)} slot_name = slot_name_from_member_name(name) ret.update({slot: {'type': 'physical'} for slot in self.status.retain_slots if not nofailover and slot not in ret and slot != slot_name}) if len(ret) < len(members): # Find which names are conflicting for a nicer error message slot_conflicts: Dict[str, List[str]] = defaultdict(list) for member in members: slot_conflicts[slot_name_from_member_name(member.name)].append(member.name) logger.error("Following cluster members share a replication slot name: %s", "; ".join(f"{', '.join(v)} map to {k}" for k, v in slot_conflicts.items() if len(v) > 1)) return ret def has_permanent_slots(self, postgresql: 'Postgresql', member: Tags) -> bool: """Check if our node has permanent replication slots configured. :param postgresql: reference to :class:`Postgresql` object. :param member: reference to an object implementing :class:`Tags` interface for the node that we are checking permanent logical replication slots for. :returns: ``True`` if there are permanent replication slots configured, otherwise ``False``. """ role = 'replica' members_slots: Dict[str, Dict[str, str]] = self._get_members_slots(postgresql.name, role, member.nofailover, postgresql.can_advance_slots) permanent_slots: Dict[str, Any] = self._get_permanent_slots(postgresql, member, role) slots = deepcopy(members_slots) self._merge_permanent_slots(slots, permanent_slots, postgresql.name, role, postgresql.can_advance_slots) return len(slots) > len(members_slots) or any(self.is_physical_slot(v) for v in permanent_slots.values()) def maybe_filter_permanent_slots(self, postgresql: 'Postgresql', slots: Dict[str, int]) -> Dict[str, int]: """Filter out all non-permanent slots from provided *slots* dict. .. note:: In case if retention of replication slots for members is enabled we will not do any filtering, because we need to publish LSN values for members replication slots, so that other nodes can use them to advance LSN, like they do it for permanent slots. :param postgresql: reference to :class:`Postgresql` object. :param slots: slot names with LSN values. :returns: a :class:`dict` object that contains only slots that are known to be permanent. """ if global_config.member_slots_ttl > 0: return slots permanent_slots: Dict[str, Any] = self._get_permanent_slots(postgresql, RemoteMember('', {}), 'replica') members_slots = {slot_name_from_member_name(m.name) for m in self.members} return {name: value for name, value in slots.items() if name in permanent_slots and (self.is_physical_slot(permanent_slots[name]) or self.is_logical_slot(permanent_slots[name]) and name not in members_slots)} def _has_permanent_logical_slots(self, postgresql: 'Postgresql', member: Tags) -> bool: """Check if the given member node has permanent ``logical`` replication slots configured. :param postgresql: reference to a :class:`Postgresql` object. :param member: reference to an object implementing :class:`Tags` interface for the node that we are checking permanent logical replication slots for. :returns: ``True`` if any detected replications slots are ``logical``, otherwise ``False``. """ slots = self.get_replication_slots(postgresql, member, role='replica').values() return any(v for v in slots if v.get("type") == "logical") def should_enforce_hot_standby_feedback(self, postgresql: 'Postgresql', member: Tags) -> bool: """Determine whether ``hot_standby_feedback`` should be enabled for the given member. The ``hot_standby_feedback`` must be enabled if the current replica has ``logical`` slots, or it is working as a cascading replica for the other node that has ``logical`` slots. :param postgresql: reference to a :class:`Postgresql` object. :param member: reference to an object implementing :class:`Tags` interface for the node that we are checking permanent logical replication slots for. :returns: ``True`` if this node or any member replicating from this node has permanent logical slots, otherwise ``False``. """ if self._has_permanent_logical_slots(postgresql, member): return True if global_config.use_slots: name = member.name if isinstance(member, Member) else postgresql.name if not self.get_slot_name_on_primary(name, member): return False members = [m for m in self.members if m.replicatefrom == name and m.name != self.leader_name] return any(self.should_enforce_hot_standby_feedback(postgresql, m) for m in members) return False def get_slot_name_on_primary(self, name: str, tags: Tags) -> Optional[str]: """Get the name of physical replication slot for this node on the primary. .. note:: P <-- I <-- L In case of cascading replication we have to check not our physical slot, but slot of the replica that connects us to the primary. :param name: name of the member node to check. :param tags: reference to an object implementing :class:`Tags` interface. :returns: the slot name on the primary that is in use for physical replication on this node. """ seen_nodes: Set[str] = set() while True: seen_nodes.add(name) if tags.nostream: return None replicatefrom = self.get_member(tags.replicatefrom, False) \ if tags.replicatefrom and tags.replicatefrom != name else None if not isinstance(replicatefrom, Member): return slot_name_from_member_name(name) if replicatefrom.name in seen_nodes: return None name, tags = replicatefrom.name, replicatefrom @property def timeline(self) -> int: """Get the cluster history index from the :attr:`~Cluster.history`. :returns: If the recorded history is empty assume timeline is ``1``, if it is not defined or the stored history is not formatted as expected ``0`` is returned and an error will be logged. Otherwise, the last number stored incremented by 1 is returned. :Example: No history provided: >>> Cluster(0, 0, 0, Status.empty(), 0, 0, 0, 0, None, {}).timeline 0 Empty history assume timeline is ``1``: >>> Cluster(0, 0, 0, Status.empty(), 0, 0, 0, TimelineHistory.from_node(1, '[]'), None, {}).timeline 1 Invalid history format, a string of ``a``, returns ``0``: >>> Cluster(0, 0, 0, Status.empty(), 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), None, {}).timeline 0 History as a list of strings: >>> history = TimelineHistory.from_node(1, '[["3", "2", "1"]]') >>> Cluster(0, 0, 0, Status.empty(), 0, 0, 0, history, None, {}).timeline 4 """ if self.history: if self.history.lines: try: return int(self.history.lines[-1][0]) + 1 except Exception: logger.error('Failed to parse cluster history from DCS: %s', self.history.lines) elif self.history.value == '[]': return 1 return 0 @property def min_version(self) -> Optional[Tuple[int, ...]]: """Lowest Patroni software version found in known members of the cluster.""" return next(iter(sorted(m.patroni_version for m in self.members if m.patroni_version)), None) class ReturnFalseException(Exception): """Exception to be caught by the :func:`catch_return_false_exception` decorator.""" def catch_return_false_exception(func: Callable[..., Any]) -> Any: """Decorator function for catching functions raising :exc:`ReturnFalseException`. :param func: function to be wrapped. :returns: wrapped function. """ def wrapper(*args: Any, **kwargs: Any): try: return func(*args, **kwargs) except ReturnFalseException: return False return wrapper class AbstractDCS(abc.ABC): """Abstract representation of DCS modules. Implementations of a concrete DCS class, using appropriate backend client interfaces, must include the following methods and properties. Functional methods that are critical in their timing, required to complete within ``retry_timeout`` period in order to prevent the DCS considered inaccessible, each perform construction of complex data objects: * :meth:`~AbstractDCS._postgresql_cluster_loader`: method which processes the structure of data stored in the DCS used to build the :class:`Cluster` object with all relevant associated data. * :meth:`~AbstractDCS._mpp_cluster_loader`: Similar to above but specifically representing MPP group and workers information. * :meth:`~AbstractDCS._load_cluster`: main method for calling specific ``loader`` method to build the :class:`Cluster` object representing the state and topology of the cluster. Functional methods that are critical in their timing and must be written with ACID transaction properties in mind: * :meth:`~AbstractDCS.attempt_to_acquire_leader`: method used in the leader race to attempt to acquire the leader lock by creating the leader key in the DCS, if it does not exist. * :meth:`~AbstractDCS._update_leader`: method to update ``leader`` key in DCS. Relies on Compare-And-Set to ensure the Primary lock key is updated. If this fails to update within the ``retry_timeout`` window the Primary will be demoted. Functional method that relies on Compare-And-Create to ensure only one member creates the relevant key: * :meth:`~AbstractDCS.initialize`: method used in the race for cluster initialization which creates the ``initialize`` key in the DCS. DCS backend getter and setter methods and properties: * :meth:`~AbstractDCS.take_leader`: method to create a new leader key in the DCS. * :meth:`~AbstractDCS.set_ttl`: method for setting TTL value in DCS. * :meth:`~AbstractDCS.ttl`: property which returns the current TTL. * :meth:`~AbstractDCS.set_retry_timeout`: method for setting ``retry_timeout`` in DCS backend. * :meth:`~AbstractDCS._write_leader_optime`: compatibility method to write WAL LSN to DCS. * :meth:`~AbstractDCS._write_status`: method to write WAL LSN for slots to the DCS. * :meth:`~AbstractDCS._write_failsafe`: method to write cluster topology to the DCS, used by failsafe mechanism. * :meth:`~AbstractDCS.touch_member`: method to update individual member key in the DCS. * :meth:`~AbstractDCS.set_history_value`: method to set the ``history`` key in the DCS. DCS setter methods using Compare-And-Set which although important are less critical if they fail, attempts can be retried or may result in warning log messages: * :meth:`~AbstractDCS.set_failover_value`: method to create and/or update the ``failover`` key in the DCS. * :meth:`~AbstractDCS.set_config_value`: method to create and/or update the ``failover`` key in the DCS. * :meth:`~AbstractDCS.set_sync_state_value`: method to set the synchronous state ``sync`` key in the DCS. DCS data and key removal methods: * :meth:`~AbstractDCS.delete_sync_state`: likewise, a method to remove synchronous state ``sync`` key from the DCS. * :meth:`~AbstractDCS.delete_cluster`: method which will remove cluster information from the DCS. Used only from `patronictl`. * :meth:`~AbstractDCS._delete_leader`: method relies on CAS, used by a member that is the current leader, to remove the ``leader`` key in the DCS. * :meth:`~AbstractDCS.cancel_initialization`: method to remove the ``initialize`` key for the cluster from the DCS. If either of the `sync_state` set or delete methods fail, although not critical, this may result in ``Synchronous replication key updated by someone else`` messages being logged. Care should be taken to consult each abstract method for any additional information and requirements such as expected exceptions that should be raised in certain conditions and the object types for arguments and return from methods and properties. """ _INITIALIZE = 'initialize' _CONFIG = 'config' _LEADER = 'leader' _FAILOVER = 'failover' _HISTORY = 'history' _MEMBERS = 'members/' _OPTIME = 'optime' _STATUS = 'status' # JSON, contains "leader_lsn" and confirmed_flush_lsn of logical "slots" on the leader _LEADER_OPTIME = _OPTIME + '/' + _LEADER # legacy _SYNC = 'sync' _FAILSAFE = 'failsafe' def __init__(self, config: Dict[str, Any], mpp: 'AbstractMPP') -> None: """Prepare DCS paths, MPP object, initial values for state information and processing dependencies. :param config: :class:`dict`, reference to config section of selected DCS. i.e.: ``zookeeper`` for zookeeper, ``etcd`` for etcd, etc... :param mpp: an object implementing :class:`AbstractMPP` interface. """ self._mpp = mpp self._name = config['name'] self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']])) self._set_loop_wait(config.get('loop_wait', 10)) self._ctl = bool(config.get('patronictl', False)) self._cluster: Optional[Cluster] = None self._cluster_valid_till: float = 0 self._cluster_thread_lock = Lock() self._last_lsn: int = 0 self._last_seen: int = 0 self._last_status: Dict[str, Any] = {'retain_slots': []} self._last_retain_slots: Dict[str, float] = {} self._last_failsafe: Optional[Dict[str, str]] = {} self.event = Event() @property def mpp(self) -> 'AbstractMPP': """Get the effective underlying MPP, if any has been configured.""" return self._mpp def client_path(self, path: str) -> str: """Construct the absolute key name from appropriate parts for the DCS type. :param path: The key name within the current Patroni cluster. :returns: absolute key name for the current Patroni cluster. """ components = [self._base_path] if self._mpp.is_enabled(): components.append(str(self._mpp.group)) components.append(path.lstrip('/')) return '/'.join(components) @property def initialize_path(self) -> str: """Get the client path for ``initialize``.""" return self.client_path(self._INITIALIZE) @property def config_path(self) -> str: """Get the client path for ``config``.""" return self.client_path(self._CONFIG) @property def members_path(self) -> str: """Get the client path for ``members``.""" return self.client_path(self._MEMBERS) @property def member_path(self) -> str: """Get the client path for ``member`` representing this node.""" return self.client_path(self._MEMBERS + self._name) @property def leader_path(self) -> str: """Get the client path for ``leader``.""" return self.client_path(self._LEADER) @property def failover_path(self) -> str: """Get the client path for ``failover``.""" return self.client_path(self._FAILOVER) @property def history_path(self) -> str: """Get the client path for ``history``.""" return self.client_path(self._HISTORY) @property def status_path(self) -> str: """Get the client path for ``status``.""" return self.client_path(self._STATUS) @property def leader_optime_path(self) -> str: """Get the client path for ``optime/leader`` (legacy key, superseded by ``status``).""" return self.client_path(self._LEADER_OPTIME) @property def sync_path(self) -> str: """Get the client path for ``sync``.""" return self.client_path(self._SYNC) @property def failsafe_path(self) -> str: """Get the client path for ``failsafe``.""" return self.client_path(self._FAILSAFE) @abc.abstractmethod def set_ttl(self, ttl: int) -> Optional[bool]: """Set the new *ttl* value for DCS keys.""" @property @abc.abstractmethod def ttl(self) -> int: """Get current ``ttl`` value.""" @abc.abstractmethod def set_retry_timeout(self, retry_timeout: int) -> None: """Set the new value for *retry_timeout*.""" def _set_loop_wait(self, loop_wait: int) -> None: """Set new *loop_wait* value. :param loop_wait: value to set. """ self._loop_wait = loop_wait def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None: """Load and set relevant values from configuration. Sets ``loop_wait``, ``ttl`` and ``retry_timeout`` properties. :param config: Loaded configuration information object or dictionary of key value pairs. """ self._set_loop_wait(config['loop_wait']) self.set_ttl(config['ttl']) self.set_retry_timeout(config['retry_timeout']) @property def loop_wait(self) -> int: """The recorded value for cluster HA loop wait time in seconds.""" return self._loop_wait @property def last_seen(self) -> int: """The time recorded when the DCS was last reachable.""" return self._last_seen @abc.abstractmethod def _postgresql_cluster_loader(self, path: Any) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ @abc.abstractmethod def _mpp_cluster_loader(self, path: Any) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ @abc.abstractmethod def _load_cluster( self, path: str, loader: Callable[[Any], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: """Main abstract method that implements the loading of :class:`Cluster` instance. .. note:: Internally this method should call the *loader* method that will build :class:`Cluster` object which represents current state and topology of the cluster in DCS. This method supposed to be called only by the :meth:`~AbstractDCS.get_cluster` method. :param path: the path in DCS where to load Cluster(s) from. :param loader: one of :meth:`~AbstractDCS._postgresql_cluster_loader` or :meth:`~AbstractDCS._mpp_cluster_loader`. :raise: :exc:`~DCSError` in case of communication problems with DCS. If the current node was running as a primary and exception raised, instance would be demoted. """ def __get_postgresql_cluster(self, path: Optional[str] = None) -> Cluster: """Low level method to load a :class:`Cluster` object from DCS. :param path: optional client path in DCS backend to load from. :returns: a loaded :class:`Cluster` instance. """ if path is None: path = self.client_path('') cluster = self._load_cluster(path, self._postgresql_cluster_loader) if TYPE_CHECKING: # pragma: no cover assert isinstance(cluster, Cluster) return cluster def is_mpp_coordinator(self) -> bool: """:class:`Cluster` instance has a Coordinator group ID. :returns: ``True`` if the given node is running as the MPP Coordinator. """ return self._mpp.is_coordinator() def get_mpp_coordinator(self) -> Optional[Cluster]: """Load the PostgreSQL cluster for the MPP Coordinator. .. note:: This method is only executed on the worker nodes to find the coordinator. :returns: Select :class:`Cluster` instance associated with the MPP Coordinator group ID. """ try: return self.__get_postgresql_cluster(f'{self._base_path}/{self._mpp.coordinator_group_id}/') except Exception as e: logger.error('Failed to load %s coordinator cluster from %s: %r', self._mpp.type, self.__class__.__name__, e) return None def _get_mpp_cluster(self) -> Cluster: """Load MPP cluster from DCS. :returns: A MPP :class:`Cluster` instance for the coordinator with workers clusters in the `Cluster.workers` dict. """ groups = self._load_cluster(self._base_path + '/', self._mpp_cluster_loader) if TYPE_CHECKING: # pragma: no cover assert isinstance(groups, dict) cluster = groups.pop(self._mpp.coordinator_group_id, Cluster.empty()) cluster.workers.update(groups) return cluster def get_cluster(self) -> Cluster: """Retrieve a fresh view of DCS. .. note:: Stores copy of time, status and failsafe values for comparison in DCS update decisions. Caching is required to avoid overhead placed upon the REST API. Returns either a PostgreSQL or MPP implementation of :class:`Cluster` depending on availability. :returns: """ try: cluster = self._get_mpp_cluster() if self.is_mpp_coordinator() else self.__get_postgresql_cluster() except Exception: self.reset_cluster() raise with self._cluster_thread_lock: self._cluster = cluster self._cluster_valid_till = time.time() + self.ttl self._last_seen = int(time.time()) self._last_status = {self._OPTIME: cluster.status.last_lsn, 'retain_slots': cluster.status.retain_slots} if cluster.status.slots: self._last_status['slots'] = cluster.status.slots self._last_failsafe = cluster.failsafe return cluster @property def cluster(self) -> Optional[Cluster]: """Cached DCS cluster information that has not yet expired.""" with self._cluster_thread_lock: return self._cluster if self._cluster_valid_till > time.time() else None def reset_cluster(self) -> None: """Clear cached state of DCS.""" with self._cluster_thread_lock: self._cluster = None self._cluster_valid_till = 0 @abc.abstractmethod def _write_leader_optime(self, last_lsn: str) -> bool: """Write current WAL LSN into ``/optime/leader`` key in DCS. :param last_lsn: absolute WAL LSN in bytes. :returns: ``True`` if successfully committed to DCS. """ def write_leader_optime(self, last_lsn: int) -> None: """Write value for WAL LSN to ``optime/leader`` key in DCS. .. note:: This method abstracts away the required data structure of :meth:`~Cluster.write_status`, so it is not needed in the caller. However, the ``optime/leader`` is only written in :meth:`~Cluster.write_status` when the cluster has members with a Patroni version that is old enough to require it (i.e. the old Patroni version doesn't understand the new format). :param last_lsn: absolute WAL LSN in bytes. """ self.write_status({self._OPTIME: last_lsn}) @abc.abstractmethod def _write_status(self, value: str) -> bool: """Write current WAL LSN and ``confirmed_flush_lsn`` of permanent slots into the ``/status`` key in DCS. :param value: status serialized in JSON format. :returns: ``True`` if successfully committed to DCS. """ def write_status(self, value: Dict[str, Any]) -> None: """Write cluster status to DCS if changed. .. note:: The DCS key ``/status`` was introduced in Patroni version 2.1.0. Previous to this the position of last known leader LSN was stored in ``optime/leader``. This method has detection for backwards compatibility of members with a version older than this. :param value: JSON serializable dictionary with current WAL LSN and ``confirmed_flush_lsn`` of permanent slots. """ # This method is always called with ``optime`` key, rest of the keys are optional. # In case if we know old values (stored in self._last_status), we will copy them over. for name in ('slots', 'retain_slots'): if name not in value and self._last_status.get(name): value[name] = self._last_status[name] # if the key is present, but the value is None, we will not write such pair. value = {k: v for k, v in value.items() if v is not None} if not deep_compare(self._last_status, value) and self._write_status(json.dumps(value, separators=(',', ':'))): self._last_status = value cluster = self.cluster min_version = cluster and cluster.min_version if min_version and min_version < (2, 1, 0) and self._last_lsn != value[self._OPTIME]: self._last_lsn = value[self._OPTIME] self._write_leader_optime(str(value[self._OPTIME])) @abc.abstractmethod def _write_failsafe(self, value: str) -> bool: """Write current cluster topology to DCS that will be used by failsafe mechanism (if enabled). :param value: failsafe topology serialized in JSON format. :returns: ``True`` if successfully committed to DCS. """ def write_failsafe(self, value: Dict[str, str]) -> None: """Write the ``/failsafe`` key in DCS. :param value: dictionary value to set, consisting of the ``name`` and ``api_url`` of members. """ if not (isinstance(self._last_failsafe, dict) and deep_compare(self._last_failsafe, value)) \ and self._write_failsafe(json.dumps(value, separators=(',', ':'))): self._last_failsafe = value @property def failsafe(self) -> Optional[Dict[str, str]]: """Stored value of :attr:`~AbstractDCS._last_failsafe`.""" return self._last_failsafe def _build_retain_slots(self, cluster: Cluster, slots: Optional[Dict[str, int]]) -> Optional[List[str]]: """Handle retention policy of physical replication slots for cluster members. When the member key is missing we want to keep its replication slot for a while, so that WAL segments will not be already absent when it comes back online. It is being solved by storing the list of replication slots representing members in the ``retain_slots`` field of the ``/status`` key. This method handles retention policy by keeping the list of such replication slots in memory and removing names when they were observed longer than ``member_slots_ttl`` ago. :param cluster: :class:`Cluster` object with information about the current cluster state. :param slots: slot names with LSN values that exist on the leader node and consists of slots for cluster members and permanent replication slots. :returns: the list of replication slots to be written to ``/status`` key or ``None``. """ timestamp = time.time() # DCS is a source of truth, therefore we take missing values from there self._last_retain_slots.update({name: timestamp for name in self._last_status['retain_slots'] if (not slots or name not in slots) and name not in self._last_retain_slots}) if slots: # if slots is not empty it implies we are running v11+ members: Set[str] = set() found_self = False for member in cluster.members: found_self = member.name == self._name if not member.nostream: members.add(slot_name_from_member_name(member.name)) if not found_self: # It could be that the member key for our node is not in DCS and we can't check tags.nostream. # In this case our name will falsely appear in `retain_slots`, but only temporary. members.add(slot_name_from_member_name(self._name)) permanent_slots = cluster.permanent_physical_slots # we want to have in ``retain_slots`` only non-permanent member slots self._last_retain_slots.update({name: timestamp for name in slots if name in members and name not in permanent_slots}) # retention for name, value in list(self._last_retain_slots.items()): if value + global_config.member_slots_ttl <= timestamp: logger.info("Replication slot '%s' for absent cluster member is expired after %d sec.", name, global_config.member_slots_ttl) del self._last_retain_slots[name] return list(sorted(self._last_retain_slots.keys())) or None @abc.abstractmethod def _update_leader(self, leader: Leader) -> bool: """Update ``leader`` key (or session) ttl. .. note:: You have to use CAS (Compare And Swap) operation in order to update leader key, for example for etcd ``prevValue`` parameter must be used. If update fails due to DCS not being accessible or because it is not able to process requests (hopefully temporary), the :exc:`DCSError` exception should be raised. :param leader: a reference to a current :class:`leader` object. :returns: ``True`` if ``leader`` key (or session) has been updated successfully. """ def update_leader(self, cluster: Cluster, last_lsn: Optional[int], slots: Optional[Dict[str, int]] = None, failsafe: Optional[Dict[str, str]] = None) -> bool: """Update ``leader`` key (or session) ttl, ``/status``, and ``/failsafe`` keys. :param cluster: :class:`Cluster` object with information about the current cluster state. :param last_lsn: absolute WAL LSN in bytes. :param slots: dictionary with permanent slots ``confirmed_flush_lsn``. :param failsafe: if defined dictionary passed to :meth:`~AbstractDCS.write_failsafe`. :returns: ``True`` if ``leader`` key (or session) has been updated successfully. """ if TYPE_CHECKING: # pragma: no cover assert isinstance(cluster.leader, Leader) ret = self._update_leader(cluster.leader) if ret and last_lsn: status: Dict[str, Any] = {self._OPTIME: last_lsn, 'slots': slots or None, 'retain_slots': self._build_retain_slots(cluster, slots)} self.write_status(status) if ret and failsafe is not None: self.write_failsafe(failsafe) return ret @abc.abstractmethod def attempt_to_acquire_leader(self) -> bool: """Attempt to acquire leader lock. .. note:: This method should create ``/leader`` key with the value :attr:`~AbstractDCS._name`. The key must be created atomically. In case the key already exists it should not be overwritten and ``False`` must be returned. If key creation fails due to DCS not being accessible or because it is not able to process requests (hopefully temporary), the :exc:`DCSError` exception should be raised. :returns: ``True`` if key has been created successfully. """ @abc.abstractmethod def set_failover_value(self, value: str, version: Optional[Any] = None) -> bool: """Create or update ``/failover`` key. :param value: value to set. :param version: for conditional update of the key/object. :returns: ``True`` if successfully committed to DCS. """ def manual_failover(self, leader: Optional[str], candidate: Optional[str], scheduled_at: Optional[datetime.datetime] = None, version: Optional[Any] = None) -> bool: """Prepare dictionary with given values and set ``/failover`` key in DCS. :param leader: value to set for ``leader``. :param candidate: value to set for ``member``. :param scheduled_at: value converted to ISO date format for ``scheduled_at``. :param version: for conditional update of the key/object. :returns: ``True`` if successfully committed to DCS. """ failover_value = {} if leader: failover_value['leader'] = leader if candidate: failover_value['member'] = candidate if scheduled_at: failover_value['scheduled_at'] = scheduled_at.isoformat() return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), version) @abc.abstractmethod def set_config_value(self, value: str, version: Optional[Any] = None) -> bool: """Create or update ``/config`` key in DCS. :param value: new value to set in the ``config`` key. :param version: for conditional update of the key/object. :returns: ``True`` if successfully committed to DCS. """ @abc.abstractmethod def touch_member(self, data: Dict[str, Any]) -> bool: """Update member key in DCS. .. note:: This method should create or update key with the name with ``/members/`` + :attr:`~AbstractDCS._name` and the value of *data* in a given DCS. :param data: information about an instance (including connection strings). :returns: ``True`` if successfully committed to DCS. """ @abc.abstractmethod def take_leader(self) -> bool: """Establish a new leader in DCS. .. note:: This method should create leader key with value of :attr:`~AbstractDCS._name` and ``ttl`` of :attr:`~AbstractDCS.ttl`. Since it could be called only on initial cluster bootstrap it could create this key regardless, overwriting the key if necessary. :returns: ``True`` if successfully committed to DCS. """ @abc.abstractmethod def initialize(self, create_new: bool = True, sysid: str = "") -> bool: """Race for cluster initialization. This method should atomically create ``initialize`` key and return ``True``, otherwise it should return ``False``. :param create_new: ``False`` if the key should already exist (in the case we are setting the system_id). :param sysid: PostgreSQL cluster system identifier, if specified, is written to the key. :returns: ``True`` if key has been created successfully. """ @abc.abstractmethod def _delete_leader(self, leader: Leader) -> bool: """Remove leader key from DCS. This method should remove leader key if current instance is the leader. :param leader: :class:`Leader` object with information about the leader. :returns: ``True`` if successfully committed to DCS. """ def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool: """Update ``optime/leader`` and voluntarily remove leader key from DCS. This method should remove leader key if current instance is the leader. :param leader: :class:`Leader` object with information about the leader. :param last_lsn: latest checkpoint location in bytes. :returns: boolean result of called abstract :meth:`~AbstractDCS._delete_leader`. """ if last_lsn: self.write_status({self._OPTIME: last_lsn}) return bool(leader) and self._delete_leader(leader) @abc.abstractmethod def cancel_initialization(self) -> bool: """Removes the ``initialize`` key for a cluster. :returns: ``True`` if successfully committed to DCS. """ @abc.abstractmethod def delete_cluster(self) -> bool: """Delete cluster from DCS. :returns: ``True`` if successfully committed to DCS. """ @staticmethod def sync_state(leader: Optional[str], sync_standby: Optional[Collection[str]], quorum: Optional[int]) -> Dict[str, Any]: """Build ``sync_state`` dictionary. :param leader: name of the leader node that manages ``/sync`` key. :param sync_standby: collection of currently known synchronous standby node names. :param quorum: if the node from :attr:`~SyncState.sync_standby` list is doing a leader race it should see at least :attr:`~SyncState.quorum` other nodes from the :attr:`~SyncState.sync_standby` + :attr:`~SyncState.leader` list :returns: dictionary that later could be serialized to JSON or saved directly to DCS. """ return {'leader': leader, 'quorum': quorum, 'sync_standby': ','.join(sorted(sync_standby)) if sync_standby else None} def write_sync_state(self, leader: Optional[str], sync_standby: Optional[Collection[str]], quorum: Optional[int], version: Optional[Any] = None) -> Optional[SyncState]: """Write the new synchronous state to DCS. Calls :meth:`~AbstractDCS.sync_state` to build a dictionary and then calls DCS specific :meth:`~AbstractDCS.set_sync_state_value`. :param leader: name of the leader node that manages ``/sync`` key. :param sync_standby: collection of currently known synchronous standby node names. :param version: for conditional update of the key/object. :param quorum: if the node from :attr:`~SyncState.sync_standby` list is doing a leader race it should see at least :attr:`~SyncState.quorum` other nodes from the :attr:`~SyncState.sync_standby` + :attr:`~SyncState.leader` list :returns: the new :class:`SyncState` object or ``None``. """ sync_value = self.sync_state(leader, sync_standby, quorum) ret = self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), version) if not isinstance(ret, bool): return SyncState.from_node(ret, sync_value) return None @abc.abstractmethod def set_history_value(self, value: str) -> bool: """Set value for ``history`` in DCS. :param value: new value of ``history`` key/object. :returns: ``True`` if successfully committed to DCS. """ @abc.abstractmethod def set_sync_state_value(self, value: str, version: Optional[Any] = None) -> Union[Any, bool]: """Set synchronous state in DCS. :param value: the new value of ``/sync`` key. :param version: for conditional update of the key/object. :returns: *version* of the new object or ``False`` in case of error. """ @abc.abstractmethod def delete_sync_state(self, version: Optional[Any] = None) -> bool: """Delete the synchronous state from DCS. :param version: for conditional deletion of the key/object. :returns: ``True`` if delete successful. """ def watch(self, leader_version: Optional[Any], timeout: float) -> bool: """Sleep if the current node is a leader, otherwise, watch for changes of leader key with a given *timeout*. :param leader_version: version of a leader key. :param timeout: timeout in seconds. :returns: if ``True`` this will reschedule the next run of the HA cycle. """ _ = leader_version self.event.wait(timeout) return self.event.is_set() patroni-4.0.4/patroni/dcs/consul.py000066400000000000000000000726201472010352700173050ustar00rootroot00000000000000from __future__ import absolute_import import json import logging import os import re import socket import ssl import time from collections import defaultdict from http.client import HTTPException from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union from urllib.parse import quote, urlencode, urlparse import urllib3 from consul import base, Check, ConsulException, NotFound from urllib3.exceptions import HTTPError from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT from . import AbstractDCS, catch_return_false_exception, Cluster, ClusterConfig, \ Failover, Leader, Member, ReturnFalseException, Status, SyncState, TimelineHistory if TYPE_CHECKING: # pragma: no cover from ..config import Config logger = logging.getLogger(__name__) class ConsulError(DCSError): pass class ConsulInternalError(ConsulException): """An internal Consul server error occurred""" class InvalidSessionTTL(ConsulException): """Session TTL is too small or too big""" class InvalidSession(ConsulException): """invalid session""" class Response(NamedTuple): code: int headers: Union[Mapping[str, str], Mapping[bytes, bytes], None] body: str content: bytes class HTTPClient(object): def __init__(self, host: str = '127.0.0.1', port: int = 8500, token: Optional[str] = None, scheme: str = 'http', verify: bool = True, cert: Optional[str] = None, ca_cert: Optional[str] = None) -> None: self.token = token self._read_timeout = 10 self.base_uri = uri(scheme, (host, port)) kwargs = {} if cert: if isinstance(cert, tuple): # Key and cert are separate kwargs['cert_file'] = cert[0] kwargs['key_file'] = cert[1] else: # combined certificate kwargs['cert_file'] = cert if ca_cert: kwargs['ca_certs'] = ca_cert kwargs['cert_reqs'] = ssl.CERT_REQUIRED if verify or ca_cert else ssl.CERT_NONE self.http = urllib3.PoolManager(num_pools=10, maxsize=10, headers={}, **kwargs) self._ttl = 30 def set_read_timeout(self, timeout: float) -> None: self._read_timeout = timeout / 3.0 @property def ttl(self) -> int: return self._ttl def set_ttl(self, ttl: int) -> bool: ret = self._ttl != ttl self._ttl = ttl return ret @staticmethod def response(response: urllib3.response.HTTPResponse) -> Response: content = response.data body = content.decode('utf-8') if response.status == 500: msg = '{0} {1}'.format(response.status, body) if body.startswith('Invalid Session TTL'): raise InvalidSessionTTL(msg) elif body.startswith('invalid session'): raise InvalidSession(msg) else: raise ConsulInternalError(msg) return Response(response.status, response.headers, body, content) def uri(self, path: str, params: Union[None, Dict[str, Any], List[Tuple[str, Any]], Tuple[Tuple[str, Any], ...]] = None) -> str: return '{0}{1}{2}'.format(self.base_uri, path, params and '?' + urlencode(params) or '') def __getattr__(self, method: str) -> Callable[[Callable[[Response], Union[bool, Any, Tuple[str, Any]]], str, Union[None, Dict[str, Any], List[Tuple[str, Any]]], str, Optional[Dict[str, str]]], Union[bool, Any, Tuple[str, Any]]]: if method not in ('get', 'post', 'put', 'delete'): raise AttributeError("HTTPClient instance has no attribute '{0}'".format(method)) def wrapper(callback: Callable[[Response], Union[bool, Any, Tuple[str, Any]]], path: str, params: Union[None, Dict[str, Any], List[Tuple[str, Any]]] = None, data: str = '', headers: Optional[Dict[str, str]] = None) -> Union[bool, Any, Tuple[str, Any]]: # python-consul doesn't allow to specify ttl smaller then 10 seconds # because session_ttl_min defaults to 10s, so we have to do this ugly dirty hack... if method == 'put' and path == '/v1/session/create': ttl = '"ttl": "{0}s"'.format(self._ttl) if not data or data == '{}': data = '{' + ttl + '}' else: data = data[:-1] + ', ' + ttl + '}' if isinstance(params, list): # starting from v1.1.0 python-consul switched from `dict` to `list` for params params = {k: v for k, v in params} kwargs: Dict[str, Any] = {'retries': 0, 'preload_content': False, 'body': data} if method == 'get' and isinstance(params, dict) and 'index' in params: timeout = float(params['wait'][:-1]) if 'wait' in params else 300 # According to the documentation a small random amount of additional wait time is added to the # supplied maximum wait time to spread out the wake up time of any concurrent requests. This adds # up to wait / 16 additional time to the maximum duration. Since our goal is actually getting a # response rather read timeout we will add to the timeout a slightly bigger value. kwargs['timeout'] = timeout + max(timeout / 15.0, 1) else: kwargs['timeout'] = self._read_timeout kwargs['headers'] = (headers or {}).copy() kwargs['headers'].update(urllib3.make_headers(user_agent=USER_AGENT)) token = params.pop('token', self.token) if isinstance(params, dict) else self.token if token: kwargs['headers']['X-Consul-Token'] = token return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs))) return wrapper class ConsulClient(base.Consul): def __init__(self, *args: Any, **kwargs: Any) -> None: """ Consul client with Patroni customisations. .. note:: Parameters, *token*, *cert* and *ca_cert* are not passed to the parent class :class:`consul.base.Consul`. Original class documentation, *token* is an optional ``ACL token``. If supplied it will be used by default for all requests made with this client session. It's still possible to override this token by passing a token explicitly for a request. *consistency* sets the consistency mode to use by default for all reads that support the consistency option. It's still possible to override this by passing explicitly for a given request. *consistency* can be either 'default', 'consistent' or 'stale'. *dc* is the datacenter that this agent will communicate with. By default, the datacenter of the host is used. *verify* is whether to verify the SSL certificate for HTTPS requests *cert* client side certificates for HTTPS requests :param args: positional arguments to pass to :class:`consul.base.Consul` :param kwargs: keyword arguments, with *cert*, *ca_cert* and *token* removed, passed to :class:`consul.base.Consul` """ self._cert = kwargs.pop('cert', None) self._ca_cert = kwargs.pop('ca_cert', None) self.token = kwargs.get('token') super(ConsulClient, self).__init__(*args, **kwargs) def http_connect(self, *args: Any, **kwargs: Any) -> HTTPClient: kwargs.update(dict(zip(['host', 'port', 'scheme', 'verify'], args))) if self._cert: kwargs['cert'] = self._cert if self._ca_cert: kwargs['ca_cert'] = self._ca_cert if self.token: kwargs['token'] = self.token return HTTPClient(**kwargs) def connect(self, *args: Any, **kwargs: Any) -> HTTPClient: return self.http_connect(*args, **kwargs) # pragma: no cover def reload_config(self, config: Dict[str, Any]) -> None: self.http.token = self.token = config.get('token') self.consistency = config.get('consistency', 'default') self.dc = config.get('dc') def catch_consul_errors(func: Callable[..., Any]) -> Callable[..., Any]: def wrapper(*args: Any, **kwargs: Any) -> Any: try: return func(*args, **kwargs) except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout): return False return wrapper def force_if_last_failed(func: Callable[..., Any]) -> Callable[..., Any]: def wrapper(*args: Any, **kwargs: Any) -> Any: if getattr(wrapper, 'last_result', None) is False: kwargs['force'] = True last_result = func(*args, **kwargs) setattr(wrapper, 'last_result', last_result) return last_result setattr(wrapper, 'last_result', None) return wrapper def service_name_from_scope_name(scope_name: str) -> str: """Translate scope name to service name which can be used in dns. 230 = 253 - len('replica.') - len('.service.consul') """ def replace_char(match: Any) -> str: c = match.group(0) return '-' if c in '. _' else "u{:04d}".format(ord(c)) service_name = re.sub(r'[^a-z0-9\-]', replace_char, scope_name.lower()) return service_name[0:230] class Consul(AbstractDCS): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: super(Consul, self).__init__(config, mpp) self._base_path = self._base_path[1:] self._scope = config['scope'] self._session = None self.__do_not_watch = False self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1, retry_exceptions=(ConsulInternalError, HTTPException, HTTPError, socket.error, socket.timeout)) if 'url' in config: url: str = config['url'] r = urlparse(url) config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500}) elif 'host' in config: host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500) config['host'] = host if 'port' not in config: config['port'] = int(port) if config.get('cacert'): config['ca_cert'] = config.pop('cacert') if config.get('key') and config.get('cert'): config['cert'] = (config['cert'], config['key']) config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc', 'consistency') kwargs: Dict[str, Any] = {p: config.get(p) for p in config_keys if config.get(p)} verify = config.get('verify') if not isinstance(verify, bool): verify = parse_bool(verify) if isinstance(verify, bool): kwargs['verify'] = verify self._client = ConsulClient(**kwargs) self.set_retry_timeout(config['retry_timeout']) self.set_ttl(config.get('ttl') or 30) self._last_session_refresh = 0 self.__session_checks = config.get('checks', []) self._register_service = config.get('register_service', False) self._previous_loop_register_service = self._register_service self._service_tags = sorted(config.get('service_tags', [])) self._previous_loop_service_tags = self._service_tags if self._register_service: self._set_service_name() self._service_check_interval = config.get('service_check_interval', '5s') self._service_check_tls_server_name = config.get('service_check_tls_server_name', None) if not self._ctl: self.create_session() self._previous_loop_token = self._client.token def retry(self, method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: return self._retry.copy()(method, *args, **kwargs) def create_session(self) -> None: while not self._session: try: self.refresh_session() except ConsulError: logger.info('waiting on consul') time.sleep(5) def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None: super(Consul, self).reload_config(config) consul_config = config.get('consul', {}) self._client.reload_config(consul_config) self._previous_loop_service_tags = self._service_tags self._service_tags: List[str] = consul_config.get('service_tags', []) self._service_tags.sort() should_register_service = consul_config.get('register_service', False) if should_register_service and not self._register_service: self._set_service_name() self._previous_loop_register_service = self._register_service self._register_service = should_register_service def set_ttl(self, ttl: int) -> Optional[bool]: if self._client.http.set_ttl(ttl / 2.0): # Consul multiplies the TTL by 2x self._session = None self.__do_not_watch = True return None @property def ttl(self) -> int: return self._client.http.ttl * 2 # we multiply the value by 2 because it was divided in the `set_ttl()` method def set_retry_timeout(self, retry_timeout: int) -> None: self._retry.deadline = retry_timeout self._client.http.set_read_timeout(retry_timeout) def adjust_ttl(self) -> None: try: settings = self._client.agent.self() min_ttl = (settings['Config']['SessionTTLMin'] or 10000000000) / 1000000000.0 logger.warning('Changing Session TTL from %s to %s', self._client.http.ttl, min_ttl) self._client.http.set_ttl(min_ttl) except Exception: logger.exception('adjust_ttl') def _do_refresh_session(self, force: bool = False) -> bool: """:returns: `!True` if it had to create new session""" if not force and self._session and self._last_session_refresh + self._loop_wait > time.time(): return False if self._session: try: self._client.session.renew(self._session) except NotFound: self._session = None ret = not self._session if ret: try: self._session = self._client.session.create(name=self._scope + '-' + self._name, checks=self.__session_checks, lock_delay=0.001, behavior='delete') except InvalidSessionTTL: logger.exception('session.create') self.adjust_ttl() raise self._last_session_refresh = time.time() return ret def refresh_session(self) -> bool: try: return self.retry(self._do_refresh_session) except (ConsulException, RetryFailedError): logger.exception('refresh_session') raise ConsulError('Failed to renew/create session') @staticmethod def member(node: Dict[str, str]) -> Member: return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value']) def _cluster_from_nodes(self, nodes: Dict[str, Any]) -> Cluster: # get initialize flag initialize = nodes.get(self._INITIALIZE) initialize = initialize and initialize['Value'] # get global dynamic configuration config = nodes.get(self._CONFIG) config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value']) # get timeline history history = nodes.get(self._HISTORY) history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value']) # get last known leader lsn and slots status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME) status = Status.from_node(status and status['Value']) # get list of members members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1] # get leader leader = nodes.get(self._LEADER) if leader: member = Member(-1, leader['Value'], None, {}) member = ([m for m in members if m.name == leader['Value']] or [member])[0] leader = Leader(leader['ModifyIndex'], leader.get('Session'), member) # failover key failover = nodes.get(self._FAILOVER) if failover: failover = Failover.from_node(failover['ModifyIndex'], failover['Value']) # get synchronization state sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value']) # get failsafe topology failsafe = nodes.get(self._FAILSAFE) try: failsafe = json.loads(failsafe['Value']) if failsafe else None except Exception: failsafe = None return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) @property def _consistency(self) -> str: return 'consistent' if self._ctl else self._client.consistency def _postgresql_cluster_loader(self, path: str) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency) if results is None: return Cluster.empty() nodes: Dict[str, Dict[str, Any]] = {} for node in results: node['Value'] = (node['Value'] or b'').decode('utf-8') nodes[node['Key'][len(path):]] = node return self._cluster_from_nodes(nodes) def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ results: Optional[List[Dict[str, Any]]] _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency) clusters: Dict[int, Dict[str, Dict[str, Any]]] = defaultdict(dict) for node in results or []: key = node['Key'][len(path):].split('/', 1) if len(key) == 2 and self._mpp.group_re.match(key[0]): node['Value'] = (node['Value'] or b'').decode('utf-8') clusters[int(key[0])][key[1]] = node return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()} def _load_cluster( self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: try: return loader(path) except Exception: logger.exception('get_cluster') raise ConsulError('Consul is not responding properly') @catch_consul_errors def touch_member(self, data: Dict[str, Any]) -> bool: cluster = self.cluster member = cluster and cluster.get_member(self._name, fallback_to_leader=False) try: create_member = self.refresh_session() except DCSError: return False if member and (create_member or member.session != self._session): self._client.kv.delete(self.member_path) create_member = True if self._register_service or self._previous_loop_register_service: try: self.update_service(not create_member and member and member.data or {}, data) except Exception: logger.exception('update_service') if not create_member and member and deep_compare(data, member.data): return True try: self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), acquire=self._session) return True except InvalidSession: self._session = None logger.error('Our session disappeared from Consul, can not "touch_member"') except Exception: logger.exception('touch_member') return False def _set_service_name(self) -> None: self._service_name = service_name_from_scope_name(self._scope) if self._scope != self._service_name: logger.warning('Using %s as consul service name instead of scope name %s', self._service_name, self._scope) @catch_consul_errors def register_service(self, service_name: str, **kwargs: Any) -> bool: logger.info('Register service %s, params %s', service_name, kwargs) return self._client.agent.service.register(service_name, **kwargs) @catch_consul_errors def deregister_service(self, service_id: str) -> bool: logger.info('Deregister service %s', service_id) # service_id can contain special characters, but is used as part of uri in deregister request service_id = quote(service_id) return self._client.agent.service.deregister(service_id) def _update_service(self, data: Dict[str, Any]) -> Optional[bool]: service_name = self._service_name role = data['role'].replace('_', '-') state = data['state'] api_url: str = data['api_url'] api_parts = urlparse(api_url) api_parts = api_parts._replace(path='/{0}'.format(role)) conn_url: str = data['conn_url'] conn_parts = urlparse(conn_url) check = Check.http(api_parts.geturl(), self._service_check_interval, deregister='{0}s'.format(self._client.http.ttl * 10)) if self._service_check_tls_server_name is not None: check['TLSServerName'] = self._service_check_tls_server_name tags = self._service_tags[:] tags.append(role) if role == 'primary': tags.append('master') self._previous_loop_service_tags = self._service_tags self._previous_loop_token = self._client.token params = { 'service_id': '{0}/{1}'.format(self._scope, self._name), 'address': conn_parts.hostname, 'port': conn_parts.port, 'check': check, 'tags': tags, 'enable_tag_override': True, } if state == 'stopped' or (not self._register_service and self._previous_loop_register_service): self._previous_loop_register_service = self._register_service return self.deregister_service(params['service_id']) self._previous_loop_register_service = self._register_service if role in ['primary', 'replica', 'standby-leader']: if state != 'running': return return self.register_service(service_name, **params) logger.warning('Could not register service: unknown role type %s', role) @force_if_last_failed def update_service(self, old_data: Dict[str, Any], new_data: Dict[str, Any], force: bool = False) -> Optional[bool]: update = False for key in ['role', 'api_url', 'conn_url', 'state']: if key not in new_data: logger.warning('Could not register service: not enough params in member data') return if old_data.get(key) != new_data[key]: update = True if ( force or update or self._register_service != self._previous_loop_register_service or self._service_tags != self._previous_loop_service_tags or self._client.token != self._previous_loop_token ): return self._update_service(new_data) def _do_attempt_to_acquire_leader(self, retry: Retry) -> bool: try: return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session) except InvalidSession: self._session = None if not retry.ensure_deadline(0): logger.error('Our session disappeared from Consul. Deadline exceeded, giving up') return False logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt') retry(self._do_refresh_session) retry.ensure_deadline(1, ConsulError('_do_attempt_to_acquire_leader timeout')) return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session) @catch_return_false_exception def attempt_to_acquire_leader(self) -> bool: retry = self._retry.copy() self._run_and_handle_exceptions(self._do_refresh_session, retry=retry) retry.ensure_deadline(1, ConsulError('attempt_to_acquire_leader timeout')) ret = self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry, retry=None) if not ret: logger.info('Could not take out TTL lock') return ret def take_leader(self) -> bool: return self.attempt_to_acquire_leader() @catch_consul_errors def set_failover_value(self, value: str, version: Optional[int] = None) -> bool: return self._client.kv.put(self.failover_path, value, cas=version) @catch_consul_errors def set_config_value(self, value: str, version: Optional[int] = None) -> bool: return self._client.kv.put(self.config_path, value, cas=version) @catch_consul_errors def _write_leader_optime(self, last_lsn: str) -> bool: return self._client.kv.put(self.leader_optime_path, last_lsn) @catch_consul_errors def _write_status(self, value: str) -> bool: return self._client.kv.put(self.status_path, value) @catch_consul_errors def _write_failsafe(self, value: str) -> bool: return self._client.kv.put(self.failsafe_path, value) @staticmethod def _run_and_handle_exceptions(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: retry = kwargs.pop('retry', None) try: return retry(method, *args, **kwargs) if retry else method(*args, **kwargs) except (RetryFailedError, InvalidSession, HTTPException, HTTPError, socket.error, socket.timeout) as e: raise ConsulError(e) except ConsulException: raise ReturnFalseException @catch_return_false_exception def _update_leader(self, leader: Leader) -> bool: retry = self._retry.copy() self._run_and_handle_exceptions(self._do_refresh_session, True, retry=retry) if self._session and leader.session != self._session: retry.ensure_deadline(1, ConsulError('update_leader timeout')) logger.warning('Recreating the leader key due to session mismatch') self._run_and_handle_exceptions(self._client.kv.delete, self.leader_path, cas=leader.version) retry.ensure_deadline(0.5, ConsulError('update_leader timeout')) self._run_and_handle_exceptions(self._client.kv.put, self.leader_path, self._name, acquire=self._session) return bool(self._session) @catch_consul_errors def initialize(self, create_new: bool = True, sysid: str = '') -> bool: kwargs = {'cas': 0} if create_new else {} return self.retry(self._client.kv.put, self.initialize_path, sysid, **kwargs) @catch_consul_errors def cancel_initialization(self) -> bool: return self.retry(self._client.kv.delete, self.initialize_path) @catch_consul_errors def delete_cluster(self) -> bool: return self.retry(self._client.kv.delete, self.client_path(''), recurse=True) @catch_consul_errors def set_history_value(self, value: str) -> bool: return self._client.kv.put(self.history_path, value) @catch_consul_errors def _delete_leader(self, leader: Leader) -> bool: return self._client.kv.delete(self.leader_path, cas=int(leader.version)) @catch_consul_errors def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]: retry = self._retry.copy() ret = retry(self._client.kv.put, self.sync_path, value, cas=version) if ret: # We have no other choice, only read after write :( if not retry.ensure_deadline(0.5): return False _, ret = self.retry(self._client.kv.get, self.sync_path, consistency='consistent') if ret and (ret.get('Value') or b'').decode('utf-8') == value: return ret['ModifyIndex'] return False @catch_consul_errors def delete_sync_state(self, version: Optional[int] = None) -> bool: return self.retry(self._client.kv.delete, self.sync_path, cas=version) def watch(self, leader_version: Optional[int], timeout: float) -> bool: self._last_session_refresh = 0 if self.__do_not_watch: self.__do_not_watch = False return True if leader_version: end_time = time.time() + timeout while timeout >= 1: try: idx, _ = self._client.kv.get(self.leader_path, index=leader_version, wait=str(timeout) + 's') return str(idx) != str(leader_version) except (ConsulException, HTTPException, HTTPError, socket.error, socket.timeout): logger.exception('watch') timeout = end_time - time.time() try: return super(Consul, self).watch(None, timeout) finally: self.event.clear() patroni-4.0.4/patroni/dcs/etcd.py000066400000000000000000001132421472010352700167150ustar00rootroot00000000000000from __future__ import absolute_import import abc import json import logging import os import random import socket import time from collections import defaultdict from copy import deepcopy from http.client import HTTPException from queue import Queue from threading import Thread from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, TYPE_CHECKING, Union from urllib.parse import urlparse import etcd import urllib3.util.connection from dns import resolver from dns.exception import DNSException from urllib3 import Timeout from urllib3.exceptions import HTTPError, ProtocolError, ReadTimeoutError from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..request import get as requests_get from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT from . import AbstractDCS, catch_return_false_exception, Cluster, ClusterConfig, \ Failover, Leader, Member, ReturnFalseException, Status, SyncState, TimelineHistory if TYPE_CHECKING: # pragma: no cover from ..config import Config logger = logging.getLogger(__name__) class EtcdRaftInternal(etcd.EtcdException): """Raft Internal Error""" class EtcdError(DCSError): pass _AddrInfo = Tuple[socket.AddressFamily, socket.SocketKind, int, str, Union[Tuple[str, int], Tuple[str, int, int, int]]] class DnsCachingResolver(Thread): def __init__(self, cache_time: float = 600.0, cache_fail_time: float = 30.0) -> None: super(DnsCachingResolver, self).__init__() self._cache: Dict[Tuple[str, int], Tuple[float, List[_AddrInfo]]] = {} self._cache_time = cache_time self._cache_fail_time = cache_fail_time self._resolve_queue: Queue[Tuple[Tuple[str, int], int]] = Queue() self.daemon = True self.start() def run(self) -> None: while True: (host, port), attempt = self._resolve_queue.get() response = self._do_resolve(host, port) if response: self._cache[(host, port)] = (time.time(), response) else: if attempt < 10: self.resolve_async(host, port, attempt + 1) time.sleep(1) def resolve(self, host: str, port: int) -> List[_AddrInfo]: current_time = time.time() cached_time, response = self._cache.get((host, port), (0, [])) time_passed = current_time - cached_time if time_passed > self._cache_time or (not response and time_passed > self._cache_fail_time): new_response = self._do_resolve(host, port) if new_response: self._cache[(host, port)] = (current_time, new_response) response = new_response return response def resolve_async(self, host: str, port: int, attempt: int = 0) -> None: self._resolve_queue.put(((host, port), attempt)) def remove(self, host: str, port: int) -> None: self._cache.pop((host, port), None) @staticmethod def _do_resolve(host: str, port: int) -> List[_AddrInfo]: try: return socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP) except Exception as e: logger.warning('failed to resolve host %s: %s', host, e) return [] class AbstractEtcdClientWithFailover(abc.ABC, etcd.Client): ERROR_CLS: Type[Exception] def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None: self._dns_resolver = dns_resolver self.set_machines_cache_ttl(cache_ttl) self._machines_cache_updated = 0 kwargs = {p: config.get(p) for p in ('host', 'port', 'protocol', 'use_proxies', 'version_prefix', 'username', 'password', 'cert', 'ca_cert') if config.get(p)} super(AbstractEtcdClientWithFailover, self).__init__(read_timeout=config['retry_timeout'], **kwargs) # For some reason python3-etcd on debian and ubuntu are not based on the latest version # Workaround for the case when https://github.com/jplana/python-etcd/pull/196 is not applied self.http.connection_pool_kw.pop('ssl_version', None) self._config = config self._load_machines_cache() self._allow_reconnect = True # allow passing retry argument to api_execute in params self._comparison_conditions.add('retry') self._read_options.add('retry') self._del_conditions.add('retry') def _calculate_timeouts(self, etcd_nodes: int, timeout: Optional[float] = None) -> Tuple[int, float, int]: """Calculate a request timeout and number of retries per single etcd node. In case if the timeout per node is too small (less than one second) we will reduce the number of nodes. For the cluster with only one node we will try to do 2 retries. For clusters with 2 nodes we will try to do 1 retry for every node. No retries for clusters with 3 or more nodes. We better rely on switching to a different node.""" per_node_timeout = timeout = float(timeout or self.read_timeout) max_retries = 4 - min(etcd_nodes, 3) per_node_retries = 1 min_timeout = 1.0 while etcd_nodes > 0: per_node_timeout = float(timeout) / etcd_nodes if per_node_timeout >= min_timeout: # for small clusters we will try to do more than on try on every node while per_node_retries < max_retries and per_node_timeout / (per_node_retries + 1) >= min_timeout: per_node_retries += 1 per_node_timeout /= per_node_retries break # if the timeout per one node is to small try to reduce number of nodes etcd_nodes -= 1 max_retries = 1 return etcd_nodes, per_node_timeout, per_node_retries - 1 def reload_config(self, config: Dict[str, Any]) -> None: self.username = config.get('username') self.password = config.get('password') def _get_headers(self) -> Dict[str, str]: basic_auth = ':'.join((self.username, self.password)) if self.username and self.password else None return urllib3.make_headers(basic_auth=basic_auth, user_agent=USER_AGENT) def _prepare_common_parameters(self, etcd_nodes: int, timeout: Optional[float] = None) -> Dict[str, Any]: kwargs: Dict[str, Any] = {'headers': self._get_headers(), 'redirect': self.allow_redirect, 'preload_content': False} if timeout is not None: kwargs.update(retries=0, timeout=timeout) else: _, per_node_timeout, per_node_retries = self._calculate_timeouts(etcd_nodes) connect_timeout = max(1.0, per_node_timeout / 2.0) kwargs.update(timeout=Timeout(connect=connect_timeout, total=per_node_timeout), retries=per_node_retries) return kwargs def set_machines_cache_ttl(self, cache_ttl: int) -> None: self._machines_cache_ttl = cache_ttl @abc.abstractmethod def _prepare_get_members(self, etcd_nodes: int) -> Dict[str, Any]: """returns: request parameters""" @abc.abstractmethod def _get_members(self, base_uri: str, **kwargs: Any) -> List[str]: """returns: list of clientURLs""" @property def machines_cache(self) -> List[str]: base_uri, cache = self._base_uri, self._machines_cache return ([base_uri] if base_uri in cache else []) + [machine for machine in cache if machine != base_uri] def _get_machines_list(self, machines_cache: List[str]) -> List[str]: """Gets list of members from Etcd cluster using API :param machines_cache: initial list of Etcd members :returns: list of clientURLs retrieved from Etcd cluster :raises EtcdConnectionFailed: if failed""" kwargs = self._prepare_get_members(len(machines_cache)) for base_uri in machines_cache: try: machines = list(set(self._get_members(base_uri, **kwargs))) logger.debug("Retrieved list of machines: %s", machines) if machines: random.shuffle(machines) if not self._use_proxies: self._update_dns_cache(self._dns_resolver.resolve_async, machines) return machines except Exception as e: self.http.clear() logger.error("Failed to get list of machines from %s%s: %r", base_uri, self.version_prefix, e) raise etcd.EtcdConnectionFailed('No more machines in the cluster') @property def machines(self) -> List[str]: """Original `machines` method(property) of `etcd.Client` class raise exception when it failed to get list of etcd cluster members. This method is being called only when request failed on one of the etcd members during `api_execute` call. For us it's more important to execute original request rather then get new topology of etcd cluster. So we will catch this exception and return empty list of machines. Later, during next `api_execute` call we will forcefully update machines_cache. Also this method implements the same timeout-retry logic as `api_execute`, because the original method was retrying 2 times with the `read_timeout` on each node. After the next refactoring the whole logic was moved to the _get_machines_list() method.""" return self._get_machines_list(self.machines_cache) def set_read_timeout(self, timeout: float) -> None: self._read_timeout = timeout def _do_http_request(self, retry: Optional[Retry], machines_cache: List[str], request_executor: Callable[..., urllib3.response.HTTPResponse], method: str, path: str, fields: Optional[Dict[str, Any]] = None, **kwargs: Any) -> urllib3.response.HTTPResponse: is_watch_request = isinstance(fields, dict) and fields.get('wait') == 'true' if fields is not None: kwargs['fields'] = fields some_request_failed = False for i, base_uri in enumerate(machines_cache): if i > 0: logger.info("Retrying on %s", base_uri) try: response = request_executor(method, base_uri + path, **kwargs) response.data.decode('utf-8') if some_request_failed: self.set_base_uri(base_uri) self._refresh_machines_cache() return response except (HTTPError, HTTPException, socket.error, socket.timeout) as e: self.http.clear() if not retry: if len(machines_cache) == 1: self.set_base_uri(self._base_uri) # trigger Etcd3 watcher restart # switch to the next etcd node because we don't know exactly what happened, # whether the key didn't received an update or there is a network problem. elif i + 1 < len(machines_cache): self.set_base_uri(machines_cache[i + 1]) if is_watch_request and isinstance(e, (ReadTimeoutError, ProtocolError)): logger.debug("Watch timed out.") raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e) logger.error("Request to server %s failed: %r", base_uri, e) logger.info("Reconnection allowed, looking for another server.") if not retry: raise etcd.EtcdException('{0} {1} request failed'.format(method, path)) some_request_failed = True raise etcd.EtcdConnectionFailed('No more machines in the cluster') @abc.abstractmethod def _prepare_request(self, kwargs: Dict[str, Any], params: Optional[Dict[str, Any]] = None, method: Optional[str] = None) -> Callable[..., urllib3.response.HTTPResponse]: """returns: request_executor""" def api_execute(self, path: str, method: str, params: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None) -> Any: retry = params.pop('retry', None) if isinstance(params, dict) else None # Update machines_cache if previous attempt of update has failed if self._update_machines_cache: self._load_machines_cache() elif not self._use_proxies and time.time() - self._machines_cache_updated > self._machines_cache_ttl: self._refresh_machines_cache() machines_cache = self.machines_cache etcd_nodes = len(machines_cache) kwargs = self._prepare_common_parameters(etcd_nodes, timeout) request_executor = self._prepare_request(kwargs, params, method) while True: try: response = self._do_http_request(retry, machines_cache, request_executor, method, path, **kwargs) return self._handle_server_response(response) except etcd.EtcdWatchTimedOut: raise except etcd.EtcdConnectionFailed as ex: try: if self._load_machines_cache(): machines_cache = self.machines_cache etcd_nodes = len(machines_cache) except Exception as e: logger.debug('Failed to update list of etcd nodes: %r', e) if TYPE_CHECKING: # pragma: no cover assert isinstance(retry, Retry) # etcd.EtcdConnectionFailed is raised only if retry is not None! sleeptime = retry.sleeptime remaining_time = retry.stoptime - sleeptime - time.time() nodes, timeout, retries = self._calculate_timeouts(etcd_nodes, remaining_time) if nodes == 0: self._update_machines_cache = True self.set_base_uri(self._base_uri) # trigger Etcd3 watcher restart raise ex retry.sleep_func(sleeptime) retry.update_delay() # We still have some time left. Partially reduce `machines_cache` and retry request kwargs.update(timeout=Timeout(connect=max(1.0, timeout / 2.0), total=timeout), retries=retries) machines_cache = machines_cache[:nodes] @staticmethod def get_srv_record(host: str) -> List[Tuple[str, int]]: try: return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')] except DNSException: return [] def _get_machines_cache_from_srv(self, srv: str, srv_suffix: Optional[str] = None) -> List[str]: """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record. This record should contain list of host and peer ports which could be used to run 'GET http://{host}:{port}/members' request (peer protocol)""" ret: List[str] = [] for r in ['-client-ssl', '-client', '-ssl', '', '-server-ssl', '-server']: r = '{0}-{1}'.format(r, srv_suffix) if srv_suffix else r protocol = 'https' if '-ssl' in r else 'http' endpoint = '/members' if '-server' in r else '' for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)): url = uri(protocol, (host, port), endpoint) if endpoint: try: response = requests_get(url, timeout=self.read_timeout, verify=False) if response.status < 400: for member in json.loads(response.data.decode('utf-8')): ret.extend(member['clientURLs']) break except Exception: logger.exception('GET %s', url) else: ret.append(url) if ret: self._protocol = protocol break else: logger.warning('Can not resolve SRV for %s', srv) return list(set(ret)) def _get_machines_cache_from_dns(self, host: str, port: int) -> List[str]: """One host might be resolved into multiple ip addresses. We will make list out of it""" if self.protocol == 'http': ret = [uri(self.protocol, res[-1][:2]) for res in self._dns_resolver.resolve(host, port)] if ret: return list(set(ret)) return [uri(self.protocol, (host, port))] def _get_machines_cache_from_config(self) -> List[str]: if 'proxy' in self._config: return [uri(self.protocol, (self._config['host'], self._config['port']))] machines_cache = [] if 'srv' in self._config: machines_cache = self._get_machines_cache_from_srv(self._config['srv'], self._config.get('srv_suffix')) if not machines_cache and 'hosts' in self._config: machines_cache = list(self._config['hosts']) if not machines_cache and 'host' in self._config: machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port']) return machines_cache @staticmethod def _update_dns_cache(func: Callable[[str, int], None], machines: List[str]) -> None: for url in machines: r = urlparse(url) if r.hostname: port = r.port or (443 if r.scheme == 'https' else 80) func(r.hostname, port) def _load_machines_cache(self) -> bool: """This method should fill up `_machines_cache` from scratch. It could happen only in two cases: 1. During class initialization 2. When all etcd members failed""" self._update_machines_cache = True if 'srv' not in self._config and 'host' not in self._config and 'hosts' not in self._config: raise Exception('Neither srv, hosts, host nor url are defined in etcd section of config') machines_cache = self._get_machines_cache_from_config() # Can not bootstrap list of etcd-cluster members, giving up if not machines_cache: raise etcd.EtcdException # enforce resolving dns name,they might get new ips self._update_dns_cache(self._dns_resolver.remove, machines_cache) # after filling up the initial list of machines_cache we should ask etcd-cluster about actual list ret = self._refresh_machines_cache(machines_cache) self._update_machines_cache = False return ret def _refresh_machines_cache(self, machines_cache: Optional[List[str]] = None) -> bool: """Get etcd cluster topology using Etcd API and put it to self._machines_cache :param machines_cache: the list of nodes we want to run through executing API request in addition to values stored in the self._machines_cache :returns: `True` if self._machines_cache was updated with new values :raises EtcdException: if failed to get topology and `machines_cache` was specified. The self._machines_cache will not be updated if nodes from the list are not accessible or if they are not returning correct results.""" if self._use_proxies: value = self._get_machines_cache_from_config() else: try: # we want to go through the list obtained from the config file + last known health topology value = self._get_machines_list(list(set((machines_cache or []) + self.machines_cache))) except etcd.EtcdConnectionFailed: value = [] if value: ret = set(self._machines_cache) != set(value) self._machines_cache = value elif machines_cache: # we are just starting or all nodes were not available at some point raise etcd.EtcdException("Could not get the list of servers, " "maybe you provided the wrong " "host(s) to connect to?") else: return False if self._base_uri not in self._machines_cache: self.set_base_uri(self._machines_cache[0]) self._machines_cache_updated = time.time() return ret def set_base_uri(self, value: str) -> None: if self._base_uri != value: logger.info('Selected new etcd server %s', value) self._base_uri = value class EtcdClient(AbstractEtcdClientWithFailover): ERROR_CLS = EtcdError def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None: super(EtcdClient, self).__init__({**config, 'version_prefix': None}, dns_resolver, cache_ttl) def __del__(self) -> None: try: self.http.clear() except (ReferenceError, TypeError, AttributeError): pass def _prepare_get_members(self, etcd_nodes: int) -> Dict[str, Any]: return self._prepare_common_parameters(etcd_nodes) def _get_members(self, base_uri: str, **kwargs: Any) -> List[str]: response = self.http.request(self._MGET, base_uri + self.version_prefix + '/machines', **kwargs) data = self._handle_server_response(response).data.decode('utf-8') return [m.strip() for m in data.split(',') if m.strip()] def _prepare_request(self, kwargs: Dict[str, Any], params: Optional[Dict[str, Any]] = None, method: Optional[str] = None) -> Callable[..., urllib3.response.HTTPResponse]: kwargs['fields'] = params if method in (self._MPOST, self._MPUT): kwargs['encode_multipart'] = False return self.http.request class AbstractEtcd(AbstractDCS): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP, client_cls: Type[AbstractEtcdClientWithFailover], retry_errors_cls: Union[Type[Exception], Tuple[Type[Exception], ...]]) -> None: super(AbstractEtcd, self).__init__(config, mpp) self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1, retry_exceptions=retry_errors_cls) self._ttl = int(config.get('ttl') or 30) self._abstract_client = self.get_etcd_client(config, client_cls) self.__do_not_watch = False self._has_failed = False @property @abc.abstractmethod def _client(self) -> AbstractEtcdClientWithFailover: """return correct type of etcd client""" def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None: super(AbstractEtcd, self).reload_config(config) self._client.reload_config(config.get(self.__class__.__name__.lower(), {})) def retry(self, method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: retry = self._retry.copy() kwargs['retry'] = retry return retry(method, *args, **kwargs) def _handle_exception(self, e: Exception, name: str = '', do_sleep: bool = False, raise_ex: Optional[Exception] = None) -> None: if not self._has_failed: logger.exception(name) else: logger.error(e) if do_sleep: time.sleep(1) self._has_failed = True if isinstance(raise_ex, Exception): raise raise_ex def handle_etcd_exceptions(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: try: retval = func(self, *args, **kwargs) self._has_failed = False return retval except (RetryFailedError, etcd.EtcdException) as e: self._handle_exception(e) return False except Exception as e: self._handle_exception(e, raise_ex=self._client.ERROR_CLS('unexpected error')) def _run_and_handle_exceptions(self, method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: retry = kwargs.pop('retry', self.retry) try: return retry(method, *args, **kwargs) if retry else method(*args, **kwargs) except (RetryFailedError, etcd.EtcdConnectionFailed) as e: raise self._client.ERROR_CLS(e) except etcd.EtcdException as e: self._handle_exception(e) raise ReturnFalseException except Exception as e: self._handle_exception(e, raise_ex=self._client.ERROR_CLS('unexpected error')) def set_socket_options(self, sock: socket.socket, socket_options: Optional[Collection[Tuple[int, int, int]]]) -> None: if socket_options: for opt in socket_options: sock.setsockopt(*opt) def get_etcd_client(self, config: Dict[str, Any], client_cls: Type[AbstractEtcdClientWithFailover]) -> AbstractEtcdClientWithFailover: config = deepcopy(config) if 'proxy' in config: config['use_proxies'] = True config['url'] = config['proxy'] if 'url' in config and isinstance(config['url'], str): r = urlparse(config['url']) config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379, 'username': r.username, 'password': r.password}) elif 'hosts' in config: hosts = config.pop('hosts') default_port = config.pop('port', 2379) protocol = config.get('protocol', 'http') if isinstance(hosts, str): hosts = hosts.split(',') config_hosts: List[str] = [] for value in hosts: if isinstance(value, str): config_hosts.append(uri(protocol, split_host_port(value.strip(), default_port))) config['hosts'] = config_hosts elif 'host' in config: host, port = split_host_port(config['host'], 2379) config['host'] = host if 'port' not in config: config['port'] = int(port) if config.get('cacert'): config['ca_cert'] = config.pop('cacert') if config.get('key') and config.get('cert'): config['cert'] = (config['cert'], config['key']) for p in ('discovery_srv', 'srv_domain'): if p in config: config['srv'] = config.pop(p) dns_resolver = DnsCachingResolver() def create_connection_patched( address: Tuple[str, int], timeout: Any = object(), source_address: Optional[Any] = None, socket_options: Optional[Collection[Tuple[int, int, int]]] = None ) -> socket.socket: host, port = address if host.startswith('['): host = host.strip('[]') err = None for af, socktype, proto, _, sa in dns_resolver.resolve(host, port): sock = None try: sock = socket.socket(af, socktype, proto) self.set_socket_options(sock, socket_options) if timeout is None or isinstance(timeout, (float, int)): sock.settimeout(timeout) if source_address: sock.bind(source_address) sock.connect(sa) return sock except socket.error as e: err = e if sock is not None: sock.close() sock = None if err is not None: raise err raise socket.error("getaddrinfo returns an empty list") urllib3.util.connection.create_connection = create_connection_patched client = None while not client: try: client = client_cls(config, dns_resolver) if 'use_proxies' in config and not client.machines: raise etcd.EtcdException except etcd.EtcdException: logger.info('waiting on etcd') time.sleep(5) return client def set_ttl(self, ttl: int) -> Optional[bool]: ttl = int(ttl) ret = self._ttl != ttl self._ttl = ttl self._client.set_machines_cache_ttl(ttl * 10) return ret @property def ttl(self) -> int: return self._ttl def set_retry_timeout(self, retry_timeout: int) -> None: self._retry.deadline = retry_timeout self._client.set_read_timeout(retry_timeout) def catch_etcd_errors(func: Callable[..., Any]) -> Any: def wrapper(self: AbstractEtcd, *args: Any, **kwargs: Any) -> Any: return self.handle_etcd_exceptions(func, *args, **kwargs) return wrapper class Etcd(AbstractEtcd): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: super(Etcd, self).__init__(config, mpp, EtcdClient, (etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal)) self.__do_not_watch = False @property def _client(self) -> EtcdClient: if TYPE_CHECKING: # pragma: no cover assert isinstance(self._abstract_client, EtcdClient) return self._abstract_client def set_ttl(self, ttl: int) -> Optional[bool]: self.__do_not_watch = super(Etcd, self).set_ttl(ttl) return None @staticmethod def member(node: etcd.EtcdResult) -> Member: return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value) def _cluster_from_nodes(self, etcd_index: int, nodes: Dict[str, etcd.EtcdResult]) -> Cluster: # get initialize flag initialize = nodes.get(self._INITIALIZE) initialize = initialize and initialize.value # get global dynamic configuration config = nodes.get(self._CONFIG) config = config and ClusterConfig.from_node(config.modifiedIndex, config.value) # get timeline history history = nodes.get(self._HISTORY) history = history and TimelineHistory.from_node(history.modifiedIndex, history.value) # get last know leader lsn and slots status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME) status = Status.from_node(status and status.value) # get list of members members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1] # get leader leader = nodes.get(self._LEADER) if leader: member = Member(-1, leader.value, None, {}) member = ([m for m in members if m.name == leader.value] or [member])[0] version = etcd_index if etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1 leader = Leader(version, leader.ttl, member) # failover key failover = nodes.get(self._FAILOVER) if failover: failover = Failover.from_node(failover.modifiedIndex, failover.value) # get synchronization state sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value) # get failsafe topology failsafe = nodes.get(self._FAILSAFE) try: failsafe = json.loads(failsafe.value) if failsafe else None except Exception: failsafe = None return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) def _postgresql_cluster_loader(self, path: str) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ try: result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl) except etcd.EtcdKeyNotFound: return Cluster.empty() nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves} return self._cluster_from_nodes(result.etcd_index, nodes) def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ try: result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl) except etcd.EtcdKeyNotFound: return {} clusters: Dict[int, Dict[str, etcd.EtcdResult]] = defaultdict(dict) for node in result.leaves: key = node.key[len(result.key):].lstrip('/').split('/', 1) if len(key) == 2 and self._mpp.group_re.match(key[0]): clusters[int(key[0])][key[1]] = node return {group: self._cluster_from_nodes(result.etcd_index, nodes) for group, nodes in clusters.items()} def _load_cluster( self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: cluster = None try: cluster = loader(path) except Exception as e: self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly')) self._has_failed = False if TYPE_CHECKING: # pragma: no cover assert cluster is not None return cluster @catch_etcd_errors def touch_member(self, data: Dict[str, Any]) -> bool: value = json.dumps(data, separators=(',', ':')) return bool(self._client.set(self.member_path, value, self._ttl)) @catch_etcd_errors def take_leader(self) -> bool: return self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl) def _do_attempt_to_acquire_leader(self) -> bool: try: return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl, prevExist=False)) except etcd.EtcdAlreadyExist: logger.info('Could not take out TTL lock') return False @catch_return_false_exception def attempt_to_acquire_leader(self) -> bool: return self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry=None) @catch_etcd_errors def set_failover_value(self, value: str, version: Optional[int] = None) -> bool: return bool(self._client.write(self.failover_path, value, prevIndex=version or 0)) @catch_etcd_errors def set_config_value(self, value: str, version: Optional[int] = None) -> bool: return bool(self._client.write(self.config_path, value, prevIndex=version or 0)) @catch_etcd_errors def _write_leader_optime(self, last_lsn: str) -> bool: return bool(self._client.set(self.leader_optime_path, last_lsn)) @catch_etcd_errors def _write_status(self, value: str) -> bool: return bool(self._client.set(self.status_path, value)) def _do_update_leader(self) -> bool: try: return self.retry(self._client.write, self.leader_path, self._name, prevValue=self._name, ttl=self._ttl) is not None except etcd.EtcdKeyNotFound: return self._do_attempt_to_acquire_leader() @catch_etcd_errors def _write_failsafe(self, value: str) -> bool: return bool(self._client.set(self.failsafe_path, value)) @catch_return_false_exception def _update_leader(self, leader: Leader) -> bool: return bool(self._run_and_handle_exceptions(self._do_update_leader, retry=None)) @catch_etcd_errors def initialize(self, create_new: bool = True, sysid: str = "") -> bool: return bool(self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new))) @catch_etcd_errors def _delete_leader(self, leader: Leader) -> bool: return bool(self._client.delete(self.leader_path, prevValue=self._name)) @catch_etcd_errors def cancel_initialization(self) -> bool: return bool(self.retry(self._client.delete, self.initialize_path)) @catch_etcd_errors def delete_cluster(self) -> bool: return bool(self.retry(self._client.delete, self.client_path(''), recursive=True)) @catch_etcd_errors def set_history_value(self, value: str) -> bool: return bool(self._client.write(self.history_path, value)) @catch_etcd_errors def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]: return self.retry(self._client.write, self.sync_path, value, prevIndex=version or 0).modifiedIndex @catch_etcd_errors def delete_sync_state(self, version: Optional[int] = None) -> bool: return bool(self.retry(self._client.delete, self.sync_path, prevIndex=version or 0)) def watch(self, leader_version: Optional[int], timeout: float) -> bool: if self.__do_not_watch: self.__do_not_watch = False return True if leader_version: end_time = time.time() + timeout while timeout >= 1: # when timeout is too small urllib3 doesn't have enough time to connect try: result = self._client.watch(self.leader_path, index=leader_version, timeout=timeout + 0.5) self._has_failed = False if result.action == 'compareAndSwap': time.sleep(0.01) # Synchronous work of all cluster members with etcd is less expensive # than reestablishing http connection every time from every replica. return True except etcd.EtcdWatchTimedOut: self._has_failed = False return False except (etcd.EtcdEventIndexCleared, etcd.EtcdWatcherCleared): # Watch failed self._has_failed = False return True # leave the loop, because watch with the same parameters will fail anyway except etcd.EtcdException as e: self._handle_exception(e, 'watch', True) timeout = end_time - time.time() try: return super(Etcd, self).watch(None, timeout) finally: self.event.clear() etcd.EtcdError.error_exceptions[300] = EtcdRaftInternal patroni-4.0.4/patroni/dcs/etcd3.py000066400000000000000000001172451472010352700170070ustar00rootroot00000000000000from __future__ import absolute_import import base64 import json import logging import os import socket import sys import time from collections import defaultdict from enum import IntEnum from threading import Condition, Lock, Thread from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union import etcd import urllib3 from urllib3.exceptions import ProtocolError, ReadTimeoutError from ..exceptions import DCSError, PatroniException from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare, enable_keepalive, iter_response_objects, RetryFailedError, USER_AGENT from . import catch_return_false_exception, Cluster, ClusterConfig, \ Failover, Leader, Member, Status, SyncState, TimelineHistory from .etcd import AbstractEtcd, AbstractEtcdClientWithFailover, catch_etcd_errors, DnsCachingResolver, Retry logger = logging.getLogger(__name__) class Etcd3Error(DCSError): pass class UnsupportedEtcdVersion(PatroniException): pass # google.golang.org/grpc/codes class GRPCCode(IntEnum): OK = 0 Canceled = 1 Unknown = 2 InvalidArgument = 3 DeadlineExceeded = 4 NotFound = 5 AlreadyExists = 6 PermissionDenied = 7 ResourceExhausted = 8 FailedPrecondition = 9 Aborted = 10 OutOfRange = 11 Unimplemented = 12 Internal = 13 Unavailable = 14 DataLoss = 15 Unauthenticated = 16 GRPCcodeToText: Dict[int, str] = {v: k for k, v in GRPCCode.__dict__['_member_map_'].items()} class Etcd3Exception(etcd.EtcdException): pass class Etcd3ClientError(Etcd3Exception): def __init__(self, code: Optional[int] = None, error: Optional[str] = None, status: Optional[int] = None) -> None: if not hasattr(self, 'error'): self.error = error and error.strip() self.codeText = GRPCcodeToText.get(code) if code is not None else None self.status = status def __repr__(self) -> str: return "<{0} error: '{1}', code: {2}>"\ .format(self.__class__.__name__, getattr(self, 'error', None), getattr(self, 'code', None)) __str__ = __repr__ def as_dict(self) -> Dict[str, Any]: return {'error': getattr(self, 'error', None), 'code': getattr(self, 'code', None), 'codeText': self.codeText, 'status': self.status} @classmethod def get_subclasses(cls) -> Iterator[Type['Etcd3ClientError']]: for subclass in cls.__subclasses__(): for subsubclass in subclass.get_subclasses(): yield subsubclass yield subclass class Unknown(Etcd3ClientError): code = GRPCCode.Unknown class InvalidArgument(Etcd3ClientError): code = GRPCCode.InvalidArgument class DeadlineExceeded(Etcd3ClientError): code = GRPCCode.DeadlineExceeded error = "context deadline exceeded" class NotFound(Etcd3ClientError): code = GRPCCode.NotFound class FailedPrecondition(Etcd3ClientError): code = GRPCCode.FailedPrecondition class Unavailable(Etcd3ClientError): code = GRPCCode.Unavailable # https://github.com/etcd-io/etcd/commits/main/api/v3rpc/rpctypes/error.go class LeaseNotFound(NotFound): error = "etcdserver: requested lease not found" class UserEmpty(InvalidArgument): error = "etcdserver: user name is empty" class AuthFailed(InvalidArgument): error = "etcdserver: authentication failed, invalid user ID or password" class AuthOldRevision(InvalidArgument): error = "etcdserver: revision of auth store is old" class PermissionDenied(Etcd3ClientError): code = GRPCCode.PermissionDenied error = "etcdserver: permission denied" class AuthNotEnabled(FailedPrecondition): error = "etcdserver: authentication is not enabled" class InvalidAuthToken(Etcd3ClientError): code = GRPCCode.Unauthenticated error = "etcdserver: invalid auth token" errStringToClientError = {getattr(s, 'error'): s for s in Etcd3ClientError.get_subclasses() if hasattr(s, 'error')} errCodeToClientError = {getattr(s, 'code'): s for s in Etcd3ClientError.__subclasses__()} def _raise_for_data(data: Union[bytes, str, Dict[str, Any]], status_code: Optional[int] = None) -> Etcd3ClientError: try: if TYPE_CHECKING: # pragma: no cover assert isinstance(data, dict) data_error: Optional[Dict[str, Any]] = data.get('error') or data.get('Error') if isinstance(data_error, dict): # streaming response status_code = data_error.get('http_code') code: Optional[int] = data_error['grpc_code'] error: str = data_error['message'] else: data_code = data.get('code') or data.get('Code') if TYPE_CHECKING: # pragma: no cover assert not isinstance(data_code, dict) code = data_code error = str(data_error) except Exception: error = str(data) code = GRPCCode.Unknown err = errStringToClientError.get(error) or errCodeToClientError.get(code) or Unknown return err(code, error, status_code) def to_bytes(v: Union[str, bytes]) -> bytes: return v if isinstance(v, bytes) else v.encode('utf-8') def prefix_range_end(v: str) -> bytes: ret = bytearray(to_bytes(v)) for i in range(len(ret) - 1, -1, -1): if ret[i] < 0xff: ret[i] += 1 break return bytes(ret) def base64_encode(v: Union[str, bytes]) -> str: return base64.b64encode(to_bytes(v)).decode('utf-8') def base64_decode(v: str) -> str: return base64.b64decode(v).decode('utf-8') def build_range_request(key: str, range_end: Union[bytes, str, None] = None) -> Dict[str, Any]: fields = {'key': base64_encode(key)} if range_end: fields['range_end'] = base64_encode(range_end) return fields def _handle_auth_errors(func: Callable[..., Any]) -> Any: def wrapper(self: 'Etcd3Client', *args: Any, **kwargs: Any) -> Any: return self.handle_auth_errors(func, *args, **kwargs) return wrapper class Etcd3Client(AbstractEtcdClientWithFailover): ERROR_CLS = Etcd3Error def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None: self._reauthenticate = False self._token = None self._cluster_version: Tuple[int, ...] = tuple() super(Etcd3Client, self).__init__({**config, 'version_prefix': '/v3beta'}, dns_resolver, cache_ttl) try: self.authenticate() except AuthFailed as e: logger.fatal('Etcd3 authentication failed: %r', e) sys.exit(1) def _get_headers(self) -> Dict[str, str]: headers = urllib3.make_headers(user_agent=USER_AGENT) if self._token and self._cluster_version >= (3, 3, 0): headers['authorization'] = self._token return headers def _prepare_request(self, kwargs: Dict[str, Any], params: Optional[Dict[str, Any]] = None, method: Optional[str] = None) -> Callable[..., urllib3.response.HTTPResponse]: if params is not None: kwargs['body'] = json.dumps(params) kwargs['headers']['Content-Type'] = 'application/json' return self.http.urlopen def _handle_server_response(self, response: urllib3.response.HTTPResponse) -> Dict[str, Any]: data = response.data try: data = data.decode('utf-8') ret: Dict[str, Any] = json.loads(data) if response.status < 400: return ret except (TypeError, ValueError, UnicodeError) as e: if response.status < 400: raise etcd.EtcdException('Server response was not valid JSON: %r' % e) ret = {} raise _raise_for_data(ret or data, response.status) def _ensure_version_prefix(self, base_uri: str, **kwargs: Any) -> None: if self.version_prefix != '/v3': response = self.http.urlopen(self._MGET, base_uri + '/version', **kwargs) response = self._handle_server_response(response) server_version_str = response['etcdserver'] server_version = tuple(int(x) for x in server_version_str.split('.')) cluster_version_str = response['etcdcluster'] self._cluster_version = tuple(int(x) for x in cluster_version_str.split('.')) if self._cluster_version < (3, 0) or server_version < (3, 0, 4): raise UnsupportedEtcdVersion('Detected Etcd version {0} is lower than 3.0.4'.format(server_version_str)) if self._cluster_version < (3, 3): if self.version_prefix != '/v3alpha': if self._cluster_version < (3, 1): logger.warning('Detected Etcd version %s is lower than 3.1.0, watches are not supported', cluster_version_str) if self.username and self.password: logger.warning('Detected Etcd version %s is lower than 3.3.0, authentication is not supported', cluster_version_str) self.version_prefix = '/v3alpha' elif self._cluster_version < (3, 4): self.version_prefix = '/v3beta' else: self.version_prefix = '/v3' def _prepare_get_members(self, etcd_nodes: int) -> Dict[str, Any]: kwargs = self._prepare_common_parameters(etcd_nodes) self._prepare_request(kwargs, {}) return kwargs def _get_members(self, base_uri: str, **kwargs: Any) -> List[str]: self._ensure_version_prefix(base_uri, **kwargs) resp = self.http.urlopen(self._MPOST, base_uri + self.version_prefix + '/cluster/member/list', **kwargs) members = self._handle_server_response(resp)['members'] return [url for member in members for url in member.get('clientURLs', [])] def call_rpc(self, method: str, fields: Dict[str, Any], retry: Optional[Retry] = None) -> Dict[str, Any]: fields['retry'] = retry return self.api_execute(self.version_prefix + method, self._MPOST, fields) def authenticate(self, *, retry: Optional[Retry] = None) -> bool: if self._use_proxies and not self._cluster_version: kwargs = self._prepare_common_parameters(1) self._ensure_version_prefix(self._base_uri, **kwargs) if not (self._cluster_version >= (3, 3) and self.username and self.password): return False logger.info('Trying to authenticate on Etcd...') old_token, self._token = self._token, None try: response = self.call_rpc('/auth/authenticate', {'name': self.username, 'password': self.password}, retry) except AuthNotEnabled: logger.info('Etcd authentication is not enabled') self._token = None except Exception: self._token = old_token raise else: self._token = response.get('token') return old_token != self._token def handle_auth_errors(self: 'Etcd3Client', func: Callable[..., Any], *args: Any, retry: Optional[Retry] = None, **kwargs: Any) -> Any: reauthenticated = False exc = None while True: if self._reauthenticate: if self.username and self.password: self.authenticate(retry=retry) self._reauthenticate = False else: msg = 'Username or password not set, authentication is not possible' logger.fatal(msg) raise exc or Etcd3Exception(msg) reauthenticated = True try: return func(self, *args, retry=retry, **kwargs) except (UserEmpty, PermissionDenied) as e: # no token provided # PermissionDenied is raised on 3.0 and 3.1 if self._cluster_version < (3, 3) and (not isinstance(e, PermissionDenied) or self._cluster_version < (3, 2)): raise UnsupportedEtcdVersion('Authentication is required by Etcd cluster but not ' 'supported on version lower than 3.3.0. Cluster version: ' '{0}'.format('.'.join(map(str, self._cluster_version)))) exc = e except InvalidAuthToken as e: logger.error('Invalid auth token: %s', self._token) exc = e except AuthOldRevision as e: logger.error('Auth token is for old revision of auth store') exc = e self._reauthenticate = True if retry: logger.error('retry = %s', retry) retry.ensure_deadline(0.5, exc) elif reauthenticated: raise exc @_handle_auth_errors def range(self, key: str, range_end: Union[bytes, str, None] = None, serializable: bool = True, *, retry: Optional[Retry] = None) -> Dict[str, Any]: params = build_range_request(key, range_end) params['serializable'] = serializable # For better performance. We can tolerate stale reads return self.call_rpc('/kv/range', params, retry) def prefix(self, key: str, serializable: bool = True, *, retry: Optional[Retry] = None) -> Dict[str, Any]: return self.range(key, prefix_range_end(key), serializable, retry=retry) @_handle_auth_errors def lease_grant(self, ttl: int, *, retry: Optional[Retry] = None) -> str: return self.call_rpc('/lease/grant', {'TTL': ttl}, retry)['ID'] def lease_keepalive(self, ID: str, *, retry: Optional[Retry] = None) -> Optional[str]: return self.call_rpc('/lease/keepalive', {'ID': ID}, retry).get('result', {}).get('TTL') @_handle_auth_errors def txn(self, compare: Dict[str, Any], success: Dict[str, Any], failure: Optional[Dict[str, Any]] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]: fields = {'compare': [compare], 'success': [success]} if failure: fields['failure'] = [failure] ret = self.call_rpc('/kv/txn', fields, retry) return ret if failure or ret.get('succeeded') else {} @_handle_auth_errors def put(self, key: str, value: str, lease: Optional[str] = None, create_revision: Optional[str] = None, mod_revision: Optional[str] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]: fields = {'key': base64_encode(key), 'value': base64_encode(value)} if lease: fields['lease'] = lease if create_revision is not None: compare = {'target': 'CREATE', 'create_revision': create_revision} elif mod_revision is not None: compare = {'target': 'MOD', 'mod_revision': mod_revision} else: return self.call_rpc('/kv/put', fields, retry) compare['key'] = fields['key'] return self.txn(compare, {'request_put': fields}, retry=retry) @_handle_auth_errors def deleterange(self, key: str, range_end: Union[bytes, str, None] = None, mod_revision: Optional[str] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]: fields = build_range_request(key, range_end) if mod_revision is None: return self.call_rpc('/kv/deleterange', fields, retry) compare = {'target': 'MOD', 'mod_revision': mod_revision, 'key': fields['key']} return self.txn(compare, {'request_delete_range': fields}, retry=retry) def deleteprefix(self, key: str, *, retry: Optional[Retry] = None) -> Dict[str, Any]: return self.deleterange(key, prefix_range_end(key), retry=retry) def watchrange(self, key: str, range_end: Union[bytes, str, None] = None, start_revision: Optional[str] = None, filters: Optional[List[Dict[str, Any]]] = None, read_timeout: Optional[float] = None) -> urllib3.response.HTTPResponse: """returns: response object""" params = build_range_request(key, range_end) if start_revision is not None: params['start_revision'] = start_revision params['filters'] = filters or [] kwargs = self._prepare_common_parameters(1, self.read_timeout) request_executor = self._prepare_request(kwargs, {'create_request': params}) kwargs.update(timeout=urllib3.Timeout(connect=kwargs['timeout'], read=read_timeout), retries=0) return request_executor(self._MPOST, self._base_uri + self.version_prefix + '/watch', **kwargs) def watchprefix(self, key: str, start_revision: Optional[str] = None, filters: Optional[List[Dict[str, Any]]] = None, read_timeout: Optional[float] = None) -> urllib3.response.HTTPResponse: return self.watchrange(key, prefix_range_end(key), start_revision, filters, read_timeout) class KVCache(Thread): def __init__(self, dcs: 'Etcd3', client: 'PatroniEtcd3Client') -> None: super(KVCache, self).__init__() self.daemon = True self._dcs = dcs self._client = client self.condition = Condition() self._config_key = base64_encode(dcs.config_path) self._leader_key = base64_encode(dcs.leader_path) self._optime_key = base64_encode(dcs.leader_optime_path) self._status_key = base64_encode(dcs.status_path) self._name = base64_encode(getattr(dcs, '_name')) # pyright self._is_ready = False self._response = None self._response_lock = Lock() self._object_cache = {} self._object_cache_lock = Lock() self.start() def set(self, value: Dict[str, Any], overwrite: bool = False) -> Tuple[bool, Optional[Dict[str, Any]]]: with self._object_cache_lock: name = value['key'] old_value = self._object_cache.get(name) ret = not old_value or int(old_value['mod_revision']) < int(value['mod_revision']) if ret or overwrite and old_value and old_value['mod_revision'] == value['mod_revision']: self._object_cache[name] = value return ret, old_value def delete(self, name: str, mod_revision: str) -> Tuple[bool, Optional[Dict[str, Any]]]: with self._object_cache_lock: old_value = self._object_cache.get(name) ret = old_value and int(old_value['mod_revision']) < int(mod_revision) if ret: del self._object_cache[name] return bool(not old_value or ret), old_value def copy(self) -> List[Dict[str, Any]]: with self._object_cache_lock: return [v.copy() for v in self._object_cache.values()] def get(self, name: str) -> Optional[Dict[str, Any]]: with self._object_cache_lock: return self._object_cache.get(name) def _process_event(self, event: Dict[str, Any]) -> None: kv = event['kv'] key = kv['key'] if event.get('type') == 'DELETE': success, old_value = self.delete(key, kv['mod_revision']) else: success, old_value = self.set(kv, True) if success: old_value = old_value and old_value.get('value') new_value = kv.get('value') value_changed = old_value != new_value and \ (key == self._leader_key or key in (self._optime_key, self._status_key) and new_value is not None or key == self._config_key and old_value is not None and new_value is not None) if value_changed: logger.debug('%s changed from %s to %s', key, old_value, new_value) # We also want to wake up HA loop on replicas if leader optime (or status key) was updated if value_changed and (key not in (self._optime_key, self._status_key) or (self.get(self._leader_key) or {}).get('value') != self._name): self._dcs.event.set() def _process_message(self, message: Dict[str, Any]) -> None: logger.debug('Received message: %s', message) if 'error' in message: raise _raise_for_data(message) events: List[Dict[str, Any]] = message.get('result', {}).get('events', []) for event in events: self._process_event(event) @staticmethod def _finish_response(response: urllib3.response.HTTPResponse) -> None: try: response.close() finally: response.release_conn() def _do_watch(self, revision: str) -> None: with self._response_lock: self._response = None # We do most of requests with timeouts. The only exception /watch requests to Etcd v3. # In order to interrupt the /watch request we do socket.shutdown() from the main thread, # which doesn't work on Windows. Therefore we want to use the last resort, `read_timeout`. # Setting it to TTL will help to partially mitigate the problem. # Setting it to lower value is not nice because for idling clusters it will increase # the numbers of interrupts and reconnects. read_timeout = self._dcs.ttl if os.name == 'nt' else None response = self._client.watchprefix(self._dcs.cluster_prefix, revision, read_timeout=read_timeout) with self._response_lock: if self._response is None: self._response = response if not self._response: return self._finish_response(response) for message in iter_response_objects(response): self._process_message(message) def _build_cache(self) -> None: result = self._dcs.retry(self._client.prefix, self._dcs.cluster_prefix) with self._object_cache_lock: self._object_cache = {node['key']: node for node in result.get('kvs', [])} with self.condition: self._is_ready = True self.condition.notify() try: self._do_watch(result['header']['revision']) except Exception as e: # Following exceptions are expected on Windows because the /watch request is done with `read_timeout` if not (os.name == 'nt' and isinstance(e, (ReadTimeoutError, ProtocolError))): logger.error('watchprefix failed: %r', e) finally: with self.condition: self._is_ready = False with self._response_lock: response, self._response = self._response, None if isinstance(response, urllib3.response.HTTPResponse): self._finish_response(response) def run(self) -> None: while True: try: self._build_cache() except Exception as e: logger.error('KVCache.run %r', e) time.sleep(1) def kill_stream(self) -> None: sock = None with self._response_lock: if isinstance(self._response, urllib3.response.HTTPResponse): try: sock = self._response.connection.sock if self._response.connection else None except Exception: sock = None else: self._response = False if sock: try: sock.shutdown(socket.SHUT_RDWR) sock.close() except Exception as e: logger.debug('Error on socket.shutdown: %r', e) def is_ready(self) -> bool: """Must be called only when holding the lock on `condition`""" return self._is_ready class PatroniEtcd3Client(Etcd3Client): def __init__(self, *args: Any, **kwargs: Any) -> None: self._kv_cache = None super(PatroniEtcd3Client, self).__init__(*args, **kwargs) def configure(self, etcd3: 'Etcd3') -> None: self._etcd3 = etcd3 def start_watcher(self) -> None: if self._cluster_version >= (3, 1): self._kv_cache = KVCache(self._etcd3, self) def _restart_watcher(self) -> None: if self._kv_cache: self._kv_cache.kill_stream() def set_base_uri(self, value: str) -> None: super(PatroniEtcd3Client, self).set_base_uri(value) self._restart_watcher() def _wait_cache(self, timeout: float) -> None: stop_time = time.time() + timeout while self._kv_cache and not self._kv_cache.is_ready(): timeout = stop_time - time.time() if timeout <= 0: raise RetryFailedError('Exceeded retry deadline') self._kv_cache.condition.wait(timeout) def get_cluster(self, path: str) -> List[Dict[str, Any]]: if self._kv_cache and path.startswith(self._etcd3.cluster_prefix): with self._kv_cache.condition: self._wait_cache(self.read_timeout) ret = self._kv_cache.copy() else: serializable = not getattr(self._etcd3, '_ctl') # use linearizable for patronictl ret = self._etcd3.retry(self.prefix, path, serializable).get('kvs', []) for node in ret: node.update({'key': base64_decode(node['key']), 'value': base64_decode(node.get('value', '')), 'lease': node.get('lease')}) return ret def call_rpc(self, method: str, fields: Dict[str, Any], retry: Optional[Retry] = None) -> Dict[str, Any]: ret = super(PatroniEtcd3Client, self).call_rpc(method, fields, retry) if self._kv_cache: value = delete = None # For the 'failure' case we only support a second (nested) transaction that attempts to # update/delete the same keys. Anything more complex than that we don't need and therefore it doesn't # make sense to write a universal response analyzer and we can just check expected JSON path. if method == '/kv/txn'\ and (ret.get('succeeded') or 'failure' in fields and 'request_txn' in fields['failure'][0] and ret.get('responses', [{'response_txn': {'succeeded': False}}])[0] .get('response_txn', {}).get('succeeded')): on_success = fields['success'][0] value = on_success.get('request_put') delete = on_success.get('request_delete_range') elif method == '/kv/put' and ret: value = fields elif method == '/kv/deleterange' and ret: delete = fields if value: value['mod_revision'] = ret['header']['revision'] self._kv_cache.set(value) elif delete and 'range_end' not in delete: self._kv_cache.delete(delete['key'], ret['header']['revision']) return ret def txn(self, compare: Dict[str, Any], success: Dict[str, Any], failure: Optional[Dict[str, Any]] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]: ret = super(PatroniEtcd3Client, self).txn(compare, success, failure, retry=retry) # Here we abuse the fact that the `failure` is only set in the call from update_leader(). # In all other cases the txn() call failure may be an indicator of a stale cache, # and therefore we want to restart watcher. if not failure and not ret: self._restart_watcher() return ret class Etcd3(AbstractEtcd): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: super(Etcd3, self).__init__(config, mpp, PatroniEtcd3Client, (DeadlineExceeded, Unavailable, FailedPrecondition)) self.__do_not_watch = False self._lease = None self._last_lease_refresh = 0 self._client.configure(self) if not self._ctl: self._client.start_watcher() self.create_lease() @property def _client(self) -> PatroniEtcd3Client: if TYPE_CHECKING: # pragma: no cover assert isinstance(self._abstract_client, PatroniEtcd3Client) return self._abstract_client def set_socket_options(self, sock: socket.socket, socket_options: Optional[Collection[Tuple[int, int, int]]]) -> None: if TYPE_CHECKING: # pragma: no cover assert self._retry.deadline is not None enable_keepalive(sock, self.ttl, int(self.loop_wait + self._retry.deadline)) def set_ttl(self, ttl: int) -> Optional[bool]: self.__do_not_watch = super(Etcd3, self).set_ttl(ttl) if self.__do_not_watch: self._lease = None return None def _do_refresh_lease(self, force: bool = False, retry: Optional[Retry] = None) -> bool: if not force and self._lease and self._last_lease_refresh + self._loop_wait > time.time(): return False if self._lease and not self._client.lease_keepalive(self._lease, retry=retry): self._lease = None ret = not self._lease if ret: self._lease = self._client.lease_grant(self._ttl, retry=retry) self._last_lease_refresh = time.time() return ret def refresh_lease(self) -> bool: try: return self.retry(self._do_refresh_lease) except (Etcd3ClientError, RetryFailedError): logger.exception('refresh_lease') raise Etcd3Error('Failed to keepalive/grant lease') def create_lease(self) -> None: while not self._lease: try: self.refresh_lease() except Etcd3Error: logger.info('waiting on etcd') time.sleep(5) @property def cluster_prefix(self) -> str: """Construct the cluster prefix for the cluster. :returns: path in the DCS under which we store information about this Patroni cluster. """ return self._base_path + '/' if self.is_mpp_coordinator() else self.client_path('') @staticmethod def member(node: Dict[str, str]) -> Member: return Member.from_node(node['mod_revision'], os.path.basename(node['key']), node['lease'], node['value']) def _cluster_from_nodes(self, nodes: Dict[str, Any]) -> Cluster: # get initialize flag initialize = nodes.get(self._INITIALIZE) initialize = initialize and initialize['value'] # get global dynamic configuration config = nodes.get(self._CONFIG) config = config and ClusterConfig.from_node(config['mod_revision'], config['value']) # get timeline history history = nodes.get(self._HISTORY) history = history and TimelineHistory.from_node(history['mod_revision'], history['value']) # get last know leader lsn and slots status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME) status = Status.from_node(status and status['value']) # get list of members members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1] # get leader leader = nodes.get(self._LEADER) if not self._ctl and leader and leader['value'] == self._name and self._lease != leader.get('lease'): logger.warning('I am the leader but not owner of the lease') if leader: member = Member(-1, leader['value'], None, {}) member = ([m for m in members if m.name == leader['value']] or [member])[0] leader = Leader(leader['mod_revision'], leader['lease'], member) # failover key failover = nodes.get(self._FAILOVER) if failover: failover = Failover.from_node(failover['mod_revision'], failover['value']) # get synchronization state sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value']) # get failsafe topology failsafe = nodes.get(self._FAILSAFE) try: failsafe = json.loads(failsafe['value']) if failsafe else None except Exception: failsafe = None return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) def _postgresql_cluster_loader(self, path: str) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ nodes = {node['key'][len(path):]: node for node in self._client.get_cluster(path) if node['key'].startswith(path)} return self._cluster_from_nodes(nodes) def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ clusters: Dict[int, Dict[str, Dict[str, Any]]] = defaultdict(dict) path = self._base_path + '/' for node in self._client.get_cluster(path): key = node['key'][len(path):].split('/', 1) if len(key) == 2 and self._mpp.group_re.match(key[0]): clusters[int(key[0])][key[1]] = node return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()} def _load_cluster( self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: cluster = None try: cluster = loader(path) except UnsupportedEtcdVersion: raise except Exception as e: self._handle_exception(e, 'get_cluster', raise_ex=Etcd3Error('Etcd is not responding properly')) self._has_failed = False if TYPE_CHECKING: # pragma: no cover assert cluster is not None return cluster @catch_etcd_errors def touch_member(self, data: Dict[str, Any]) -> bool: try: self.refresh_lease() except Etcd3Error: return False cluster = self.cluster member = cluster and cluster.get_member(self._name, fallback_to_leader=False) if member and member.session == self._lease and deep_compare(data, member.data): return True value = json.dumps(data, separators=(',', ':')) try: return bool(self._client.put(self.member_path, value, self._lease)) except LeaseNotFound: self._lease = None logger.error('Our lease disappeared from Etcd, can not "touch_member"') return False @catch_etcd_errors def take_leader(self) -> bool: return self.retry(self._client.put, self.leader_path, self._name, self._lease) def _do_attempt_to_acquire_leader(self, retry: Retry) -> bool: def _retry(*args: Any, **kwargs: Any) -> Any: kwargs['retry'] = retry return retry(*args, **kwargs) try: return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0') except LeaseNotFound: self._lease = None if not retry.ensure_deadline(0): logger.error('Our lease disappeared from Etcd. Deadline exceeded, giving up') return False logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt') _retry(self._do_refresh_lease) retry.ensure_deadline(1, Etcd3Error('_do_attempt_to_acquire_leader timeout')) return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0') @catch_return_false_exception def attempt_to_acquire_leader(self) -> bool: retry = self._retry.copy() def _retry(*args: Any, **kwargs: Any) -> Any: kwargs['retry'] = retry return retry(*args, **kwargs) self._run_and_handle_exceptions(self._do_refresh_lease, retry=_retry) retry.ensure_deadline(1, Etcd3Error('attempt_to_acquire_leader timeout')) ret = self._run_and_handle_exceptions(self._do_attempt_to_acquire_leader, retry, retry=None) if not ret: logger.info('Could not take out TTL lock') return ret @catch_etcd_errors def set_failover_value(self, value: str, version: Optional[str] = None) -> bool: return bool(self._client.put(self.failover_path, value, mod_revision=version)) @catch_etcd_errors def set_config_value(self, value: str, version: Optional[str] = None) -> bool: return bool(self._client.put(self.config_path, value, mod_revision=version)) @catch_etcd_errors def _write_leader_optime(self, last_lsn: str) -> bool: return bool(self._client.put(self.leader_optime_path, last_lsn)) @catch_etcd_errors def _write_status(self, value: str) -> bool: return bool(self._client.put(self.status_path, value)) @catch_etcd_errors def _write_failsafe(self, value: str) -> bool: return bool(self._client.put(self.failsafe_path, value)) @catch_return_false_exception def _update_leader(self, leader: Leader) -> bool: retry = self._retry.copy() def _retry(*args: Any, **kwargs: Any) -> Any: kwargs['retry'] = retry return retry(*args, **kwargs) self._run_and_handle_exceptions(self._do_refresh_lease, True, retry=_retry) if self._lease and leader.session != self._lease: retry.ensure_deadline(1, Etcd3Error('update_leader timeout')) fields = {'key': base64_encode(self.leader_path), 'value': base64_encode(self._name), 'lease': self._lease} # First we try to update lease on existing leader key "hoping" that we still owning it compare1 = {'key': fields['key'], 'target': 'VALUE', 'value': fields['value']} request_put = {'request_put': fields} # If the first comparison failed we will try to create the new leader key in a transaction compare2 = {'key': fields['key'], 'target': 'CREATE', 'create_revision': '0'} request_txn = {'request_txn': {'compare': [compare2], 'success': [request_put]}} ret = self._run_and_handle_exceptions(self._client.txn, compare1, request_put, request_txn, retry=_retry) return ret.get('succeeded', False)\ or ret.get('responses', [{}])[0].get('response_txn', {}).get('succeeded', False) return bool(self._lease) @catch_etcd_errors def initialize(self, create_new: bool = True, sysid: str = ""): return self.retry(self._client.put, self.initialize_path, sysid, create_revision='0' if create_new else None) @catch_etcd_errors def _delete_leader(self, leader: Leader) -> bool: fields = build_range_request(self.leader_path) compare = {'key': fields['key'], 'target': 'VALUE', 'value': base64_encode(self._name)} return bool(self._client.txn(compare, {'request_delete_range': fields})) @catch_etcd_errors def cancel_initialization(self) -> bool: return self.retry(self._client.deleterange, self.initialize_path) @catch_etcd_errors def delete_cluster(self) -> bool: return self.retry(self._client.deleteprefix, self.client_path('')) @catch_etcd_errors def set_history_value(self, value: str) -> bool: return bool(self._client.put(self.history_path, value)) @catch_etcd_errors def set_sync_state_value(self, value: str, version: Optional[str] = None) -> Union[str, bool]: return self.retry(self._client.put, self.sync_path, value, mod_revision=version)\ .get('header', {}).get('revision', False) @catch_etcd_errors def delete_sync_state(self, version: Optional[str] = None) -> bool: return self.retry(self._client.deleterange, self.sync_path, mod_revision=version) def watch(self, leader_version: Optional[str], timeout: float) -> bool: if self.__do_not_watch: self.__do_not_watch = False return True # We want to give a bit more time to non-leader nodes to synchronize HA loops if leader_version: timeout += 0.5 try: return super(Etcd3, self).watch(None, timeout) finally: self.event.clear() patroni-4.0.4/patroni/dcs/exhibitor.py000066400000000000000000000060311472010352700177700ustar00rootroot00000000000000import json import logging import random import time from typing import Any, Callable, cast, Dict, List, Union from ..postgresql.mpp import AbstractMPP from ..request import get as requests_get from ..utils import uri from . import Cluster from .zookeeper import ZooKeeper logger = logging.getLogger(__name__) class ExhibitorEnsembleProvider(object): TIMEOUT = 3.1 def __init__(self, hosts: List[str], port: int, uri_path: str = '/exhibitor/v1/cluster/list', poll_interval: int = 300) -> None: self._exhibitor_port = port self._uri_path = uri_path self._poll_interval = poll_interval self._exhibitors: List[str] = hosts self._boot_exhibitors = hosts self._zookeeper_hosts = '' self._next_poll = None while not self.poll(): logger.info('waiting on exhibitor') time.sleep(5) def poll(self) -> bool: if self._next_poll and self._next_poll > time.time(): return False json = self._query_exhibitors(self._exhibitors) if not json: json = self._query_exhibitors(self._boot_exhibitors) if isinstance(json, dict) and 'servers' in json and 'port' in json: self._next_poll = time.time() + self._poll_interval servers: List[str] = cast(Dict[str, Any], json)['servers'] port = str(cast(Dict[str, Any], json)['port']) zookeeper_hosts = ','.join([h + ':' + port for h in sorted(servers)]) if self._zookeeper_hosts != zookeeper_hosts: logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts) self._zookeeper_hosts = zookeeper_hosts self._exhibitors = json['servers'] return True return False def _query_exhibitors(self, exhibitors: List[str]) -> Any: random.shuffle(exhibitors) for host in exhibitors: try: response = requests_get(uri('http', (host, self._exhibitor_port), self._uri_path), timeout=self.TIMEOUT) return json.loads(response.data.decode('utf-8')) except Exception: logging.debug('Request to %s failed', host) return None @property def zookeeper_hosts(self) -> str: return self._zookeeper_hosts class Exhibitor(ZooKeeper): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: interval = config.get('poll_interval', 300) self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval) super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts}, mpp) def _load_cluster( self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: if self._ensemble_provider.poll(): self._client.set_hosts(self._ensemble_provider.zookeeper_hosts) return super(Exhibitor, self)._load_cluster(path, loader) patroni-4.0.4/patroni/dcs/kubernetes.py000066400000000000000000002046331472010352700201520ustar00rootroot00000000000000import atexit import base64 import datetime import functools import json import logging import os import random import socket import tempfile import time from collections import defaultdict from copy import deepcopy from http.client import HTTPException from threading import Condition, Lock, Thread from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, TYPE_CHECKING, Union import urllib3 import yaml from urllib3.exceptions import HTTPError from ..collections import EMPTY_DICT from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare, iter_response_objects, \ keepalive_socket_options, Retry, RetryFailedError, tzutc, uri, USER_AGENT from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory if TYPE_CHECKING: # pragma: no cover from ..config import Config logger = logging.getLogger(__name__) KUBE_CONFIG_DEFAULT_LOCATION = os.environ.get('KUBECONFIG', '~/.kube/config') SERVICE_HOST_ENV_NAME = 'KUBERNETES_SERVICE_HOST' SERVICE_PORT_ENV_NAME = 'KUBERNETES_SERVICE_PORT' SERVICE_TOKEN_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/token' SERVICE_CERT_FILENAME = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' __temp_files: List[str] = [] class KubernetesError(DCSError): pass def _cleanup_temp_files() -> None: global __temp_files for temp_file in __temp_files: try: os.remove(temp_file) except OSError: pass __temp_files = [] def _create_temp_file(content: bytes) -> str: if len(__temp_files) == 0: atexit.register(_cleanup_temp_files) fd, name = tempfile.mkstemp() os.write(fd, content) os.close(fd) __temp_files.append(name) return name # this function does the same mapping of snake_case => camelCase for > 97% of cases as autogenerated swagger code def to_camel_case(value: str) -> str: reserved = {'api', 'apiv3', 'cidr', 'cpu', 'csi', 'id', 'io', 'ip', 'ipc', 'pid', 'tls', 'uri', 'url', 'uuid'} words = value.split('_') return words[0] + ''.join(w.upper() if w in reserved else w.title() for w in words[1:]) class K8sConfig(object): class ConfigException(Exception): pass def __init__(self) -> None: self.pool_config: Dict[str, Any] = {'maxsize': 10, 'num_pools': 10} # urllib3.PoolManager config self._token_expires_at = datetime.datetime.max self._headers: Dict[str, str] = {} self._make_headers() def _set_token(self, token: str) -> None: self._headers['authorization'] = 'Bearer ' + token def _make_headers(self, token: Optional[str] = None, **kwargs: Any) -> None: self._headers = urllib3.make_headers(user_agent=USER_AGENT, **kwargs) if token: self._set_token(token) def _read_token_file(self) -> str: if not os.path.isfile(SERVICE_TOKEN_FILENAME): raise self.ConfigException('Service token file does not exists.') with open(SERVICE_TOKEN_FILENAME) as f: token = f.read() if not token: raise self.ConfigException('Token file exists but empty.') self._token_expires_at = datetime.datetime.now() + self._token_refresh_interval return token def load_incluster_config(self, ca_certs: str = SERVICE_CERT_FILENAME, token_refresh_interval: datetime.timedelta = datetime.timedelta(minutes=1)) -> None: if SERVICE_HOST_ENV_NAME not in os.environ or SERVICE_PORT_ENV_NAME not in os.environ: raise self.ConfigException('Service host/port is not set.') if not os.environ[SERVICE_HOST_ENV_NAME] or not os.environ[SERVICE_PORT_ENV_NAME]: raise self.ConfigException('Service host/port is set but empty.') if not os.path.isfile(ca_certs): raise self.ConfigException('Service certificate file does not exists.') with open(ca_certs) as f: if not f.read(): raise self.ConfigException('Cert file exists but empty.') self.pool_config['ca_certs'] = ca_certs self._token_refresh_interval = token_refresh_interval token = self._read_token_file() self._make_headers(token=token) self._server = uri('https', (os.environ[SERVICE_HOST_ENV_NAME], os.environ[SERVICE_PORT_ENV_NAME])) @staticmethod def _get_by_name(config: Dict[str, List[Dict[str, Any]]], section: str, name: str) -> Optional[Dict[str, Any]]: for c in config[section + 's']: if c['name'] == name: return c[section] def _pool_config_from_file_or_data(self, config: Dict[str, str], file_key_name: str, pool_key_name: str) -> None: data_key_name = file_key_name + '-data' if data_key_name in config: self.pool_config[pool_key_name] = _create_temp_file(base64.b64decode(config[data_key_name])) elif file_key_name in config: self.pool_config[pool_key_name] = config[file_key_name] def load_kube_config(self, context: Optional[str] = None) -> None: with open(os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)) as f: config: Dict[str, Any] = yaml.safe_load(f) context = context or config['current-context'] if TYPE_CHECKING: # pragma: no cover assert isinstance(context, str) context_value = self._get_by_name(config, 'context', context) if TYPE_CHECKING: # pragma: no cover assert isinstance(context_value, dict) cluster = self._get_by_name(config, 'cluster', context_value['cluster']) if TYPE_CHECKING: # pragma: no cover assert isinstance(cluster, dict) user = self._get_by_name(config, 'user', context_value['user']) if TYPE_CHECKING: # pragma: no cover assert isinstance(user, dict) self._server = cluster['server'].rstrip('/') if self._server.startswith('https'): self._pool_config_from_file_or_data(user, 'client-certificate', 'cert_file') self._pool_config_from_file_or_data(user, 'client-key', 'key_file') self._pool_config_from_file_or_data(cluster, 'certificate-authority', 'ca_certs') self.pool_config['cert_reqs'] = 'CERT_NONE' if cluster.get('insecure-skip-tls-verify') else 'CERT_REQUIRED' if user.get('token'): self._make_headers(token=user['token']) elif 'username' in user and 'password' in user: self._make_headers(basic_auth=':'.join((user['username'], user['password']))) @property def server(self) -> str: return self._server @property def headers(self) -> Dict[str, str]: if self._token_expires_at <= datetime.datetime.now(): try: self._set_token(self._read_token_file()) except Exception as e: logger.error('Failed to refresh service account token: %r', e) return self._headers.copy() class K8sObject(object): def __init__(self, kwargs: Dict[str, Any]) -> None: self._dict = {k: self._wrap(k, v) for k, v in kwargs.items()} def get(self, name: str, default: Optional[Any] = None) -> Optional[Any]: return self._dict.get(name, default) def __getattr__(self, name: str) -> Any: return self.get(to_camel_case(name)) @classmethod def _wrap(cls, parent: Optional[str], value: Any) -> Any: if isinstance(value, dict): data_dict: Dict[str, Any] = value # we know that `annotations` and `labels` are dicts and therefore don't want to convert them into K8sObject return data_dict if parent in {'annotations', 'labels'} and \ all(isinstance(v, str) for v in data_dict.values()) else cls(data_dict) elif isinstance(value, list): data_list: List[Any] = value return [cls._wrap(None, v) for v in data_list] else: return value def to_dict(self) -> Dict[str, Any]: return self._dict def __repr__(self) -> str: return json.dumps(self, indent=4, default=lambda o: o.to_dict()) class K8sException(Exception): pass class K8sConnectionFailed(K8sException): pass class K8sClient(object): class rest(object): class ApiException(Exception): def __init__(self, status: Optional[int] = None, reason: Optional[str] = None, http_resp: Optional[urllib3.HTTPResponse] = None) -> None: self.status = http_resp.status if http_resp else status self.reason = http_resp.reason if http_resp else reason self.body = http_resp.data if http_resp else None self.headers = http_resp.headers if http_resp else None def __str__(self) -> str: error_message = "({0})\nReason: {1}\n".format(self.status, self.reason) if self.headers: error_message += "HTTP response headers: {0}\n".format(self.headers) if self.body: error_message += "HTTP response body: {0}\n".format(self.body) return error_message class ApiClient(object): _API_URL_PREFIX = '/api/v1/namespaces/' def __init__(self, bypass_api_service: Optional[bool] = False) -> None: self._bypass_api_service = bypass_api_service self.pool_manager = urllib3.PoolManager(**k8s_config.pool_config) self._base_uri = k8s_config.server self._api_servers_cache = [k8s_config.server] self._api_servers_cache_updated = 0 self.set_api_servers_cache_ttl(10) self.set_read_timeout(10) try: self._load_api_servers_cache() except K8sException: pass def set_read_timeout(self, timeout: Union[int, float]) -> None: self._read_timeout = timeout def set_api_servers_cache_ttl(self, ttl: int) -> None: self._api_servers_cache_ttl = ttl - 0.5 def set_base_uri(self, value: str) -> None: logger.info('Selected new K8s API server endpoint %s', value) # We will connect by IP of the K8s master node which is not listed as alternative name self.pool_manager.connection_pool_kw['assert_hostname'] = False self._base_uri = value @staticmethod def _handle_server_response(response: urllib3.HTTPResponse, _preload_content: bool) -> Union[urllib3.HTTPResponse, K8sObject]: if response.status not in range(200, 206): raise k8s_client.rest.ApiException(http_resp=response) return K8sObject(json.loads(response.data.decode('utf-8'))) if _preload_content else response @staticmethod def _make_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]: ret = k8s_config.headers ret.update(headers or {}) return ret @property def api_servers_cache(self) -> List[str]: base_uri, cache = self._base_uri, self._api_servers_cache return ([base_uri] if base_uri in cache else []) + [machine for machine in cache if machine != base_uri] def _get_api_servers(self, api_servers_cache: List[str]) -> List[str]: _, per_node_timeout, per_node_retries = self._calculate_timeouts(len(api_servers_cache)) headers = self._make_headers({}) kwargs = {'preload_content': True, 'retries': per_node_retries, 'timeout': urllib3.Timeout(connect=max(1.0, per_node_timeout / 2.0), total=per_node_timeout)} path = self._API_URL_PREFIX + 'default/endpoints/kubernetes' for base_uri in api_servers_cache: try: response = self.pool_manager.request('GET', base_uri + path, headers=headers, **kwargs) endpoint = self._handle_server_response(response, True) if TYPE_CHECKING: # pragma: no cover assert isinstance(endpoint, K8sObject) for subset in endpoint.subsets: for port in subset.ports: if port.name == 'https' and port.protocol == 'TCP': addresses = [uri('https', (a.ip, port.port)) for a in subset.addresses] if addresses: random.shuffle(addresses) return addresses except Exception as e: if isinstance(e, k8s_client.rest.ApiException) and e.status == 403: raise self.pool_manager.clear() logger.error('Failed to get "kubernetes" endpoint from %s: %r', base_uri, e) raise K8sConnectionFailed('No more K8s API server nodes in the cluster') def _refresh_api_servers_cache(self, updating_cache: Optional[bool] = False) -> None: if self._bypass_api_service: try: api_servers_cache = [k8s_config.server] if updating_cache else self.api_servers_cache self._api_servers_cache = self._get_api_servers(api_servers_cache) if updating_cache: self.pool_manager.clear() except k8s_client.rest.ApiException: # 403 Permission denied logger.warning("Kubernetes RBAC doesn't allow GET access to the 'kubernetes' " "endpoint in the 'default' namespace. Disabling 'bypass_api_service'.") self._bypass_api_service = False self._api_servers_cache = [k8s_config.server] if not updating_cache: self.pool_manager.clear() except K8sConnectionFailed: if updating_cache: raise K8sException("Could not get the list of K8s API server nodes") return else: self._api_servers_cache = [k8s_config.server] if self._base_uri not in self._api_servers_cache: self.set_base_uri(self._api_servers_cache[0]) self._api_servers_cache_updated = time.time() def refresh_api_servers_cache(self) -> None: if self._bypass_api_service and time.time() - self._api_servers_cache_updated > self._api_servers_cache_ttl: self._refresh_api_servers_cache() def _load_api_servers_cache(self) -> None: self._update_api_servers_cache = True self._refresh_api_servers_cache(True) self._update_api_servers_cache = False def _calculate_timeouts(self, api_servers: int, timeout: Optional[float] = None) -> Tuple[int, float, int]: """Calculate a request timeout and number of retries per single K8s API server node. In case if the timeout per node is too small (less than one second) we will reduce the number of nodes. For the cluster with only one API server node we will try to do 1 retry. No retries for clusters with 2 or more API server nodes. We better rely on switching to a different node.""" per_node_timeout = timeout = float(timeout or self._read_timeout) max_retries = 3 - min(api_servers, 2) per_node_retries = 1 min_timeout = 1.0 while api_servers > 0: per_node_timeout = float(timeout) / api_servers if per_node_timeout >= min_timeout: # for small clusters we will try to do more than one try on every node while per_node_retries < max_retries and per_node_timeout / (per_node_retries + 1) >= min_timeout: per_node_retries += 1 per_node_timeout /= per_node_retries break # if the timeout per one node is to small try to reduce number of nodes api_servers -= 1 max_retries = 1 return api_servers, per_node_timeout, per_node_retries - 1 def _do_http_request(self, retry: Optional[Retry], api_servers_cache: List[str], method: str, path: str, **kwargs: Any) -> urllib3.HTTPResponse: some_request_failed = False for i, base_uri in enumerate(api_servers_cache): if i > 0: logger.info('Retrying on %s', base_uri) try: response = self.pool_manager.request(method, base_uri + path, **kwargs) if some_request_failed: self.set_base_uri(base_uri) self._refresh_api_servers_cache() return response except (HTTPError, HTTPException, socket.error, socket.timeout) as e: self.pool_manager.clear() if not retry: # switch to the next node if request failed and retry is not allowed if i + 1 < len(api_servers_cache): self.set_base_uri(api_servers_cache[i + 1]) raise K8sException('{0} {1} request failed'.format(method, path)) logger.error('Request to server %s failed: %r', base_uri, e) some_request_failed = True raise K8sConnectionFailed('No more API server nodes in the cluster') def request( self, retry: Optional[Retry], method: str, path: str, timeout: Union[int, float, Tuple[Union[int, float], Union[int, float]], urllib3.Timeout, None] = None, **kwargs: Any) -> urllib3.HTTPResponse: if self._update_api_servers_cache: self._load_api_servers_cache() api_servers_cache = self.api_servers_cache api_servers = len(api_servers_cache) if timeout: if isinstance(timeout, (int, float)): timeout = urllib3.Timeout(total=timeout) elif isinstance(timeout, tuple) and len(timeout) == 2: timeout = urllib3.Timeout(connect=timeout[0], read=timeout[1]) retries = 0 else: _, timeout, retries = self._calculate_timeouts(api_servers) timeout = urllib3.Timeout(connect=max(1.0, timeout / 2.0), total=timeout) kwargs.update(retries=retries, timeout=timeout) while True: try: return self._do_http_request(retry, api_servers_cache, method, path, **kwargs) except K8sConnectionFailed as ex: try: self._load_api_servers_cache() api_servers_cache = self.api_servers_cache api_servers = len(api_servers_cache) except Exception as e: logger.debug('Failed to update list of K8s master nodes: %r', e) if TYPE_CHECKING: # pragma: no cover assert isinstance(retry, Retry) # K8sConnectionFailed is raised only if retry is not None! sleeptime = retry.sleeptime remaining_time = (retry.stoptime or time.time()) - sleeptime - time.time() nodes, timeout, retries = self._calculate_timeouts(api_servers, remaining_time) if nodes == 0: self._update_api_servers_cache = True raise ex retry.sleep_func(sleeptime) retry.update_delay() # We still have some time left. Partially reduce `api_servers_cache` and retry request kwargs.update(timeout=urllib3.Timeout(connect=max(1.0, timeout / 2.0), total=timeout), retries=retries) api_servers_cache = api_servers_cache[:nodes] def call_api(self, method: str, path: str, headers: Optional[Dict[str, str]] = None, body: Optional[Any] = None, _retry: Optional[Retry] = None, _preload_content: bool = True, _request_timeout: Optional[float] = None, **kwargs: Any) -> Union[urllib3.HTTPResponse, K8sObject]: headers = self._make_headers(headers) fields = {to_camel_case(k): v for k, v in kwargs.items()} # resource_version => resourceVersion body = json.dumps(body, default=lambda o: o.to_dict()) if body is not None else None response = self.request(_retry, method, self._API_URL_PREFIX + path, headers=headers, fields=fields, body=body, preload_content=_preload_content, timeout=_request_timeout) return self._handle_server_response(response, _preload_content) class CoreV1Api(object): def __init__(self, api_client: Optional['K8sClient.ApiClient'] = None) -> None: self._api_client = api_client or k8s_client.ApiClient() def __getattr__(self, func: str) -> Callable[..., Any]: # `func` name pattern: (action)_namespaced_(kind) action, kind = func.split('_namespaced_') # (read|list|create|patch|replace|delete|delete_collection) kind = kind.replace('_', '') + ('s' * int(kind[-1] != 's')) # plural, single word def wrapper(*args: Any, **kwargs: Any) -> Union[urllib3.HTTPResponse, K8sObject]: method = {'read': 'GET', 'list': 'GET', 'create': 'POST', 'replace': 'PUT'}.get(action, action.split('_')[0]).upper() if action == 'create' or len(args) == 1: # namespace is a first argument and name in not in arguments path = '/'.join([args[0], kind]) else: # name, namespace followed by optional body path = '/'.join([args[1], kind, args[0]]) headers = {'Content-Type': 'application/strategic-merge-patch+json'} if action == 'patch' else {} if len(args) == 3: # name, namespace, body body = args[2] elif action == 'create': # namespace, body body = args[1] # pyright: ignore [reportGeneralTypeIssues] elif action == 'delete': # name, namespace body = kwargs.pop('body', None) else: body = None return self._api_client.call_api(method, path, headers, body, **kwargs) return wrapper class _K8sObjectTemplate(K8sObject): """The template for objects which we create locally, e.g. k8s_client.V1ObjectMeta & co""" def __init__(self, **kwargs: Any) -> None: self._dict = {to_camel_case(k): v for k, v in kwargs.items()} def __init__(self) -> None: self.__cls_cache: Dict[str, Type['K8sClient._K8sObjectTemplate']] = {} self.__cls_lock = Lock() def __getattr__(self, name: str) -> Type['K8sClient._K8sObjectTemplate']: with self.__cls_lock: if name not in self.__cls_cache: self.__cls_cache[name] = type(name, (self._K8sObjectTemplate,), {}) return self.__cls_cache[name] k8s_client = K8sClient() k8s_config = K8sConfig() class KubernetesRetriableException(k8s_client.rest.ApiException): def __init__(self, orig: K8sClient.rest.ApiException) -> None: super(KubernetesRetriableException, self).__init__(orig.status, orig.reason) self.body = orig.body self.headers = orig.headers @property def sleeptime(self) -> Optional[int]: try: return int((self.headers or EMPTY_DICT).get('retry-after', '')) except Exception: return None class CoreV1ApiProxy(object): """Proxy class to work with k8s_client.CoreV1Api() object""" _DEFAULT_RETRIABLE_HTTP_CODES = frozenset([500, 503, 504]) def __init__(self, use_endpoints: Optional[bool] = False, bypass_api_service: Optional[bool] = False) -> None: self._api_client = k8s_client.ApiClient(bypass_api_service) self._core_v1_api = k8s_client.CoreV1Api(self._api_client) self._use_endpoints = bool(use_endpoints) self._retriable_http_codes = set(self._DEFAULT_RETRIABLE_HTTP_CODES) def configure_timeouts(self, loop_wait: int, retry_timeout: Union[int, float], ttl: int) -> None: # Normally every loop_wait seconds we should have receive something from the socket. # If we didn't received anything after the loop_wait + retry_timeout it is a time # to start worrying (send keepalive messages). Finally, the connection should be # considered as dead if we received nothing from the socket after the ttl seconds. self._api_client.pool_manager.connection_pool_kw['socket_options'] = \ list(keepalive_socket_options(ttl, int(loop_wait + retry_timeout))) self._api_client.set_read_timeout(retry_timeout) self._api_client.set_api_servers_cache_ttl(loop_wait) def configure_retriable_http_codes(self, retriable_http_codes: List[int]) -> None: self._retriable_http_codes = self._DEFAULT_RETRIABLE_HTTP_CODES | set(retriable_http_codes) def refresh_api_servers_cache(self) -> None: self._api_client.refresh_api_servers_cache() def __getattr__(self, func: str) -> Callable[..., Any]: """Intercepts calls to `CoreV1Api` methods. Handles two important cases: 1. Depending on whether Patroni is configured to work with `ConfigMaps` or `Endpoints` it remaps "virtual" method names from `*_kind` to `*_endpoints` or `*_config_map`. 2. It handles HTTP error codes and raises `KubernetesRetriableException` if the given error is supposed to be handled with retry.""" if func.endswith('_kind'): func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map') def wrapper(*args: Any, **kwargs: Any) -> Any: try: return getattr(self._core_v1_api, func)(*args, **kwargs) except k8s_client.rest.ApiException as e: if e.status in self._retriable_http_codes or e.headers and 'retry-after' in e.headers: raise KubernetesRetriableException(e) raise return wrapper @property def use_endpoints(self) -> bool: return self._use_endpoints def _run_and_handle_exceptions(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: try: return method(*args, **kwargs) except k8s_client.rest.ApiException as e: if e.status == 403: logger.exception('Permission denied') elif e.status != 409: # Object exists or conflict in resource_version logger.exception('Unexpected error from Kubernetes API') return False except (RetryFailedError, K8sException) as e: raise KubernetesError(e) def catch_kubernetes_errors(func: Callable[..., Any]) -> Callable[..., Any]: def wrapper(self: 'Kubernetes', *args: Any, **kwargs: Any) -> Any: try: return _run_and_handle_exceptions(func, self, *args, **kwargs) except KubernetesError: return False return wrapper class ObjectCache(Thread): def __init__(self, dcs: 'Kubernetes', func: Callable[..., Any], retry: Retry, condition: Condition, name: Optional[str] = None) -> None: super(ObjectCache, self).__init__() self.daemon = True self._dcs = dcs self._func = func self._retry = retry self._condition = condition self._name = name # name of this pod self._is_ready = False self._response: Union[urllib3.HTTPResponse, bool, None] = None # needs to be accessible from the `kill_stream` self._response_lock = Lock() # protect the `self._response` from concurrent access self._object_cache: Dict[str, K8sObject] = {} self._object_cache_lock = Lock() self._annotations_map = {self._dcs.leader_path: getattr(self._dcs, '_LEADER'), self._dcs.config_path: getattr(self._dcs, '_CONFIG')} # pyright self.start() def _list(self) -> K8sObject: try: return self._func(_retry=self._retry.copy()) except Exception: time.sleep(1) raise def _watch(self, resource_version: str) -> urllib3.HTTPResponse: return self._func(_request_timeout=(self._retry.deadline, urllib3.Timeout.DEFAULT_TIMEOUT), _preload_content=False, watch=True, resource_version=resource_version) def set(self, name: str, value: K8sObject) -> Tuple[bool, Optional[K8sObject]]: with self._object_cache_lock: old_value = self._object_cache.get(name) ret = not old_value or int(old_value.metadata.resource_version) < int(value.metadata.resource_version) if ret: self._object_cache[name] = value return ret, old_value def delete(self, name: str, resource_version: str) -> Tuple[bool, Optional[K8sObject]]: with self._object_cache_lock: old_value = self._object_cache.get(name) ret = old_value and int(old_value.metadata.resource_version) < int(resource_version) if ret: del self._object_cache[name] return bool(not old_value or ret), old_value def copy(self) -> Dict[str, K8sObject]: with self._object_cache_lock: return self._object_cache.copy() def get(self, name: str) -> Optional[K8sObject]: with self._object_cache_lock: return self._object_cache.get(name) def _process_event(self, event: Dict[str, Any]) -> None: ev_type = event['type'] obj = event['object'] name = obj['metadata']['name'] new_value = None if ev_type in ('ADDED', 'MODIFIED'): obj = K8sObject(obj) success, old_value = self.set(name, obj) if success: new_value = (obj.metadata.annotations or EMPTY_DICT).get(self._annotations_map.get(name, '')) elif ev_type == 'DELETED': success, old_value = self.delete(name, obj['metadata']['resourceVersion']) else: return logger.warning('Unexpected event type: %s', ev_type) if success and obj.get('kind') != 'Pod': if old_value: old_value = (old_value.metadata.annotations or EMPTY_DICT).get(self._annotations_map.get(name, '')) value_changed = old_value != new_value and \ (name != self._dcs.config_path or old_value is not None and new_value is not None) if value_changed: logger.debug('%s changed from %s to %s', name, old_value, new_value) # Do not wake up HA loop if we run as leader and received leader object update event if value_changed or name == self._dcs.leader_path and self._name != new_value: self._dcs.event.set() @staticmethod def _finish_response(response: urllib3.HTTPResponse) -> None: try: response.close() finally: response.release_conn() def _do_watch(self, resource_version: str) -> None: with self._response_lock: self._response = None response = self._watch(resource_version) with self._response_lock: if self._response is None: self._response = response if not self._response: return self._finish_response(response) for event in iter_response_objects(response): if event['object'].get('code') == 410: break self._process_event(event) def _build_cache(self) -> None: objects = self._list() with self._object_cache_lock: self._object_cache = {item.metadata.name: item for item in objects.items} with self._condition: self._is_ready = True self._condition.notify() try: self._do_watch(objects.metadata.resource_version) finally: with self._condition: self._is_ready = False with self._response_lock: response, self._response = self._response, None if isinstance(response, urllib3.HTTPResponse): self._finish_response(response) def kill_stream(self) -> None: sock = None with self._response_lock: if isinstance(self._response, urllib3.HTTPResponse): try: sock = self._response.connection.sock if self._response.connection else None except Exception: sock = None else: self._response = False if sock: try: sock.shutdown(socket.SHUT_RDWR) sock.close() except Exception as e: logger.debug('Error on socket.shutdown: %r', e) def run(self) -> None: while True: try: self._build_cache() except Exception as e: logger.error('ObjectCache.run %r', e) def is_ready(self) -> bool: """Must be called only when holding the lock on `_condition`""" return self._is_ready class Kubernetes(AbstractDCS): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: self._labels = deepcopy(config['labels']) self._labels[config.get('scope_label', 'cluster-name')] = config['scope'] self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items()) self._namespace = config.get('namespace') or 'default' self._role_label = config.get('role_label', 'role') self._leader_label_value = config.get('leader_label_value', 'primary') self._follower_label_value = config.get('follower_label_value', 'replica') self._standby_leader_label_value = config.get('standby_leader_label_value', 'primary') self._tmp_role_label = config.get('tmp_role_label') self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME super(Kubernetes, self).__init__({**config, 'namespace': ''}, mpp) if self._mpp.is_enabled(): self._labels[self._mpp.k8s_group_label] = str(self._mpp.group) self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1, retry_exceptions=KubernetesRetriableException) self._ttl = int(config.get('ttl') or 30) try: k8s_config.load_incluster_config(ca_certs=self._ca_certs) except k8s_config.ConfigException: k8s_config.load_kube_config(context=config.get('context', 'kind-kind')) self.__ips: List[str] = [] if self._ctl else [config.get('pod_ip', '')] self.__ports: List[K8sObject] = [] ports: List[Dict[str, Any]] = config.get('ports', [{}]) for p in ports: port: Dict[str, Any] = {'port': int(p.get('port', '5432'))} port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)}) self.__ports.append(k8s_client.V1EndpointPort(**port)) bypass_api_service = not self._ctl and config.get('bypass_api_service') self._api = CoreV1ApiProxy(config.get('use_endpoints'), bypass_api_service) self._should_create_config_service = self._api.use_endpoints self.reload_config(config) # leader_observed_record, leader_resource_version, and leader_observed_time are used only for leader race! self._leader_observed_record: Dict[str, str] = {} self._leader_observed_time = None self._leader_resource_version = None self.__do_not_watch = False self._condition = Condition() pods_func = functools.partial(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector) self._pods = ObjectCache(self, pods_func, self._retry, self._condition) kinds_func = functools.partial(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector) self._kinds = ObjectCache(self, kinds_func, self._retry, self._condition, self._name) def retry(self, method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: retry = self._retry.copy() kwargs['_retry'] = retry return retry(method, *args, **kwargs) def client_path(self, path: str) -> str: return super(Kubernetes, self).client_path(path)[1:].replace('/', '-') @property def leader_path(self) -> str: return super(Kubernetes, self).leader_path[:-7 if self._api.use_endpoints else None] def set_ttl(self, ttl: int) -> Optional[bool]: ttl = int(ttl) self.__do_not_watch = self._ttl != ttl self._ttl = ttl return None @property def ttl(self) -> int: return self._ttl def set_retry_timeout(self, retry_timeout: int) -> None: self._retry.deadline = retry_timeout def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None: """Handles dynamic config changes. Either cause by changes in the local configuration file + SIGHUP or by changes of dynamic configuration""" super(Kubernetes, self).reload_config(config) if TYPE_CHECKING: # pragma: no cover assert self._retry.deadline is not None self._api.configure_timeouts(self.loop_wait, self._retry.deadline, self.ttl) # retriable_http_codes supposed to be either int, list of integers or comma-separated string with integers. retriable_http_codes: Union[str, List[Union[str, int]]] = config.get('retriable_http_codes', []) if not isinstance(retriable_http_codes, list): retriable_http_codes = [c.strip() for c in str(retriable_http_codes).split(',')] try: self._api.configure_retriable_http_codes([int(c) for c in retriable_http_codes]) except Exception as e: logger.warning('Invalid value of retriable_http_codes = %s: %r', config['retriable_http_codes'], e) @staticmethod def member(pod: K8sObject) -> Member: annotations = pod.metadata.annotations or EMPTY_DICT member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', '')) member.data['pod_labels'] = pod.metadata.labels return member def _wait_caches(self, stop_time: float) -> None: while not (self._pods.is_ready() and self._kinds.is_ready()): timeout = stop_time - time.time() if timeout <= 0: raise RetryFailedError('Exceeded retry deadline') self._condition.wait(timeout) def _cluster_from_nodes(self, group: str, nodes: Dict[str, K8sObject], pods: Collection[K8sObject]) -> Cluster: members = [self.member(pod) for pod in pods] path = self._base_path[1:] + '-' if group: path += group + '-' config = nodes.get(path + self._CONFIG) metadata = config and config.metadata annotations = metadata and metadata.annotations or {} # get initialize flag initialize = annotations.get(self._INITIALIZE) # get global dynamic configuration config = metadata and ClusterConfig.from_node(metadata.resource_version, annotations.get(self._CONFIG) or '{}', metadata.resource_version if self._CONFIG in annotations else 0) # get timeline history history = metadata and TimelineHistory.from_node(metadata.resource_version, annotations.get(self._HISTORY) or '[]') leader_path = path[:-1] if self._api.use_endpoints else path + self._LEADER leader = nodes.get(leader_path) metadata = leader and leader.metadata if leader_path == self.leader_path: # We want to memorize leader_resource_version only for our cluster self._leader_resource_version = metadata.resource_version if metadata else None annotations: Dict[str, str] = metadata and metadata.annotations or {} # get last known leader lsn and slots status = Status.from_node(annotations) # get failsafe topology try: failsafe = json.loads(annotations.get(self._FAILSAFE, '')) except Exception: failsafe = None # get leader leader_record: Dict[str, str] = {n: annotations[n] for n in (self._LEADER, 'acquireTime', 'ttl', 'renewTime', 'transitions') if n in annotations} # We want to memorize leader_observed_record and update leader_observed_time only for our cluster if leader_path == self.leader_path and (leader_record or self._leader_observed_record)\ and leader_record != self._leader_observed_record: self._leader_observed_record = leader_record self._leader_observed_time = time.time() leader = leader_record.get(self._LEADER) try: ttl = int(leader_record.get('ttl', self._ttl)) or self._ttl except (TypeError, ValueError): ttl = self._ttl # We want to check validity of the leader record only for our own cluster if leader_path == self.leader_path and\ not (metadata and self._leader_observed_time and self._leader_observed_time + ttl >= time.time()): leader = None if metadata: member = Member(-1, leader or '', None, {}) member = ([m for m in members if m.name == leader] or [member])[0] leader = Leader(metadata.resource_version, None, member) else: leader = None # failover key failover = nodes.get(path + self._FAILOVER) metadata = failover and failover.metadata failover = metadata and Failover.from_node(metadata.resource_version, (metadata.annotations or EMPTY_DICT).copy()) # get synchronization state sync = nodes.get(path + self._SYNC) metadata = sync and sync.metadata sync = SyncState.from_node(metadata and metadata.resource_version, metadata and metadata.annotations) return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) def _postgresql_cluster_loader(self, path: Dict[str, Any]) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ return self._cluster_from_nodes(path['group'], path['nodes'], path['pods'].values()) def _mpp_cluster_loader(self, path: Dict[str, Any]) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ clusters: Dict[str, Dict[str, Dict[str, K8sObject]]] = defaultdict(lambda: defaultdict(dict)) for name, pod in path['pods'].items(): group = pod.metadata.labels.get(self._mpp.k8s_group_label) if group and self._mpp.group_re.match(group): clusters[group]['pods'][name] = pod for name, kind in path['nodes'].items(): group = kind.metadata.labels.get(self._mpp.k8s_group_label) if group and self._mpp.group_re.match(group): clusters[group]['nodes'][name] = kind return {int(group): self._cluster_from_nodes(group, value['nodes'], value['pods'].values()) for group, value in clusters.items()} def __load_cluster( self, group: Optional[str], loader: Callable[[Dict[str, Any]], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: if TYPE_CHECKING: # pragma: no cover assert self._retry.deadline is not None stop_time = time.time() + self._retry.deadline self._api.refresh_api_servers_cache() try: with self._condition: self._wait_caches(stop_time) pods = {name: pod for name, pod in self._pods.copy().items() if not group or pod.metadata.labels.get(self._mpp.k8s_group_label) == group} nodes = {name: kind for name, kind in self._kinds.copy().items() if not group or kind.metadata.labels.get(self._mpp.k8s_group_label) == group} return loader({'group': group, 'pods': pods, 'nodes': nodes}) except Exception: logger.exception('get_cluster') raise KubernetesError('Kubernetes API is not responding properly') def _load_cluster( self, path: str, loader: Callable[[Any], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: group = str(self._mpp.group) if self._mpp.is_enabled() and path == self.client_path('') else None return self.__load_cluster(group, loader) def get_mpp_coordinator(self) -> Optional[Cluster]: """Load the PostgreSQL cluster for the MPP Coordinator. .. note:: This method is only executed on the worker nodes to find the coordinator. :returns: Select :class:`Cluster` instance associated with the MPP Coordinator group ID. """ try: ret = self.__load_cluster(str(self._mpp.coordinator_group_id), self._postgresql_cluster_loader) if TYPE_CHECKING: # pragma: no cover assert isinstance(ret, Cluster) return ret except Exception as e: logger.error('Failed to load %s coordinator cluster from Kubernetes: %r', self._mpp.type, e) @staticmethod def compare_ports(p1: K8sObject, p2: K8sObject) -> bool: return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP') @staticmethod def subsets_changed(last_observed_subsets: List[K8sObject], ip: str, ports: List[K8sObject]) -> bool: """ >>> ip = '1.2.3.4' >>> a = [k8s_client.V1EndpointAddress(ip=ip)] >>> s = [k8s_client.V1EndpointSubset(addresses=a)] >>> Kubernetes.subsets_changed(s, '1.2.3.5', []) True >>> s = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(protocol='TCP', port=1)])] >>> Kubernetes.subsets_changed(s, '1.2.3.4', [k8s_client.V1EndpointPort(port=5432)]) True >>> p1 = k8s_client.V1EndpointPort(name='port1', port=1) >>> p2 = k8s_client.V1EndpointPort(name='port2', port=2) >>> p3 = k8s_client.V1EndpointPort(name='port3', port=3) >>> s = [k8s_client.V1EndpointSubset(addresses=a, ports=[p1, p2])] >>> Kubernetes.subsets_changed(s, ip, [p2, p3]) True >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p1])] >>> Kubernetes.subsets_changed(s, ip, [p2, p1]) False """ if len(last_observed_subsets) != 1: return True if len(last_observed_subsets[0].addresses or []) != 1 or \ last_observed_subsets[0].addresses[0].ip != ip or \ len(last_observed_subsets[0].ports) != len(ports): return True if len(ports) == 1: return not Kubernetes.compare_ports(last_observed_subsets[0].ports[0], ports[0]) observed_ports = {p.name: p for p in last_observed_subsets[0].ports} for p in ports: if p.name not in observed_ports or not Kubernetes.compare_ports(p, observed_ports.pop(p.name)): return True return False def __target_ref(self, leader_ip: str, latest_subsets: List[K8sObject], pod: K8sObject) -> K8sObject: # we want to reuse existing target_ref if possible empty_addresses: List[K8sObject] = [] for subset in latest_subsets: for address in subset.addresses or empty_addresses: if address.ip == leader_ip and address.target_ref and address.target_ref.name == self._name: return address.target_ref return k8s_client.V1ObjectReference(kind='Pod', uid=pod.metadata.uid, namespace=self._namespace, name=self._name, resource_version=pod.metadata.resource_version) def _map_subsets(self, endpoints: Dict[str, Any], ips: List[str]) -> None: leader = self._kinds.get(self.leader_path) empty_addresses: List[K8sObject] = [] latest_subsets = leader and leader.subsets or empty_addresses if not ips: # We want to have subsets empty if latest_subsets: endpoints['subsets'] = [] return pod = self._pods.get(self._name) leader_ip = ips[0] or pod and pod.status.pod_ip # don't touch subsets if our (leader) ip is unknown or subsets is valid if leader_ip and self.subsets_changed(latest_subsets, leader_ip, self.__ports): kwargs = {'hostname': pod.spec.hostname, 'node_name': pod.spec.node_name, 'target_ref': self.__target_ref(leader_ip, latest_subsets, pod)} if pod else {} address = k8s_client.V1EndpointAddress(ip=leader_ip, **kwargs) endpoints['subsets'] = [k8s_client.V1EndpointSubset(addresses=[address], ports=self.__ports)] def _patch_or_create(self, name: str, annotations: Dict[str, Any], resource_version: Optional[str] = None, patch: bool = False, retry: Optional[Callable[..., Any]] = None, ips: Optional[List[str]] = None) -> K8sObject: """Patch or create K8s object, Endpoint or ConfigMap. :param name: the name of the object. :param annotations: mapping of annotations that we want to create/update. :param resource_version: object should be updated only if the ``resource_version`` matches provided value. :param patch: ``True`` if we know in advance that the object already exists and we should patch it. :param retry: a callable that will take care of retries :param ips: IP address that we want to put to the subsets of the endpoint. Could have following values: * ``None`` - when we don't need to touch subset; * ``[]`` - to set subsets to the empty list, when :meth:`delete_leader` method is called; * ``['ip.add.re.ss']`` - when we want to make sure that the subsets of the leader endpoint contains the IP address of the leader, that we get from the ``kubernetes.pod_ip``; * ``['']`` - when we want to make sure that the subsets of the leader endpoint contains the IP address of the leader, but ``kubernetes.pod_ip`` configuration is missing. In this case we will try to take the IP address of the Pod which name matches ``name`` from the config file. :returns: the new :class:`V1Endpoints` or :class:`V1ConfigMap` object, that was created or updated. """ metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations} if patch or resource_version: if resource_version is not None: metadata['resource_version'] = resource_version func = functools.partial(self._api.patch_namespaced_kind, name) metadata['annotations'] = annotations else: func = functools.partial(self._api.create_namespaced_kind) # skip annotations with null values metadata['annotations'] = {k: v for k, v in annotations.items() if v is not None} metadata = k8s_client.V1ObjectMeta(**metadata) if self._api.use_endpoints: endpoints = {'metadata': metadata} if ips is not None: self._map_subsets(endpoints, ips) body = k8s_client.V1Endpoints(**endpoints) else: body = k8s_client.V1ConfigMap(metadata=metadata) ret = retry(func, self._namespace, body) if retry else func(self._namespace, body) if ret: self._kinds.set(name, ret) return ret @catch_kubernetes_errors def patch_or_create(self, name: str, annotations: Dict[str, Any], resource_version: Optional[str] = None, patch: bool = False, retry: bool = True, ips: Optional[List[str]] = None) -> K8sObject: try: return self._patch_or_create(name, annotations, resource_version, patch, self.retry if retry else None, ips) except k8s_client.rest.ApiException as e: if e.status == 409 and resource_version: # Conflict in resource_version # Terminate watchers, it could be a sign that K8s API is in a failed state self._kinds.kill_stream() self._pods.kill_stream() raise e def patch_or_create_config(self, annotations: Dict[str, Any], resource_version: Optional[str] = None, patch: bool = False, retry: bool = True) -> bool: # SCOPE-config endpoint requires corresponding service otherwise it might be "cleaned" by k8s master if self._api.use_endpoints and not patch and not resource_version: self._should_create_config_service = True self._create_config_service() return bool(self.patch_or_create(self.config_path, annotations, resource_version, patch, retry)) def _create_config_service(self) -> None: metadata = k8s_client.V1ObjectMeta(namespace=self._namespace, name=self.config_path, labels=self._labels) body = k8s_client.V1Service(metadata=metadata, spec=k8s_client.V1ServiceSpec(cluster_ip='None')) try: if not self._api.create_namespaced_service(self._namespace, body): return except Exception as e: # 409 - service already exists, 403 - creation forbidden if not isinstance(e, k8s_client.rest.ApiException) or e.status not in (409, 403): return logger.exception('create_config_service failed') self._should_create_config_service = False def _write_leader_optime(self, last_lsn: str) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def _write_status(self, value: str) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def _write_failsafe(self, value: str) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def _update_leader(self, leader: Leader) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def write_leader_optime(self, last_lsn: int) -> None: """Write value for WAL LSN to ``optime`` annotation of the leader object. :param last_lsn: absolute WAL LSN in bytes. """ self.patch_or_create(self.leader_path, {self._OPTIME: str(last_lsn)}, patch=True, retry=False) def _update_leader_with_retry(self, annotations: Dict[str, Any], resource_version: Optional[str], ips: List[str]) -> bool: retry = self._retry.copy() def _retry(*args: Any, **kwargs: Any) -> Any: kwargs['_retry'] = retry return retry(*args, **kwargs) try: return bool(self._patch_or_create(self.leader_path, annotations, resource_version, ips=ips, retry=_retry)) except k8s_client.rest.ApiException as e: if e.status == 409: logger.warning('Concurrent update of %s', self.leader_path) else: logger.exception('Permission denied' if e.status == 403 else 'Unexpected error from Kubernetes API') return False except (RetryFailedError, K8sException) as e: raise KubernetesError(e) # if we are here, that means update failed with 409 if not retry.ensure_deadline(1): return False # No time for retry. Tell ha.py that we have to demote due to failed update. # Try to get the latest version directly from K8s API instead of relying on async cache try: kind = _retry(self._api.read_namespaced_kind, self.leader_path, self._namespace) except (RetryFailedError, K8sException) as e: raise KubernetesError(e) except Exception as e: logger.error('Failed to get the leader object "%s": %r', self.leader_path, e) return False self._kinds.set(self.leader_path, kind) kind_annotations = kind and kind.metadata.annotations or EMPTY_DICT kind_resource_version = kind and kind.metadata.resource_version # There is different leader or resource_version in cache didn't change if kind and (kind_annotations.get(self._LEADER) != self._name or kind_resource_version == resource_version): return False # We can get 409 because we do at least one retry, and the first update might have succeeded, # therefore we will check if annotations on the read object match expectations. if all(kind_annotations.get(k) == v for k, v in annotations.items()): return True if not retry.ensure_deadline(0.5): return False return bool(_run_and_handle_exceptions(self._patch_or_create, self.leader_path, annotations, kind_resource_version, ips=ips, retry=_retry)) def update_leader(self, cluster: Cluster, last_lsn: Optional[int], slots: Optional[Dict[str, int]] = None, failsafe: Optional[Dict[str, str]] = None) -> bool: kind = self._kinds.get(self.leader_path) kind_annotations = kind and kind.metadata.annotations or EMPTY_DICT if kind and kind_annotations.get(self._LEADER) != self._name: return False now = datetime.datetime.now(tzutc).isoformat() leader_observed_record = kind_annotations or self._leader_observed_record annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now, 'acquireTime': leader_observed_record.get('acquireTime') or now, 'transitions': leader_observed_record.get('transitions') or '0'} if last_lsn: annotations[self._OPTIME] = str(last_lsn) annotations['slots'] = json.dumps(slots, separators=(',', ':')) if slots else None retain_slots = self._build_retain_slots(cluster, slots) annotations['retain_slots'] = json.dumps(retain_slots) if retain_slots else None if failsafe is not None: annotations[self._FAILSAFE] = json.dumps(failsafe, separators=(',', ':')) if failsafe else None resource_version = kind and kind.metadata.resource_version return self._update_leader_with_retry(annotations, resource_version, self.__ips) def attempt_to_acquire_leader(self) -> bool: now = datetime.datetime.now(tzutc).isoformat() annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now, 'acquireTime': now, 'transitions': '0'} if self._leader_observed_record: try: transitions = int(self._leader_observed_record.get('transitions', '')) except (TypeError, ValueError): transitions = 0 if self._leader_observed_record.get(self._LEADER) != self._name: transitions += 1 else: annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now annotations['transitions'] = str(transitions) try: ret = bool(self._patch_or_create(self.leader_path, annotations, self._leader_resource_version, retry=self.retry, ips=self.__ips)) except k8s_client.rest.ApiException as e: if e.status == 409 and self._leader_resource_version: # Conflict in resource_version # Terminate watchers, it could be a sign that K8s API is in a failed state self._kinds.kill_stream() self._pods.kill_stream() ret = False except (RetryFailedError, K8sException) as e: raise KubernetesError(e) if not ret: logger.info('Could not take out TTL lock') return ret def take_leader(self) -> bool: return self.attempt_to_acquire_leader() def set_failover_value(self, value: str, version: Optional[str] = None) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def manual_failover(self, leader: Optional[str], candidate: Optional[str], scheduled_at: Optional[datetime.datetime] = None, version: Optional[str] = None) -> bool: annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at and scheduled_at.isoformat()} patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.version) return bool(self.patch_or_create(self.failover_path, annotations, version, bool(version or patch), False)) @property def _config_resource_version(self) -> Optional[str]: config = self._kinds.get(self.config_path) return config and config.metadata.resource_version def set_config_value(self, value: str, version: Optional[str] = None) -> bool: return self.patch_or_create_config({self._CONFIG: value}, version, bool(self._config_resource_version), False) @catch_kubernetes_errors def touch_member(self, data: Dict[str, Any]) -> bool: cluster = self.cluster if cluster and cluster.leader and cluster.leader.name == self._name: role = self._standby_leader_label_value if data['role'] == 'standby_leader' else self._leader_label_value tmp_role = 'primary' elif data['state'] == 'running' and data['role'] != 'primary': role = {'replica': self._follower_label_value}.get(data['role'], data['role']) tmp_role = data['role'] else: role = None tmp_role = None role_labels = {self._role_label: role} if self._tmp_role_label: role_labels[self._tmp_role_label] = tmp_role member = cluster and cluster.get_member(self._name, fallback_to_leader=False) pod_labels = member and member.data.pop('pod_labels', None) ret = member and pod_labels is not None\ and all(pod_labels.get(k) == v for k, v in role_labels.items())\ and deep_compare(data, member.data) if not ret: metadata = {'namespace': self._namespace, 'name': self._name, 'labels': role_labels, 'annotations': {'status': json.dumps(data, separators=(',', ':'))}} body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata)) ret = self._api.patch_namespaced_pod(self._name, self._namespace, body) if ret: self._pods.set(self._name, ret) if self._should_create_config_service: self._create_config_service() return bool(ret) def initialize(self, create_new: bool = True, sysid: str = "") -> bool: cluster = self.cluster resource_version = str(cluster.config.version)\ if cluster and cluster.config and cluster.config.version else None return self.patch_or_create_config({self._INITIALIZE: sysid}, resource_version) def _delete_leader(self, leader: Leader) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool: ret = False kind = self._kinds.get(self.leader_path) if kind and (kind.metadata.annotations or EMPTY_DICT).get(self._LEADER) == self._name: annotations: Dict[str, Optional[str]] = {self._LEADER: None} if last_lsn: annotations[self._OPTIME] = str(last_lsn) ret = self.patch_or_create(self.leader_path, annotations, kind.metadata.resource_version, True, False, []) self.reset_cluster() return ret def cancel_initialization(self) -> bool: return self.patch_or_create_config({self._INITIALIZE: None}, None, True) @catch_kubernetes_errors def delete_cluster(self) -> bool: return bool(self.retry(self._api.delete_collection_namespaced_kind, self._namespace, label_selector=self._label_selector)) def set_history_value(self, value: str) -> bool: return self.patch_or_create_config({self._HISTORY: value}, None, bool(self._config_resource_version), False) def set_sync_state_value(self, value: str, version: Optional[str] = None) -> bool: """Unused""" raise NotImplementedError # pragma: no cover def write_sync_state(self, leader: Optional[str], sync_standby: Optional[Collection[str]], quorum: Optional[int], version: Optional[str] = None) -> Optional[SyncState]: """Prepare and write annotations to $SCOPE-sync Endpoint or ConfigMap. :param leader: name of the leader node that manages /sync key :param sync_standby: collection of currently known synchronous standby node names :param quorum: if the node from sync_standby list is doing a leader race it should see at least quorum other nodes from the sync_standby + leader list :param version: last known `resource_version` for conditional update of the object :returns: the new :class:`SyncState` object or None """ sync_state = self.sync_state(leader, sync_standby, quorum) sync_state['quorum'] = str(sync_state['quorum']) if sync_state['quorum'] is not None else None ret = self.patch_or_create(self.sync_path, sync_state, version, False) if not isinstance(ret, bool): return SyncState.from_node(ret.metadata.resource_version, sync_state) def delete_sync_state(self, version: Optional[str] = None) -> bool: """Patch annotations of $SCOPE-sync Endpoint or ConfigMap with empty values. Effectively it removes "leader" and "sync_standby" annotations from the object. :param version: last known `resource_version` for conditional update of the object :returns: `True` if "delete" was successful """ return self.write_sync_state(None, None, None, version=version) is not None def watch(self, leader_version: Optional[str], timeout: float) -> bool: if self.__do_not_watch: self.__do_not_watch = False return True # We want to give a bit more time to non-leader nodes to synchronize HA loops if leader_version: timeout += 0.5 try: return super(Kubernetes, self).watch(None, timeout) finally: self.event.clear() patroni-4.0.4/patroni/dcs/raft.py000066400000000000000000000464331472010352700167410ustar00rootroot00000000000000import json import logging import os import threading import time from collections import defaultdict from typing import Any, Callable, Collection, Dict, List, Optional, Set, TYPE_CHECKING, Union from pysyncobj import FAIL_REASON, replicated, SyncObj, SyncObjConf from pysyncobj.dns_resolver import globalDnsResolver from pysyncobj.node import TCPNode from pysyncobj.transport import CONNECTION_STATE, TCPTransport from pysyncobj.utility import TcpUtility from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import validate_directory from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory if TYPE_CHECKING: # pragma: no cover from ..config import Config logger = logging.getLogger(__name__) class RaftError(DCSError): pass class _TCPTransport(TCPTransport): def __init__(self, syncObj: 'DynMemberSyncObj', selfNode: Optional[TCPNode], otherNodes: Collection[TCPNode]) -> None: super(_TCPTransport, self).__init__(syncObj, selfNode, otherNodes) self.setOnUtilityMessageCallback('members', syncObj.getMembers) def _connectIfNecessarySingle(self, node: TCPNode) -> bool: try: return super(_TCPTransport, self)._connectIfNecessarySingle(node) except Exception as e: logger.debug('Connection to %s failed: %r', node, e) return False def resolve_host(self: TCPNode) -> Optional[str]: return globalDnsResolver().resolve(self.host) setattr(TCPNode, 'ip', property(resolve_host)) class SyncObjUtility(object): def __init__(self, otherNodes: Collection[Union[str, TCPNode]], conf: SyncObjConf, retry_timeout: int = 10) -> None: self._nodes = otherNodes self._utility = TcpUtility(conf.password, retry_timeout / max(1, len(otherNodes))) self.__node = next(iter(otherNodes), None) def executeCommand(self, command: List[Any]) -> Any: try: if self.__node: return self._utility.executeCommand(self.__node, command) except Exception: return None def getMembers(self) -> Optional[List[str]]: for self.__node in self._nodes: response = self.executeCommand(['members']) if response: return [member['addr'] for member in response] class DynMemberSyncObj(SyncObj): def __init__(self, selfAddress: Optional[str], partnerAddrs: Collection[str], conf: SyncObjConf, retry_timeout: int = 10) -> None: self.__early_apply_local_log = selfAddress is not None self.applied_local_log = False utility = SyncObjUtility(partnerAddrs, conf, retry_timeout) members = utility.getMembers() add_self = members and selfAddress not in members partnerAddrs = [member for member in (members or partnerAddrs) if member != selfAddress] super(DynMemberSyncObj, self).__init__(selfAddress, partnerAddrs, conf, transportClass=_TCPTransport) if add_self: thread = threading.Thread(target=utility.executeCommand, args=(['add', selfAddress],)) thread.daemon = True thread.start() def getMembers(self, args: Any, callback: Callable[[Any, Any], Any]) -> None: callback([{'addr': node.id, 'leader': node == self._getLeader(), 'status': CONNECTION_STATE.CONNECTED if self.isNodeConnected(node) else CONNECTION_STATE.DISCONNECTED} for node in self.otherNodes] + [{'addr': self.selfNode.id, 'leader': self._isLeader(), 'status': CONNECTION_STATE.CONNECTED}], None) def _onTick(self, timeToWait: float = 0.0): super(DynMemberSyncObj, self)._onTick(timeToWait) # The SyncObj calls onReady callback only when cluster got the leader and is ready for writes. # In some cases for us it is safe to "signal" the Raft object when the local log is fully applied. # We are using the `applied_local_log` property for that, but not calling the callback function. if self.__early_apply_local_log and not self.applied_local_log and self.raftLastApplied == self.raftCommitIndex: self.applied_local_log = True class KVStoreTTL(DynMemberSyncObj): def __init__(self, on_ready: Optional[Callable[..., Any]], on_set: Optional[Callable[[str, Dict[str, Any]], None]], on_delete: Optional[Callable[[str], None]], **config: Any) -> None: self.__thread = None self.__on_set = on_set self.__on_delete = on_delete self.__limb: Dict[str, Dict[str, Any]] = {} self.set_retry_timeout(int(config.get('retry_timeout') or 10)) self_addr = config.get('self_addr') partner_addrs: Set[str] = set(config.get('partner_addrs', [])) if config.get('patronictl'): if self_addr: partner_addrs.add(self_addr) self_addr = None # Create raft data_dir if necessary raft_data_dir = config.get('data_dir', '') if raft_data_dir != '': validate_directory(raft_data_dir) file_template = (self_addr or '') file_template = file_template.replace(':', '_') if os.name == 'nt' else file_template file_template = os.path.join(raft_data_dir, file_template) conf = SyncObjConf(password=config.get('password'), autoTick=False, appendEntriesUseBatch=False, bindAddress=config.get('bind_addr'), dnsFailCacheTime=(config.get('loop_wait') or 10), dnsCacheTime=(config.get('ttl') or 30), commandsWaitLeader=config.get('commandsWaitLeader'), fullDumpFile=(file_template + '.dump' if self_addr else None), journalFile=(file_template + '.journal' if self_addr else None), onReady=on_ready, dynamicMembershipChange=True) super(KVStoreTTL, self).__init__(self_addr, partner_addrs, conf, self.__retry_timeout) self.__data: Dict[str, Dict[str, Any]] = {} @staticmethod def __check_requirements(old_value: Dict[str, Any], **kwargs: Any) -> bool: return bool(('prevExist' not in kwargs or bool(kwargs['prevExist']) == bool(old_value)) and ('prevValue' not in kwargs or old_value and old_value['value'] == kwargs['prevValue']) and (kwargs.get('prevIndex') is None or old_value and old_value['index'] == kwargs['prevIndex'])) def set_retry_timeout(self, retry_timeout: int) -> None: self.__retry_timeout = retry_timeout def retry(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: event = threading.Event() ret = {'result': None, 'error': -1} def callback(result: Any, error: Any) -> None: ret.update(result=result, error=error) event.set() kwargs['callback'] = callback timeout = kwargs.pop('timeout', None) or self.__retry_timeout deadline = timeout and time.time() + timeout while True: event.clear() func(*args, **kwargs) event.wait(timeout) if ret['error'] == FAIL_REASON.SUCCESS: return ret['result'] elif ret['error'] == FAIL_REASON.REQUEST_DENIED: break elif deadline: timeout = deadline - time.time() if timeout <= 0: raise RaftError('timeout') time.sleep(1) return False @replicated def _set(self, key: str, value: Dict[str, Any], **kwargs: Any) -> Union[bool, Dict[str, Any]]: old_value = self.__data.get(key, {}) if not self.__check_requirements(old_value, **kwargs): return False if old_value and old_value['created'] != value['created']: value['created'] = value['updated'] value['index'] = self.raftLastApplied + 1 self.__data[key] = value if self.__on_set: self.__on_set(key, value) return value def set(self, key: str, value: str, ttl: Optional[int] = None, handle_raft_error: bool = True, **kwargs: Any) -> Union[bool, Dict[str, Any]]: old_value = self.__data.get(key, {}) if not self.__check_requirements(old_value, **kwargs): return False data: Dict[str, Any] = {'value': value, 'updated': time.time()} data['created'] = old_value.get('created', data['updated']) if ttl: data['expire'] = data['updated'] + ttl try: return self.retry(self._set, key, data, **kwargs) except RaftError: if not handle_raft_error: raise return False def __pop(self, key: str) -> None: self.__data.pop(key) if self.__on_delete: self.__on_delete(key) @replicated def _delete(self, key: str, recursive: bool = False, **kwargs: Any) -> bool: if recursive: for k in list(self.__data.keys()): if k.startswith(key): self.__pop(k) elif not self.__check_requirements(self.__data.get(key, {}), **kwargs): return False else: self.__pop(key) return True def delete(self, key: str, recursive: bool = False, **kwargs: Any) -> bool: if not recursive and not self.__check_requirements(self.__data.get(key, {}), **kwargs): return False try: return self.retry(self._delete, key, recursive=recursive, **kwargs) except RaftError: return False @staticmethod def __values_match(old: Dict[str, Any], new: Dict[str, Any]) -> bool: return all(old.get(n) == new.get(n) for n in ('created', 'updated', 'expire', 'value')) @replicated def _expire(self, key: str, value: Dict[str, Any], callback: Optional[Callable[..., Any]] = None) -> None: current = self.__data.get(key) if current and self.__values_match(current, value): self.__pop(key) def __expire_keys(self) -> None: for key, value in self.__data.items(): if value and 'expire' in value and value['expire'] <= time.time() and \ not (key in self.__limb and self.__values_match(self.__limb[key], value)): self.__limb[key] = value def callback(*args: Any) -> None: if key in self.__limb and self.__values_match(self.__limb[key], value): self.__limb.pop(key) self._expire(key, value, callback=callback) def get(self, key: str, recursive: bool = False) -> Optional[Dict[str, Any]]: if not recursive: return self.__data.get(key) return {k: v for k, v in self.__data.items() if k.startswith(key)} def _onTick(self, timeToWait: float = 0.0) -> None: super(KVStoreTTL, self)._onTick(timeToWait) if self._isLeader(): self.__expire_keys() else: self.__limb.clear() def _autoTickThread(self) -> None: self.__destroying = False while not self.__destroying: self.doTick(self.conf.autoTickPeriod) def startAutoTick(self) -> None: self.__thread = threading.Thread(target=self._autoTickThread) self.__thread.daemon = True self.__thread.start() def destroy(self) -> None: if self.__thread: self.__destroying = True self.__thread.join() super(KVStoreTTL, self).destroy() class Raft(AbstractDCS): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: super(Raft, self).__init__(config, mpp) self._ttl = int(config.get('ttl') or 30) ready_event = threading.Event() self._sync_obj = KVStoreTTL(ready_event.set, self._on_set, self._on_delete, commandsWaitLeader=False, **config) self._sync_obj.startAutoTick() while True: ready_event.wait(5) if ready_event.is_set() or self._sync_obj.applied_local_log: break else: logger.info('waiting on raft') def _on_set(self, key: str, value: Dict[str, Any]) -> None: leader = (self._sync_obj.get(self.leader_path) or {}).get('value') if key == value['created'] == value['updated'] and \ (key.startswith(self.members_path) or key == self.leader_path and leader != self._name) or \ key in (self.leader_optime_path, self.status_path) and leader != self._name or \ key in (self.config_path, self.sync_path): self.event.set() def _on_delete(self, key: str) -> None: if key == self.leader_path: self.event.set() def set_ttl(self, ttl: int) -> Optional[bool]: self._ttl = ttl @property def ttl(self) -> int: return self._ttl def set_retry_timeout(self, retry_timeout: int) -> None: self._sync_obj.set_retry_timeout(retry_timeout) def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None: super(Raft, self).reload_config(config) globalDnsResolver().setTimeouts(self.ttl, self.loop_wait) @staticmethod def member(key: str, value: Dict[str, Any]) -> Member: return Member.from_node(value['index'], os.path.basename(key), None, value['value']) def _cluster_from_nodes(self, nodes: Dict[str, Any]) -> Cluster: # get initialize flag initialize = nodes.get(self._INITIALIZE) initialize = initialize and initialize['value'] # get global dynamic configuration config = nodes.get(self._CONFIG) config = config and ClusterConfig.from_node(config['index'], config['value']) # get timeline history history = nodes.get(self._HISTORY) history = history and TimelineHistory.from_node(history['index'], history['value']) # get last know leader lsn and slots status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME) status = Status.from_node(status and status['value']) # get list of members members = [self.member(k, n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1] # get leader leader = nodes.get(self._LEADER) if leader: member = Member(-1, leader['value'], None, {}) member = ([m for m in members if m.name == leader['value']] or [member])[0] leader = Leader(leader['index'], None, member) # failover key failover = nodes.get(self._FAILOVER) if failover: failover = Failover.from_node(failover['index'], failover['value']) # get synchronization state sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync['index'], sync and sync['value']) # get failsafe topology failsafe = nodes.get(self._FAILSAFE) try: failsafe = json.loads(failsafe['value']) if failsafe else None except Exception: failsafe = None return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) def _postgresql_cluster_loader(self, path: str) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ response = self._sync_obj.get(path, recursive=True) if not response: return Cluster.empty() nodes = {key[len(path):]: value for key, value in response.items()} return self._cluster_from_nodes(nodes) def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ clusters: Dict[int, Dict[str, Any]] = defaultdict(dict) response = self._sync_obj.get(path, recursive=True) for key, value in (response or {}).items(): key = key[len(path):].split('/', 1) if len(key) == 2 and self._mpp.group_re.match(key[0]): clusters[int(key[0])][key[1]] = value return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()} def _load_cluster( self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: return loader(path) def _write_leader_optime(self, last_lsn: str) -> bool: return self._sync_obj.set(self.leader_optime_path, last_lsn, timeout=1) is not False def _write_status(self, value: str) -> bool: return self._sync_obj.set(self.status_path, value, timeout=1) is not False def _write_failsafe(self, value: str) -> bool: return self._sync_obj.set(self.failsafe_path, value, timeout=1) is not False def _update_leader(self, leader: Leader) -> bool: ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, handle_raft_error=False, prevValue=self._name) is not False if not ret and self._sync_obj.get(self.leader_path) is None: ret = self.attempt_to_acquire_leader() return ret def attempt_to_acquire_leader(self) -> bool: return self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, handle_raft_error=False, prevExist=False) is not False def set_failover_value(self, value: str, version: Optional[int] = None) -> bool: return self._sync_obj.set(self.failover_path, value, prevIndex=version) is not False def set_config_value(self, value: str, version: Optional[int] = None) -> bool: return self._sync_obj.set(self.config_path, value, prevIndex=version) is not False def touch_member(self, data: Dict[str, Any]) -> bool: value = json.dumps(data, separators=(',', ':')) return self._sync_obj.set(self.member_path, value, self._ttl, timeout=2) is not False def take_leader(self) -> bool: return self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl) is not False def initialize(self, create_new: bool = True, sysid: str = '') -> bool: return self._sync_obj.set(self.initialize_path, sysid, prevExist=(not create_new)) is not False def _delete_leader(self, leader: Leader) -> bool: return self._sync_obj.delete(self.leader_path, prevValue=self._name, timeout=1) def cancel_initialization(self) -> bool: return self._sync_obj.delete(self.initialize_path) def delete_cluster(self) -> bool: return self._sync_obj.delete(self.client_path(''), recursive=True) def set_history_value(self, value: str) -> bool: return self._sync_obj.set(self.history_path, value) is not False def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]: ret = self._sync_obj.set(self.sync_path, value, prevIndex=version) if isinstance(ret, dict): return ret['index'] return ret def delete_sync_state(self, version: Optional[int] = None) -> bool: return self._sync_obj.delete(self.sync_path, prevIndex=version) def watch(self, leader_version: Optional[int], timeout: float) -> bool: try: return super(Raft, self).watch(leader_version, timeout) finally: self.event.clear() patroni-4.0.4/patroni/dcs/zookeeper.py000066400000000000000000000511601472010352700200010ustar00rootroot00000000000000import json import logging import select import socket import time from typing import Any, Callable, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union from kazoo.client import KazooClient, KazooRetry, KazooState from kazoo.exceptions import ConnectionClosedError, NodeExistsError, NoNodeError, SessionExpiredError from kazoo.handlers.threading import AsyncResult, SequentialThreadingHandler from kazoo.protocol.states import KeeperState, WatchedEvent, ZnodeStat from kazoo.retry import RetryFailedError from kazoo.security import ACL, make_acl from ..exceptions import DCSError from ..postgresql.mpp import AbstractMPP from ..utils import deep_compare from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory if TYPE_CHECKING: # pragma: no cover from ..config import Config logger = logging.getLogger(__name__) class ZooKeeperError(DCSError): pass class PatroniSequentialThreadingHandler(SequentialThreadingHandler): def __init__(self, connect_timeout: Union[int, float]) -> None: super(PatroniSequentialThreadingHandler, self).__init__() self.set_connect_timeout(connect_timeout) def set_connect_timeout(self, connect_timeout: Union[int, float]) -> None: self._connect_timeout = max(1.0, connect_timeout / 2.0) # try to connect to zookeeper node during loop_wait/2 def create_connection(self, *args: Any, **kwargs: Any) -> socket.socket: """This method is trying to establish connection with one of the zookeeper nodes. Somehow strategy "fail earlier and retry more often" works way better comparing to the original strategy "try to connect with specified timeout". Since we want to try connect to zookeeper more often (with the smaller connect_timeout), he have to override `create_connection` method in the `SequentialThreadingHandler` class (which is used by `kazoo.Client`). :param args: always contains `tuple(host, port)` as the first element and could contain `connect_timeout` (negotiated session timeout) as the second element.""" args_list: List[Any] = list(args) if len(args_list) == 0: # kazoo 2.6.0 slightly changed the way how it calls create_connection method kwargs['timeout'] = max(self._connect_timeout, kwargs.get('timeout', self._connect_timeout * 10) / 10.0) elif len(args_list) == 1: args_list.append(self._connect_timeout) else: args_list[1] = max(self._connect_timeout, args_list[1] / 10.0) return super(PatroniSequentialThreadingHandler, self).create_connection(*args_list, **kwargs) def select(self, *args: Any, **kwargs: Any) -> Any: """ Python 3.XY may raise following exceptions if select/poll are called with an invalid socket: - `ValueError`: because fd == -1 - `TypeError`: Invalid file descriptor: -1 (starting from kazoo 2.9) Python 2.7 may raise the `IOError` instead of `socket.error` (starting from kazoo 2.9) When it is appropriate we map these exceptions to `socket.error`. """ try: return super(PatroniSequentialThreadingHandler, self).select(*args, **kwargs) except (TypeError, ValueError) as e: raise select.error(9, str(e)) class PatroniKazooClient(KazooClient): def _call(self, request: Tuple[Any], async_object: AsyncResult) -> Optional[bool]: # Before kazoo==2.7.0 it wasn't possible to send requests to zookeeper if # the connection is in the SUSPENDED state and Patroni was strongly relying on it. # The https://github.com/python-zk/kazoo/pull/588 changed it, and now such requests are queued. # We override the `_call()` method in order to keep the old behavior. if self._state == KeeperState.CONNECTING: async_object.set_exception(SessionExpiredError()) return False return super(PatroniKazooClient, self)._call(request, async_object) class ZooKeeper(AbstractDCS): def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: super(ZooKeeper, self).__init__(config, mpp) hosts: Union[str, List[str]] = config.get('hosts', []) if isinstance(hosts, list): hosts = ','.join(hosts) mapping = {'use_ssl': 'use_ssl', 'verify': 'verify_certs', 'cacert': 'ca', 'cert': 'certfile', 'key': 'keyfile', 'key_password': 'keyfile_password'} kwargs = {v: config[k] for k, v in mapping.items() if k in config} if 'set_acls' in config: default_acl: List[ACL] = [] for principal, permissions in config['set_acls'].items(): normalizedPermissions = [p.upper() for p in permissions] default_acl.append(make_acl(scheme='x509', credential=principal, read='READ' in normalizedPermissions, write='WRITE' in normalizedPermissions, create='CREATE' in normalizedPermissions, delete='DELETE' in normalizedPermissions, admin='ADMIN' in normalizedPermissions, all='ALL' in normalizedPermissions)) kwargs['default_acl'] = default_acl self._client = PatroniKazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']), timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1, sleep_func=time.sleep), command_retry=KazooRetry(max_delay=1, max_tries=-1, deadline=config['retry_timeout'], sleep_func=time.sleep), auth_data=list(config.get('auth_data', {}).items()), **kwargs) self.__last_member_data: Optional[Dict[str, Any]] = None self._orig_kazoo_connect = self._client._connection._connect self._client._connection._connect = self._kazoo_connect self._client.start() def _kazoo_connect(self, *args: Any) -> Tuple[Union[int, float], Union[int, float]]: """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no response on Ping after Ping interval (1/2 from read_timeout) it will consider current connection dead and try to connect to another node. Without this "magic" it was taking up to 2/3 from session timeout (ttl) to figure out that connection was dead and we had only small time for reconnect and retry. This method is needed to return different value of read_timeout, which is not calculated from negotiated session timeout but from value of `loop_wait`. And it is 2 sec smaller than loop_wait, because we can spend up to 2 seconds when calling `touch_member()` and `write_leader_optime()` methods, which also may hang...""" ret = self._orig_kazoo_connect(*args) return max(self.loop_wait - 2, 2) * 1000, ret[1] def _watcher(self, event: WatchedEvent) -> None: if event.state != KazooState.CONNECTED or event.path.startswith(self.client_path('')): self.event.set() def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None: self.set_retry_timeout(config['retry_timeout']) loop_wait = config['loop_wait'] loop_wait_changed = self._loop_wait != loop_wait self._loop_wait = loop_wait if isinstance(self._client.handler, PatroniSequentialThreadingHandler): self._client.handler.set_connect_timeout(loop_wait) # We need to reestablish connection to zookeeper if we want to change # read_timeout (and Ping interval respectively), because read_timeout # is calculated in `_kazoo_connect` method. If we are changing ttl at # the same time, set_ttl method will reestablish connection and return # `!True`, otherwise we will close existing connection and let kazoo # open the new one. if not self.set_ttl(config['ttl']) and loop_wait_changed: self._client._connection._socket.close() def set_ttl(self, ttl: int) -> Optional[bool]: """It is not possible to change ttl (session_timeout) in zookeeper without destroying old session and creating the new one. This method returns `!True` if session_timeout has been changed (`restart()` has been called).""" ttl = int(ttl * 1000) if self._client._session_timeout != ttl: self._client._session_timeout = ttl self._client.restart() return True @property def ttl(self) -> int: return int(self._client._session_timeout / 1000.0) def set_retry_timeout(self, retry_timeout: int) -> None: old_kazoo = isinstance(self._client.retry, KazooRetry) # pyright: ignore [reportUnnecessaryIsInstance] retry = cast(KazooRetry, self._client.retry) if old_kazoo else self._client._retry retry.deadline = retry_timeout def get_node( self, key: str, watch: Optional[Callable[[WatchedEvent], None]] = None ) -> Optional[Tuple[str, ZnodeStat]]: try: ret = self._client.get(key, watch) return (ret[0].decode('utf-8'), ret[1]) except NoNodeError: return None def get_status(self, path: str, leader: Optional[Leader]) -> Status: status = self.get_node(path + self._STATUS) if not status: status = self.get_node(path + self._LEADER_OPTIME) return Status.from_node(status and status[0]) @staticmethod def member(name: str, value: str, znode: ZnodeStat) -> Member: return Member.from_node(znode.version, name, znode.ephemeralOwner, value) def get_children(self, key: str) -> List[str]: try: return self._client.get_children(key) except NoNodeError: return [] def load_members(self, path: str) -> List[Member]: members: List[Member] = [] for member in self.get_children(path + self._MEMBERS): data = self.get_node(path + self._MEMBERS + member) if data is not None: members.append(self.member(member, *data)) return members def _postgresql_cluster_loader(self, path: str) -> Cluster: """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster. :param path: the path in DCS where to load :class:`Cluster` from. :returns: :class:`Cluster` instance. """ nodes = set(self.get_children(path)) # get initialize flag initialize = (self.get_node(path + self._INITIALIZE) or [None])[0] if self._INITIALIZE in nodes else None # get global dynamic configuration config = self.get_node(path + self._CONFIG, watch=self._watcher) if self._CONFIG in nodes else None config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid) # get timeline history history = self.get_node(path + self._HISTORY) if self._HISTORY in nodes else None history = history and TimelineHistory.from_node(history[1].mzxid, history[0]) # get synchronization state sync = self.get_node(path + self._SYNC) if self._SYNC in nodes else None sync = SyncState.from_node(sync and sync[1].version, sync and sync[0]) # get list of members members = self.load_members(path) if self._MEMBERS[:-1] in nodes else [] # get leader leader = self.get_node(path + self._LEADER, watch=self._watcher) if self._LEADER in nodes else None if leader: member = Member(-1, leader[0], None, {}) member = ([m for m in members if m.name == leader[0]] or [member])[0] leader = Leader(leader[1].version, leader[1].ephemeralOwner, member) # get last known leader lsn and slots status = self.get_status(path, leader) # failover key failover = self.get_node(path + self._FAILOVER) if self._FAILOVER in nodes else None failover = failover and Failover.from_node(failover[1].version, failover[0]) # get failsafe topology failsafe = self.get_node(path + self._FAILSAFE) if self._FAILSAFE in nodes else None try: failsafe = json.loads(failsafe[0]) if failsafe else None except Exception: failsafe = None return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]: """Load and build all PostgreSQL clusters from a single MPP cluster. :param path: the path in DCS where to load Cluster(s) from. :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values. """ ret: Dict[int, Cluster] = {} for node in self.get_children(path): if self._mpp.group_re.match(node): ret[int(node)] = self._postgresql_cluster_loader(path + node + '/') return ret def _load_cluster( self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]] ) -> Union[Cluster, Dict[int, Cluster]]: try: return self._client.retry(loader, path) except Exception: logger.exception('get_cluster') raise ZooKeeperError('ZooKeeper in not responding properly') def _create(self, path: str, value: bytes, retry: bool = False, ephemeral: bool = False) -> bool: try: if retry: self._client.retry(self._client.create, path, value, makepath=True, ephemeral=ephemeral) else: self._client.create_async(path, value, makepath=True, ephemeral=ephemeral).get(timeout=1) return True except Exception: logger.exception('Failed to create %s', path) return False def attempt_to_acquire_leader(self) -> bool: try: self._client.retry(self._client.create, self.leader_path, self._name.encode('utf-8'), makepath=True, ephemeral=True) return True except (ConnectionClosedError, RetryFailedError) as e: raise ZooKeeperError(e) except Exception as e: if not isinstance(e, NodeExistsError): logger.error('Failed to create %s: %r', self.leader_path, e) logger.info('Could not take out TTL lock') return False def _set_or_create(self, key: str, value: str, version: Optional[int] = None, retry: bool = False, do_not_create_empty: bool = False) -> Union[int, bool]: value_bytes = value.encode('utf-8') try: if retry: ret = self._client.retry(self._client.set, key, value_bytes, version=version or -1) else: ret = self._client.set_async(key, value_bytes, version=version or -1).get(timeout=1) return ret.version except NoNodeError: if do_not_create_empty and not value_bytes: return True elif version is None: if self._create(key, value_bytes, retry): return 0 else: return False except Exception: logger.exception('Failed to update %s', key) return False def set_failover_value(self, value: str, version: Optional[int] = None) -> bool: return self._set_or_create(self.failover_path, value, version) is not False def set_config_value(self, value: str, version: Optional[int] = None) -> bool: return self._set_or_create(self.config_path, value, version, retry=True) is not False def initialize(self, create_new: bool = True, sysid: str = "") -> bool: sysid_bytes = sysid.encode('utf-8') return self._create(self.initialize_path, sysid_bytes, retry=True) if create_new \ else self._client.retry(self._client.set, self.initialize_path, sysid_bytes) def touch_member(self, data: Dict[str, Any]) -> bool: cluster = self.cluster member = cluster and cluster.get_member(self._name, fallback_to_leader=False) member_data = self.__last_member_data or member and member.data if member and member_data: # We want delete the member ZNode if our session doesn't match with session id on our member key if self._client.client_id is not None and member.session != self._client.client_id[0]: logger.warning('Recreating the member ZNode due to ownership mismatch') try: self._client.delete_async(self.member_path).get(timeout=1) except NoNodeError: pass except Exception: return False member = None encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8') if member and member_data: if deep_compare(data, member_data): return True else: try: self._client.create_async(self.member_path, encoded_data, makepath=True, ephemeral=True).get(timeout=1) self.__last_member_data = data return True except Exception as e: if not isinstance(e, NodeExistsError): logger.exception('touch_member') return False try: self._client.set_async(self.member_path, encoded_data).get(timeout=1) self.__last_member_data = data return True except Exception: logger.exception('touch_member') return False def take_leader(self) -> bool: return self.attempt_to_acquire_leader() def _write_leader_optime(self, last_lsn: str) -> bool: return self._set_or_create(self.leader_optime_path, last_lsn) is not False def _write_status(self, value: str) -> bool: return self._set_or_create(self.status_path, value) is not False def _write_failsafe(self, value: str) -> bool: return self._set_or_create(self.failsafe_path, value) is not False def _update_leader(self, leader: Leader) -> bool: if self._client.client_id and self._client.client_id[0] != leader.session: logger.warning('Recreating the leader ZNode due to ownership mismatch') try: self._client.retry(self._client.delete, self.leader_path) except NoNodeError: pass except (ConnectionClosedError, RetryFailedError) as e: raise ZooKeeperError(e) except Exception as e: logger.error('Failed to remove %s: %r', self.leader_path, e) return False try: self._client.retry(self._client.create, self.leader_path, self._name.encode('utf-8'), makepath=True, ephemeral=True) except (ConnectionClosedError, RetryFailedError) as e: raise ZooKeeperError(e) except Exception as e: logger.error('Failed to create %s: %r', self.leader_path, e) return False return True def _delete_leader(self, leader: Leader) -> bool: self._client.restart() return True def _cancel_initialization(self) -> None: node = self.get_node(self.initialize_path) if node: self._client.delete(self.initialize_path, version=node[1].version) def cancel_initialization(self) -> bool: try: self._client.retry(self._cancel_initialization) return True except Exception: logger.exception("Unable to delete initialize key") return False def delete_cluster(self) -> bool: try: return self._client.retry(self._client.delete, self.client_path(''), recursive=True) except NoNodeError: return True def set_history_value(self, value: str) -> bool: return self._set_or_create(self.history_path, value) is not False def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]: return self._set_or_create(self.sync_path, value, version, retry=True, do_not_create_empty=True) def delete_sync_state(self, version: Optional[int] = None) -> bool: return self.set_sync_state_value("{}", version) is not False def watch(self, leader_version: Optional[int], timeout: float) -> bool: if leader_version: timeout += 0.5 try: return super(ZooKeeper, self).watch(leader_version, timeout) finally: self.event.clear() patroni-4.0.4/patroni/dynamic_loader.py000066400000000000000000000103421472010352700201740ustar00rootroot00000000000000"""Helper functions to search for implementations of specific abstract interface in a package.""" import importlib import inspect import logging import os import pkgutil import sys from types import ModuleType from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Type, TYPE_CHECKING, TypeVar, Union if TYPE_CHECKING: # pragma: no cover from .config import Config logger = logging.getLogger(__name__) def iter_modules(package: str) -> List[str]: """Get names of modules from *package*, depending on execution environment. .. note:: If being packaged with PyInstaller, modules aren't discoverable dynamically by scanning source directory because :class:`importlib.machinery.FrozenImporter` doesn't implement :func:`iter_modules`. But it is still possible to find all potential modules by iterating through ``toc``, which contains list of all "frozen" resources. :param package: a package name to search modules in, e.g. ``patroni.dcs``. :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``. """ module_prefix = package + '.' if getattr(sys, 'frozen', False): toc: Set[str] = set() # dirname may contain a few dots, which causes pkgutil.iter_importers() # to misinterpret the path as a package name. This can be avoided # altogether by not passing a path at all, because PyInstaller's # FrozenImporter is a singleton and registered as top-level finder. for importer in pkgutil.iter_importers(): if hasattr(importer, 'toc'): toc |= getattr(importer, 'toc') dots = module_prefix.count('.') # search for modules only on the same level return [module for module in toc if module.startswith(module_prefix) and module.count('.') == dots] # here we are making an assumption that the package which is calling this function is already imported pkg_file = sys.modules[package].__file__ if TYPE_CHECKING: # pragma: no cover assert isinstance(pkg_file, str) return [name for _, name, is_pkg in pkgutil.iter_modules([os.path.dirname(pkg_file)], module_prefix) if not is_pkg] ClassType = TypeVar("ClassType") def find_class_in_module(module: ModuleType, cls_type: Type[ClassType]) -> Optional[Type[ClassType]]: """Try to find the implementation of *cls_type* class interface in *module* matching the *module* name. :param module: imported module. :param cls_type: a class type we are looking for. :returns: class with a name matching the name of *module* that implements *cls_type* or ``None`` if not found. """ module_name = module.__name__.rpartition('.')[2] return next( (obj for obj_name, obj in module.__dict__.items() if (obj_name.lower() == module_name and inspect.isclass(obj) and issubclass(obj, cls_type))), None) def iter_classes( package: str, cls_type: Type[ClassType], config: Optional[Union['Config', Dict[str, Any]]] = None ) -> Iterator[Tuple[str, Type[ClassType]]]: """Attempt to import modules and find implementations of *cls_type* that are present in the given configuration. .. note:: If a module successfully imports we can assume that all its requirements are installed. :param package: a package name to search modules in, e.g. ``patroni.dcs``. :param cls_type: a class type we are looking for. :param config: configuration information with possible module names as keys. If given, only attempt to import modules defined in the configuration. Else, if ``None``, attempt to import any supported module. :yields: a tuple containing the module ``name`` and the imported class object. """ for mod_name in iter_modules(package): name = mod_name.rpartition('.')[2] if config is None or name in config: try: module = importlib.import_module(mod_name) module_cls = find_class_in_module(module, cls_type) if module_cls: yield name, module_cls except ImportError: logger.log(logging.DEBUG if config is not None else logging.INFO, 'Failed to import %s', mod_name) patroni-4.0.4/patroni/exceptions.py000066400000000000000000000024341472010352700174060ustar00rootroot00000000000000"""Implement high-level Patroni exceptions. More specific exceptions can be found in other modules, as subclasses of any exception defined in this module. """ from typing import Any class PatroniException(Exception): """Parent class for all kind of Patroni exceptions. :ivar value: description of the exception. """ def __init__(self, value: Any) -> None: """Create a new instance of :class:`PatroniException` with the given description. :param value: description of the exception. """ self.value = value class PatroniFatalException(PatroniException): """Catastrophic exception that prevents Patroni from performing its job.""" pass class PostgresException(PatroniException): """Any exception related with Postgres management.""" pass class DCSError(PatroniException): """Parent class for all kind of DCS related exceptions.""" pass class PostgresConnectionException(PostgresException): """Any problem faced while connecting to a Postgres instance.""" pass class WatchdogError(PatroniException): """Any problem faced while managing a watchdog device.""" pass class ConfigParseError(PatroniException): """Any issue identified while loading or validating the Patroni configuration.""" pass patroni-4.0.4/patroni/file_perm.py000066400000000000000000000073511472010352700171720ustar00rootroot00000000000000"""Helper object that helps with figuring out file and directory permissions based on permissions of PGDATA. :var logger: logger of this module. :var pg_perm: instance of the :class:`__FilePermissions` object. """ import logging import os import stat logger = logging.getLogger(__name__) class __FilePermissions: """Helper class for managing permissions of directories and files under PGDATA. Execute :meth:`set_permissions_from_data_directory` to figure out which permissions should be used for files and directories under PGDATA based on permissions of PGDATA root directory. """ # Mode mask for data directory permissions that only allows the owner to # read/write directories and files -- mask 077. __PG_MODE_MASK_OWNER = stat.S_IRWXG | stat.S_IRWXO # Mode mask for data directory permissions that also allows group read/execute -- mask 027. __PG_MODE_MASK_GROUP = stat.S_IWGRP | stat.S_IRWXO # Default mode for creating directories -- mode 700. __PG_DIR_MODE_OWNER = stat.S_IRWXU # Mode for creating directories that allows group read/execute -- mode 750. __PG_DIR_MODE_GROUP = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP # Default mode for creating files -- mode 600. __PG_FILE_MODE_OWNER = stat.S_IRUSR | stat.S_IWUSR # Mode for creating files that allows group read -- mode 640. __PG_FILE_MODE_GROUP = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP def __init__(self) -> None: """Create a :class:`__FilePermissions` object and set default permissions.""" self.__set_owner_permissions() self.__orig_umask = self.__set_umask() def __set_umask(self) -> int: """Set umask value based on calculations. .. note:: Should only be called once either :meth:`__set_owner_permissions` or :meth:`__set_group_permissions` has been executed. :returns: the previous value of the umask or ``0022`` if umask call failed. """ try: return os.umask(self.__pg_mode_mask) except Exception as e: logger.error('Can not set umask to %03o: %r', self.__pg_mode_mask, e) return 0o22 @property def orig_umask(self) -> int: """Original umask value.""" return self.__orig_umask def __set_owner_permissions(self) -> None: """Make directories/files accessible only by the owner.""" self.__pg_dir_create_mode = self.__PG_DIR_MODE_OWNER self.__pg_file_create_mode = self.__PG_FILE_MODE_OWNER self.__pg_mode_mask = self.__PG_MODE_MASK_OWNER def __set_group_permissions(self) -> None: """Make directories/files accessible by the owner and readable by group.""" self.__pg_dir_create_mode = self.__PG_DIR_MODE_GROUP self.__pg_file_create_mode = self.__PG_FILE_MODE_GROUP self.__pg_mode_mask = self.__PG_MODE_MASK_GROUP def set_permissions_from_data_directory(self, data_dir: str) -> None: """Set new permissions based on provided *data_dir*. :param data_dir: reference to PGDATA to calculate permissions from. """ try: st = os.stat(data_dir) if (st.st_mode & self.__PG_DIR_MODE_GROUP) == self.__PG_DIR_MODE_GROUP: self.__set_group_permissions() else: self.__set_owner_permissions() except Exception as e: logger.error('Can not check permissions on %s: %r', data_dir, e) else: self.__set_umask() @property def dir_create_mode(self) -> int: """Directory permissions.""" return self.__pg_dir_create_mode @property def file_create_mode(self) -> int: """File permissions.""" return self.__pg_file_create_mode pg_perm = __FilePermissions() patroni-4.0.4/patroni/global_config.py000066400000000000000000000226401472010352700200130ustar00rootroot00000000000000"""Implements *global_config* facilities. The :class:`GlobalConfig` object is instantiated on import and replaces ``patroni.global_config`` module in :data:`sys.modules`, what allows to use its properties and methods like they were module variables and functions. """ import sys import types from copy import deepcopy from typing import Any, cast, Dict, List, Optional, TYPE_CHECKING from .collections import EMPTY_DICT from .utils import parse_bool, parse_int if TYPE_CHECKING: # pragma: no cover from .dcs import Cluster def __getattr__(mod: types.ModuleType, name: str) -> Any: """This function exists just to make pyright happy. Without it pyright complains about access to unknown members of global_config module. """ return getattr(sys.modules[__name__], name) # pragma: no cover class GlobalConfig(types.ModuleType): """A class that wraps global configuration and provides convenient methods to access/check values.""" __file__ = __file__ # just to make unittest and pytest happy def __init__(self) -> None: """Initialize :class:`GlobalConfig` object.""" super().__init__(__name__) self.__config = {} @staticmethod def _cluster_has_valid_config(cluster: Optional['Cluster']) -> bool: """Check if provided *cluster* object has a valid global configuration. :param cluster: the currently known cluster state from DCS. :returns: ``True`` if provided *cluster* object has a valid global configuration, otherwise ``False``. """ return bool(cluster and cluster.config and cluster.config.modify_version) def update(self, cluster: Optional['Cluster'], default: Optional[Dict[str, Any]] = None) -> None: """Update with the new global configuration from the :class:`Cluster` object view. .. note:: Update happens in-place and is executed only from the main heartbeat thread. :param cluster: the currently known cluster state from DCS. :param default: default configuration, which will be used if there is no valid *cluster.config*. """ # Try to protect from the case when DCS was wiped out if self._cluster_has_valid_config(cluster): self.__config = cluster.config.data # pyright: ignore [reportOptionalMemberAccess] elif default: self.__config = default def from_cluster(self, cluster: Optional['Cluster']) -> 'GlobalConfig': """Return :class:`GlobalConfig` instance from the provided :class:`Cluster` object view. .. note:: If the provided *cluster* object doesn't have a valid global configuration we return the last known valid state of the :class:`GlobalConfig` object. This method is used when we need to have the most up-to-date values in the global configuration, but we don't want to update the global object. :param cluster: the currently known cluster state from DCS. :returns: :class:`GlobalConfig` object. """ if not self._cluster_has_valid_config(cluster): return self ret = GlobalConfig() ret.update(cluster) return ret def get(self, name: str) -> Any: """Gets global configuration value by *name*. :param name: parameter name. :returns: configuration value or ``None`` if it is missing. """ return self.__config.get(name) def check_mode(self, mode: str) -> bool: """Checks whether the certain parameter is enabled. :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and so on. :returns: ``True`` if parameter *mode* is enabled in the global configuration. """ return bool(parse_bool(self.__config.get(mode))) @property def is_paused(self) -> bool: """``True`` if cluster is in maintenance mode.""" return self.check_mode('pause') @property def is_quorum_commit_mode(self) -> bool: """:returns: ``True`` if quorum commit replication is requested""" return str(self.get('synchronous_mode')).lower() == 'quorum' @property def is_synchronous_mode(self) -> bool: """``True`` if synchronous replication is requested and it is not a standby cluster config.""" return (self.check_mode('synchronous_mode') is True or self.is_quorum_commit_mode) \ and not self.is_standby_cluster @property def is_synchronous_mode_strict(self) -> bool: """``True`` if at least one synchronous node is required.""" return self.check_mode('synchronous_mode_strict') def get_standby_cluster_config(self) -> Any: """Get ``standby_cluster`` configuration. :returns: a copy of ``standby_cluster`` configuration. """ return deepcopy(self.get('standby_cluster')) @property def is_standby_cluster(self) -> bool: """``True`` if global configuration has a valid ``standby_cluster`` section.""" config = self.get_standby_cluster_config() return isinstance(config, dict) and\ any(cast(Dict[str, Any], config).get(p) for p in ('host', 'port', 'restore_command')) def get_int(self, name: str, default: int = 0, base_unit: Optional[str] = None) -> int: """Gets current value of *name* from the global configuration and try to return it as :class:`int`. :param name: name of the parameter. :param default: default value if *name* is not in the configuration or invalid. :param base_unit: an optional base unit to convert value of *name* parameter to. Not used if the value does not contain a unit. :returns: currently configured value of *name* from the global configuration or *default* if it is not set or invalid. """ ret = parse_int(self.get(name), base_unit) return default if ret is None else ret @property def min_synchronous_nodes(self) -> int: """The minimum number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not.""" return 1 if self.is_synchronous_mode_strict else 0 @property def synchronous_node_count(self) -> int: """Currently configured value of ``synchronous_node_count`` from the global configuration. Assume ``1`` if it is not set or invalid. """ return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes) @property def maximum_lag_on_failover(self) -> int: """Currently configured value of ``maximum_lag_on_failover`` from the global configuration. Assume ``1048576`` if it is not set or invalid. """ return self.get_int('maximum_lag_on_failover', 1048576) @property def maximum_lag_on_syncnode(self) -> int: """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration. Assume ``-1`` if it is not set or invalid. """ return self.get_int('maximum_lag_on_syncnode', -1) @property def primary_start_timeout(self) -> int: """Currently configured value of ``primary_start_timeout`` from the global configuration. Assume ``300`` if it is not set or invalid. .. note:: ``master_start_timeout`` is still supported to keep backward compatibility. """ default = 300 return self.get_int('primary_start_timeout', default)\ if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default) @property def primary_stop_timeout(self) -> int: """Currently configured value of ``primary_stop_timeout`` from the global configuration. Assume ``0`` if it is not set or invalid. .. note:: ``master_stop_timeout`` is still supported to keep backward compatibility. """ default = 0 return self.get_int('primary_stop_timeout', default)\ if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default) @property def ignore_slots_matchers(self) -> List[Dict[str, Any]]: """Currently configured value of ``ignore_slots`` from the global configuration. Assume an empty :class:`list` if not set. """ return self.get('ignore_slots') or [] @property def max_timelines_history(self) -> int: """Currently configured value of ``max_timelines_history`` from the global configuration. Assume ``0`` if not set or invalid. """ return self.get_int('max_timelines_history', 0) @property def use_slots(self) -> bool: """``True`` if cluster is configured to use replication slots.""" return bool(parse_bool((self.get('postgresql') or EMPTY_DICT).get('use_slots', True))) @property def permanent_slots(self) -> Dict[str, Any]: """Dictionary of permanent slots information from the global configuration.""" return deepcopy(self.get('permanent_replication_slots') or self.get('permanent_slots') or self.get('slots') or EMPTY_DICT.copy()) @property def member_slots_ttl(self) -> int: """Currently configured value of ``member_slots_ttl`` from the global configuration converted to seconds. Assume ``1800`` if it is not set or invalid. """ return self.get_int('member_slots_ttl', 1800, base_unit='s') sys.modules[__name__] = GlobalConfig() patroni-4.0.4/patroni/ha.py000066400000000000000000003716071472010352700156300ustar00rootroot00000000000000import datetime import functools import json import logging import sys import time import uuid from multiprocessing.pool import ThreadPool from threading import RLock from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union from . import global_config, psycopg from .__main__ import Patroni from .async_executor import AsyncExecutor, CriticalTask from .collections import CaseInsensitiveSet from .dcs import AbstractDCS, Cluster, Leader, Member, RemoteMember, Status, SyncState from .exceptions import DCSError, PatroniFatalException, PostgresConnectionException from .postgresql.callback_executor import CallbackAction from .postgresql.misc import postgres_version_to_int from .postgresql.postmaster import PostmasterProcess from .postgresql.rewind import Rewind from .quorum import QuorumStateResolver from .tags import Tags from .utils import parse_int, polling_loop, tzutc logger = logging.getLogger(__name__) class _MemberStatus(Tags, NamedTuple('_MemberStatus', [('member', Member), ('reachable', bool), ('in_recovery', Optional[bool]), ('wal_position', int), ('data', Dict[str, Any])])): """Node status distilled from API response. Consists of the following fields: :ivar member: :class:`~patroni.dcs.Member` object of the node. :ivar reachable: ``False`` if the node is not reachable or is not responding with correct JSON. :ivar in_recovery: ``False`` if the node is running as a primary (`if pg_is_in_recovery() == true`). :ivar wal_position: maximum value of ``replayed_location`` or ``received_location`` from JSON. :ivar data: the whole JSON response for future usage. """ @classmethod def from_api_response(cls, member: Member, json: Dict[str, Any]) -> '_MemberStatus': """ :param member: dcs.Member object :param json: RestApiHandler.get_postgresql_status() result :returns: _MemberStatus object """ # If one of those is not in a response we want to count the node as not healthy/reachable wal: Dict[str, Any] = json.get('wal') or json['xlog'] # abuse difference in primary/replica response format in_recovery = not (bool(wal.get('location')) or json.get('role') in ('master', 'primary')) lsn = int(in_recovery and max(wal.get('received_location', 0), wal.get('replayed_location', 0))) return cls(member, True, in_recovery, lsn, json) @property def tags(self) -> Dict[str, Any]: """Dictionary with values of different tags (i.e. nofailover).""" return self.data.get('tags', {}) @property def timeline(self) -> int: """Timeline value from JSON.""" return self.data.get('timeline', 0) @property def watchdog_failed(self) -> bool: """Indicates that watchdog is required by configuration but not available or failed.""" return self.data.get('watchdog_failed', False) @classmethod def unknown(cls, member: Member) -> '_MemberStatus': """Create a new class instance with empty or null values.""" return cls(member, False, None, 0, {}) def failover_limitation(self) -> Optional[str]: """Returns reason why this node can't promote or None if everything is ok.""" if not self.reachable: return 'not reachable' if self.nofailover: return 'not allowed to promote' if self.watchdog_failed: return 'not watchdog capable' return None class _FailsafeResponse(NamedTuple): """Response on POST ``/failsafe`` API request. Consists of the following fields: :ivar member_name: member name. :ivar accepted: ``True`` if the member agrees that the current primary will continue running, ``False`` otherwise. :ivar lsn: absolute position of received/replayed location in bytes. """ member_name: str accepted: bool lsn: Optional[int] class Failsafe(object): """Object that represents failsafe state of the cluster.""" def __init__(self, dcs: AbstractDCS) -> None: """Initialize the :class:`Failsafe` object. :param dcs: current DCS object, is used only to get current value of ``ttl``. """ self._lock = RLock() self._dcs = dcs self._reset_state() def update_slots(self, slots: Dict[str, int]) -> None: """Assign value to :attr:`_slots`. .. note:: This method is only called on the primary node. :param slots: a :class:`dict` object with member names as keys and received/replayed LSNs as values. """ with self._lock: self._slots = slots def update(self, data: Dict[str, Any]) -> None: """Update the :class:`Failsafe` object state. The last update time is stored and object will be invalidated after ``ttl`` seconds. .. note:: This method is only called as a result of `POST /failsafe` REST API call. :param data: deserialized JSON document from REST API call that contains information about current leader. """ with self._lock: self._last_update = time.time() self._name = data['name'] self._conn_url = data['conn_url'] self._api_url = data['api_url'] self._slots = data.get('slots') def _reset_state(self) -> None: """Reset state of the :class:`Failsafe` object.""" self._last_update = 0 # holds information when failsafe was triggered last time. self._name = '' # name of the cluster leader self._conn_url = None # PostgreSQL conn_url of the leader self._api_url = None # Patroni REST api_url of the leader self._slots = None # state of replication slots on the leader @property def leader(self) -> Optional[Leader]: """Return information about current cluster leader if the failsafe mode is active.""" with self._lock: if self._last_update + self._dcs.ttl > time.time(): return Leader('', '', RemoteMember(self._name, {'api_url': self._api_url, 'conn_url': self._conn_url, 'slots': self._slots})) def update_cluster(self, cluster: Cluster) -> Cluster: """Update and return provided :class:`Cluster` object with fresh values. .. note:: This method is called when failsafe mode is active and is used to update cluster state with fresh values of replication ``slots`` status and ``xlog_location`` on member nodes. :returns: :class:`Cluster` object, either unchanged or updated. """ # Enreach cluster with the real leader if there was a ping from it leader = self.leader if leader: # We rely on the strict order of fields in the namedtuple status = Status(cluster.status[0], leader.member.data['slots'], *cluster.status[2:]) cluster = Cluster(*cluster[0:2], leader, status, *cluster[4:]) # To advance LSN of replication slots on the primary for nodes that are doing cascading # replication from other nodes we need to update `xlog_location` on respective members. for member in cluster.members: if member.replicatefrom and status.slots and member.name in status.slots: member.data['xlog_location'] = status.slots[member.name] return cluster def is_active(self) -> bool: """Check whether the failsafe mode is active. .. note: This method is called from the REST API to report whether the failsafe mode was activated. On primary the :attr:`_last_update` is updated from the :func:`set_is_active` method and always returns the correct value. On replicas the :attr:`_last_update` is updated at the moment when the primary performs ``POST /failsafe`` REST API calls. The side-effect - it is possible that replicas will show ``failsafe_is_active`` values different from the primary. :returns: ``True`` if failsafe mode is active, ``False`` otherwise. """ with self._lock: return self._last_update + self._dcs.ttl > time.time() def set_is_active(self, value: float) -> None: """Update :attr:`_last_update` value. .. note:: This method is only called on the primary. Effectively it sets expiration time of failsafe mode. If the provided value is ``0``, it disables failsafe mode. :param value: time of the last update. """ with self._lock: self._last_update = value if not value: self._reset_state() class Ha(object): def __init__(self, patroni: Patroni): self.patroni = patroni self.state_handler = patroni.postgresql self._rewind = Rewind(self.state_handler) self.dcs = patroni.dcs self.cluster = Cluster.empty() self.old_cluster = Cluster.empty() self._leader_expiry = 0 self._leader_expiry_lock = RLock() self._failsafe = Failsafe(patroni.dcs) self._was_paused = False self._promote_timestamp = 0 self._leader_timeline = None self.recovering = False self._async_response = CriticalTask() self._crash_recovery_started = 0 self._start_timeout = None self._async_executor = AsyncExecutor(self.state_handler.cancellable, self.wakeup) self.watchdog = patroni.watchdog # Each member publishes various pieces of information to the DCS using touch_member. This lock protects # the state and publishing procedure to have consistent ordering and avoid publishing stale values. self._member_state_lock = RLock() # The last know value of current receive/flush/replay LSN. # We update this value from update_lock() and touch_member() methods, because they fetch it anyway. # This value is used to notify the leader when the failsafe_mode is active without performing any queries. self._last_wal_lsn = None # Count of concurrent sync disabling requests. Value above zero means that we don't want to be synchronous # standby. Changes protected by _member_state_lock. self._disable_sync = 0 # Remember the last known member role and state written to the DCS in order to notify MPP coordinator self._last_state = None # We need following property to avoid shutdown of postgres when join of Patroni to the postgres # already running as replica was aborted due to cluster not being initialized in DCS. self._join_aborted = False # used only in backoff after failing a pre_promote script self._released_leader_key_timestamp = 0 # Initialize global config global_config.update(None, self.patroni.config.dynamic_configuration) def primary_stop_timeout(self) -> Union[int, None]: """:returns: "primary_stop_timeout" from the global configuration or `None` when not in synchronous mode.""" ret = global_config.primary_stop_timeout return ret if ret > 0 and self.is_synchronous_mode() else None def is_paused(self) -> bool: """:returns: `True` if in maintenance mode.""" return global_config.is_paused def check_timeline(self) -> bool: """:returns: `True` if should check whether the timeline is latest during the leader race.""" return global_config.check_mode('check_timeline') def is_standby_cluster(self) -> bool: """:returns: `True` if global configuration has a valid "standby_cluster" section.""" return global_config.is_standby_cluster def is_leader(self) -> bool: """:returns: `True` if the current node is the leader, based on expiration set when it last held the key.""" with self._leader_expiry_lock: return self._leader_expiry > time.time() def set_is_leader(self, value: bool) -> None: """Update the current node's view of it's own leadership status. Will update the expiry timestamp to match the dcs ttl if setting leadership to true, otherwise will set the expiry to the past to immediately invalidate. :param value: is the current node the leader. """ with self._leader_expiry_lock: self._leader_expiry = time.time() + self.dcs.ttl if value else 0 if not value: self._promote_timestamp = 0 def sync_mode_is_active(self) -> bool: """Check whether synchronous replication is requested and already active. :returns: ``True`` if the primary already put its name into the ``/sync`` in DCS. """ return self.is_synchronous_mode() and not self.cluster.sync.is_empty def quorum_commit_mode_is_active(self) -> bool: """Checks whether quorum replication is requested and already active. :returns: ``True`` if the primary already put its name into the ``/sync`` in DCS. """ return self.is_quorum_commit_mode() and not self.cluster.sync.is_empty def _get_failover_action_name(self) -> str: """Return the currently requested manual failover action name or the default ``failover``. :returns: :class:`str` representing the manually requested action (``manual failover`` if no leader is specified in the ``/failover`` in DCS, ``switchover`` otherwise) or ``failover`` if ``/failover`` is empty. """ if not self.cluster.failover: return 'failover' return 'switchover' if self.cluster.failover.leader else 'manual failover' def load_cluster_from_dcs(self) -> None: cluster = self.dcs.get_cluster() # We want to keep the state of cluster when it was healthy if not cluster.is_unlocked() or not self.old_cluster: self.old_cluster = cluster self.cluster = cluster if self.cluster.is_unlocked() and self.is_failsafe_mode(): # If failsafe mode is enabled we want to inject the "real" leader to the cluster self.cluster = cluster = self._failsafe.update_cluster(cluster) if not self.has_lock(False): self.set_is_leader(False) self._leader_timeline = cluster.leader.timeline if cluster.leader else None def acquire_lock(self) -> bool: try: ret = self.dcs.attempt_to_acquire_leader() except DCSError: raise except Exception: logger.exception('Unexpected exception raised from attempt_to_acquire_leader, please report it as a BUG') ret = False self.set_is_leader(ret) return ret def _failsafe_config(self) -> Optional[Dict[str, str]]: if self.is_failsafe_mode(): ret = {m.name: m.api_url for m in self.cluster.members if m.api_url} if self.state_handler.name not in ret: ret[self.state_handler.name] = self.patroni.api.connection_string return ret def update_lock(self, update_status: bool = False) -> bool: """Update the leader lock in DCS. .. note:: After successful update of the leader key the :meth:`AbstractDCS.update_leader` method could also optionally update the ``/status`` and ``/failsafe`` keys. The ``/status`` key contains the last known LSN on the leader node and the last known state of permanent replication slots including permanent physical replication slot for the leader. Last, but not least, this method calls a :meth:`Watchdog.keepalive` method after the leader key was successfully updated. :param update_status: ``True`` if we also need to update the ``/status`` key in DCS, otherwise ``False``. :returns: ``True`` if the leader key was successfully updated and we can continue to run postgres as a ``primary`` or as a ``standby_leader``, otherwise ``False``. """ last_lsn = slots = None if update_status: try: last_lsn = self._last_wal_lsn = self.state_handler.last_operation() slots = self.cluster.maybe_filter_permanent_slots(self.state_handler, self.state_handler.slots()) except Exception: logger.exception('Exception when called state_handler.last_operation()') try: ret = self.dcs.update_leader(self.cluster, last_lsn, slots, self._failsafe_config()) except DCSError: raise except Exception: logger.exception('Unexpected exception raised from update_leader, please report it as a BUG') ret = False self.set_is_leader(ret) if ret: self.watchdog.keepalive() return ret def has_lock(self, info: bool = True) -> bool: lock_owner = self.cluster.leader and self.cluster.leader.name if info: logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name) return lock_owner == self.state_handler.name def get_effective_tags(self) -> Dict[str, Any]: """Return configuration tags merged with dynamically applied tags.""" tags = self.patroni.tags.copy() # _disable_sync could be modified concurrently, but we don't care as attribute get and set are atomic. if self._disable_sync > 0: tags['nosync'] = True return tags def notify_mpp_coordinator(self, event: str) -> None: """Send an event to the MPP coordinator. :param event: the type of event for coordinator to parse. """ mpp_handler = self.state_handler.mpp_handler if mpp_handler.is_worker(): coordinator = self.dcs.get_mpp_coordinator() if coordinator and coordinator.leader and coordinator.leader.conn_url: try: data = {'type': event, 'group': mpp_handler.group, 'leader': self.state_handler.name, 'timeout': self.dcs.ttl, 'cooldown': self.patroni.config['retry_timeout']} timeout = self.dcs.ttl if event == 'before_demote' else 2 endpoint = 'citus' if mpp_handler.type == 'Citus' else 'mpp' self.patroni.request(coordinator.leader.member, 'post', endpoint, data, timeout=timeout, retries=0) except Exception as e: logger.warning('Request to %s coordinator leader %s %s failed: %r', mpp_handler.type, coordinator.leader.name, coordinator.leader.member.api_url, e) def touch_member(self) -> bool: with self._member_state_lock: data: Dict[str, Any] = { 'conn_url': self.state_handler.connection_string, 'api_url': self.patroni.api.connection_string, 'state': self.state_handler.state, 'role': self.state_handler.role, 'version': self.patroni.version } proxy_url = self.state_handler.proxy_url if proxy_url: data['proxy_url'] = proxy_url if self.is_leader() and not self._rewind.checkpoint_after_promote(): data['checkpoint_after_promote'] = False tags = self.get_effective_tags() if tags: data['tags'] = tags if self.state_handler.pending_restart_reason: data['pending_restart'] = True data['pending_restart_reason'] = dict(self.state_handler.pending_restart_reason) if self._async_executor.scheduled_action in (None, 'promote') \ and data['state'] in ['running', 'restarting', 'starting']: try: timeline, wal_position, pg_control_timeline = self.state_handler.timeline_wal_position() data['xlog_location'] = self._last_wal_lsn = wal_position if not timeline: # running as a standby replication_state = self.state_handler.replication_state() if replication_state: data['replication_state'] = replication_state # try pg_stat_wal_receiver to get the timeline timeline = self.state_handler.received_timeline() if not timeline: # So far the only way to get the current timeline on the standby is from # the replication connection. In order to avoid opening the replication # connection on every iteration of HA loop we will do it only when noticed # that the timeline on the primary has changed. # Unfortunately such optimization isn't possible on the standby_leader, # therefore we will get the timeline from pg_control, either by calling # pg_control_checkpoint() on 9.6+ or by parsing the output of pg_controldata. if self.state_handler.role == 'standby_leader': timeline = pg_control_timeline or self.state_handler.pg_control_timeline() else: timeline = self.state_handler.replica_cached_timeline(self._leader_timeline) or 0 if timeline: data['timeline'] = timeline except Exception: pass if self.patroni.scheduled_restart: scheduled_restart_data = self.patroni.scheduled_restart.copy() scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat() data['scheduled_restart'] = scheduled_restart_data if self.is_paused(): data['pause'] = True ret = self.dcs.touch_member(data) if ret: new_state = (data['state'], data['role']) if self._last_state != new_state and new_state == ('running', 'primary'): self.notify_mpp_coordinator('after_promote') self._last_state = new_state return ret def clone(self, clone_member: Union[Leader, Member, None] = None, msg: str = '(without leader)') -> Optional[bool]: if self.is_standby_cluster() and not isinstance(clone_member, RemoteMember): clone_member = self.get_remote_member(clone_member) self._rewind.reset_state() if self.state_handler.bootstrap.clone(clone_member): logger.info('bootstrapped %s', msg) cluster = self.dcs.get_cluster() node_to_follow = self._get_node_to_follow(cluster) return self.state_handler.follow(node_to_follow) is not False else: logger.error('failed to bootstrap %s', msg) self.state_handler.remove_data_directory() def bootstrap(self) -> str: # no initialize key and node is allowed to be primary and has 'bootstrap' section in a configuration file if self.cluster.is_unlocked() and self.cluster.initialize is None\ and not self.patroni.nofailover and 'bootstrap' in self.patroni.config: if self.dcs.initialize(create_new=True): # race for initialization self.state_handler.bootstrapping = True with self._async_response: self._async_response.reset() if self.is_standby_cluster(): ret = self._async_executor.try_run_async('bootstrap_standby_leader', self.bootstrap_standby_leader) return ret or 'trying to bootstrap a new standby leader' else: ret = self._async_executor.try_run_async('bootstrap', self.state_handler.bootstrap.bootstrap, args=(self.patroni.config['bootstrap'],)) return ret or 'trying to bootstrap a new cluster' else: return 'failed to acquire initialize lock' clone_member = self.cluster.get_clone_member(self.state_handler.name) # cluster already has a leader, we can bootstrap from it or from one of replicas (if they allow) if not self.cluster.is_unlocked() and clone_member: member_role = 'leader' if clone_member == self.cluster.leader else 'replica' msg = "from {0} '{1}'".format(member_role, clone_member.name) ret = self._async_executor.try_run_async('bootstrap {0}'.format(msg), self.clone, args=(clone_member, msg)) return ret or 'trying to bootstrap {0}'.format(msg) # no leader, but configuration may allowed replica creation using backup tools create_replica_methods = global_config.get_standby_cluster_config().get('create_replica_methods', []) \ if self.is_standby_cluster() else None can_bootstrap = self.state_handler.can_create_replica_without_replication_connection(create_replica_methods) concurrent_bootstrap = self.cluster.initialize == "" if can_bootstrap and not concurrent_bootstrap: msg = 'bootstrap (without leader)' return self._async_executor.try_run_async(msg, self.clone) or 'trying to ' + msg return 'waiting for {0}leader to bootstrap'.format('standby_' if self.is_standby_cluster() else '') def bootstrap_standby_leader(self) -> Optional[bool]: """ If we found 'standby' key in the configuration, we need to bootstrap not a real primary, but a 'standby leader', that will take base backup from a remote member and start follow it. """ clone_source = self.get_remote_member() msg = 'clone from remote member {0}'.format(clone_source.conn_url) result = self.clone(clone_source, msg) with self._async_response: # pretend that post_bootstrap was already executed self._async_response.complete(result) if result: self.state_handler.set_role('standby_leader') return result def _handle_crash_recovery(self) -> Optional[str]: if self._crash_recovery_started == 0 and (self.cluster.is_unlocked() or self._rewind.can_rewind): self._crash_recovery_started = time.time() msg = 'doing crash recovery in a single user mode' return self._async_executor.try_run_async(msg, self._rewind.ensure_clean_shutdown) or msg def _handle_rewind_or_reinitialize(self) -> Optional[str]: leader = self.get_remote_member() if self.is_standby_cluster() else self.cluster.leader if not self._rewind.rewind_or_reinitialize_needed_and_possible(leader) or not leader: return None if self._rewind.can_rewind: # rewind is required, but postgres wasn't shut down cleanly. if not self.state_handler.is_running() and \ self.state_handler.controldata().get('Database cluster state') == 'in archive recovery': msg = self._handle_crash_recovery() if msg: return msg msg = 'running pg_rewind from ' + leader.name return self._async_executor.try_run_async(msg, self._rewind.execute, args=(leader,)) or msg if self._rewind.should_remove_data_directory_on_diverged_timelines and not self.is_standby_cluster(): msg = 'reinitializing due to diverged timelines' return self._async_executor.try_run_async(msg, self._do_reinitialize, args=(self.cluster,)) or msg def recover(self) -> str: """Handle the case when postgres isn't running. Depending on the state of Patroni, DCS cluster view, and pg_controldata the following could happen: - if ``primary_start_timeout`` is 0 and this node owns the leader lock, the lock will be voluntarily released if there are healthy replicas to take it over. - if postgres was running as a ``primary`` and this node owns the leader lock, postgres is started as primary. - crash recover in a single-user mode is executed in the following cases: - postgres was running as ``primary`` wasn't ``shut down`` cleanly and there is no leader in DCS - postgres was running as ``replica`` wasn't ``shut down in recovery`` (cleanly) and we need to run ``pg_rewind`` to join back to the cluster. - ``pg_rewind`` is executed if it is necessary, or optionally, the data directory could be removed if it is allowed by configuration. - after ``crash recovery`` and/or ``pg_rewind`` are executed, postgres is started in recovery. :returns: action message, describing what was performed. """ if self.has_lock() and self.update_lock(): timeout = global_config.primary_start_timeout if timeout == 0: # We are requested to prefer failing over to restarting primary. But see first if there # is anyone to fail over to. if self.is_failover_possible(): self.watchdog.disable() logger.info("Primary crashed. Failing over.") self.demote('immediate') return 'stopped PostgreSQL to fail over after a crash' else: timeout = None data = self.state_handler.controldata() logger.info('pg_controldata:\n%s\n', '\n'.join(' {0}: {1}'.format(k, v) for k, v in data.items())) # timeout > 0 indicates that we still have the leader lock, and it was just updated if timeout\ and data.get('Database cluster state') in ('in production', 'in crash recovery', 'shutting down', 'shut down')\ and self.state_handler.state == 'crashed'\ and self.state_handler.role == 'primary'\ and not self.state_handler.config.recovery_conf_exists(): # We know 100% that we were running as a primary a few moments ago, therefore could just start postgres msg = 'starting primary after failure' if self._async_executor.try_run_async(msg, self.state_handler.start, args=(timeout, self._async_executor.critical_task)) is None: self.recovering = True return msg # Postgres is not running, and we will restart in standby mode. Watchdog is not needed until we promote. self.watchdog.disable() if data.get('Database cluster state') in ('in production', 'shutting down', 'in crash recovery'): msg = self._handle_crash_recovery() if msg: return msg self.load_cluster_from_dcs() role = 'replica' if self.has_lock() and not self.is_standby_cluster(): self._rewind.reset_state() # we want to later trigger CHECKPOINT after promote msg = "starting as readonly because i had the session lock" node_to_follow = None else: if not self._rewind.executed: self._rewind.trigger_check_diverged_lsn() msg = self._handle_rewind_or_reinitialize() if msg: return msg if self.has_lock(): # in standby cluster msg = "starting as a standby leader because i had the session lock" role = 'standby_leader' node_to_follow = self._get_node_to_follow(self.cluster) elif self.is_standby_cluster() and self.cluster.is_unlocked(): msg = "trying to follow a remote member because standby cluster is unhealthy" node_to_follow = self.get_remote_member() else: msg = "starting as a secondary" node_to_follow = self._get_node_to_follow(self.cluster) if self.is_synchronous_mode(): self.state_handler.sync_handler.set_synchronous_standby_names(CaseInsensitiveSet()) if self._async_executor.try_run_async('restarting after failure', self.state_handler.follow, args=(node_to_follow, role, timeout)) is None: self.recovering = True return msg def _get_node_to_follow(self, cluster: Cluster) -> Union[Leader, Member, None]: """Determine the node to follow. :param cluster: the currently known cluster state from DCS. :returns: the node which we should be replicating from. """ # nostream is set, the node must not use WAL streaming if self.patroni.nostream: return None # The standby leader or when there is no standby leader we want to follow # the remote member, except when there is no standby leader in pause. elif self.is_standby_cluster() \ and (cluster.leader and cluster.leader.name and cluster.leader.name == self.state_handler.name or cluster.is_unlocked() and not self.is_paused()): node_to_follow = self.get_remote_member() # If replicatefrom tag is set, try to follow the node mentioned there, otherwise, follow the leader. elif self.patroni.replicatefrom and self.patroni.replicatefrom != self.state_handler.name: node_to_follow = cluster.get_member(self.patroni.replicatefrom) else: node_to_follow = cluster.leader if cluster.leader and cluster.leader.name else None node_to_follow = node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None if node_to_follow and not isinstance(node_to_follow, RemoteMember): # we are going to abuse Member.data to pass following parameters params = ('restore_command', 'archive_cleanup_command') for param in params: # It is highly unlikely to happen, but we want to protect from the case node_to_follow.data.pop(param, None) # when above-mentioned params came from outside. if self.is_standby_cluster(): standby_config = global_config.get_standby_cluster_config() node_to_follow.data.update({p: standby_config[p] for p in params if standby_config.get(p)}) return node_to_follow def follow(self, demote_reason: str, follow_reason: str, refresh: bool = True) -> str: if refresh: self.load_cluster_from_dcs() is_leader = self.state_handler.is_primary() node_to_follow = self._get_node_to_follow(self.cluster) if self.is_paused(): if not (self._rewind.is_needed and self._rewind.can_rewind_or_reinitialize_allowed)\ or self.cluster.is_unlocked(): if is_leader: self.state_handler.set_role('primary') return 'continue to run as primary without lock' elif self.state_handler.role != 'standby_leader': self.state_handler.set_role('replica') if not node_to_follow: return 'no action. I am ({0})'.format(self.state_handler.name) elif is_leader: self.demote('immediate-nolock') return demote_reason if self.is_standby_cluster() and self._leader_timeline and \ self.state_handler.get_history(self._leader_timeline + 1): self._rewind.trigger_check_diverged_lsn() if not self.state_handler.is_starting(): msg = self._handle_rewind_or_reinitialize() if msg: return msg if not self.is_paused(): self.state_handler.handle_parameter_change() role = 'standby_leader' if isinstance(node_to_follow, RemoteMember) and self.has_lock(False) else 'replica' # It might happen that leader key in the standby cluster references non-exiting member. # In this case it is safe to continue running without changing recovery.conf if self.is_standby_cluster() and role == 'replica' and not (node_to_follow and node_to_follow.conn_url): return 'continue following the old known standby leader' else: change_required, restart_required = self.state_handler.config.check_recovery_conf(node_to_follow) if change_required: if restart_required: self._async_executor.try_run_async('changing primary_conninfo and restarting', self.state_handler.follow, args=(node_to_follow, role)) else: self.state_handler.follow(node_to_follow, role, do_reload=True) self._rewind.trigger_check_diverged_lsn() elif role == 'standby_leader' and self.state_handler.role != role: self.state_handler.set_role(role) self.state_handler.call_nowait(CallbackAction.ON_ROLE_CHANGE) return follow_reason def is_synchronous_mode(self) -> bool: """:returns: `True` if synchronous replication is requested.""" return global_config.is_synchronous_mode def is_quorum_commit_mode(self) -> bool: """``True`` if quorum commit replication is requested and "supported".""" return global_config.is_quorum_commit_mode and self.state_handler.supports_multiple_sync def is_failsafe_mode(self) -> bool: """:returns: `True` if failsafe_mode is enabled in global configuration.""" return global_config.check_mode('failsafe_mode') def _maybe_enable_synchronous_mode(self) -> Optional[SyncState]: """Explicitly enable synchronous mode if not yet enabled. We are trying to solve a corner case: synchronous mode needs to be explicitly enabled by updating the ``/sync`` key with the current leader name and empty members. In opposite case it will never be automatically enabled if there are no eligible candidates. :returns: the latest version of :class:`~patroni.dcs.SyncState` object. """ sync = self.cluster.sync if sync.is_empty: sync = self.dcs.write_sync_state(self.state_handler.name, None, 0, version=sync.version) if sync: logger.info("Enabled synchronous replication") else: logger.warning("Updating sync state failed") return sync def disable_synchronous_replication(self) -> None: """Cleans up ``/sync`` key in DCS and updates ``synchronous_standby_names``. .. note:: We fall back to using the value configured by the user for ``synchronous_standby_names``, if any. """ # If synchronous_mode was turned off, we need to update synchronous_standby_names in Postgres if not self.cluster.sync.is_empty and self.dcs.delete_sync_state(version=self.cluster.sync.version): logger.info("Disabled synchronous replication") self.state_handler.sync_handler.set_synchronous_standby_names(CaseInsensitiveSet()) # As synchronous_mode is off, check if the user configured Postgres synchronous replication instead ssn = self.state_handler.config.synchronous_standby_names self.state_handler.config.set_synchronous_standby_names(ssn) def _process_quorum_replication(self) -> None: """Process synchronous replication state when quorum commit is requested. Synchronous standbys are registered in two places: ``postgresql.conf`` and DCS. The order of updating them must keep the invariant that ``quorum + sync >= len(set(quorum pool)|set(sync pool))``. This is done using :class:`QuorumStateResolver` that given a current state and set of desired synchronous nodes and replication level outputs changes to DCS and synchronous replication in correct order to reach the desired state. In case any of those steps causes an error we can just bail out and let next iteration rediscover the state and retry necessary transitions. """ start_time = time.time() min_sync = global_config.min_synchronous_nodes sync_wanted = global_config.synchronous_node_count sync = self._maybe_enable_synchronous_mode() if not sync or not sync.leader: return leader = sync.leader def _check_timeout(offset: float = 0) -> bool: return time.time() - start_time + offset >= self.dcs.loop_wait while True: transition = 'break' # we need define transition value if `QuorumStateResolver` produced no changes sync_state = self.state_handler.sync_handler.current_state(self.cluster) for transition, leader, num, nodes in QuorumStateResolver(leader=leader, quorum=sync.quorum, voters=sync.voters, numsync=sync_state.numsync, sync=sync_state.sync, numsync_confirmed=sync_state.numsync_confirmed, active=sync_state.active, sync_wanted=sync_wanted, leader_wanted=self.state_handler.name): if _check_timeout(): return if transition == 'quorum': logger.info("Setting leader to %s, quorum to %d of %d (%s)", leader, num, len(nodes), ", ".join(sorted(nodes))) sync = self.dcs.write_sync_state(leader, nodes, num, version=sync.version) if not sync: return logger.info('Synchronous replication key updated by someone else.') elif transition == 'sync': logger.info("Setting synchronous replication to %d of %d (%s)", num, len(nodes), ", ".join(sorted(nodes))) # Bump up number of num nodes to meet minimum replication factor. Commits will have to wait until # we have enough nodes to meet replication target. if num < min_sync: logger.warning("Replication factor %d requested, but %d synchronous standbys available." " Commits will be delayed.", min_sync + 1, num) num = min_sync self.state_handler.sync_handler.set_synchronous_standby_names(nodes, num) if transition != 'restart' or _check_timeout(1): return # synchronous_standby_names was transitioned from empty to non-empty and it may take # some time for nodes to become synchronous. In this case we want to restart state machine # hoping that we can update /sync key earlier than in loop_wait seconds. time.sleep(1) self.state_handler.reset_cluster_info_state(None) def _process_multisync_replication(self) -> None: """Process synchronous replication state with one or more sync standbys. Synchronous standbys are registered in two places postgresql.conf and DCS. The order of updating them must be right. The invariant that should be kept is that if a node is primary and sync_standby is set in DCS, then that node must have synchronous_standby set to that value. Or more simple, first set in postgresql.conf and then in DCS. When removing, first remove in DCS, then in postgresql.conf. This is so we only consider promoting standbys that were guaranteed to be replicating synchronously. """ sync = self._maybe_enable_synchronous_mode() if not sync: return current_state = self.state_handler.sync_handler.current_state(self.cluster) picked = current_state.active allow_promote = current_state.sync voters = CaseInsensitiveSet(sync.voters) if picked == voters and voters != allow_promote: logger.warning('Inconsistent state between synchronous_standby_names = %s and /sync = %s key ' 'detected, updating synchronous replication key...', list(allow_promote), list(voters)) sync = self.dcs.write_sync_state(self.state_handler.name, allow_promote, 0, version=sync.version) if not sync: return logger.warning("Updating sync state failed") voters = CaseInsensitiveSet(sync.voters) if picked == voters: return # update synchronous standby list in dcs temporarily to point to common nodes in current and picked sync_common = voters & allow_promote if sync_common != voters: logger.info("Updating synchronous privilege temporarily from %s to %s", list(voters), list(sync_common)) sync = self.dcs.write_sync_state(self.state_handler.name, sync_common, 0, version=sync.version) if not sync: return logger.info('Synchronous replication key updated by someone else.') # When strict mode and no suitable replication connections put "*" to synchronous_standby_names if global_config.is_synchronous_mode_strict and not picked: picked = CaseInsensitiveSet('*') logger.warning("No standbys available!") # Update postgresql.conf and wait 2 secs for changes to become active logger.info("Assigning synchronous standby status to %s", list(picked)) self.state_handler.sync_handler.set_synchronous_standby_names(picked) if picked and picked != CaseInsensitiveSet('*') and allow_promote != picked: # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby time.sleep(2) allow_promote = self.state_handler.sync_handler.current_state(self.cluster).sync if allow_promote and allow_promote != sync_common: if self.dcs.write_sync_state(self.state_handler.name, allow_promote, 0, version=sync.version): logger.info("Synchronous standby status assigned to %s", list(allow_promote)) else: logger.info("Synchronous replication key updated by someone else") def process_sync_replication(self) -> None: """Process synchronous replication behavior on the primary.""" if self.is_quorum_commit_mode(): # The synchronous_standby_names was adjusted right before promote. # After that, when postgres has become a primary, we need to reflect this change # in the /sync key. Further changes of synchronous_standby_names and /sync key should # be postponed for `loop_wait` seconds, to give a chance to some replicas to start streaming. # In opposite case the /sync key will end up without synchronous nodes. if self.state_handler.is_primary(): if self._promote_timestamp == 0 or time.time() - self._promote_timestamp > self.dcs.loop_wait: self._process_quorum_replication() if self._promote_timestamp == 0: self._promote_timestamp = time.time() elif self.is_synchronous_mode(): self._process_multisync_replication() else: self.disable_synchronous_replication() def process_sync_replication_prepromote(self) -> bool: """Handle sync replication state before promote. If quorum replication is requested, and we can keep syncing to enough nodes satisfying the quorum invariant we can promote immediately and let normal quorum resolver process handle any membership changes later. Otherwise, we will just reset DCS state to ourselves and add replicas as they connect. :returns: ``True`` if on success or ``False`` if failed to update /sync key in DCS. """ if not self.is_synchronous_mode(): self.disable_synchronous_replication() return True if self.quorum_commit_mode_is_active(): sync = CaseInsensitiveSet(self.cluster.sync.members) numsync = len(sync) - self.cluster.sync.quorum - 1 if self.state_handler.name not in sync: # Node outside voters achieved quorum and got leader numsync += 1 else: sync.discard(self.state_handler.name) else: sync = CaseInsensitiveSet() numsync = global_config.min_synchronous_nodes if not self.is_quorum_commit_mode() or not self.state_handler.supports_multiple_sync and numsync > 1: sync = CaseInsensitiveSet() numsync = global_config.min_synchronous_nodes # Just set ourselves as the authoritative source of truth for now. We don't want to wait for standbys # to connect. We will try finding a synchronous standby in the next cycle. if not self.dcs.write_sync_state(self.state_handler.name, None, 0, version=self.cluster.sync.version): return False self.state_handler.sync_handler.set_synchronous_standby_names(sync, numsync) return True def is_sync_standby(self, cluster: Cluster) -> bool: """:returns: `True` if the current node is a synchronous standby.""" return bool(cluster.leader) and cluster.sync.leader_matches(cluster.leader.name) \ and cluster.sync.matches(self.state_handler.name) def while_not_sync_standby(self, func: Callable[..., Any]) -> Any: """Runs specified action while trying to make sure that the node is not assigned synchronous standby status. When running in ``synchronous_mode`` with ``synchronous_node_count = 2``, shutdown or restart of a synchronous standby may cause a write downtime. Therefore we need to signal a primary that we don't want to by synchronous anymore and wait until it will replace our name from ``synchronous_standby_names`` and ``/sync`` key in DCS with some other node. Once current node is not synchronous we will run the *func*. .. note:: If the connection to DCS fails we run the *func* anyway, as this is only a hint. There is a small race window where this function runs between a primary picking us the sync standby and publishing it to the DCS. As the window is rather tiny consequences are holding up commits for one cycle period we don't worry about it here. :param func: the function to be executed. :returns: a return value of the *func*. """ if self.is_leader() or not self.is_synchronous_mode() or self.patroni.nosync: return func() with self._member_state_lock: self._disable_sync += 1 try: if self.touch_member(): # Primary should notice the updated value during the next cycle. We will wait double that, if primary # hasn't noticed the value by then not disabling sync replication is not likely to matter. for _ in polling_loop(timeout=self.dcs.loop_wait * 2, interval=2): try: if not self.is_sync_standby(self.dcs.get_cluster()): break except DCSError: logger.warning("Could not get cluster state, skipping synchronous standby disable") break logger.info("Waiting for primary to release us from synchronous standby") else: logger.warning("Updating member state failed, skipping synchronous standby disable") return func() finally: with self._member_state_lock: self._disable_sync -= 1 def update_cluster_history(self) -> None: primary_timeline = self.state_handler.get_primary_timeline() cluster_history = self.cluster.history.lines if self.cluster.history else [] if primary_timeline == 1: if cluster_history: self.dcs.set_history_value('[]') elif not cluster_history or cluster_history[-1][0] != primary_timeline - 1 or len(cluster_history[-1]) != 5: cluster_history_dict: Dict[int, List[Any]] = {line[0]: list(line) for line in cluster_history} history: List[List[Any]] = list(map(list, self.state_handler.get_history(primary_timeline))) if self.cluster.config: history = history[-global_config.max_timelines_history:] for line in history: # enrich current history with promotion timestamps stored in DCS cluster_history_line = cluster_history_dict.get(line[0], []) if len(line) == 3 and len(cluster_history_line) >= 4 and cluster_history_line[1] == line[1]: line.append(cluster_history_line[3]) if len(cluster_history_line) == 5: line.append(cluster_history_line[4]) if history: self.dcs.set_history_value(json.dumps(history, separators=(',', ':'))) def enforce_follow_remote_member(self, message: str) -> str: demote_reason = 'cannot be a real primary in standby cluster' return self.follow(demote_reason, message) def enforce_primary_role(self, message: str, promote_message: str) -> str: """ Ensure the node that has won the race for the leader key meets criteria for promoting its PG server to the 'primary' role. """ if not self.is_paused(): if not self.watchdog.is_running and not self.watchdog.activate(): if self.state_handler.is_primary(): self.demote('immediate') return 'Demoting self because watchdog could not be activated' else: self.release_leader_key_voluntarily() return 'Not promoting self because watchdog could not be activated' with self._async_response: if self._async_response.result is False: logger.warning("Releasing the leader key voluntarily because the pre-promote script failed") self._released_leader_key_timestamp = time.time() self.release_leader_key_voluntarily() # discard the result of the failed pre-promote script to be able to re-try promote self._async_response.reset() return 'Promotion cancelled because the pre-promote script failed' if self.state_handler.is_primary(): # Inform the state handler about its primary role. # It may be unaware of it if postgres is promoted manually. self.state_handler.set_role('primary') self.process_sync_replication() self.update_cluster_history() self.state_handler.mpp_handler.sync_meta_data(self.cluster) return message elif self.state_handler.role in ('primary', 'promoted'): self.process_sync_replication() return message else: if not self.process_sync_replication_prepromote(): # Somebody else updated sync state, it may be due to us losing the lock. To be safe, # postpone promotion until next cycle. TODO: trigger immediate retry of run_cycle. return 'Postponing promotion because synchronous replication state was updated by somebody else' if self.state_handler.role not in ('primary', 'promoted'): # reset failsafe state when promote self._failsafe.set_is_active(0) def before_promote(): self.notify_mpp_coordinator('before_promote') with self._async_response: self._async_response.reset() self._async_executor.try_run_async('promote', self.state_handler.promote, args=(self.dcs.loop_wait, self._async_response, before_promote)) return promote_message def fetch_node_status(self, member: Member) -> _MemberStatus: """Perform http get request on member.api_url to fetch its status. Usually this happens during the leader race and we can't afford to wait an indefinite time for a response, therefore the request timeout is hardcoded to 2 seconds, which seems to be a good compromise. The node which is slow to respond is most likely unhealthy. :returns: :class:`_MemberStatus` object """ try: response = self.patroni.request(member, timeout=2, retries=0) data = response.data.decode('utf-8') logger.info('Got response from %s %s: %s', member.name, member.api_url, data) return _MemberStatus.from_api_response(member, json.loads(data)) except Exception as e: logger.warning("Request failed to %s: GET %s (%s)", member.name, member.api_url, e) return _MemberStatus.unknown(member) def fetch_nodes_statuses(self, members: List[Member]) -> List[_MemberStatus]: if not members: return [] pool = ThreadPool(len(members)) results = pool.map(self.fetch_node_status, members) # Run API calls on members in parallel pool.close() pool.join() return results def update_failsafe(self, data: Dict[str, Any]) -> Union[int, str, None]: """Update failsafe state. :param data: deserialized JSON document from REST API call that contains information about current leader. :returns: the reason why caller shouldn't continue as a primary or the current value of received/replayed LSN. """ if self.state_handler.state == 'running' and self.state_handler.role == 'primary': return 'Running as a leader' self._failsafe.update(data) return self._last_wal_lsn def failsafe_is_active(self) -> bool: return self._failsafe.is_active() def call_failsafe_member(self, data: Dict[str, Any], member: Member) -> _FailsafeResponse: """Call ``POST /failsafe`` REST API request on provided member. :param data: data to be send in the POST request. :returns: a :class:`_FailsafeResponse` object. """ endpoint = 'failsafe' url = member.get_endpoint_url(endpoint) try: response = self.patroni.request(member, 'post', endpoint, data, timeout=2, retries=1) response_data = response.data.decode('utf-8') logger.info('Got response from %s %s: %s', member.name, url, response_data) accepted = response.status == 200 and response_data == 'Accepted' # member may return its current received/replayed LSN in the "lsn" header. return _FailsafeResponse(member.name, accepted, parse_int(response.headers.get('lsn'))) except Exception as e: logger.warning("Request failed to %s: POST %s (%s)", member.name, url, e) return _FailsafeResponse(member.name, False, None) def check_failsafe_topology(self) -> bool: """Check whether we could continue to run as a primary by calling all members from the failsafe topology. .. note:: If the ``/failsafe`` key contains invalid data or if the ``name`` of our node is missing in the ``/failsafe`` key, we immediately give up and return ``False``. We send the JSON document in the POST request with the following fields: * ``name`` - the name of our node; * ``conn_url`` - connection URL to the postgres, which is reachable from other nodes; * ``api_url`` - connection URL to Patroni REST API on this node reachable from other nodes; * ``slots`` - a :class:`dict` with replication slots that exist on the leader node, including the primary itself with the last known LSN, because there could be a permanent physical slot on standby nodes. Standby nodes are using information from the ``slots`` dict to advance position of permanent replication slots while DCS is not accessible in order to avoid indefinite growth of ``pg_wal``. Standby nodes are returning their received/replayed location in the ``lsn`` header, which later are used by the primary to advance position of replication slots that for nodes that are doing cascading replication from other nodes. It is required to avoid indefinite growth of ``pg_wal``. :returns: ``True`` if all members from the ``/failsafe`` topology agree that this node could continue to run as a ``primary``, or ``False`` if some of standby nodes are not accessible or don't agree. """ failsafe = self.dcs.failsafe if not isinstance(failsafe, dict) or self.state_handler.name not in failsafe: return False data: Dict[str, Any] = { 'name': self.state_handler.name, 'conn_url': self.state_handler.connection_string, 'api_url': self.patroni.api.connection_string, } try: data['slots'] = self.state_handler.slots() except Exception: logger.exception('Exception when called state_handler.slots()') members = [RemoteMember(name, {'api_url': url}) for name, url in failsafe.items() if name != self.state_handler.name] if not members: # A single node cluster return True pool = ThreadPool(len(members)) call_failsafe_member = functools.partial(self.call_failsafe_member, data) results: List[_FailsafeResponse] = pool.map(call_failsafe_member, members) pool.close() pool.join() ret = all(r.accepted for r in results) if ret: # The LSN feedback will be later used to advance position of replication slots # for nodes that are doing cascading replication from other nodes. self._failsafe.update_slots({r.member_name: r.lsn for r in results if r.lsn}) return ret def is_lagging(self, wal_position: int) -> bool: """Check if node should consider itself unhealthy to be promoted due to replication lag. :param wal_position: Current wal position. :returns: ``True`` when node is lagging """ lag = self.cluster.status.last_lsn - wal_position return lag > global_config.maximum_lag_on_failover def _is_healthiest_node(self, members: Collection[Member], check_replication_lag: bool = True) -> bool: """Determine whether the current node is healthy enough to become a new leader candidate. :param members: the list of nodes to check against :param check_replication_lag: whether to take the replication lag into account. If the lag exceeds configured threshold the node disqualifies itself. :returns: ``True`` if the node is eligible to become the new leader. Since this method is executed on multiple nodes independently it is possible that multiple nodes could count themselves as the healthiest because they received/replayed up to the same LSN, but this is totally fine. """ my_wal_position = self.state_handler.last_operation() if check_replication_lag and self.is_lagging(my_wal_position): logger.info('My wal position exceeds maximum replication lag') return False # Too far behind last reported wal position on primary if not self.is_standby_cluster() and self.check_timeline(): cluster_timeline = self.cluster.timeline my_timeline = self.state_handler.replica_cached_timeline(cluster_timeline) if my_timeline is None: logger.info('Can not figure out my timeline') return False if my_timeline < cluster_timeline: logger.info('My timeline %s is behind last known cluster timeline %s', my_timeline, cluster_timeline) return False if self.quorum_commit_mode_is_active(): quorum = self.cluster.sync.quorum voting_set = CaseInsensitiveSet(self.cluster.sync.members) else: quorum = 0 voting_set = CaseInsensitiveSet() # Prepare list of nodes to run check against. If quorum commit is enabled # we also include members with nofailover tag if they are listed in voters. members = [m for m in members if m.name != self.state_handler.name and m.api_url and (not m.nofailover or m.name in voting_set)] # If there is a quorum active then at least one of the quorum contains latest commit. A quorum member saying # their WAL position is not ahead counts as a vote saying we may become new leader. Note that a node doesn't # have to be a member of the voting set to gather the necessary votes. # Regardless of voting, if we observe a node that can become a leader and is ahead, we defer to that node. # This can lead to failure to act on quorum if there is asymmetric connectivity. quorum_votes = 0 if self.state_handler.name in voting_set else -1 nodes_ahead = 0 for st in self.fetch_nodes_statuses(members): if st.failover_limitation() is None: if st.in_recovery is False: logger.warning('Primary (%s) is still alive', st.member.name) return False if my_wal_position < st.wal_position: nodes_ahead += 1 logger.info('Wal position of %s is ahead of my wal position', st.member.name) # In synchronous mode the former leader might be still accessible and even be ahead of us. # We should not disqualify himself from the leader race in such a situation. if not self.sync_mode_is_active() or not self.cluster.sync.leader_matches(st.member.name): return False logger.info('Ignoring the former leader being ahead of us') elif st.wal_position > 0: # we want to count votes only from nodes with postgres up and running! quorum_vote = st.member.name in voting_set low_priority = my_wal_position == st.wal_position \ and self.patroni.failover_priority < st.failover_priority if low_priority and (not self.sync_mode_is_active() or quorum_vote): # There's a higher priority non-lagging replica logger.info( '%s has equally tolerable WAL position and priority %s, while this node has priority %s', st.member.name, st.failover_priority, self.patroni.failover_priority) return False if quorum_vote: logger.info('Got quorum vote from %s', st.member.name) quorum_votes += 1 # When not in quorum commit we just want to return `True`. # In quorum commit the former leader is special and counted healthy even when there are no other nodes. # Otherwise check that the number of votes exceeds the quorum field from the /sync key. return not self.quorum_commit_mode_is_active() or quorum_votes >= quorum\ or nodes_ahead == 0 and self.cluster.sync.leader == self.state_handler.name def is_failover_possible(self, *, cluster_lsn: int = 0, exclude_failover_candidate: bool = False) -> bool: """Checks whether any of the cluster members is allowed to promote and is healthy enough for that. :param cluster_lsn: to calculate replication lag and exclude member if it is lagging. :param exclude_failover_candidate: if ``True``, exclude :attr:`failover.candidate` from the members list against which the failover possibility checks are run. :returns: `True` if there are members eligible to become the new leader. """ candidates = self.get_failover_candidates(exclude_failover_candidate) action = self._get_failover_action_name() if self.is_synchronous_mode() and self.cluster.failover and self.cluster.failover.candidate and not candidates: logger.warning('%s candidate=%s does not match with sync_standbys=%s', action.title(), self.cluster.failover.candidate, self.cluster.sync.sync_standby) elif not candidates: logger.warning('%s: candidates list is empty', action) ret = False cluster_timeline = self.cluster.timeline for st in self.fetch_nodes_statuses(candidates): not_allowed_reason = st.failover_limitation() if not_allowed_reason: logger.info('Member %s is %s', st.member.name, not_allowed_reason) elif cluster_lsn and st.wal_position < cluster_lsn or \ not cluster_lsn and self.is_lagging(st.wal_position): logger.info('Member %s exceeds maximum replication lag', st.member.name) elif self.check_timeline() and (not st.timeline or st.timeline < cluster_timeline): logger.info('Timeline %s of member %s is behind the cluster timeline %s', st.timeline, st.member.name, cluster_timeline) else: ret = True return ret def manual_failover_process_no_leader(self) -> Optional[bool]: """Handles manual failover/switchover when the old leader already stepped down. :returns: - `True` if the current node is the best candidate to become the new leader - `None` if the current node is running as a primary and requested candidate doesn't exist """ failover = self.cluster.failover if TYPE_CHECKING: # pragma: no cover assert failover is not None action = self._get_failover_action_name() if failover.candidate: # manual failover/switchover to specific member if failover.candidate == self.state_handler.name: # manual failover/switchover to me return True elif self.is_paused(): # Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely # In order to avoid attempts to delete this key from all nodes only the primary is allowed to do it. if not self.cluster.get_member(failover.candidate, fallback_to_leader=False)\ and self.state_handler.is_primary(): logger.warning("%s: removing failover key because failover candidate is not running", action) self.dcs.manual_failover('', '', version=failover.version) return None return False # in synchronous mode (except quorum commit!) when our name is not in the # /sync key we shouldn't take any action even if the candidate is unhealthy if self.is_synchronous_mode() and not self.is_quorum_commit_mode()\ and not self.cluster.sync.matches(self.state_handler.name, True): return False # find specific node and check that it is healthy member = self.cluster.get_member(failover.candidate, fallback_to_leader=False) if isinstance(member, Member): st = self.fetch_node_status(member) not_allowed_reason = st.failover_limitation() if not_allowed_reason is None: # node is healthy logger.info('%s: to %s, i am %s', action, st.member.name, self.state_handler.name) return False # we wanted to failover/switchover to specific member but it is not healthy logger.warning('%s: member %s is %s', action, st.member.name, not_allowed_reason) # at this point we should consider all members as a candidates for failover/switchover # i.e. we assume that failover.candidate is None elif self.is_paused(): return False # try to pick some other members for switchover and check that they are healthy if failover.leader: if self.state_handler.name == failover.leader: # I was the leader # exclude desired member which is unhealthy if it was specified if self.is_failover_possible(exclude_failover_candidate=bool(failover.candidate)): return False else: # I was the leader and it looks like currently I am the only healthy member return True # at this point we assume that our node is a candidate for a failover among all nodes except former leader # exclude former leader from the list (failover.leader can be None) members = [m for m in self.cluster.members if m.name != failover.leader] return self._is_healthiest_node(members, check_replication_lag=False) def is_healthiest_node(self) -> bool: """Performs a series of checks to determine that the current node is the best candidate. In case if manual failover/switchover is requested it calls :func:`manual_failover_process_no_leader` method. :returns: `True` if the current node is among the best candidates to become the new leader. """ if time.time() - self._released_leader_key_timestamp < self.dcs.ttl: logger.info('backoff: skip leader race after pre_promote script failure and releasing the lock voluntarily') return False if self.is_paused() and not self.patroni.nofailover and \ self.cluster.failover and not self.cluster.failover.scheduled_at: ret = self.manual_failover_process_no_leader() if ret is not None: # continue if we just deleted the stale failover key as a leader return ret if self.state_handler.is_primary(): if self.is_paused(): # in pause leader is the healthiest only when no initialize or sysid matches with initialize! return not self.cluster.initialize or self.state_handler.sysid == self.cluster.initialize # We want to protect from the following scenario: # 1. node1 is stressed so much that heart-beat isn't running regularly and the leader lock expires. # 2. node2 promotes, gets heavy load and the situation described in 1 repeats. # 3. Patroni on node1 comes back, notices that Postgres is running as primary but there is # no leader key and "happily" acquires the leader lock. # That is, node1 discarded promotion of node2. To avoid it we want to detect timeline change. my_timeline = self.state_handler.get_primary_timeline() if my_timeline < self.cluster.timeline: logger.warning('My timeline %s is behind last known cluster timeline %s', my_timeline, self.cluster.timeline) return False return True if self.is_paused(): return False if self.patroni.nofailover: # nofailover tag makes node always unhealthy return False if self.cluster.failover: # When doing a switchover in synchronous mode only synchronous nodes and former leader are allowed to race if self.cluster.failover.leader and self.sync_mode_is_active() \ and not self.cluster.sync.matches(self.state_handler.name, True): return False return self.manual_failover_process_no_leader() or False if not self.watchdog.is_healthy: logger.warning('Watchdog device is not usable') return False all_known_members = self.old_cluster.members if self.is_failsafe_mode(): failsafe_members = self.dcs.failsafe # We want to discard failsafe_mode if the /failsafe key contains garbage or empty. if isinstance(failsafe_members, dict): # If current node is missing in the /failsafe key we immediately disqualify it from the race. if failsafe_members and self.state_handler.name not in failsafe_members: return False # Race among not only existing cluster members, but also all known members from the failsafe config all_known_members += [RemoteMember(name, {'api_url': url}) for name, url in failsafe_members.items()] all_known_members += self.cluster.members # Special handling if synchronous mode was requested and activated (the leader in /sync is not empty) if self.sync_mode_is_active(): # In quorum commit mode we allow nodes outside of "voters" to take part in # the leader race. They just need to get enough votes to `reach quorum + 1`. if not self.is_quorum_commit_mode() and not self.cluster.sync.matches(self.state_handler.name, True): return False # pick between synchronous candidates so we minimize unnecessary failovers/demotions members = {m.name: m for m in all_known_members if self.cluster.sync.matches(m.name, True)} else: # run usual health check members = {m.name: m for m in all_known_members} return self._is_healthiest_node(members.values()) def _delete_leader(self, last_lsn: Optional[int] = None) -> None: self.set_is_leader(False) self.dcs.delete_leader(self.cluster.leader, last_lsn) self.dcs.reset_cluster() def release_leader_key_voluntarily(self, last_lsn: Optional[int] = None) -> None: self._delete_leader(last_lsn) self.touch_member() logger.info("Leader key released") def demote(self, mode: str) -> Optional[bool]: """Demote PostgreSQL running as primary. :param mode: One of offline, graceful, immediate or immediate-nolock. ``offline`` is used when connection to DCS is not available. ``graceful`` is used when failing over to another node due to user request. May only be called running async. ``immediate`` is used when we determine that we are not suitable for primary and want to failover quickly without regard for data durability. May only be called synchronously. ``immediate-nolock`` is used when find out that we have lost the lock to be primary. Need to bring down PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously. """ mode_control = { 'offline': dict(stop='fast', checkpoint=False, release=False, offline=True, async_req=False), # noqa: E241,E501 'graceful': dict(stop='fast', checkpoint=True, release=True, offline=False, async_req=False), # noqa: E241,E501 'immediate': dict(stop='immediate', checkpoint=False, release=True, offline=False, async_req=True), # noqa: E241,E501 'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async_req=True), # noqa: E241,E501 }[mode] logger.info('Demoting self (%s)', mode) self._rewind.trigger_check_diverged_lsn() status = {'released': False} def on_shutdown(checkpoint_location: int, prev_location: int) -> None: # Postmaster is still running, but pg_control already reports clean "shut down". # It could happen if Postgres is still archiving the backlog of WAL files. # If we know that there are replicas that received the shutdown checkpoint # location, we can remove the leader key and allow them to start leader race. time.sleep(1) # give replicas some more time to catch up if self.is_failover_possible(cluster_lsn=checkpoint_location): self.state_handler.set_role('demoted') with self._async_executor: self.release_leader_key_voluntarily(prev_location) status['released'] = True def before_shutdown() -> None: if self.state_handler.mpp_handler.is_coordinator(): self.state_handler.mpp_handler.on_demote() else: self.notify_mpp_coordinator('before_demote') self.state_handler.stop(str(mode_control['stop']), checkpoint=bool(mode_control['checkpoint']), on_safepoint=self.watchdog.disable if self.watchdog.is_running else None, on_shutdown=on_shutdown if mode_control['release'] else None, before_shutdown=before_shutdown if mode == 'graceful' else None, stop_timeout=self.primary_stop_timeout()) self.state_handler.set_role('demoted') self.set_is_leader(False) if mode_control['release']: if not status['released']: checkpoint_location = self.state_handler.latest_checkpoint_location() if mode == 'graceful' else None with self._async_executor: self.release_leader_key_voluntarily(checkpoint_location) time.sleep(2) # Give a time to somebody to take the leader lock if mode_control['offline']: node_to_follow, leader = None, None else: try: cluster = self.dcs.get_cluster() node_to_follow, leader = self._get_node_to_follow(cluster), cluster.leader except Exception: node_to_follow, leader = None, None if self.is_synchronous_mode(): self.state_handler.sync_handler.set_synchronous_standby_names(CaseInsensitiveSet()) # FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress # there could be an async action already running, calling follow from here will lead # to racy state handler state updates. if mode_control['async_req']: self._async_executor.try_run_async('starting after demotion', self.state_handler.follow, (node_to_follow,)) else: if self._rewind.rewind_or_reinitialize_needed_and_possible(leader): return False # do not start postgres, but run pg_rewind on the next iteration self.state_handler.follow(node_to_follow) def should_run_scheduled_action(self, action_name: str, scheduled_at: Optional[datetime.datetime], cleanup_fn: Callable[..., Any]) -> bool: if scheduled_at and not self.is_paused(): # If the scheduled action is in the far future, we shouldn't do anything and just return. # If the scheduled action is in the past, we consider the value to be stale and we remove # the value. # If the value is close to now, we initiate the scheduled action # Additionally, if the scheduled action cannot be executed altogether, i.e. there is an error # or the action is in the past - we take care of cleaning it up. now = datetime.datetime.now(tzutc) try: delta = (scheduled_at - now).total_seconds() if delta > self.dcs.loop_wait: logger.info('Awaiting %s at %s (in %.0f seconds)', action_name, scheduled_at.isoformat(), delta) return False elif delta < - int(self.dcs.loop_wait * 1.5): # This means that if run_cycle gets delayed for 2.5x loop_wait we skip the # scheduled action. Probably not a problem, if things are that bad we don't # want to be restarting or failing over anyway. logger.warning('Found a stale %s value, cleaning up: %s', action_name, scheduled_at.isoformat()) cleanup_fn() return False # The value is very close to now time.sleep(max(delta, 0)) logger.info('Manual scheduled {0} at %s'.format(action_name), scheduled_at.isoformat()) return True except TypeError: logger.warning('Incorrect value of scheduled_at: %s', scheduled_at) cleanup_fn() return False def process_manual_failover_from_leader(self) -> Optional[str]: """Checks if manual failover is requested and takes action if appropriate. Cleans up failover key if failover conditions are not matched. :returns: action message if demote was initiated, None if no action was taken""" failover = self.cluster.failover # if there is no failover key or # I am holding the lock but am not primary = I am the standby leader, # then do nothing if not failover or (self.is_paused() and not self.state_handler.is_primary()): return action = self._get_failover_action_name() bare_action = action.replace('manual ', '') # it is not the time for the scheduled switchover yet, do nothing if (failover.scheduled_at and not self.should_run_scheduled_action(bare_action, failover.scheduled_at, lambda: self.dcs.manual_failover('', '', version=failover.version))): return if not failover.leader or failover.leader == self.state_handler.name: if not failover.candidate or failover.candidate != self.state_handler.name: if not failover.candidate and self.is_paused(): logger.warning('%s is possible only to a specific candidate in a paused state', action.title()) elif self.is_failover_possible(): ret = self._async_executor.try_run_async(f'{action}: demote', self.demote, ('graceful',)) return ret or f'{action}: demoting myself' else: logger.warning('%s: no healthy members found, %s is not possible', action, bare_action) else: logger.warning('%s: I am already the leader, no need to %s', action, bare_action) else: logger.warning('%s: leader name does not match: %s != %s', action, failover.leader, self.state_handler.name) logger.info('Cleaning up failover key') self.dcs.manual_failover('', '', version=failover.version) def process_unhealthy_cluster(self) -> str: """Cluster has no leader key""" if self.is_healthiest_node(): if self.acquire_lock(): failover = self.cluster.failover if failover: if self.is_paused() and failover.leader and failover.candidate: logger.info('Updating failover key after acquiring leader lock...') self.dcs.manual_failover('', failover.candidate, failover.scheduled_at, failover.version) else: logger.info('Cleaning up failover key after acquiring leader lock...') self.dcs.manual_failover('', '') self.load_cluster_from_dcs() if self.is_standby_cluster(): # standby leader disappeared, and this is the healthiest # replica, so it should become a new standby leader. # This implies we need to start following a remote member msg = 'promoted self to a standby leader by acquiring session lock' return self.enforce_follow_remote_member(msg) else: return self.enforce_primary_role( 'acquired session lock as a leader', 'promoted self to leader by acquiring session lock' ) else: return self.follow('demoted self after trying and failing to obtain lock', 'following new leader after trying and failing to obtain lock') else: # when we are doing manual failover there is no guaranty that new leader is ahead of any other node # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections if bool(self.cluster.failover) or self.patroni.nofailover: self._rewind.trigger_check_diverged_lsn() time.sleep(2) # Give a time to somebody to take the leader lock if self.patroni.nofailover: return self.follow('demoting self because I am not allowed to become primary', 'following a different leader because I am not allowed to promote') return self.follow('demoting self because i am not the healthiest node', 'following a different leader because i am not the healthiest node') def process_healthy_cluster(self) -> str: if self.has_lock(): if self.is_paused() and not self.state_handler.is_primary(): if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name: return 'waiting to become primary after promote...' if not self.is_standby_cluster(): self._delete_leader() return 'removed leader lock because postgres is not running as primary' # update lock to avoid split-brain if self.update_lock(True): msg = self.process_manual_failover_from_leader() if msg is not None: return msg # check if the node is ready to be used by pg_rewind self._rewind.ensure_checkpoint_after_promote(self.wakeup) if self.is_standby_cluster(): # in case of standby cluster we don't really need to # enforce anything, since the leader is not a primary # So just remind the role. msg = 'no action. I am ({0}), the standby leader with the lock'.format(self.state_handler.name) \ if self.state_handler.role == 'standby_leader' else \ 'promoted self to a standby leader because i had the session lock' return self.enforce_follow_remote_member(msg) else: return self.enforce_primary_role( 'no action. I am ({0}), the leader with the lock'.format(self.state_handler.name), 'promoted self to leader because I had the session lock' ) else: # Either there is no connection to DCS or someone else acquired the lock logger.error('failed to update leader lock') if self.state_handler.is_primary(): if self.is_paused(): return 'continue to run as primary after failing to update leader lock in DCS' self.demote('immediate-nolock') return 'demoted self because failed to update leader lock in DCS' else: return 'not promoting because failed to update leader lock in DCS' else: logger.debug('does not have lock') lock_owner = self.cluster.leader and self.cluster.leader.name if self.is_standby_cluster(): return self.follow('cannot be a real primary in a standby cluster', 'no action. I am ({0}), a secondary, and following a standby leader ({1})'.format( self.state_handler.name, lock_owner), refresh=False) return self.follow('demoting self because I do not have the lock and I was a leader', 'no action. I am ({0}), a secondary, and following a leader ({1})'.format( self.state_handler.name, lock_owner), refresh=False) def evaluate_scheduled_restart(self) -> Optional[str]: if self._async_executor.busy: # Restart already in progress return None # restart if we need to restart_data = self.future_restart_scheduled() if restart_data: recent_time = self.state_handler.postmaster_start_time() request_time = restart_data['postmaster_start_time'] # check if postmaster start time has changed since the last restart if recent_time and request_time and recent_time != request_time: logger.info("Cancelling scheduled restart: postgres restart has already happened at %s", recent_time) self.delete_future_restart() return None if restart_data\ and self.should_run_scheduled_action('restart', restart_data['schedule'], self.delete_future_restart): try: ret, message = self.restart(restart_data, run_async=True) if not ret: logger.warning("Scheduled restart: %s", message) return None return message finally: self.delete_future_restart() def restart_matches(self, role: Optional[str], postgres_version: Optional[str], pending_restart: bool) -> bool: reason_to_cancel = "" # checking the restart filters here seem to be less ugly than moving them into the # run_scheduled_action. if role and role != self.state_handler.role: reason_to_cancel = "host role mismatch" if postgres_version and postgres_version_to_int(postgres_version) <= int(self.state_handler.server_version): reason_to_cancel = "postgres version mismatch" if pending_restart and not self.state_handler.pending_restart_reason: reason_to_cancel = "pending restart flag is not set" if not reason_to_cancel: return True else: logger.info("not proceeding with the restart: %s", reason_to_cancel) return False def schedule_future_restart(self, restart_data: Dict[str, Any]) -> bool: with self._async_executor: restart_data['postmaster_start_time'] = self.state_handler.postmaster_start_time() if not self.patroni.scheduled_restart: self.patroni.scheduled_restart = restart_data self.touch_member() return True return False def delete_future_restart(self) -> bool: ret = False with self._async_executor: if self.patroni.scheduled_restart: self.patroni.scheduled_restart = {} self.touch_member() ret = True return ret def future_restart_scheduled(self) -> Dict[str, Any]: return self.patroni.scheduled_restart.copy() def restart_scheduled(self) -> bool: return self._async_executor.scheduled_action == 'restart' def restart(self, restart_data: Dict[str, Any], run_async: bool = False) -> Tuple[bool, str]: """ conditional and unconditional restart """ assert isinstance(restart_data, dict) if (not self.restart_matches(restart_data.get('role'), restart_data.get('postgres_version'), ('restart_pending' in restart_data))): return (False, "restart conditions are not satisfied") with self._async_executor: prev = self._async_executor.schedule('restart') if prev is not None: return (False, prev + ' already in progress') # Make the main loop to think that we were recovering dead postgres. If we fail # to start postgres after a specified timeout (see below), we need to remove # leader key (if it belong to us) rather than trying to start postgres once again. self.recovering = True # Now that restart is scheduled we can set timeout for startup, it will get reset # once async executor runs and main loop notices PostgreSQL as up. timeout = restart_data.get('timeout', global_config.primary_start_timeout) self.set_start_timeout(timeout) def before_shutdown() -> None: self.notify_mpp_coordinator('before_demote') def after_start() -> None: self.notify_mpp_coordinator('after_promote') # For non async cases we want to wait for restart to complete or timeout before returning. do_restart = functools.partial(self.state_handler.restart, timeout, self._async_executor.critical_task, before_shutdown=before_shutdown if self.has_lock() else None, after_start=after_start if self.has_lock() else None) if self.is_synchronous_mode() and not self.has_lock(): do_restart = functools.partial(self.while_not_sync_standby, do_restart) if run_async: self._async_executor.run_async(do_restart) return (True, 'restart initiated') else: res = self._async_executor.run(do_restart) if res: return (True, 'restarted successfully') elif res is None: return (False, 'postgres is still starting') else: return (False, 'restart failed') def _do_reinitialize(self, cluster: Cluster) -> Optional[bool]: self.state_handler.stop('immediate', stop_timeout=self.patroni.config['retry_timeout']) # Commented redundant data directory cleanup here # self.state_handler.remove_data_directory() clone_member = cluster.get_clone_member(self.state_handler.name) if clone_member: member_role = 'leader' if clone_member == cluster.leader else 'replica' return self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name)) def reinitialize(self, force: bool = False) -> Optional[str]: with self._async_executor: self.load_cluster_from_dcs() if self.cluster.is_unlocked(): return 'Cluster has no leader, can not reinitialize' if self.has_lock(False): return 'I am the leader, can not reinitialize' cluster = self.cluster if force: self._async_executor.cancel() with self._async_executor: action = self._async_executor.schedule('reinitialize') if action is not None: return '{0} already in progress'.format(action) self._async_executor.run_async(self._do_reinitialize, args=(cluster, )) def handle_long_action_in_progress(self) -> str: """Figure out what to do with the task AsyncExecutor is performing.""" if self.has_lock() and self.update_lock(): if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode': time_left = global_config.primary_start_timeout - (time.time() - self._crash_recovery_started) if time_left <= 0 and self.is_failover_possible(): logger.info("Demoting self because crash recovery is taking too long") self.state_handler.cancellable.cancel(True) self.demote('immediate') return 'terminated crash recovery because of startup timeout' return 'updated leader lock during {0}'.format(self._async_executor.scheduled_action) elif not self.state_handler.bootstrapping and not self.is_paused(): # Don't have lock, make sure we are not promoting or starting up a primary in the background if self._async_executor.scheduled_action == 'promote': with self._async_response: cancel = self._async_response.cancel() if cancel: self.state_handler.cancellable.cancel() return 'lost leader before promote' if self.state_handler.role == 'primary': logger.info('Demoting primary during %s', self._async_executor.scheduled_action) if self._async_executor.scheduled_action in ('restart', 'starting primary after failure'): # Restart needs a special interlocking cancel because postmaster may be just started in a # background thread and has not even written a pid file yet. with self._async_executor.critical_task as task: if not task.cancel() and isinstance(task.result, PostmasterProcess): self.state_handler.terminate_starting_postmaster(postmaster=task.result) self.demote('immediate-nolock') return 'lost leader lock during {0}'.format(self._async_executor.scheduled_action) if self.cluster.is_unlocked(): logger.info('not healthy enough for leader race') return '{0} in progress'.format(self._async_executor.scheduled_action) @staticmethod def sysid_valid(sysid: Optional[str]) -> bool: # sysid does tv_sec << 32, where tv_sec is the number of seconds sine 1970, # so even 1 << 32 would have 10 digits. sysid = str(sysid) return len(sysid) >= 10 and sysid.isdigit() def post_recover(self) -> Optional[str]: if not self.state_handler.is_running(): self.watchdog.disable() if self.has_lock(): if self.state_handler.role in ('primary', 'standby_leader'): self.state_handler.set_role('demoted') self.state_handler.call_nowait(CallbackAction.ON_ROLE_CHANGE) self._delete_leader() return 'removed leader key after trying and failing to start postgres' return 'failed to start postgres' return None def cancel_initialization(self) -> None: logger.info('removing initialize key after failed attempt to bootstrap the cluster') self.dcs.cancel_initialization() self.state_handler.stop('immediate', stop_timeout=self.patroni.config['retry_timeout']) self.state_handler.move_data_directory() raise PatroniFatalException('Failed to bootstrap cluster') def post_bootstrap(self) -> str: with self._async_response: result = self._async_response.result # bootstrap has failed if postgres is not running if not self.state_handler.is_running() or result is False: self.cancel_initialization() if result is None: if not self.state_handler.is_primary(): return 'waiting for end of recovery after bootstrap' self.state_handler.set_role('primary') ret = self._async_executor.try_run_async('post_bootstrap', self.state_handler.bootstrap.post_bootstrap, args=(self.patroni.config['bootstrap'], self._async_response)) return ret or 'running post_bootstrap' self.state_handler.bootstrapping = False if not self.watchdog.activate(): logger.error('Cancelling bootstrap because watchdog activation failed') self.cancel_initialization() self._rewind.ensure_checkpoint_after_promote(self.wakeup) self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid) self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':'))) self.dcs.take_leader() self.set_is_leader(True) if self.is_synchronous_mode(): self.state_handler.sync_handler.set_synchronous_standby_names( CaseInsensitiveSet('*') if global_config.is_synchronous_mode_strict else CaseInsensitiveSet()) self.state_handler.call_nowait(CallbackAction.ON_START) self.load_cluster_from_dcs() return 'initialized a new cluster' def handle_starting_instance(self) -> Optional[str]: """Starting up PostgreSQL may take a long time. In case we are the leader we may want to fail over to.""" # Check if we are in startup, when paused defer to main loop for manual failovers. if not self.state_handler.check_for_startup() or self.is_paused(): self.set_start_timeout(None) if self.is_paused(): self.state_handler.set_state(self.state_handler.is_running() and 'running' or 'stopped') return None # state_handler.state == 'starting' here if self.has_lock(): if not self.update_lock(): logger.info("Lost lock while starting up. Demoting self.") self.demote('immediate-nolock') return 'stopped PostgreSQL while starting up because leader key was lost' timeout = self._start_timeout or global_config.primary_start_timeout time_left = timeout - self.state_handler.time_in_state() if time_left <= 0: if self.is_failover_possible(): logger.info("Demoting self because primary startup is taking too long") self.demote('immediate') return 'stopped PostgreSQL because of startup timeout' else: return 'primary start has timed out, but continuing to wait because failover is not possible' else: msg = self.process_manual_failover_from_leader() if msg is not None: return msg return 'PostgreSQL is still starting up, {0:.0f} seconds until timeout'.format(time_left) else: # Use normal processing for standbys logger.info("Still starting up as a standby.") return None def set_start_timeout(self, value: Optional[int]) -> None: """Sets timeout for starting as primary before eligible for failover. Must be called when async_executor is busy or in the main thread. """ self._start_timeout = value def _run_cycle(self) -> str: dcs_failed = False try: try: self.load_cluster_from_dcs() global_config.update(self.cluster) self.state_handler.reset_cluster_info_state(self.cluster, self.patroni) except Exception as exc1: self.state_handler.reset_cluster_info_state(None) if self.is_failsafe_mode(): # If DCS is not accessible we want to get the latest value of received/replayed LSN # in order to have it immediately available if the failsafe mode is enabled. try: self._last_wal_lsn = self.state_handler.last_operation() except Exception as exc2: logger.debug('Failed to fetch current wal lsn: %r', exc2) raise exc1 if self.is_paused(): self.watchdog.disable() self._was_paused = True else: if self._was_paused: self.state_handler.schedule_sanity_checks_after_pause() # during pause people could manually do something with Postgres, therefore we want # to double check rewind conditions on replicas and maybe run CHECKPOINT on the primary self._rewind.reset_state() self._was_paused = False if not self.cluster.has_member(self.state_handler.name): self.touch_member() # cluster has leader key but not initialize key if self.has_lock(False) and not self.sysid_valid(self.cluster.initialize): self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid) if self.has_lock(False) and not (self.cluster.config and self.cluster.config.data): self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':'))) self.cluster = self.dcs.get_cluster() if self._async_executor.busy: return self.handle_long_action_in_progress() msg = self.handle_starting_instance() if msg is not None: return msg # we've got here, so any async action has finished. if self.state_handler.bootstrapping: return self.post_bootstrap() if self.recovering: self.recovering = False if not self._rewind.is_needed: # Check if we tried to recover from postgres crash and failed msg = self.post_recover() if msg is not None: return msg # Reset some states after postgres successfully started up self._crash_recovery_started = 0 if self._rewind.executed and not self._rewind.failed: self._rewind.reset_state() # The Raft cluster without a quorum takes a bit of time to stabilize. # Therefore we want to postpone the leader race if we just started up. if self.cluster.is_unlocked() and self.dcs.__class__.__name__ == 'Raft': return 'started as a secondary' # is data directory empty? data_directory_error = '' data_directory_is_empty = None try: data_directory_is_empty = self.state_handler.data_directory_empty() data_directory_is_accessible = True except OSError as e: data_directory_is_accessible = False data_directory_error = e if not data_directory_is_accessible or data_directory_is_empty: self.state_handler.set_role('uninitialized') self.state_handler.stop('immediate', stop_timeout=self.patroni.config['retry_timeout']) # In case datadir went away while we were primary self.watchdog.disable() # is this instance the leader? if self.has_lock(): self.release_leader_key_voluntarily() return 'released leader key voluntarily as data dir {0} and currently leader'.format( 'empty' if data_directory_is_accessible else 'not accessible') if not data_directory_is_accessible: return 'data directory is not accessible: {0}'.format(data_directory_error) if self.is_paused(): return 'running with empty data directory' return self.bootstrap() # new node else: # check if we are allowed to join data_sysid = self.state_handler.sysid if not self.sysid_valid(data_sysid): # data directory is not empty, but no valid sysid, cluster must be broken, suggest reinit return ("data dir for the cluster is not empty, " "but system ID is invalid; consider doing reinitialize") if self.sysid_valid(self.cluster.initialize): if self.cluster.initialize != data_sysid: if self.is_paused(): logger.warning('system ID has changed while in paused mode. Patroni will exit when resuming' ' unless system ID is reset: %s != %s', self.cluster.initialize, data_sysid) if self.has_lock(): self.release_leader_key_voluntarily() return 'released leader key voluntarily due to the system ID mismatch' else: logger.fatal('system ID mismatch, node %s belongs to a different cluster: %s != %s', self.state_handler.name, self.cluster.initialize, data_sysid) sys.exit(1) elif self.cluster.is_unlocked() and not self.is_paused() and not self.state_handler.cb_called: # "bootstrap", but data directory is not empty if self.state_handler.is_running() and not self.state_handler.is_primary(): self._join_aborted = True logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start') logger.error('Please first start Patroni on the node running as primary') sys.exit(1) self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=data_sysid) if not self.state_handler.is_healthy(): if self.is_paused(): self.state_handler.set_state('stopped') if self.has_lock(): self._delete_leader() return 'removed leader lock because postgres is not running' # Normally we don't start Postgres in a paused state. We make an exception for the demoted primary # that needs to be started after it had been stopped by demote. When there is no need to call rewind # the demote code follows through to starting Postgres right away, however, in the rewind case # it returns from demote and reaches this point to start PostgreSQL again after rewind. In that # case it makes no sense to continue to recover() unless rewind has finished successfully. elif self._rewind.failed or not self._rewind.executed and not \ (self._rewind.is_needed and self._rewind.can_rewind_or_reinitialize_allowed): return 'postgres is not running' if self.state_handler.state in ('running', 'starting'): self.state_handler.set_state('crashed') # try to start dead postgres return self.recover() if self.cluster.is_unlocked(): ret = self.process_unhealthy_cluster() else: msg = self.process_healthy_cluster() ret = self.evaluate_scheduled_restart() or msg # We might not have a valid PostgreSQL connection here if AsyncExecutor is doing # something with PostgreSQL. Therefore we will sync replication slots only if no # asynchronous processes are running or we know that this is a standby being promoted. # But, we don't want to run pg_rewind checks or copy logical slots from itself, # therefore we have a couple additional `not is_promoting` checks. is_promoting = self._async_executor.scheduled_action == 'promote' if (not self._async_executor.busy or is_promoting) and not self.state_handler.is_starting(): create_slots = self._sync_replication_slots(False) if not self.state_handler.cb_called: if not is_promoting and not self.state_handler.is_primary(): self._rewind.trigger_check_diverged_lsn() self.state_handler.call_nowait(CallbackAction.ON_START) if not is_promoting and create_slots and self.cluster.leader: err = self._async_executor.try_run_async('copy_logical_slots', self.state_handler.slots_handler.copy_logical_slots, args=(self.cluster, self.patroni, create_slots)) if not err: ret = 'Copying logical slots {0} from the primary'.format(create_slots) return ret except DCSError: dcs_failed = True logger.error('Error communicating with DCS') return self._handle_dcs_error() except (psycopg.Error, PostgresConnectionException): return 'Error communicating with PostgreSQL. Will try again later' finally: if not dcs_failed: if self.is_leader(): self._failsafe.set_is_active(0) self.touch_member() def _handle_dcs_error(self) -> str: if not self.is_paused() and self.state_handler.is_running(): if self.state_handler.is_primary(): if self.is_failsafe_mode() and self.check_failsafe_topology(): self.set_is_leader(True) self._failsafe.set_is_active(time.time()) self.watchdog.keepalive() self._sync_replication_slots(True) return 'continue to run as a leader because failsafe mode is enabled and all members are accessible' self._failsafe.set_is_active(0) msg = 'demoting self because DCS is not accessible and I was a leader' if not self._async_executor.try_run_async(msg, self.demote, ('offline',)): return msg logger.warning('AsyncExecutor is busy, demoting from the main thread') self.demote('offline') return 'demoted self because DCS is not accessible and I was a leader' else: self._sync_replication_slots(True) return 'DCS is not accessible' def _sync_replication_slots(self, dcs_failed: bool) -> List[str]: """Handles replication slots. :param dcs_failed: bool, indicates that communication with DCS failed (get_cluster() or update_leader()) :returns: list[str], replication slots names that should be copied from the primary """ slots: List[str] = [] # If dcs_failed we don't want to touch replication slots on a leader or replicas if failsafe_mode isn't enabled. if not self.cluster or dcs_failed and not self.is_failsafe_mode(): return slots # It could be that DCS is read-only, or only the leader can't access it. # Only the second one could be handled by `load_cluster_from_dcs()`. # The first one affects advancing logical replication slots on replicas, therefore we rely on # Failsafe.update_cluster(), that will return "modified" Cluster if failsafe mode is active. cluster = self._failsafe.update_cluster(self.cluster) if self.is_failsafe_mode() else self.cluster if cluster: slots = self.state_handler.slots_handler.sync_replication_slots(cluster, self.patroni) # Don't copy replication slots if failsafe_mode is active return [] if self.failsafe_is_active() else slots def run_cycle(self) -> str: with self._async_executor: try: info = self._run_cycle() return (self.is_paused() and 'PAUSE: ' or '') + info except PatroniFatalException: raise except Exception: logger.exception('Unexpected exception') return 'Unexpected exception raised, please report it as a BUG' def shutdown(self) -> None: if self.is_paused(): logger.info('Leader key is not deleted and Postgresql is not stopped due paused state') self.watchdog.disable() elif not self._join_aborted: # FIXME: If stop doesn't reach safepoint quickly enough keepalive is triggered. If shutdown checkpoint # takes longer than ttl, then leader key is lost and replication might not have sent out all WAL. # This might not be the desired behavior of users, as a graceful shutdown of the host can mean lost data. # We probably need to something smarter here. disable_wd = self.watchdog.disable if self.watchdog.is_running else None status = {'deleted': False} def _on_shutdown(checkpoint_location: int, prev_location: int) -> None: if self.is_leader(): # Postmaster is still running, but pg_control already reports clean "shut down". # It could happen if Postgres is still archiving the backlog of WAL files. # If we know that there are replicas that received the shutdown checkpoint # location, we can remove the leader key and allow them to start leader race. time.sleep(1) # give replicas some more time to catch up if self.is_failover_possible(cluster_lsn=checkpoint_location): self.dcs.delete_leader(self.cluster.leader, prev_location) status['deleted'] = True else: self.dcs.write_leader_optime(prev_location) def _before_shutdown() -> None: self.notify_mpp_coordinator('before_demote') on_shutdown = _on_shutdown if self.is_leader() else None before_shutdown = _before_shutdown if self.is_leader() else None self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd, on_shutdown=on_shutdown, before_shutdown=before_shutdown, stop_timeout=self.primary_stop_timeout())) if not self.state_handler.is_running(): if self.is_leader() and not status['deleted']: checkpoint_location = self.state_handler.latest_checkpoint_location() self.dcs.delete_leader(self.cluster.leader, checkpoint_location) self.touch_member() else: # XXX: what about when Patroni is started as the wrong user that has access to the watchdog device # but cannot shut down PostgreSQL. Root would be the obvious example. Would be nice to not kill the # system due to a bad config. logger.error("PostgreSQL shutdown failed, leader key not removed.%s", (" Leaving watchdog running." if self.watchdog.is_running else "")) def watch(self, timeout: float) -> bool: # watch on leader key changes if the postgres is running and leader is known and current node is not lock owner if self._async_executor.busy or not self.cluster or self.cluster.is_unlocked() or self.has_lock(False): leader_version = None else: leader_version = self.cluster.leader.version if self.cluster.leader else None return self.dcs.watch(leader_version, timeout) def wakeup(self) -> None: """Trigger the next run of HA loop if there is no "active" leader watch request in progress. This usually happens on the leader or if the node is running async action""" self.dcs.event.set() def get_remote_member(self, member: Union[Leader, Member, None] = None) -> RemoteMember: """Get remote member node to stream from. In case of standby cluster this will tell us from which remote member to stream. Config can be both patroni config or cluster.config.data. """ data: Dict[str, Any] = {} cluster_params = global_config.get_standby_cluster_config() if cluster_params: data.update({k: v for k, v in cluster_params.items() if k in RemoteMember.ALLOWED_KEYS}) data['no_replication_slot'] = 'primary_slot_name' not in cluster_params conn_kwargs = member.conn_kwargs() if member else \ {k: cluster_params[k] for k in ('host', 'port') if k in cluster_params} if conn_kwargs: data['conn_kwargs'] = conn_kwargs name = member.name if member else 'remote_member:{}'.format(uuid.uuid1()) return RemoteMember(name, data) def get_failover_candidates(self, exclude_failover_candidate: bool) -> List[Member]: """Return a list of candidates for either manual or automatic failover. Exclude non-sync members when in synchronous mode, the current node (its checks are always performed earlier) and the candidate if required. If failover candidate exclusion is not requested and a candidate is specified in the /failover key, return the candidate only. The result is further evaluated in the caller :func:`Ha.is_failover_possible` to check if any member is actually healthy enough and is allowed to poromote. :param exclude_failover_candidate: if ``True``, exclude :attr:`failover.candidate` from the candidates. :returns: a list of :class:`Member` objects or an empty list if there is no candidate available. """ failover = self.cluster.failover exclude = [self.state_handler.name] + ([failover.candidate] if failover and exclude_failover_candidate else []) def is_eligible(node: Member) -> bool: # If quorum commit is requested we want to check all nodes (even not voters), # because they could get enough votes and reach necessary quorum + 1. # in synchronous mode we allow failover (not switchover!) to async node if self.sync_mode_is_active()\ and not (self.is_quorum_commit_mode() or self.cluster.sync.matches(node.name))\ and not (failover and not failover.leader): return False # Don't spend time on "nofailover" nodes checking. # We also don't need nodes which we can't query with the api in the list. return node.name not in exclude and \ not node.nofailover and bool(node.api_url) and \ (not failover or not failover.candidate or node.name == failover.candidate) return list(filter(is_eligible, self.cluster.members)) patroni-4.0.4/patroni/log.py000066400000000000000000000565261472010352700160210ustar00rootroot00000000000000"""Patroni logging facilities. Daemon processes will use a 2-step logging handler. Whenever a log message is issued it is initially enqueued in-memory and is later asynchronously flushed by a thread to the final destination. """ import logging import os import sys from copy import deepcopy from io import TextIOWrapper from logging.handlers import RotatingFileHandler from queue import Full, Queue from threading import Lock, Thread from typing import Any, cast, Dict, List, Optional, TYPE_CHECKING, Union from .file_perm import pg_perm from .utils import deep_compare, parse_int type_logformat = Union[List[Union[str, Dict[str, Any], Any]], str, Any] _LOGGER = logging.getLogger(__name__) class PatroniFileHandler(RotatingFileHandler): """Wrapper of :class:`RotatingFileHandler` to handle permissions of log files. """ def __init__(self, filename: str, mode: Optional[int]) -> None: """Create a new :class:`PatroniFileHandler` instance. :param filename: basename for log files. :param mode: permissions for log files. """ self.set_log_file_mode(mode) super(PatroniFileHandler, self).__init__(filename) def set_log_file_mode(self, mode: Optional[int]) -> None: """Set mode for Patroni log files. :param mode: permissions for log files. .. note:: If *mode* is not specified, we calculate it from the `umask` value. """ self._log_file_mode = 0o666 & ~pg_perm.orig_umask if mode is None else mode def _open(self) -> TextIOWrapper: """Open a new log file and assign permissions. :returns: the resulting stream. """ ret = super(PatroniFileHandler, self)._open() os.chmod(self.baseFilename, self._log_file_mode) return ret def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None: """Add full stack trace info to debug log messages and partial to others. Handle :func:`~self.exception` calls for *self*. .. note:: * If *self* log level is set to ``DEBUG``, then issue a ``DEBUG`` message with the complete stack trace; * If *self* log level is ``INFO`` or higher, then issue an ``ERROR`` message with only the last line of the stack trace. :param self: logger for which :func:`~self.exception` will be processed. :param msg: the message related to the exception to be logged. :param args: positional arguments to be passed to :func:`~self.debug` or :func:`~self.error`. :param kwargs: keyword arguments to be passed to :func:`~self.debug` or :func:`~self.error`. """ kwargs.pop("exc_info", False) if self.isEnabledFor(logging.DEBUG): self.debug(msg, *args, exc_info=True, **kwargs) else: msg = "{0}, DETAIL: '{1}'".format(msg, sys.exc_info()[1]) self.error(msg, *args, exc_info=False, **kwargs) def error_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None: """Add full stack trace info to error messages. Handle :func:`~self.exception` calls for *self*. .. note:: * By default issue an ``ERROR`` message with the complete stack trace. If you do not want to show the complete stack trace, call with ``exc_info=False``. :param self: logger for which :func:`~self.exception` will be processed. :param msg: the message related to the exception to be logged. :param args: positional arguments to be passed to :func:`~self.error`. :param kwargs: keyword arguments to be passed to :func:`~self.error`. """ exc_info = kwargs.pop("exc_info", True) self.error(msg, *args, exc_info=exc_info, **kwargs) def _type(value: Any) -> str: """Get type of the *value*. :param value: any arbitrary value. :returns: a string with a type name. """ return value.__class__.__name__ class QueueHandler(logging.Handler): """Queue-based logging handler. :ivar queue: queue to hold log messages that are pending to be flushed to the final destination. """ def __init__(self) -> None: """Queue initialised and initial records_lost established.""" super().__init__() self.queue: Queue[Union[logging.LogRecord, None]] = Queue() self._records_lost = 0 def _put_record(self, record: logging.LogRecord) -> None: """Asynchronously enqueue a log record. :param record: the record to be logged. """ self.format(record) record.msg = record.message record.args = None record.exc_info = None self.queue.put_nowait(record) def _try_to_report_lost_records(self) -> None: """Report the number of log messages that have been lost and reset the counter. .. note:: It will issue an ``WARNING`` message in the logs with the number of lost log messages. """ if self._records_lost: try: record = _LOGGER.makeRecord(_LOGGER.name, logging.WARNING, __file__, 0, 'QueueHandler has lost %s log records', (self._records_lost,), None, 'emit') self._put_record(record) self._records_lost = 0 except Exception: pass def emit(self, record: logging.LogRecord) -> None: """Handle each log record that is emitted. Call :func:`_put_record` to enqueue the emitted log record. Also check if we have previously lost any log record, and if so, log a ``WARNING`` message. :param record: the record that was emitted. """ try: self._put_record(record) self._try_to_report_lost_records() except Exception: self._records_lost += 1 @property def records_lost(self) -> int: """Number of log messages that have been lost while the queue was full.""" return self._records_lost class ProxyHandler(logging.Handler): """Handle log records in place of pending log handlers. .. note:: This is used to handle log messages while the logger thread has not started yet, in which case the queue-based handler is not yet started. :ivar patroni_logger: the logger thread. """ def __init__(self, patroni_logger: 'PatroniLogger') -> None: """Create a new :class:`ProxyHandler` instance. :param patroni_logger: the logger thread. """ super().__init__() self.patroni_logger = patroni_logger def emit(self, record: logging.LogRecord) -> None: """Emit each log record that is handled. Will push the log record down to :func:`~logging.Handler.handle` method of the currently configured log handler. :param record: the record that was emitted. """ if self.patroni_logger.log_handler is not None: self.patroni_logger.log_handler.handle(record) class PatroniLogger(Thread): """Logging thread for the Patroni daemon process. It is a 2-step logging approach. Any time a log message is issued it is initially enqueued in-memory, and then asynchronously flushed to the final destination by the logging thread. .. seealso:: :class:`QueueHandler`: object used for enqueueing messages in-memory. :cvar DEFAULT_TYPE: default type of log format (``plain``). :cvar DEFAULT_LEVEL: default logging level (``INFO``). :cvar DEFAULT_TRACEBACK_LEVEL: default traceback logging level (``ERROR``). :cvar DEFAULT_FORMAT: default format of log messages (``%(asctime)s %(levelname)s: %(message)s``). :cvar NORMAL_LOG_QUEUE_SIZE: expected number of log messages per HA loop when operating under a normal situation. :cvar DEFAULT_MAX_QUEUE_SIZE: default maximum queue size for holding a backlog of log messages that are pending to be flushed. :cvar LOGGING_BROKEN_EXIT_CODE: exit code to be used if it detects(``5``). :ivar log_handler: log handler that is currently being used by the thread. :ivar log_handler_lock: lock used to modify ``log_handler``. """ DEFAULT_TYPE = 'plain' DEFAULT_LEVEL = 'INFO' DEFAULT_TRACEBACK_LEVEL = 'ERROR' DEFAULT_FORMAT = '%(asctime)s %(levelname)s: %(message)s' NORMAL_LOG_QUEUE_SIZE = 2 # When everything goes normal Patroni writes only 2 messages per HA loop DEFAULT_MAX_QUEUE_SIZE = 1000 LOGGING_BROKEN_EXIT_CODE = 5 def __init__(self) -> None: """Prepare logging queue and proxy handlers as they become ready during daemon startup. .. note:: While Patroni is starting up it keeps ``DEBUG`` log level, and writes log messages through a proxy handler. Once the logger thread is finally started, it switches from that proxy handler to the queue based logger, and applies the configured log settings. The switching is used to avoid that the logger thread prevents Patroni from shutting down if any issue occurs in the meantime until the thread is properly started. """ super(PatroniLogger, self).__init__() self._queue_handler = QueueHandler() self._root_logger = logging.getLogger() self._config: Optional[Dict[str, Any]] = None self.log_handler = None self.log_handler_lock = Lock() self._old_handlers: List[logging.Handler] = [] # initially set log level to ``DEBUG`` while the logger thread has not started running yet. The daemon process # will later adjust all log related settings with what was provided through the user configuration file. self.reload_config({'level': 'DEBUG'}) # We will switch to the QueueHandler only when thread was started. # This is necessary to protect from the cases when Patroni constructor # failed and PatroniLogger thread remain running and prevent shutdown. self._proxy_handler = ProxyHandler(self) self._root_logger.addHandler(self._proxy_handler) def update_loggers(self, config: Dict[str, Any]) -> None: """Configure custom loggers' log levels. .. note:: It creates logger objects that are not defined yet in the log manager. :param config: :class:`dict` object with custom loggers configuration, is set either from: * ``log.loggers`` section of Patroni configuration; or * from the method that is trying to make sure that the node name isn't duplicated (to silence annoying ``urllib3`` WARNING's). :Example: .. code-block:: python update_loggers({'urllib3.connectionpool': 'WARNING'}) """ loggers = deepcopy(config) for name, logger in self._root_logger.manager.loggerDict.items(): # ``Placeholder`` is a node in the log manager for which no logger has been defined. We are interested only # in the ones that were defined if not isinstance(logger, logging.PlaceHolder): # if this logger is present in *config*, use the configured level, otherwise # use ``logging.NOTSET``, which means it will inherit the level # from any parent node up to the root for which log level is defined. level = loggers.pop(name, logging.NOTSET) logger.setLevel(level) # define loggers that do not exist yet and set level as configured in the *config* for name, level in loggers.items(): logger = self._root_logger.manager.getLogger(name) logger.setLevel(level) def _is_config_changed(self, config: Dict[str, Any]) -> bool: """Checks if the given config is different from the current one. :param config: ``log`` section from Patroni configuration. :returns: ``True`` if the config is changed, ``False`` otherwise. """ old_config = self._config or {} oldlogtype = old_config.get('type', PatroniLogger.DEFAULT_TYPE) logtype = config.get('type', PatroniLogger.DEFAULT_TYPE) oldlogformat: type_logformat = old_config.get('format', PatroniLogger.DEFAULT_FORMAT) logformat: type_logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT) olddateformat = old_config.get('dateformat') or None dateformat = config.get('dateformat') or None # Convert empty string to `None` old_static_fields = old_config.get('static_fields', {}) static_fields = config.get('static_fields', {}) old_log_config = { 'type': oldlogtype, 'format': oldlogformat, 'dateformat': olddateformat, 'static_fields': old_static_fields } log_config = { 'type': logtype, 'format': logformat, 'dateformat': dateformat, 'static_fields': static_fields } return not deep_compare(old_log_config, log_config) def _get_plain_formatter(self, logformat: type_logformat, dateformat: Optional[str]) -> logging.Formatter: """Returns a logging formatter with the specified format and date format. .. note:: If the log format isn't a string, prints a warning message and uses the default log format instead. :param logformat: The format of the log messages. :param dateformat: The format of the timestamp in the log messages. :returns: A logging formatter object that can be used to format log records. """ if not isinstance(logformat, str): _LOGGER.warning('Expected log format to be a string when log type is plain, but got "%s"', _type(logformat)) logformat = PatroniLogger.DEFAULT_FORMAT return logging.Formatter(logformat, dateformat) def _get_json_formatter(self, logformat: type_logformat, dateformat: Optional[str], static_fields: Dict[str, Any]) -> logging.Formatter: """Returns a logging formatter that outputs JSON formatted messages. .. note:: If :mod:`pythonjsonlogger` library is not installed, prints an error message and returns a plain log formatter instead. :param logformat: Specifies the log fields and their key names in the JSON log message. :param dateformat: The format of the timestamp in the log messages. :param static_fields: A dictionary of static fields that are added to every log message. :returns: A logging formatter object that can be used to format log records as JSON strings. """ if isinstance(logformat, str): jsonformat = logformat rename_fields = {} elif isinstance(logformat, list): logformat = cast(List[Any], logformat) log_fields: List[str] = [] rename_fields: Dict[str, str] = {} for field in logformat: if isinstance(field, str): log_fields.append(field) elif isinstance(field, dict): field = cast(Dict[str, Any], field) for original_field, renamed_field in field.items(): if isinstance(renamed_field, str): log_fields.append(original_field) rename_fields[original_field] = renamed_field else: _LOGGER.warning( 'Expected renamed log field to be a string, but got "%s"', _type(renamed_field) ) else: _LOGGER.warning( 'Expected each item of log format to be a string or dictionary, but got "%s"', _type(field) ) if len(log_fields) > 0: jsonformat = ' '.join([f'%({field})s' for field in log_fields]) else: jsonformat = PatroniLogger.DEFAULT_FORMAT else: jsonformat = PatroniLogger.DEFAULT_FORMAT rename_fields = {} _LOGGER.warning('Expected log format to be a string or a list, but got "%s"', _type(logformat)) try: from pythonjsonlogger import jsonlogger if hasattr(jsonlogger, 'RESERVED_ATTRS') \ and 'taskName' not in jsonlogger.RESERVED_ATTRS: # pyright: ignore [reportUnnecessaryContains] # compatibility with python 3.12, that added a new attribute to LogRecord jsonlogger.RESERVED_ATTRS += ('taskName',) return jsonlogger.JsonFormatter( jsonformat, dateformat, rename_fields=rename_fields, static_fields=static_fields ) except ImportError as e: _LOGGER.error('Failed to import "python-json-logger" library: %r. Falling back to the plain logger', e) except Exception as e: _LOGGER.error('Failed to initialize JsonFormatter: %r. Falling back to the plain logger', e) return self._get_plain_formatter(jsonformat, dateformat) def _get_formatter(self, config: Dict[str, Any]) -> logging.Formatter: """Returns a logging formatter based on the type of logger in the given configuration. :param config: ``log`` section from Patroni configuration. :returns: A :class:`logging.Formatter` object that can be used to format log records. """ logtype = config.get('type', PatroniLogger.DEFAULT_TYPE) logformat: type_logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT) dateformat = config.get('dateformat') or None # Convert empty string to `None` static_fields = config.get('static_fields', {}) if dateformat is not None and not isinstance(dateformat, str): _LOGGER.warning('Expected log dateformat to be a string, but got "%s"', _type(dateformat)) dateformat = None if logtype == 'json': formatter = self._get_json_formatter(logformat, dateformat, static_fields) else: formatter = self._get_plain_formatter(logformat, dateformat) return formatter def reload_config(self, config: Dict[str, Any]) -> None: """Apply log related configuration. .. note:: It is also able to deal with runtime configuration changes. :param config: ``log`` section from Patroni configuration. """ if self._config is None or not deep_compare(self._config, config): with self._queue_handler.queue.mutex: self._queue_handler.queue.maxsize = config.get('max_queue_size', self.DEFAULT_MAX_QUEUE_SIZE) self._root_logger.setLevel(config.get('level', PatroniLogger.DEFAULT_LEVEL)) if config.get('traceback_level', PatroniLogger.DEFAULT_TRACEBACK_LEVEL).lower() == 'debug': # show stack traces only if ``log.traceback_level`` is ``DEBUG`` logging.Logger.exception = debug_exception else: # show stack traces as ``ERROR`` log messages logging.Logger.exception = error_exception handler = self.log_handler if 'dir' in config: mode = parse_int(config.get('mode')) if not isinstance(handler, PatroniFileHandler): handler = PatroniFileHandler(os.path.join(config['dir'], __name__), mode) handler.set_log_file_mode(mode) max_file_size = int(config.get('file_size', 25000000)) handler.maxBytes = max_file_size # pyright: ignore [reportAttributeAccessIssue] handler.backupCount = int(config.get('file_num', 4)) # we can't use `if not isinstance(handler, logging.StreamHandler)` below, # because RotatingFileHandler and PatroniFileHandler are children of StreamHandler!!! elif handler is None or isinstance(handler, PatroniFileHandler): handler = logging.StreamHandler() is_new_handler = handler != self.log_handler if (self._is_config_changed(config) or is_new_handler) and handler: formatter = self._get_formatter(config) handler.setFormatter(formatter) if is_new_handler: with self.log_handler_lock: if self.log_handler: self._old_handlers.append(self.log_handler) self.log_handler = handler self._config = config.copy() self.update_loggers(config.get('loggers') or {}) def _close_old_handlers(self) -> None: """Close old log handlers. .. note:: It is used to remove different handlers that were configured previous to a reload in the configuration, e.g. if we are switching from :class:`PatroniFileHandler` to class:`~logging.StreamHandler` and vice-versa. """ while True: with self.log_handler_lock: if not self._old_handlers: break handler = self._old_handlers.pop() try: handler.close() except Exception: _LOGGER.exception('Failed to close the old log handler %s', handler) def run(self) -> None: """Run logger's thread main loop. Keep consuming log queue until requested to quit through ``None`` special log record. """ # switch to QueueHandler only when the thread was started with self.log_handler_lock: self._root_logger.addHandler(self._queue_handler) self._root_logger.removeHandler(self._proxy_handler) prev_record = None while True: self._close_old_handlers() if TYPE_CHECKING: # pragma: no cover assert self.log_handler is not None record = self._queue_handler.queue.get(True) # special message that indicates Patroni is shutting down if record is None: break if self._root_logger.level == logging.INFO: # messages like ``Lock owner: postgresql0; I am postgresql1`` will be shown only when stream doesn't # look normal. This is used to reduce chattiness of Patroni logs. if record.msg.startswith('Lock owner: '): prev_record, record = record, None else: if prev_record and prev_record.thread == record.thread: if not (record.msg.startswith('no action. ') or record.msg.startswith('PAUSE: no action')): self.log_handler.handle(prev_record) prev_record = None if record: self.log_handler.handle(record) self._queue_handler.queue.task_done() def shutdown(self) -> None: """Shut down the logger thread.""" try: # ``None`` is a special message indicating to queue handler that it should quit its main loop. self._queue_handler.queue.put_nowait(None) except Full: # Queue is full. # It seems that logging is not working, exiting with non-standard exit-code is the best we can do. sys.exit(self.LOGGING_BROKEN_EXIT_CODE) self.join() logging.shutdown() @property def queue_size(self) -> int: """Number of log records in the queue.""" return self._queue_handler.queue.qsize() @property def records_lost(self) -> int: """Number of logging records that have been lost while the queue was full.""" return self._queue_handler.records_lost patroni-4.0.4/patroni/postgresql/000077500000000000000000000000001472010352700170535ustar00rootroot00000000000000patroni-4.0.4/patroni/postgresql/__init__.py000066400000000000000000001745121472010352700211760ustar00rootroot00000000000000import logging import os import re import shlex import shutil import subprocess import time from contextlib import contextmanager from copy import deepcopy from datetime import datetime from threading import current_thread, Lock from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union from dateutil import tz from psutil import TimeoutExpired from .. import global_config, psycopg from ..async_executor import CriticalTask from ..collections import CaseInsensitiveDict, CaseInsensitiveSet, EMPTY_DICT from ..dcs import Cluster, Leader, Member, slot_name_from_member_name from ..exceptions import PostgresConnectionException from ..tags import Tags from ..utils import data_directory_is_empty, parse_int, polling_loop, Retry, RetryFailedError from .bootstrap import Bootstrap from .callback_executor import CallbackAction, CallbackExecutor from .cancellable import CancellableSubprocess from .config import ConfigHandler, mtime from .connection import ConnectionPool, get_connection_cursor from .misc import parse_history, parse_lsn, postgres_major_version_to_int from .mpp import AbstractMPP from .postmaster import PostmasterProcess from .slots import SlotsHandler from .sync import SyncHandler if TYPE_CHECKING: # pragma: no cover from psycopg import Connection as Connection3, Cursor from psycopg2 import connection as connection3, cursor logger = logging.getLogger(__name__) STATE_RUNNING = 'running' STATE_REJECT = 'rejecting connections' STATE_NO_RESPONSE = 'not responding' STATE_UNKNOWN = 'unknown' STOP_POLLING_INTERVAL = 1 @contextmanager def null_context(): yield class Postgresql(object): POSTMASTER_START_TIME = "pg_catalog.pg_postmaster_start_time()" TL_LSN = ("CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 " "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name(" "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, " # primary timeline "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 ELSE " "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}{2}_{1}(), '0/0')::bigint END, " # wal(_flush)?_lsn "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint, " "pg_catalog.pg_{0}_{1}_diff(COALESCE(pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint, " "pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused()") def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None: self.name: str = config['name'] self.scope: str = config['scope'] self._data_dir: str = config['data_dir'] self._database = config.get('database', 'postgres') self._version_file = os.path.join(self._data_dir, 'PG_VERSION') self._pg_control = os.path.join(self._data_dir, 'global', 'pg_control') self.connection_string: str self.proxy_url: Optional[str] self._major_version = self.get_major_version() self._state_lock = Lock() self.set_state('stopped') self._pending_restart_reason = CaseInsensitiveDict() self.connection_pool = ConnectionPool() self._connection = self.connection_pool.get('heartbeat') self.mpp_handler = mpp.get_handler_impl(self) self._bin_dir = config.get('bin_dir') or '' self.config = ConfigHandler(self, config) self.config.check_directories() self.bootstrap = Bootstrap(self) self.bootstrapping = False self.__thread_ident = current_thread().ident self.slots_handler = SlotsHandler(self) self.sync_handler = SyncHandler(self) self._callback_executor = CallbackExecutor() self.__cb_called = False self.__cb_pending = None self.cancellable = CancellableSubprocess() self._sysid = '' self.retry = Retry(max_tries=-1, deadline=config['retry_timeout'] / 2.0, max_delay=1, retry_exceptions=PostgresConnectionException) # Retry 'pg_is_in_recovery()' only once self._is_leader_retry = Retry(max_tries=1, deadline=config['retry_timeout'] / 2.0, max_delay=1, retry_exceptions=PostgresConnectionException) self._role_lock = Lock() self.set_role(self.get_postgres_role_from_data_directory()) self._state_entry_timestamp = 0 self._cluster_info_state = {} self._should_query_slots = True self._enforce_hot_standby_feedback = False self._cached_replica_timeline = None # Last known running process self._postmaster_proc = None self._available_gucs = None if self.is_running(): # If we found postmaster process we need to figure out whether postgres is accepting connections self.set_state('starting') self.check_startup_state_changed() if self.state == 'running': # we are "joining" already running postgres # we know that PostgreSQL is accepting connections and can read some GUC's from pg_settings self.config.load_current_server_parameters() self.set_role('primary' if self.is_primary() else 'replica') hba_saved = self.config.replace_pg_hba() ident_saved = self.config.replace_pg_ident() if self.major_version < 120000 or self.role == 'primary': # If PostgreSQL is running as a primary or we run PostgreSQL that is older than 12 we can # call reload_config() once again (the first call happened in the ConfigHandler constructor), # so that it can figure out if config files should be updated and pg_ctl reload executed. self.config.reload_config(config, sighup=bool(hba_saved or ident_saved)) elif hba_saved or ident_saved: self.reload() elif not self.is_running() and self.role == 'primary': self.set_role('demoted') @property def create_replica_methods(self) -> List[str]: return self.config.get('create_replica_methods', []) or self.config.get('create_replica_method', []) or [] @property def major_version(self) -> int: return self._major_version @property def database(self) -> str: return self._database @property def data_dir(self) -> str: return self._data_dir @property def callback(self) -> Dict[str, str]: return self.config.get('callbacks', {}) or {} @property def wal_dir(self) -> str: return os.path.join(self._data_dir, 'pg_' + self.wal_name) @property def wal_name(self) -> str: return 'wal' if self._major_version >= 100000 else 'xlog' @property def wal_flush(self) -> str: """For PostgreSQL 9.6 onwards we want to use pg_current_wal_flush_lsn()/pg_current_xlog_flush_location().""" return '_flush' if self._major_version >= 90600 else '' @property def lsn_name(self) -> str: return 'lsn' if self._major_version >= 100000 else 'location' @property def supports_quorum_commit(self) -> bool: """``True`` if quorum commit is supported by Postgres.""" return self._major_version >= 100000 @property def supports_multiple_sync(self) -> bool: """:returns: `True` if Postgres version supports more than one synchronous node.""" return self._major_version >= 90600 @property def can_advance_slots(self) -> bool: """``True`` if :attr:``major_version`` is greater than 110000.""" return self.major_version >= 110000 @property def cluster_info_query(self) -> str: """Returns the monitoring query with a fixed number of fields. The query text is constructed based on current state in DCS and PostgreSQL version: 1. function names depend on version. wal/lsn for v10+ and xlog/location for pre v10. 2. for primary we query timeline_id (extracted from pg_walfile_name()) and pg_current_wal_lsn() 3. for replicas we query pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn(), and pg_is_wal_replay_paused() 4. for v9.6+ we query primary_slot_name and primary_conninfo from pg_stat_get_wal_receiver() 5. for v11+ with permanent logical slots we query from pg_replication_slots and aggregate the result 6. for standby_leader node running v9.6+ we also query pg_control_checkpoint to fetch timeline_id 7. if sync replication is enabled we query pg_stat_replication and aggregate the result. In addition to that we get current values of synchronous_commit and synchronous_standby_names GUCs. If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on. """ extra = ", " + (("pg_catalog.current_setting('synchronous_commit'), " "pg_catalog.current_setting('synchronous_standby_names'), " "(SELECT pg_catalog.json_agg(r.*) FROM (SELECT w.pid as pid, application_name, sync_state," " pg_catalog.pg_{0}_{1}_diff(write_{1}, '0/0')::bigint AS write_lsn," " pg_catalog.pg_{0}_{1}_diff(flush_{1}, '0/0')::bigint AS flush_lsn," " pg_catalog.pg_{0}_{1}_diff(replay_{1}, '0/0')::bigint AS replay_lsn " "FROM pg_catalog.pg_stat_get_wal_senders() w," " pg_catalog.pg_stat_get_activity(w.pid)" " WHERE w.state = 'streaming') r)").format(self.wal_name, self.lsn_name) if global_config.is_synchronous_mode and self.role in ('primary', 'promoted') else "'on', '', NULL") if self._major_version >= 90600: extra = ("pg_catalog.current_setting('restore_command')" if self._major_version >= 120000 else "NULL") +\ ", " + ("(SELECT pg_catalog.json_agg(s.*) FROM (SELECT slot_name, slot_type as type, datoid::bigint, " "plugin, catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" " AS confirmed_flush_lsn, pg_catalog.pg_wal_lsn_diff(restart_lsn, '0/0')::bigint" " AS restart_lsn, xmin FROM pg_catalog.pg_get_replication_slots()) AS s)" if self._should_query_slots and self.can_advance_slots else "NULL") + extra extra = (", CASE WHEN latest_end_lsn IS NULL THEN NULL ELSE received_tli END," " slot_name, conninfo, status, {0} FROM pg_catalog.pg_stat_get_wal_receiver()").format(extra) if self.role == 'standby_leader': extra = "timeline_id" + extra + ", pg_catalog.pg_control_checkpoint()" else: extra = "0" + extra else: extra = "0, NULL, NULL, NULL, NULL, NULL, NULL" + extra return ("SELECT " + self.TL_LSN + ", {3}").format(self.wal_name, self.lsn_name, self.wal_flush, extra) @property def available_gucs(self) -> CaseInsensitiveSet: """GUCs available in this Postgres server.""" if not self._available_gucs: self._available_gucs = self._get_gucs() return self._available_gucs def _version_file_exists(self) -> bool: return not self.data_directory_empty() and os.path.isfile(self._version_file) def get_major_version(self) -> int: """Reads major version from PG_VERSION file :returns: major PostgreSQL version in integer format or 0 in case of missing file or errors""" if self._version_file_exists(): try: with open(self._version_file) as f: return postgres_major_version_to_int(f.read().strip()) except Exception: logger.exception('Failed to read PG_VERSION from %s', self._data_dir) return 0 def pgcommand(self, cmd: str) -> str: """Return path to the specified PostgreSQL command. .. note:: If ``postgresql.bin_name.*cmd*`` was configured by the user then that binary name is used, otherwise the default binary name *cmd* is used. :param cmd: the Postgres binary name to get path to. :returns: path to Postgres binary named *cmd*. """ return os.path.join(self._bin_dir, (self.config.get('bin_name', {}) or EMPTY_DICT).get(cmd, cmd)) def pg_ctl(self, cmd: str, *args: str, **kwargs: Any) -> bool: """Builds and executes pg_ctl command :returns: `!True` when return_code == 0, otherwise `!False`""" pg_ctl = [self.pgcommand('pg_ctl'), cmd] return subprocess.call(pg_ctl + ['-D', self._data_dir] + list(args), **kwargs) == 0 def initdb(self, *args: str, **kwargs: Any) -> bool: """Builds and executes the initdb command. :param args: List of arguments to be joined into the initdb command. :param kwargs: Keyword arguments to pass to ``subprocess.call``. :returns: ``True`` if the result of ``subprocess.call`, the exit code, is ``0``. """ initdb = [self.pgcommand('initdb')] + list(args) + [self.data_dir] return subprocess.call(initdb, **kwargs) == 0 def pg_isready(self) -> str: """Runs pg_isready to see if PostgreSQL is accepting connections. :returns: 'ok' if PostgreSQL is up, 'reject' if starting up, 'no_response' if not up.""" r = self.connection_pool.conn_kwargs cmd = [self.pgcommand('pg_isready'), '-p', r['port'], '-d', self._database] # Host is not set if we are connecting via default unix socket if 'host' in r: cmd.extend(['-h', r['host']]) # We only need the username because pg_isready does not try to authenticate if 'user' in r: cmd.extend(['-U', r['user']]) ret = subprocess.call(cmd) return_codes = {0: STATE_RUNNING, 1: STATE_REJECT, 2: STATE_NO_RESPONSE, 3: STATE_UNKNOWN} return return_codes.get(ret, STATE_UNKNOWN) def reload_config(self, config: Dict[str, Any], sighup: bool = False) -> None: self.config.reload_config(config, sighup) self._is_leader_retry.deadline = self.retry.deadline = config['retry_timeout'] / 2.0 @property def pending_restart_reason(self) -> CaseInsensitiveDict: """Get :attr:`_pending_restart_reason` value. :attr:`_pending_restart_reason` is a :class:`CaseInsensitiveDict` object of the PG parameters that are causing pending restart state. Every key is a parameter name, value - a dictionary containing the old and the new value (see :func:`~patroni.postgresql.config.get_param_diff`). """ return self._pending_restart_reason def set_pending_restart_reason(self, diff_dict: CaseInsensitiveDict) -> None: """Set new or update current :attr:`_pending_restart_reason`. :param diff_dict: :class:``CaseInsensitiveDict`` object with the parameters that are causing pending restart state with the diff of their values. Used to reset/update the :attr:`_pending_restart_reason`. """ self._pending_restart_reason = diff_dict @property def sysid(self) -> str: if not self._sysid and not self.bootstrapping: data = self.controldata() self._sysid = data.get('Database system identifier', '') return self._sysid def get_postgres_role_from_data_directory(self) -> str: if self.data_directory_empty() or not self.controldata(): return 'uninitialized' elif self.config.recovery_conf_exists(): return 'replica' else: return 'primary' @property def server_version(self) -> int: return self._connection.server_version def connection(self) -> Union['connection3', 'Connection3[Any]']: return self._connection.get() def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]: """Execute *sql* query with *params* and optionally return results. :param sql: SQL statement to execute. :param params: parameters to pass. :returns: a query response as a list of tuples if there is any. :raises: :exc:`~psycopg.Error` if had issues while executing *sql*. :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database. :exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL restart. """ try: return self._connection.query(sql, *params) except PostgresConnectionException as exc: if self.state == 'restarting': raise RetryFailedError('cluster is being restarted') from exc raise def query(self, sql: str, *params: Any, retry: bool = True) -> List[Tuple[Any, ...]]: """Execute *sql* query with *params* and optionally return results. :param sql: SQL statement to execute. :param params: parameters to pass. :param retry: whether the query should be retried upon failure or given up immediately. :returns: a query response as a list of tuples if there is any. :raises: :exc:`~psycopg.Error` if had issues while executing *sql*. :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database. :exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL restart or if retry deadline was exceeded. """ if not retry: return self._query(sql, *params) try: return self.retry(self._query, sql, *params) except RetryFailedError as exc: raise PostgresConnectionException(str(exc)) from exc def pg_control_exists(self) -> bool: return os.path.isfile(self._pg_control) def data_directory_empty(self) -> bool: if self.pg_control_exists(): return False return data_directory_is_empty(self._data_dir) def replica_method_options(self, method: str) -> Dict[str, Any]: return deepcopy(self.config.get(method, {}) or EMPTY_DICT.copy()) def replica_method_can_work_without_replication_connection(self, method: str) -> bool: return method != 'basebackup' and bool(self.replica_method_options(method).get('no_leader')) def can_create_replica_without_replication_connection(self, replica_methods: Optional[List[str]]) -> bool: """ go through the replication methods to see if there are ones that does not require a working replication connection. """ if replica_methods is None: replica_methods = self.create_replica_methods return any(self.replica_method_can_work_without_replication_connection(m) for m in replica_methods) @property def enforce_hot_standby_feedback(self) -> bool: return self._enforce_hot_standby_feedback def set_enforce_hot_standby_feedback(self, value: bool) -> None: # If we enable or disable the hot_standby_feedback we need to update postgresql.conf and reload if self._enforce_hot_standby_feedback != value: self._enforce_hot_standby_feedback = value if self.is_running(): self.config.write_postgresql_conf() self.reload() def reset_cluster_info_state(self, cluster: Optional[Cluster], tags: Optional[Tags] = None) -> None: """Reset monitoring query cache. .. note:: It happens in the beginning of heart-beat loop and on change of `synchronous_standby_names`. :param cluster: currently known cluster state from DCS :param tags: reference to an object implementing :class:`Tags` interface. """ self._cluster_info_state = {} if not tags: return if global_config.is_standby_cluster: # Standby cluster can't have logical replication slots, and we don't need to enforce hot_standby_feedback self.set_enforce_hot_standby_feedback(False) if cluster and cluster.config and cluster.config.modify_version: # We want to enable hot_standby_feedback if the replica is supposed # to have a logical slot or in case if it is the cascading replica. self.set_enforce_hot_standby_feedback(not global_config.is_standby_cluster and self.can_advance_slots and cluster.should_enforce_hot_standby_feedback(self, tags)) self._should_query_slots = global_config.member_slots_ttl > 0 or cluster.has_permanent_slots(self, tags) def _cluster_info_state_get(self, name: str) -> Optional[Any]: if not self._cluster_info_state: try: result = self._is_leader_retry(self._query, self.cluster_info_query)[0] cluster_info_state = dict(zip(['timeline', 'wal_position', 'replayed_location', 'received_location', 'replay_paused', 'pg_control_timeline', 'received_tli', 'slot_name', 'conninfo', 'receiver_state', 'restore_command', 'slots', 'synchronous_commit', 'synchronous_standby_names', 'pg_stat_replication'], result)) if self._should_query_slots and self.can_advance_slots: cluster_info_state['slots'] =\ self.slots_handler.process_permanent_slots(cluster_info_state['slots']) self._cluster_info_state = cluster_info_state except RetryFailedError as e: # SELECT failed two times self._cluster_info_state = {'error': str(e)} if not self.is_starting() and self.pg_isready() == STATE_REJECT: self.set_state('starting') if 'error' in self._cluster_info_state: raise PostgresConnectionException(self._cluster_info_state['error']) return self._cluster_info_state.get(name) def replayed_location(self) -> Optional[int]: return self._cluster_info_state_get('replayed_location') def received_location(self) -> Optional[int]: return self._cluster_info_state_get('received_location') def slots(self) -> Dict[str, int]: """Get replication slots state. ..note:: Since this methods is supposed to be used only by the leader and only to publish state of replication slots to DCS so that other nodes can advance LSN on respective replication slots, we are also adding our own name to the list. All slots that shouldn't be published to DCS later will be filtered out by :meth:`~Cluster.maybe_filter_permanent_slots` method. :returns: A :class:`dict` object with replication slot names and LSNs as absolute values. """ return {**(self._cluster_info_state_get('slots') or {}), slot_name_from_member_name(self.name): self.last_operation()} \ if self.can_advance_slots else {} def primary_slot_name(self) -> Optional[str]: return self._cluster_info_state_get('slot_name') def primary_conninfo(self) -> Optional[str]: return self._cluster_info_state_get('conninfo') def received_timeline(self) -> Optional[int]: return self._cluster_info_state_get('received_tli') def synchronous_commit(self) -> str: """:returns: "synchronous_commit" GUC value.""" return self._cluster_info_state_get('synchronous_commit') or 'on' def synchronous_standby_names(self) -> str: """:returns: "synchronous_standby_names" GUC value.""" return self._cluster_info_state_get('synchronous_standby_names') or '' def pg_stat_replication(self) -> List[Dict[str, Any]]: """:returns: a result set of 'SELECT * FROM pg_stat_replication'.""" return self._cluster_info_state_get('pg_stat_replication') or [] def replication_state_from_parameters(self, is_primary: bool, receiver_state: Optional[str], restore_command: Optional[str]) -> Optional[str]: """Figure out the replication state from input parameters. .. note:: This method could be only called when Postgres is up, running and queries are successfully executed. :is_primary: `True` is postgres is not running in recovery :receiver_state: value from `pg_stat_get_wal_receiver.state` or None if Postgres is older than 9.6 :restore_command: value of ``restore_command`` GUC for PostgreSQL 12+ or `postgresql.recovery_conf.restore_command` if it is set in Patroni configuration :returns: - `None` for the primary and for Postgres older than 9.6; - 'streaming' if replica is streaming according to the `pg_stat_wal_receiver` view; - 'in archive recovery' if replica isn't streaming and there is a `restore_command` """ if self._major_version >= 90600 and not is_primary: if receiver_state == 'streaming': return 'streaming' # For Postgres older than 12 we get `restore_command` from Patroni config, otherwise we check GUC if self._major_version < 120000 and self.config.restore_command() or restore_command: return 'in archive recovery' def replication_state(self) -> Optional[str]: """Checks replication state from `pg_stat_get_wal_receiver()`. .. note:: Available only since 9.6 :returns: ``streaming``, ``in archive recovery``, or ``None`` """ return self.replication_state_from_parameters(self.is_primary(), self._cluster_info_state_get('receiver_state'), self._cluster_info_state_get('restore_command')) def is_primary(self) -> bool: try: return bool(self._cluster_info_state_get('timeline')) except PostgresConnectionException: logger.warning('Failed to determine PostgreSQL state from the connection, falling back to cached role') return bool(self.is_running() and self.role == 'primary') def replay_paused(self) -> bool: return self._cluster_info_state_get('replay_paused') or False def resume_wal_replay(self) -> None: self._query('SELECT pg_catalog.pg_{0}_replay_resume()'.format(self.wal_name)) def handle_parameter_change(self) -> None: if self.major_version >= 140000 and not self.is_starting() and self.replay_paused(): logger.info('Resuming paused WAL replay for PostgreSQL 14+') self.resume_wal_replay() def pg_control_timeline(self) -> Optional[int]: try: return int(self.controldata().get("Latest checkpoint's TimeLineID", "")) except (TypeError, ValueError): logger.exception('Failed to parse timeline from pg_controldata output') def parse_wal_record(self, timeline: str, lsn: str) -> Union[Tuple[str, str, str, str], Tuple[None, None, None, None]]: out, err = self.waldump(timeline, lsn, 1) if out and not err: match = re.match(r'^rmgr:\s+(.+?)\s+len \(rec/tot\):\s+\d+/\s+\d+, tx:\s+\d+, ' r'lsn: ([0-9A-Fa-f]+/[0-9A-Fa-f]+), prev ([0-9A-Fa-f]+/[0-9A-Fa-f]+), ' r'.*?desc: (.+)', out.decode('utf-8')) if match: return match.group(1), match.group(2), match.group(3), match.group(4) return None, None, None, None def _checkpoint_locations_from_controldata(self, data: Dict[str, str]) -> Optional[Tuple[int, int]]: """Get shutdown checkpoint location. :param data: :class:`dict` object with values returned by `pg_controldata` tool. :returns: a tuple of checkpoint LSN for the cleanly shut down primary, and LSN of prev wal record (SWITCH) if we know that the checkpoint was written to the new WAL file due to the archive_mode=on. """ timeline = data.get("Latest checkpoint's TimeLineID") lsn = checkpoint_lsn = data.get('Latest checkpoint location') prev_lsn = None if data.get('Database cluster state') == 'shut down' and lsn and timeline and checkpoint_lsn: try: checkpoint_lsn = parse_lsn(checkpoint_lsn) rm_name, lsn, prev, desc = self.parse_wal_record(timeline, lsn) desc = str(desc).strip().lower() if rm_name == 'XLOG' and lsn and parse_lsn(lsn) == checkpoint_lsn and prev and\ desc.startswith('checkpoint') and desc.endswith('shutdown'): _, lsn, _, desc = self.parse_wal_record(timeline, prev) prev = parse_lsn(prev) # If the cluster is shutdown with archive_mode=on, WAL is switched before writing the checkpoint. # In this case we want to take the LSN of previous record (SWITCH) as the last known WAL location. if lsn and parse_lsn(lsn) == prev and str(desc).strip() in ('xlog switch', 'SWITCH'): prev_lsn = prev except Exception as e: logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e) if isinstance(checkpoint_lsn, int): return checkpoint_lsn, (prev_lsn or checkpoint_lsn) def latest_checkpoint_location(self) -> Optional[int]: """Get shutdown checkpoint location. .. note:: In case if checkpoint was written to the new WAL file due to the archive_mode=on we return LSN of the previous wal record (SWITCH). :returns: checkpoint LSN for the cleanly shut down primary. """ checkpoint_locations = self._checkpoint_locations_from_controldata(self.controldata()) if checkpoint_locations: return checkpoint_locations[1] def is_running(self) -> Optional[PostmasterProcess]: """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process is running updates the cached process based on pid file.""" if self._postmaster_proc: if self._postmaster_proc.is_running(): return self._postmaster_proc self._postmaster_proc = None self._available_gucs = None # we noticed that postgres was restarted, force syncing of replication slots and check of logical slots self.slots_handler.schedule() self._postmaster_proc = PostmasterProcess.from_pidfile(self._data_dir) return self._postmaster_proc @property def cb_called(self) -> bool: return self.__cb_called def call_nowait(self, cb_type: CallbackAction) -> None: """pick a callback command and call it without waiting for it to finish """ if self.bootstrapping: return if cb_type in (CallbackAction.ON_START, CallbackAction.ON_STOP, CallbackAction.ON_RESTART, CallbackAction.ON_ROLE_CHANGE): self.__cb_called = True if self.callback and cb_type in self.callback: cmd = self.callback[cb_type] role = 'primary' if self.role == 'promoted' else self.role try: cmd = shlex.split(self.callback[cb_type]) + [cb_type, role, self.scope] self._callback_executor.call(cmd) except Exception: logger.exception('callback %s %r %s %s failed', cmd, cb_type, role, self.scope) @property def role(self) -> str: with self._role_lock: return self._role def set_role(self, value: str) -> None: with self._role_lock: self._role = value @property def state(self) -> str: with self._state_lock: return self._state def set_state(self, value: str) -> None: with self._state_lock: self._state = value self._state_entry_timestamp = time.time() def time_in_state(self) -> float: return time.time() - self._state_entry_timestamp def is_starting(self) -> bool: return self.state == 'starting' def wait_for_port_open(self, postmaster: PostmasterProcess, timeout: float) -> bool: """Waits until PostgreSQL opens ports.""" for _ in polling_loop(timeout): if self.cancellable.is_cancelled: return False if not postmaster.is_running(): logger.error('postmaster is not running') self.set_state('start failed') return False isready = self.pg_isready() if isready != STATE_NO_RESPONSE: if isready not in [STATE_REJECT, STATE_RUNNING]: logger.warning("Can't determine PostgreSQL startup status, assuming running") return True logger.warning("Timed out waiting for PostgreSQL to start") return False def start(self, timeout: Optional[float] = None, task: Optional[CriticalTask] = None, block_callbacks: bool = False, role: Optional[str] = None, after_start: Optional[Callable[..., Any]] = None) -> Optional[bool]: """Start PostgreSQL Waits for postmaster to open ports or terminate so pg_isready can be used to check startup completion or failure. :returns: True if start was initiated and postmaster ports are open, False if start failed, and None if postgres is still starting up""" # make sure we close all connections established against # the former node, otherwise, we might get a stalled one # after kill -9, which would report incorrect data to # patroni. self.connection_pool.close() if self.is_running(): logger.error('Cannot start PostgreSQL because one is already running.') self.set_state('starting') return True if not block_callbacks: self.__cb_pending = CallbackAction.ON_START self.set_role(role or self.get_postgres_role_from_data_directory()) self.set_state('starting') self.set_pending_restart_reason(CaseInsensitiveDict()) try: if not self.ensure_major_version_is_known(): return None configuration = self.config.effective_configuration except Exception: return None self.config.check_directories() self.config.write_postgresql_conf(configuration) self.config.resolve_connection_addresses() self.config.replace_pg_hba() self.config.replace_pg_ident() options = ['--{0}={1}'.format(p, configuration[p]) for p in self.config.CMDLINE_OPTIONS if p in configuration and p not in ('wal_keep_segments', 'wal_keep_size')] if self.cancellable.is_cancelled: return False with task or null_context(): if task and task.is_cancelled: logger.info("PostgreSQL start cancelled.") return False self._postmaster_proc = PostmasterProcess.start(self.pgcommand('postgres'), self._data_dir, self.config.postgresql_conf, options) if task: task.complete(self._postmaster_proc) start_timeout = timeout if not start_timeout: try: start_timeout = float(self.config.get('pg_ctl_timeout', 60) or 0) except ValueError: start_timeout = 60 # We want postmaster to open ports before we continue if not self._postmaster_proc or not self.wait_for_port_open(self._postmaster_proc, start_timeout): return False ret = self.wait_for_startup(start_timeout) if ret is not None: if ret and after_start: after_start() return ret elif timeout is not None: return False else: return None def checkpoint(self, connect_kwargs: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None) -> Optional[str]: check_not_is_in_recovery = connect_kwargs is not None connect_kwargs = connect_kwargs or self.connection_pool.conn_kwargs for p in ['connect_timeout', 'options']: connect_kwargs.pop(p, None) if timeout: connect_kwargs['connect_timeout'] = timeout try: with get_connection_cursor(**connect_kwargs) as cur: cur.execute("SET statement_timeout = 0") if check_not_is_in_recovery: cur.execute('SELECT pg_catalog.pg_is_in_recovery()') row = cur.fetchone() if not row or row[0]: return 'is_in_recovery=true' cur.execute('CHECKPOINT') except psycopg.Error: logger.exception('Exception during CHECKPOINT') return 'not accessible or not healthy' def stop(self, mode: str = 'fast', block_callbacks: bool = False, checkpoint: Optional[bool] = None, on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int, int], Any]] = None, before_shutdown: Optional[Callable[..., Any]] = None, stop_timeout: Optional[int] = None) -> bool: """Stop PostgreSQL Supports a callback when a safepoint is reached. A safepoint is when no user backend can return a successful commit to users. Currently this means we wait for user backends to close. But in the future alternate mechanisms could be added. :param on_safepoint: This callback is called when no user backends are running. :param on_shutdown: is called when pg_controldata starts reporting `Database cluster state: shut down` :param before_shutdown: is called after running optional CHECKPOINT and before running pg_ctl stop """ if checkpoint is None: checkpoint = False if mode == 'immediate' else True success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint, on_shutdown, before_shutdown, stop_timeout) if success: # block_callbacks is used during restart to avoid # running start/stop callbacks in addition to restart ones if not block_callbacks: self.set_state('stopped') if pg_signaled: self.call_nowait(CallbackAction.ON_STOP) else: logger.warning('pg_ctl stop failed') self.set_state('stop failed') return success def _do_stop(self, mode: str, block_callbacks: bool, checkpoint: bool, on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[[int, int], Any]], before_shutdown: Optional[Callable[..., Any]], stop_timeout: Optional[int]) -> Tuple[bool, bool]: postmaster = self.is_running() if not postmaster: if on_safepoint: on_safepoint() return True, False if checkpoint and not self.is_starting(): self.checkpoint(timeout=stop_timeout) if not block_callbacks: self.set_state('stopping') # invoke user-directed before stop script self._before_stop() if before_shutdown: before_shutdown() # Send signal to postmaster to stop success = postmaster.signal_stop(mode, self.pgcommand('pg_ctl')) if success is not None: if success and on_safepoint: on_safepoint() return success, True # We can skip safepoint detection if we don't have a callback if on_safepoint: # Wait for our connection to terminate so we can be sure that no new connections are being initiated self._wait_for_connection_close(postmaster) postmaster.wait_for_user_backends_to_close(stop_timeout) on_safepoint() if on_shutdown and mode in ('fast', 'smart'): i = 0 # Wait for pg_controldata `Database cluster state:` to change to "shut down" while postmaster.is_running(): data = self.controldata() if data.get('Database cluster state', '') == 'shut down': checkpoint_locations = self._checkpoint_locations_from_controldata(data) if checkpoint_locations: on_shutdown(*checkpoint_locations) break elif data.get('Database cluster state', '').startswith('shut down'): # shut down in recovery break elif stop_timeout and i >= stop_timeout: stop_timeout = 0 break time.sleep(STOP_POLLING_INTERVAL) i += STOP_POLLING_INTERVAL try: postmaster.wait(timeout=stop_timeout) except TimeoutExpired: logger.warning("Timeout during postmaster stop, aborting Postgres.") if not self.terminate_postmaster(postmaster, mode, stop_timeout): postmaster.wait() return True, True def terminate_postmaster(self, postmaster: PostmasterProcess, mode: str, stop_timeout: Optional[int]) -> Optional[bool]: if mode in ['fast', 'smart']: try: success = postmaster.signal_stop('immediate', self.pgcommand('pg_ctl')) if success: return True postmaster.wait(timeout=stop_timeout) return True except TimeoutExpired: pass logger.warning("Sending SIGKILL to Postmaster and its children") return postmaster.signal_kill() def terminate_starting_postmaster(self, postmaster: PostmasterProcess) -> None: """Terminates a postmaster that has not yet opened ports or possibly even written a pid file. Blocks until the process goes away.""" postmaster.signal_stop('immediate', self.pgcommand('pg_ctl')) postmaster.wait() def _wait_for_connection_close(self, postmaster: PostmasterProcess) -> None: try: while postmaster.is_running(): # Need a timeout here? self._connection.query("SELECT 1") time.sleep(STOP_POLLING_INTERVAL) except (psycopg.Error, PostgresConnectionException): pass def reload(self, block_callbacks: bool = False) -> bool: ret = self.pg_ctl('reload') if ret and not block_callbacks: self.call_nowait(CallbackAction.ON_RELOAD) return ret def check_for_startup(self) -> bool: """Checks PostgreSQL status and returns if PostgreSQL is in the middle of startup.""" return self.is_starting() and not self.check_startup_state_changed() def check_startup_state_changed(self) -> bool: """Checks if PostgreSQL has completed starting up or failed or still starting. Should only be called when state == 'starting' :returns: True if state was changed from 'starting' """ ready = self.pg_isready() if ready == STATE_REJECT: return False elif ready == STATE_NO_RESPONSE: ret = not self.is_running() if ret: self.set_state('start failed') self.slots_handler.schedule(False) # TODO: can remove this? self.config.save_configuration_files(True) # TODO: maybe remove this? return ret else: if ready != STATE_RUNNING: # Bad configuration or unexpected OS error. No idea of PostgreSQL status. # Let the main loop of run cycle clean up the mess. logger.warning("%s status returned from pg_isready", "Unknown" if ready == STATE_UNKNOWN else "Invalid") self.set_state('running') self.slots_handler.schedule() self.config.save_configuration_files(True) # TODO: __cb_pending can be None here after PostgreSQL restarts on its own. Do we want to call the callback? # Previously we didn't even notice. action = self.__cb_pending or CallbackAction.ON_START self.call_nowait(action) self.__cb_pending = None return True def wait_for_startup(self, timeout: float = 0) -> Optional[bool]: """Waits for PostgreSQL startup to complete or fail. :returns: True if start was successful, False otherwise""" if not self.is_starting(): # Should not happen logger.warning("wait_for_startup() called when not in starting state") while not self.check_startup_state_changed(): if self.cancellable.is_cancelled or timeout and self.time_in_state() > timeout: return None time.sleep(1) return self.state == 'running' def restart(self, timeout: Optional[float] = None, task: Optional[CriticalTask] = None, block_callbacks: bool = False, role: Optional[str] = None, before_shutdown: Optional[Callable[..., Any]] = None, after_start: Optional[Callable[..., Any]] = None) -> Optional[bool]: """Restarts PostgreSQL. When timeout parameter is set the call will block either until PostgreSQL has started, failed to start or timeout arrives. :returns: True when restart was successful and timeout did not expire when waiting. """ self.set_state('restarting') if not block_callbacks: self.__cb_pending = CallbackAction.ON_RESTART ret = self.stop(block_callbacks=True, before_shutdown=before_shutdown)\ and self.start(timeout, task, True, role, after_start) if not ret and not self.is_starting(): self.set_state('restart failed ({0})'.format(self.state)) return ret def is_healthy(self) -> bool: if not self.is_running(): logger.warning('Postgresql is not running.') return False return True def get_guc_value(self, name: str) -> Optional[str]: cmd = [self.pgcommand('postgres'), '-D', self._data_dir, '-C', name, '--config-file={}'.format(self.config.postgresql_conf)] try: data = subprocess.check_output(cmd) if data: return data.decode('utf-8').strip() except Exception as e: logger.error('Failed to execute %s: %r', cmd, e) def controldata(self) -> Dict[str, str]: """ return the contents of pg_controldata, or non-True value if pg_controldata call failed """ # Don't try to call pg_controldata during backup restore if self._version_file_exists() and self.state != 'creating replica': try: env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C'} data = subprocess.check_output([self.pgcommand('pg_controldata'), self._data_dir], env=env) if data: data = filter(lambda e: ':' in e, data.decode('utf-8').splitlines()) # pg_controldata output depends on major version. Some of parameters are prefixed by 'Current ' return {k.replace('Current ', '', 1): v.strip() for k, v in map(lambda e: e.split(':', 1), data)} except subprocess.CalledProcessError: logger.exception("Error when calling pg_controldata") return {} def waldump(self, timeline: Union[int, str], lsn: str, limit: int) -> Tuple[Optional[bytes], Optional[bytes]]: cmd = self.pgcommand('pg_{0}dump'.format(self.wal_name)) env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C', 'PGDATA': self._data_dir} try: waldump = subprocess.Popen([cmd, '-t', str(timeline), '-s', lsn, '-n', str(limit)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) out, err = waldump.communicate() waldump.wait() return out, err except Exception as e: logger.error('Failed to execute `%s -t %s -s %s -n %s`: %r', cmd, timeline, lsn, limit, e) return None, None @contextmanager def get_replication_connection_cursor(self, host: Optional[str] = None, port: Union[int, str] = 5432, **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]: conn_kwargs = self.config.replication.copy() conn_kwargs.update(host=host, port=int(port) if port else None, user=conn_kwargs.pop('username'), connect_timeout=3, replication=1, options='-c statement_timeout=2000') with get_connection_cursor(**conn_kwargs) as cur: yield cur def get_replica_timeline(self) -> Optional[int]: try: with self.get_replication_connection_cursor(**self.config.local_replication_address) as cur: cur.execute('IDENTIFY_SYSTEM') row = cur.fetchone() return row[1] if row else None except Exception: logger.exception('Can not fetch local timeline and lsn from replication connection') def replica_cached_timeline(self, primary_timeline: Optional[int]) -> Optional[int]: if not self._cached_replica_timeline or not primary_timeline\ or self._cached_replica_timeline != primary_timeline: self._cached_replica_timeline = self.get_replica_timeline() return self._cached_replica_timeline def get_primary_timeline(self) -> int: """:returns: current timeline if postgres is running as a primary or 0.""" return self._cluster_info_state_get('timeline') or 0 def get_history(self, timeline: int) -> List[Union[Tuple[int, int, str], Tuple[int, int, str, str, str]]]: history_path = os.path.join(self.wal_dir, '{0:08X}.history'.format(timeline)) history_mtime = mtime(history_path) history: List[Union[Tuple[int, int, str], Tuple[int, int, str, str, str]]] = [] if history_mtime: try: with open(history_path, 'r') as f: history_content = f.read() history = list(parse_history(history_content)) if history[-1][0] == timeline - 1: history_mtime = datetime.fromtimestamp(history_mtime).replace(tzinfo=tz.tzlocal()) history[-1] = history[-1][:3] + (history_mtime.isoformat(), self.name) except Exception: logger.exception('Failed to read and parse %s', (history_path,)) return history def follow(self, member: Union[Leader, Member, None], role: str = 'replica', timeout: Optional[float] = None, do_reload: bool = False) -> Optional[bool]: """Reconfigure postgres to follow a new member or use different recovery parameters. Method may call `on_role_change` callback if role is changing. :param member: The member to follow :param role: The desired role, normally 'replica', but could also be a 'standby_leader' :param timeout: start timeout, how long should the `start()` method wait for postgres accepting connections :param do_reload: indicates that after updating postgresql.conf we just need to do a reload instead of restart :returns: True - if restart/reload were successfully performed, False - if restart/reload failed None - if nothing was done or if Postgres is still in starting state after `timeout` seconds.""" if not self.ensure_major_version_is_known(): return None recovery_params = self.config.build_recovery_params(member) self.config.write_recovery_conf(recovery_params) # When we demoting the primary or standby_leader to replica or promoting replica to a standby_leader # and we know for sure that postgres was already running before, we will only execute on_role_change # callback and prevent execution of on_restart/on_start callback. # If the role remains the same (replica or standby_leader), we will execute on_start or on_restart change_role = self.cb_called and (self.role in ('primary', 'demoted') or not {'standby_leader', 'replica'} - {self.role, role}) if change_role: self.__cb_pending = CallbackAction.NOOP ret = True if self.is_running(): if do_reload: self.config.write_postgresql_conf() ret = self.reload(block_callbacks=change_role) if ret and change_role: self.set_role(role) else: ret = self.restart(block_callbacks=change_role, role=role) else: ret = self.start(timeout=timeout, block_callbacks=change_role, role=role) or None if change_role: # TODO: postpone this until start completes, or maybe do even earlier self.call_nowait(CallbackAction.ON_ROLE_CHANGE) return ret def _wait_promote(self, wait_seconds: int) -> Optional[bool]: for _ in polling_loop(wait_seconds): data = self.controldata() if data.get('Database cluster state') == 'in production': self.set_role('primary') return True def _pre_promote(self) -> bool: """ Runs a fencing script after the leader lock is acquired but before the replica is promoted. If the script exits with a non-zero code, promotion does not happen and the leader key is removed from DCS. """ cmd = self.config.get('pre_promote') if not cmd: return True ret = self.cancellable.call(shlex.split(cmd)) if ret is not None: logger.info('pre_promote script `%s` exited with %s', cmd, ret) return ret == 0 def _before_stop(self) -> None: """Synchronously run a script prior to stopping postgres.""" cmd = self.config.get('before_stop') if cmd: self._do_before_stop(cmd) def _do_before_stop(self, cmd: str) -> None: try: ret = self.cancellable.call(shlex.split(cmd)) if ret is not None: logger.info('before_stop script `%s` exited with %s', cmd, ret) except Exception as e: logger.error('Exception when calling `%s`: %r', cmd, e) def promote(self, wait_seconds: int, task: CriticalTask, before_promote: Optional[Callable[..., Any]] = None) -> Optional[bool]: if self.role in ('promoted', 'primary'): return True ret = self._pre_promote() with task: if task.is_cancelled: return False task.complete(ret) if ret is False: return False if self.cancellable.is_cancelled: logger.info("PostgreSQL promote cancelled.") return False if before_promote is not None: before_promote() self.slots_handler.on_promote() self.mpp_handler.schedule_cache_rebuild() ret = self.pg_ctl('promote', '-W') if ret: self.set_role('promoted') self.call_nowait(CallbackAction.ON_ROLE_CHANGE) ret = self._wait_promote(wait_seconds) return ret @staticmethod def _wal_position(is_primary: bool, wal_position: int, received_location: Optional[int], replayed_location: Optional[int]) -> int: return wal_position if is_primary else max(received_location or 0, replayed_location or 0) def timeline_wal_position(self) -> Tuple[int, int, Optional[int]]: # This method could be called from different threads (simultaneously with some other `_query` calls). # If it is called not from main thread we will create a new cursor to execute statement. if current_thread().ident == self.__thread_ident: timeline = self._cluster_info_state_get('timeline') or 0 wal_position = self._cluster_info_state_get('wal_position') or 0 replayed_location = self.replayed_location() received_location = self.received_location() pg_control_timeline = self._cluster_info_state_get('pg_control_timeline') else: timeline, wal_position, replayed_location, received_location, _, pg_control_timeline = \ self._query(self.cluster_info_query)[0][:6] wal_position = self._wal_position(bool(timeline), wal_position, received_location, replayed_location) return timeline, wal_position, pg_control_timeline def postmaster_start_time(self) -> Optional[str]: try: sql = "SELECT " + self.POSTMASTER_START_TIME return self.query(sql, retry=current_thread().ident == self.__thread_ident)[0][0].isoformat(sep=' ') except psycopg.Error: return None def last_operation(self) -> int: return self._wal_position(self.is_primary(), self._cluster_info_state_get('wal_position') or 0, self.received_location(), self.replayed_location()) def configure_server_parameters(self) -> None: self._major_version = self.get_major_version() self.config.setup_server_parameters() def ensure_major_version_is_known(self) -> bool: """Calls configure_server_parameters() if `_major_version` is not known :returns: `True` if `_major_version` is set, otherwise `False`""" if not self._major_version: self.configure_server_parameters() return self._major_version > 0 def pg_wal_realpath(self) -> Dict[str, str]: """Returns a dict containing the symlink (key) and target (value) for the wal directory""" links: Dict[str, str] = {} for pg_wal_dir in ('pg_xlog', 'pg_wal'): pg_wal_path = os.path.join(self._data_dir, pg_wal_dir) if os.path.exists(pg_wal_path) and os.path.islink(pg_wal_path): pg_wal_realpath = os.path.realpath(pg_wal_path) links[pg_wal_path] = pg_wal_realpath return links def pg_tblspc_realpaths(self) -> Dict[str, str]: """Returns a dict containing the symlink (key) and target (values) for the tablespaces""" links: Dict[str, str] = {} pg_tblsp_dir = os.path.join(self._data_dir, 'pg_tblspc') if os.path.exists(pg_tblsp_dir): for tsdn in os.listdir(pg_tblsp_dir): pg_tsp_path = os.path.join(pg_tblsp_dir, tsdn) if parse_int(tsdn) and os.path.islink(pg_tsp_path): pg_tsp_rpath = os.path.realpath(pg_tsp_path) links[pg_tsp_path] = pg_tsp_rpath return links def move_data_directory(self) -> None: if os.path.isdir(self._data_dir) and not self.is_running(): try: postfix = 'failed' # let's see if the wal directory is a symlink, in this case we # should move the target for (source, pg_wal_realpath) in self.pg_wal_realpath().items(): logger.info('renaming WAL directory and updating symlink: %s', pg_wal_realpath) new_name = '{0}.{1}'.format(pg_wal_realpath, postfix) if os.path.exists(new_name): shutil.rmtree(new_name) os.rename(pg_wal_realpath, new_name) os.unlink(source) os.symlink(new_name, source) # Move user defined tablespace directory for (source, pg_tsp_rpath) in self.pg_tblspc_realpaths().items(): logger.info('renaming user defined tablespace directory and updating symlink: %s', pg_tsp_rpath) new_name = '{0}.{1}'.format(pg_tsp_rpath, postfix) if os.path.exists(new_name): shutil.rmtree(new_name) os.rename(pg_tsp_rpath, new_name) os.unlink(source) os.symlink(new_name, source) new_name = '{0}.{1}'.format(self._data_dir, postfix) logger.info('renaming data directory to %s', new_name) if os.path.exists(new_name): shutil.rmtree(new_name) os.rename(self._data_dir, new_name) except OSError: logger.exception("Could not rename data directory %s", self._data_dir) def remove_data_directory(self) -> None: self.set_role('uninitialized') logger.info('Removing data directory: %s', self._data_dir) try: if os.path.islink(self._data_dir): os.unlink(self._data_dir) elif not os.path.exists(self._data_dir): return elif os.path.isfile(self._data_dir): os.remove(self._data_dir) elif os.path.isdir(self._data_dir): # let's see if wal directory is a symlink, in this case we # should clean the target for pg_wal_realpath in self.pg_wal_realpath().values(): logger.info('Removing WAL directory: %s', pg_wal_realpath) shutil.rmtree(pg_wal_realpath) # Remove user defined tablespace directories for pg_tsp_rpath in self.pg_tblspc_realpaths().values(): logger.info('Removing user defined tablespace directory: %s', pg_tsp_rpath) shutil.rmtree(pg_tsp_rpath, ignore_errors=True) shutil.rmtree(self._data_dir) except (IOError, OSError): logger.exception('Could not remove data directory %s', self._data_dir) self.move_data_directory() def schedule_sanity_checks_after_pause(self) -> None: """ After coming out of pause we have to: 1. configure server parameters if necessary 2. sync replication slots, because it might happen that slots were removed 3. get new 'Database system identifier' to make sure that it wasn't changed """ self.ensure_major_version_is_known() self.slots_handler.schedule() self.mpp_handler.schedule_cache_rebuild() self._sysid = '' def _get_gucs(self) -> CaseInsensitiveSet: """Get all available GUCs based on ``postgres --describe-config`` output. :returns: all available GUCs in the local Postgres server. """ cmd = [self.pgcommand('postgres'), '--describe-config'] return CaseInsensitiveSet({ line.split('\t')[0] for line in subprocess.check_output(cmd).decode('utf-8').strip().split('\n') }) patroni-4.0.4/patroni/postgresql/available_parameters/000077500000000000000000000000001472010352700232165ustar00rootroot00000000000000patroni-4.0.4/patroni/postgresql/available_parameters/0_postgres.yml000066400000000000000000001120251472010352700260270ustar00rootroot00000000000000parameters: allow_alter_system: - type: Bool version_from: 170000 allow_in_place_tablespaces: - type: Bool version_from: 150000 - type: Bool version_from: 140005 version_till: 140099 - type: Bool version_from: 130008 version_till: 130099 - type: Bool version_from: 120012 version_till: 120099 - type: Bool version_from: 110017 version_till: 110099 - type: Bool version_from: 100022 version_till: 100099 allow_system_table_mods: - type: Bool version_from: 90300 application_name: - type: String version_from: 90300 archive_command: - type: String version_from: 90300 archive_library: - type: String version_from: 150000 archive_mode: - type: Bool version_from: 90300 version_till: 90500 - type: EnumBool version_from: 90500 possible_values: - always archive_timeout: - type: Integer version_from: 90300 min_val: 0 max_val: 1073741823 unit: s array_nulls: - type: Bool version_from: 90300 authentication_timeout: - type: Integer version_from: 90300 min_val: 1 max_val: 600 unit: s autovacuum: - type: Bool version_from: 90300 autovacuum_analyze_scale_factor: - type: Real version_from: 90300 min_val: 0 max_val: 100 autovacuum_analyze_threshold: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 autovacuum_freeze_max_age: - type: Integer version_from: 90300 min_val: 100000 max_val: 2000000000 autovacuum_max_workers: - type: Integer version_from: 90300 version_till: 90600 min_val: 1 max_val: 8388607 - type: Integer version_from: 90600 min_val: 1 max_val: 262143 autovacuum_multixact_freeze_max_age: - type: Integer version_from: 90300 min_val: 10000 max_val: 2000000000 autovacuum_naptime: - type: Integer version_from: 90300 min_val: 1 max_val: 2147483 unit: s autovacuum_vacuum_cost_delay: - type: Integer version_from: 90300 version_till: 120000 min_val: -1 max_val: 100 unit: ms - type: Real version_from: 120000 min_val: -1 max_val: 100 unit: ms autovacuum_vacuum_cost_limit: - type: Integer version_from: 90300 min_val: -1 max_val: 10000 autovacuum_vacuum_insert_scale_factor: - type: Real version_from: 130000 min_val: 0 max_val: 100 autovacuum_vacuum_insert_threshold: - type: Integer version_from: 130000 min_val: -1 max_val: 2147483647 autovacuum_vacuum_scale_factor: - type: Real version_from: 90300 min_val: 0 max_val: 100 autovacuum_vacuum_threshold: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 autovacuum_work_mem: - type: Integer version_from: 90400 min_val: -1 max_val: 2147483647 unit: kB backend_flush_after: - type: Integer version_from: 90600 min_val: 0 max_val: 256 unit: 8kB backslash_quote: - type: EnumBool version_from: 90300 possible_values: - safe_encoding backtrace_functions: - type: String version_from: 130000 bgwriter_delay: - type: Integer version_from: 90300 min_val: 10 max_val: 10000 unit: ms bgwriter_flush_after: - type: Integer version_from: 90600 min_val: 0 max_val: 256 unit: 8kB bgwriter_lru_maxpages: - type: Integer version_from: 90300 version_till: 100000 min_val: 0 max_val: 1000 - type: Integer version_from: 100000 min_val: 0 max_val: 1073741823 bgwriter_lru_multiplier: - type: Real version_from: 90300 min_val: 0 max_val: 10 bonjour: - type: Bool version_from: 90300 bonjour_name: - type: String version_from: 90300 bytea_output: - type: Enum version_from: 90300 possible_values: - escape - hex check_function_bodies: - type: Bool version_from: 90300 checkpoint_completion_target: - type: Real version_from: 90300 min_val: 0 max_val: 1 checkpoint_flush_after: - type: Integer version_from: 90600 min_val: 0 max_val: 256 unit: 8kB checkpoint_segments: - type: Integer version_from: 90300 version_till: 90500 min_val: 1 max_val: 2147483647 checkpoint_timeout: - type: Integer version_from: 90300 version_till: 90600 min_val: 30 max_val: 3600 unit: s - type: Integer version_from: 90600 min_val: 30 max_val: 86400 unit: s checkpoint_warning: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: s client_connection_check_interval: - type: Integer version_from: 140000 min_val: 0 max_val: 2147483647 unit: ms client_encoding: - type: String version_from: 90300 client_min_messages: - type: Enum version_from: 90300 possible_values: - debug5 - debug4 - debug3 - debug2 - debug1 - log - notice - warning - error cluster_name: - type: String version_from: 90500 commit_delay: - type: Integer version_from: 90300 min_val: 0 max_val: 100000 commit_siblings: - type: Integer version_from: 90300 min_val: 0 max_val: 1000 commit_timestamp_buffers: - type: Integer version_from: 170000 min_val: 0 max_val: 131072 unit: 8kB compute_query_id: - type: EnumBool version_from: 140000 version_till: 150000 possible_values: - auto - type: EnumBool version_from: 150000 possible_values: - auto - regress config_file: - type: String version_from: 90300 constraint_exclusion: - type: EnumBool version_from: 90300 possible_values: - partition cpu_index_tuple_cost: - type: Real version_from: 90300 min_val: 0 max_val: 1.79769e+308 cpu_operator_cost: - type: Real version_from: 90300 min_val: 0 max_val: 1.79769e+308 cpu_tuple_cost: - type: Real version_from: 90300 min_val: 0 max_val: 1.79769e+308 createrole_self_grant: - type: String version_from: 160000 cursor_tuple_fraction: - type: Real version_from: 90300 min_val: 0 max_val: 1 data_directory: - type: String version_from: 90300 data_sync_retry: - type: Bool version_from: 90400 DateStyle: - type: String version_from: 90300 db_user_namespace: - type: Bool version_from: 90300 version_till: 170000 deadlock_timeout: - type: Integer version_from: 90300 min_val: 1 max_val: 2147483647 unit: ms debug_discard_caches: - type: Integer version_from: 150000 min_val: 0 max_val: 0 debug_io_direct: - type: String version_from: 160000 debug_logical_replication_streaming: - type: Enum version_from: 170000 possible_values: - buffered - immediate debug_parallel_query: - type: EnumBool version_from: 160000 possible_values: - regress debug_pretty_print: - type: Bool version_from: 90300 debug_print_parse: - type: Bool version_from: 90300 debug_print_plan: - type: Bool version_from: 90300 debug_print_rewritten: - type: Bool version_from: 90300 default_statistics_target: - type: Integer version_from: 90300 min_val: 1 max_val: 10000 default_table_access_method: - type: String version_from: 120000 default_tablespace: - type: String version_from: 90300 default_text_search_config: - type: String version_from: 90300 default_toast_compression: - type: Enum version_from: 140000 possible_values: - pglz - lz4 default_transaction_deferrable: - type: Bool version_from: 90300 default_transaction_isolation: - type: Enum version_from: 90300 possible_values: - serializable - repeatable read - read committed - read uncommitted default_transaction_read_only: - type: Bool version_from: 90300 default_with_oids: - type: Bool version_from: 90300 version_till: 120000 dynamic_library_path: - type: String version_from: 90300 dynamic_shared_memory_type: - type: Enum version_from: 90400 version_till: 120000 possible_values: - posix - sysv - mmap - none - type: Enum version_from: 120000 possible_values: - posix - sysv - mmap effective_cache_size: - type: Integer version_from: 90300 min_val: 1 max_val: 2147483647 unit: 8kB effective_io_concurrency: - type: Integer version_from: 90300 min_val: 0 max_val: 1000 enable_async_append: - type: Bool version_from: 140000 enable_bitmapscan: - type: Bool version_from: 90300 enable_gathermerge: - type: Bool version_from: 100000 enable_group_by_reordering: - type: Bool version_from: 170000 enable_hashagg: - type: Bool version_from: 90300 enable_hashjoin: - type: Bool version_from: 90300 enable_incremental_sort: - type: Bool version_from: 130000 enable_indexonlyscan: - type: Bool version_from: 90300 enable_indexscan: - type: Bool version_from: 90300 enable_material: - type: Bool version_from: 90300 enable_memoize: - type: Bool version_from: 150000 enable_mergejoin: - type: Bool version_from: 90300 enable_nestloop: - type: Bool version_from: 90300 enable_parallel_append: - type: Bool version_from: 110000 enable_parallel_hash: - type: Bool version_from: 110000 enable_partition_pruning: - type: Bool version_from: 110000 enable_partitionwise_aggregate: - type: Bool version_from: 110000 enable_partitionwise_join: - type: Bool version_from: 110000 enable_presorted_aggregate: - type: Bool version_from: 160000 enable_seqscan: - type: Bool version_from: 90300 enable_sort: - type: Bool version_from: 90300 enable_tidscan: - type: Bool version_from: 90300 escape_string_warning: - type: Bool version_from: 90300 event_source: - type: String version_from: 90300 event_triggers: - type: Bool version_from: 170000 exit_on_error: - type: Bool version_from: 90300 extension_destdir: - type: String version_from: 140000 external_pid_file: - type: String version_from: 90300 extra_float_digits: - type: Integer version_from: 90300 min_val: -15 max_val: 3 force_parallel_mode: - type: EnumBool version_from: 90600 version_till: 160000 possible_values: - regress from_collapse_limit: - type: Integer version_from: 90300 min_val: 1 max_val: 2147483647 fsync: - type: Bool version_from: 90300 full_page_writes: - type: Bool version_from: 90300 geqo: - type: Bool version_from: 90300 geqo_effort: - type: Integer version_from: 90300 min_val: 1 max_val: 10 geqo_generations: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 geqo_pool_size: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 geqo_seed: - type: Real version_from: 90300 min_val: 0 max_val: 1 geqo_selection_bias: - type: Real version_from: 90300 min_val: 1.5 max_val: 2 geqo_threshold: - type: Integer version_from: 90300 min_val: 2 max_val: 2147483647 gin_fuzzy_search_limit: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 gin_pending_list_limit: - type: Integer version_from: 90500 min_val: 64 max_val: 2147483647 unit: kB gss_accept_delegation: - type: Bool version_from: 160000 hash_mem_multiplier: - type: Real version_from: 130000 min_val: 1 max_val: 1000 hba_file: - type: String version_from: 90300 hot_standby: - type: Bool version_from: 90300 hot_standby_feedback: - type: Bool version_from: 90300 huge_pages: - type: EnumBool version_from: 90400 possible_values: - try huge_page_size: - type: Integer version_from: 140000 min_val: 0 max_val: 2147483647 unit: kB icu_validation_level: - type: Enum version_from: 160000 possible_values: - disabled - debug5 - debug4 - debug3 - debug2 - debug1 - log - notice - warning - error ident_file: - type: String version_from: 90300 idle_in_transaction_session_timeout: - type: Integer version_from: 90600 min_val: 0 max_val: 2147483647 unit: ms idle_session_timeout: - type: Integer version_from: 140000 min_val: 0 max_val: 2147483647 unit: ms ignore_checksum_failure: - type: Bool version_from: 90300 ignore_invalid_pages: - type: Bool version_from: 130000 ignore_system_indexes: - type: Bool version_from: 90300 io_combine_limit: - type: Integer version_from: 170000 min_val: 1 max_val: 32 unit: 8kB IntervalStyle: - type: Enum version_from: 90300 possible_values: - postgres - postgres_verbose - sql_standard - iso_8601 jit: - type: Bool version_from: 110000 jit_above_cost: - type: Real version_from: 110000 min_val: -1 max_val: 1.79769e+308 jit_debugging_support: - type: Bool version_from: 110000 jit_dump_bitcode: - type: Bool version_from: 110000 jit_expressions: - type: Bool version_from: 110000 jit_inline_above_cost: - type: Real version_from: 110000 min_val: -1 max_val: 1.79769e+308 jit_optimize_above_cost: - type: Real version_from: 110000 min_val: -1 max_val: 1.79769e+308 jit_profiling_support: - type: Bool version_from: 110000 jit_provider: - type: String version_from: 110000 jit_tuple_deforming: - type: Bool version_from: 110000 join_collapse_limit: - type: Integer version_from: 90300 min_val: 1 max_val: 2147483647 krb_caseins_users: - type: Bool version_from: 90300 krb_server_keyfile: - type: String version_from: 90300 krb_srvname: - type: String version_from: 90300 version_till: 90400 lc_messages: - type: String version_from: 90300 lc_monetary: - type: String version_from: 90300 lc_numeric: - type: String version_from: 90300 lc_time: - type: String version_from: 90300 listen_addresses: - type: String version_from: 90300 local_preload_libraries: - type: String version_from: 90300 lock_timeout: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: ms lo_compat_privileges: - type: Bool version_from: 90300 log_autovacuum_min_duration: - type: Integer version_from: 90300 min_val: -1 max_val: 2147483647 unit: ms log_checkpoints: - type: Bool version_from: 90300 log_connections: - type: Bool version_from: 90300 log_destination: - type: String version_from: 90300 log_directory: - type: String version_from: 90300 log_disconnections: - type: Bool version_from: 90300 log_duration: - type: Bool version_from: 90300 log_error_verbosity: - type: Enum version_from: 90300 possible_values: - terse - default - verbose log_executor_stats: - type: Bool version_from: 90300 log_file_mode: - type: Integer version_from: 90300 min_val: 0 max_val: 511 log_filename: - type: String version_from: 90300 logging_collector: - type: Bool version_from: 90300 log_hostname: - type: Bool version_from: 90300 logical_decoding_work_mem: - type: Integer version_from: 130000 min_val: 64 max_val: 2147483647 unit: kB log_line_prefix: - type: String version_from: 90300 log_lock_waits: - type: Bool version_from: 90300 log_min_duration_sample: - type: Integer version_from: 130000 min_val: -1 max_val: 2147483647 unit: ms log_min_duration_statement: - type: Integer version_from: 90300 min_val: -1 max_val: 2147483647 unit: ms log_min_error_statement: - type: Enum version_from: 90300 possible_values: - debug5 - debug4 - debug3 - debug2 - debug1 - info - notice - warning - error - log - fatal - panic log_min_messages: - type: Enum version_from: 90300 possible_values: - debug5 - debug4 - debug3 - debug2 - debug1 - info - notice - warning - error - log - fatal - panic log_parameter_max_length: - type: Integer version_from: 130000 min_val: -1 max_val: 1073741823 unit: B log_parameter_max_length_on_error: - type: Integer version_from: 130000 min_val: -1 max_val: 1073741823 unit: B log_parser_stats: - type: Bool version_from: 90300 log_planner_stats: - type: Bool version_from: 90300 log_recovery_conflict_waits: - type: Bool version_from: 140000 log_replication_commands: - type: Bool version_from: 90500 log_rotation_age: - type: Integer version_from: 90300 min_val: 0 max_val: 35791394 unit: min log_rotation_size: - type: Integer version_from: 90300 min_val: 0 max_val: 2097151 unit: kB log_startup_progress_interval: - type: Integer version_from: 150000 min_val: 0 max_val: 2147483647 unit: ms log_statement: - type: Enum version_from: 90300 possible_values: - none - ddl - mod - all log_statement_sample_rate: - type: Real version_from: 130000 min_val: 0 max_val: 1 log_statement_stats: - type: Bool version_from: 90300 log_temp_files: - type: Integer version_from: 90300 min_val: -1 max_val: 2147483647 unit: kB log_timezone: - type: String version_from: 90300 log_transaction_sample_rate: - type: Real version_from: 120000 min_val: 0 max_val: 1 log_truncate_on_rotation: - type: Bool version_from: 90300 logical_replication_mode: - type: Enum version_from: 160000 version_till: 170000 possible_values: - buffered - immediate maintenance_io_concurrency: - type: Integer version_from: 130000 min_val: 0 max_val: 1000 maintenance_work_mem: - type: Integer version_from: 90300 min_val: 1024 max_val: 2147483647 unit: kB max_connections: - type: Integer version_from: 90300 version_till: 90600 min_val: 1 max_val: 8388607 - type: Integer version_from: 90600 min_val: 1 max_val: 262143 max_files_per_process: - type: Integer version_from: 90300 version_till: 130000 min_val: 25 max_val: 2147483647 - type: Integer version_from: 130000 min_val: 64 max_val: 2147483647 max_locks_per_transaction: - type: Integer version_from: 90300 min_val: 10 max_val: 2147483647 max_logical_replication_workers: - type: Integer version_from: 100000 min_val: 0 max_val: 262143 max_notify_queue_pages: - type: Integer version_from: 170000 min_val: 64 max_val: 2147483647 max_parallel_apply_workers_per_subscription: - type: Integer version_from: 160000 min_val: 0 max_val: 1024 max_parallel_maintenance_workers: - type: Integer version_from: 110000 min_val: 0 max_val: 1024 max_parallel_workers: - type: Integer version_from: 100000 min_val: 0 max_val: 1024 max_parallel_workers_per_gather: - type: Integer version_from: 90600 min_val: 0 max_val: 1024 max_pred_locks_per_page: - type: Integer version_from: 100000 min_val: 0 max_val: 2147483647 max_pred_locks_per_relation: - type: Integer version_from: 100000 min_val: -2147483648 max_val: 2147483647 max_pred_locks_per_transaction: - type: Integer version_from: 90300 min_val: 10 max_val: 2147483647 max_prepared_transactions: - type: Integer version_from: 90300 version_till: 90600 min_val: 0 max_val: 8388607 - type: Integer version_from: 90600 min_val: 0 max_val: 262143 max_replication_slots: - type: Integer version_from: 90400 version_till: 90600 min_val: 0 max_val: 8388607 - type: Integer version_from: 90600 min_val: 0 max_val: 262143 max_slot_wal_keep_size: - type: Integer version_from: 130000 min_val: -1 max_val: 2147483647 unit: MB max_stack_depth: - type: Integer version_from: 90300 min_val: 100 max_val: 2147483647 unit: kB max_standby_archive_delay: - type: Integer version_from: 90300 min_val: -1 max_val: 2147483647 unit: ms max_standby_streaming_delay: - type: Integer version_from: 90300 min_val: -1 max_val: 2147483647 unit: ms max_sync_workers_per_subscription: - type: Integer version_from: 100000 min_val: 0 max_val: 262143 max_wal_senders: - type: Integer version_from: 90300 version_till: 90600 min_val: 0 max_val: 8388607 - type: Integer version_from: 90600 min_val: 0 max_val: 262143 max_wal_size: - type: Integer version_from: 90500 version_till: 100000 min_val: 2 max_val: 2147483647 unit: 16MB - type: Integer version_from: 100000 min_val: 2 max_val: 2147483647 unit: MB max_worker_processes: - type: Integer version_from: 90400 version_till: 90600 min_val: 1 max_val: 8388607 - type: Integer version_from: 90600 min_val: 0 max_val: 262143 min_dynamic_shared_memory: - type: Integer version_from: 140000 min_val: 0 max_val: 2147483647 unit: MB min_parallel_index_scan_size: - type: Integer version_from: 100000 min_val: 0 max_val: 715827882 unit: 8kB min_parallel_relation_size: - type: Integer version_from: 90600 version_till: 100000 min_val: 0 max_val: 715827882 unit: 8kB min_parallel_table_scan_size: - type: Integer version_from: 100000 min_val: 0 max_val: 715827882 unit: 8kB min_wal_size: - type: Integer version_from: 90500 version_till: 100000 min_val: 2 max_val: 2147483647 unit: 16MB - type: Integer version_from: 100000 min_val: 2 max_val: 2147483647 unit: MB multixact_member_buffers: - type: Integer version_from: 170000 min_val: 16 max_val: 131072 unit: 8kB multixact_offset_buffers: - type: Integer version_from: 170000 min_val: 16 max_val: 131072 unit: 8kB notify_buffers: - type: Integer version_from: 170000 min_val: 16 max_val: 131072 unit: 8kB old_snapshot_threshold: - type: Integer version_from: 90600 version_till: 170000 min_val: -1 max_val: 86400 unit: min operator_precedence_warning: - type: Bool version_from: 90500 version_till: 140000 parallel_leader_participation: - type: Bool version_from: 110000 parallel_setup_cost: - type: Real version_from: 90600 min_val: 0 max_val: 1.79769e+308 parallel_tuple_cost: - type: Real version_from: 90600 min_val: 0 max_val: 1.79769e+308 password_encryption: - type: Bool version_from: 90300 version_till: 100000 - type: Enum version_from: 100000 possible_values: - md5 - scram-sha-256 plan_cache_mode: - type: Enum version_from: 120000 possible_values: - auto - force_generic_plan - force_custom_plan port: - type: Integer version_from: 90300 min_val: 1 max_val: 65535 post_auth_delay: - type: Integer version_from: 90300 min_val: 0 max_val: 2147 unit: s pre_auth_delay: - type: Integer version_from: 90300 min_val: 0 max_val: 60 unit: s quote_all_identifiers: - type: Bool version_from: 90300 random_page_cost: - type: Real version_from: 90300 min_val: 0 max_val: 1.79769e+308 recovery_init_sync_method: - type: Enum version_from: 140000 possible_values: - fsync - syncfs recovery_prefetch: - type: EnumBool version_from: 150000 possible_values: - try recursive_worktable_factor: - type: Real version_from: 150000 min_val: 0.001 max_val: 1000000.0 remove_temp_files_after_crash: - type: Bool version_from: 140000 replacement_sort_tuples: - type: Integer version_from: 90600 version_till: 110000 min_val: 0 max_val: 2147483647 reserved_connections: - type: Integer version_from: 160000 min_val: 0 max_val: 262143 restart_after_crash: - type: Bool version_from: 90300 restrict_nonsystem_relation_kind: - type: String version_from: 170000 - type: String version_from: 160004 version_till: 160099 - type: String version_from: 150008 version_till: 150099 - type: String version_from: 140013 version_till: 140099 - type: String version_from: 130016 version_till: 130099 - type: String version_from: 120020 version_till: 120099 row_security: - type: Bool version_from: 90500 scram_iterations: - type: Integer version_from: 160000 min_val: 1 max_val: 2147483647 search_path: - type: String version_from: 90300 send_abort_for_crash: - type: Bool version_from: 160000 send_abort_for_kill: - type: Bool version_from: 160000 seq_page_cost: - type: Real version_from: 90300 min_val: 0 max_val: 1.79769e+308 serializable_buffers: - type: Integer version_from: 170000 min_val: 16 max_val: 131072 unit: 8kB session_preload_libraries: - type: String version_from: 90400 session_replication_role: - type: Enum version_from: 90300 possible_values: - origin - replica - local shared_buffers: - type: Integer version_from: 90300 min_val: 16 max_val: 1073741823 unit: 8kB shared_memory_type: - type: Enum version_from: 120000 possible_values: - sysv - mmap shared_preload_libraries: - type: String version_from: 90300 sql_inheritance: - type: Bool version_from: 90300 version_till: 100000 ssl: - type: Bool version_from: 90300 ssl_ca_file: - type: String version_from: 90300 ssl_cert_file: - type: String version_from: 90300 ssl_ciphers: - type: String version_from: 90300 ssl_crl_dir: - type: String version_from: 140000 ssl_crl_file: - type: String version_from: 90300 ssl_dh_params_file: - type: String version_from: 100000 ssl_ecdh_curve: - type: String version_from: 90400 ssl_key_file: - type: String version_from: 90300 ssl_max_protocol_version: - type: Enum version_from: 120000 possible_values: - '' - tlsv1 - tlsv1.1 - tlsv1.2 - tlsv1.3 ssl_min_protocol_version: - type: Enum version_from: 120000 possible_values: - tlsv1 - tlsv1.1 - tlsv1.2 - tlsv1.3 ssl_passphrase_command: - type: String version_from: 110000 ssl_passphrase_command_supports_reload: - type: Bool version_from: 110000 ssl_prefer_server_ciphers: - type: Bool version_from: 90400 ssl_renegotiation_limit: - type: Integer version_from: 90300 version_till: 90500 min_val: 0 max_val: 2147483647 unit: kB standard_conforming_strings: - type: Bool version_from: 90300 statement_timeout: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: ms stats_fetch_consistency: - type: Enum version_from: 150000 possible_values: - none - cache - snapshot stats_temp_directory: - type: String version_from: 90300 version_till: 150000 subtransaction_buffers: - type: Integer version_from: 170000 min_val: 0 max_val: 131072 unit: 8kB summarize_wal: - type: Bool version_from: 170000 superuser_reserved_connections: - type: Integer version_from: 90300 version_till: 90600 min_val: 0 max_val: 8388607 - type: Integer version_from: 90600 min_val: 0 max_val: 262143 sync_replication_slots: - type: Bool version_from: 170000 synchronize_seqscans: - type: Bool version_from: 90300 synchronized_standby_slots: - type: String version_from: 170000 synchronous_commit: - type: EnumBool version_from: 90300 version_till: 90600 possible_values: - local - remote_write - type: EnumBool version_from: 90600 possible_values: - local - remote_write - remote_apply synchronous_standby_names: - type: String version_from: 90300 syslog_facility: - type: Enum version_from: 90300 possible_values: - local0 - local1 - local2 - local3 - local4 - local5 - local6 - local7 syslog_ident: - type: String version_from: 90300 syslog_sequence_numbers: - type: Bool version_from: 90600 syslog_split_messages: - type: Bool version_from: 90600 tcp_keepalives_count: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 tcp_keepalives_idle: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: s tcp_keepalives_interval: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: s tcp_user_timeout: - type: Integer version_from: 120000 min_val: 0 max_val: 2147483647 unit: ms temp_buffers: - type: Integer version_from: 90300 min_val: 100 max_val: 1073741823 unit: 8kB temp_file_limit: - type: Integer version_from: 90300 min_val: -1 max_val: 2147483647 unit: kB temp_tablespaces: - type: String version_from: 90300 TimeZone: - type: String version_from: 90300 timezone_abbreviations: - type: String version_from: 90300 trace_connection_negotiation: - type: Bool version_from: 170000 trace_notify: - type: Bool version_from: 90300 trace_recovery_messages: - type: Enum version_from: 90300 version_till: 170000 possible_values: - debug5 - debug4 - debug3 - debug2 - debug1 - log - notice - warning - error trace_sort: - type: Bool version_from: 90300 track_activities: - type: Bool version_from: 90300 track_activity_query_size: - type: Integer version_from: 90300 version_till: 110000 min_val: 100 max_val: 102400 - type: Integer version_from: 110000 version_till: 130000 min_val: 100 max_val: 102400 unit: B - type: Integer version_from: 130000 min_val: 100 max_val: 1048576 unit: B track_commit_timestamp: - type: Bool version_from: 90500 track_counts: - type: Bool version_from: 90300 track_functions: - type: Enum version_from: 90300 possible_values: - none - pl - all track_io_timing: - type: Bool version_from: 90300 track_wal_io_timing: - type: Bool version_from: 140000 transaction_buffers: - type: Integer version_from: 170000 min_val: 0 max_val: 131072 unit: 8kB transaction_deferrable: - type: Bool version_from: 90300 transaction_isolation: - type: Enum version_from: 90300 possible_values: - serializable - repeatable read - read committed - read uncommitted transaction_read_only: - type: Bool version_from: 90300 transaction_timeout: - type: Integer version_from: 170000 min_val: 0 max_val: 2147483647 unit: ms transform_null_equals: - type: Bool version_from: 90300 unix_socket_directories: - type: String version_from: 90300 unix_socket_group: - type: String version_from: 90300 unix_socket_permissions: - type: Integer version_from: 90300 min_val: 0 max_val: 511 update_process_title: - type: Bool version_from: 90300 vacuum_buffer_usage_limit: - type: Integer version_from: 160000 min_val: 0 max_val: 16777216 unit: kB vacuum_cleanup_index_scale_factor: - type: Real version_from: 110000 version_till: 140000 min_val: 0 max_val: 10000000000.0 vacuum_cost_delay: - type: Integer version_from: 90300 version_till: 120000 min_val: 0 max_val: 100 unit: ms - type: Real version_from: 120000 min_val: 0 max_val: 100 unit: ms vacuum_cost_limit: - type: Integer version_from: 90300 min_val: 1 max_val: 10000 vacuum_cost_page_dirty: - type: Integer version_from: 90300 min_val: 0 max_val: 10000 vacuum_cost_page_hit: - type: Integer version_from: 90300 min_val: 0 max_val: 10000 vacuum_cost_page_miss: - type: Integer version_from: 90300 min_val: 0 max_val: 10000 vacuum_defer_cleanup_age: - type: Integer version_from: 90300 version_till: 160000 min_val: 0 max_val: 1000000 vacuum_failsafe_age: - type: Integer version_from: 140000 min_val: 0 max_val: 2100000000 vacuum_freeze_min_age: - type: Integer version_from: 90300 min_val: 0 max_val: 1000000000 vacuum_freeze_table_age: - type: Integer version_from: 90300 min_val: 0 max_val: 2000000000 vacuum_multixact_failsafe_age: - type: Integer version_from: 140000 min_val: 0 max_val: 2100000000 vacuum_multixact_freeze_min_age: - type: Integer version_from: 90300 min_val: 0 max_val: 1000000000 vacuum_multixact_freeze_table_age: - type: Integer version_from: 90300 min_val: 0 max_val: 2000000000 wal_buffers: - type: Integer version_from: 90300 min_val: -1 max_val: 262143 unit: 8kB wal_compression: - type: Bool version_from: 90500 version_till: 150000 - type: EnumBool version_from: 150000 possible_values: - pglz - lz4 - zstd wal_consistency_checking: - type: String version_from: 100000 wal_decode_buffer_size: - type: Integer version_from: 150000 min_val: 65536 max_val: 1073741823 unit: B wal_init_zero: - type: Bool version_from: 120000 wal_keep_segments: - type: Integer version_from: 90300 version_till: 130000 min_val: 0 max_val: 2147483647 wal_keep_size: - type: Integer version_from: 130000 min_val: 0 max_val: 2147483647 unit: MB wal_level: - type: Enum version_from: 90300 version_till: 90400 possible_values: - minimal - archive - hot_standby - type: Enum version_from: 90400 version_till: 90600 possible_values: - minimal - archive - hot_standby - logical - type: Enum version_from: 90600 possible_values: - minimal - replica - logical wal_log_hints: - type: Bool version_from: 90400 wal_receiver_create_temp_slot: - type: Bool version_from: 130000 wal_receiver_status_interval: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483 unit: s wal_receiver_timeout: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: ms wal_recycle: - type: Bool version_from: 120000 wal_retrieve_retry_interval: - type: Integer version_from: 90500 min_val: 1 max_val: 2147483647 unit: ms wal_sender_timeout: - type: Integer version_from: 90300 min_val: 0 max_val: 2147483647 unit: ms wal_skip_threshold: - type: Integer version_from: 130000 min_val: 0 max_val: 2147483647 unit: kB wal_summary_keep_time: - type: Integer version_from: 170000 min_val: 0 max_val: 35791394 unit: min wal_sync_method: - type: Enum version_from: 90300 possible_values: - fsync - fdatasync - open_sync - open_datasync wal_writer_delay: - type: Integer version_from: 90300 min_val: 1 max_val: 10000 unit: ms wal_writer_flush_after: - type: Integer version_from: 90600 min_val: 0 max_val: 2147483647 unit: 8kB work_mem: - type: Integer version_from: 90300 min_val: 64 max_val: 2147483647 unit: kB xmlbinary: - type: Enum version_from: 90300 possible_values: - base64 - hex xmloption: - type: Enum version_from: 90300 possible_values: - content - document zero_damaged_pages: - type: Bool version_from: 90300 recovery_parameters: archive_cleanup_command: - type: String version_from: 90300 pause_at_recovery_target: - type: Bool version_from: 90300 version_till: 90500 primary_conninfo: - type: String version_from: 90300 primary_slot_name: - type: String version_from: 90400 promote_trigger_file: - type: String version_from: 120000 version_till: 160000 recovery_end_command: - type: String version_from: 90300 recovery_min_apply_delay: - type: Integer version_from: 90400 min_val: 0 max_val: 2147483647 unit: ms recovery_target: - type: Enum version_from: 90400 possible_values: - immediate - '' recovery_target_action: - type: Enum version_from: 90500 possible_values: - pause - promote - shutdown recovery_target_inclusive: - type: Bool version_from: 90300 recovery_target_lsn: - type: String version_from: 100000 recovery_target_name: - type: String version_from: 90400 recovery_target_time: - type: String version_from: 90300 recovery_target_timeline: - type: String version_from: 90300 recovery_target_xid: - type: String version_from: 90300 restore_command: - type: String version_from: 90300 standby_mode: - type: Bool version_from: 90300 version_till: 120000 trigger_file: - type: String version_from: 90300 version_till: 120000 patroni-4.0.4/patroni/postgresql/available_parameters/__init__.py000066400000000000000000000041751472010352700253360ustar00rootroot00000000000000import logging import sys from typing import Iterator logger = logging.getLogger(__name__) if sys.version_info < (3, 9): # pragma: no cover from pathlib import Path PathLikeObj = Path conf_dir = Path(__file__).parent else: from importlib.resources import files if sys.version_info < (3, 11): # pragma: no cover from importlib.abc import Traversable else: # pragma: no cover from importlib.resources.abc import Traversable PathLikeObj = Traversable conf_dir = files(__name__) def get_validator_files() -> Iterator[PathLikeObj]: """Recursively find YAML files from the current package directory. :returns: an iterator of :class:`PathLikeObj` objects representing validator files. """ return _traversable_walk(conf_dir.iterdir()) def _traversable_walk(tvbs: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]: """Recursively walk through Path/Traversable objects, yielding all YAML files in deterministic order. :param tvbs: An iterator over :class:`PathLikeObj` objects, where each object is a file or directory that potentially contains YAML files. :yields: :class:`PathLikeObj` objects representing YAML files found during the traversal. """ for tvb in _filter_and_sort_files(tvbs): if tvb.is_file(): yield tvb elif tvb.is_dir(): try: yield from _traversable_walk(tvb.iterdir()) except Exception as e: logger.debug("Can't list directory %s: %r", tvb, e) def _filter_and_sort_files(files: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]: """Sort files by name, and filter out non-YAML files and Python files. :param files: A list of files and/or directories to be filtered and sorted. :yields: filtered and sorted objects. """ for file in sorted(files, key=lambda x: x.name): if file.name.lower().endswith((".yml", ".yaml")) or file.is_dir(): yield file elif not file.name.lower().endswith((".py", ".pyc")): logger.info("Ignored a non-YAML file found under `%s` directory: `%s`.", __name__.split('.')[-1], file) patroni-4.0.4/patroni/postgresql/bootstrap.py000066400000000000000000000563141472010352700214530ustar00rootroot00000000000000import logging import os import shlex import tempfile import time from typing import Any, Callable, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union from ..async_executor import CriticalTask from ..collections import EMPTY_DICT from ..dcs import Leader, Member, RemoteMember from ..psycopg import quote_ident, quote_literal from ..utils import deep_compare, unquote if TYPE_CHECKING: # pragma: no cover from . import Postgresql logger = logging.getLogger(__name__) class Bootstrap(object): def __init__(self, postgresql: 'Postgresql') -> None: self._postgresql = postgresql self._running_custom_bootstrap = False @property def running_custom_bootstrap(self) -> bool: return self._running_custom_bootstrap @property def keep_existing_recovery_conf(self) -> bool: return self._running_custom_bootstrap and self._keep_existing_recovery_conf @staticmethod def process_user_options(tool: str, options: Any, not_allowed_options: Tuple[str, ...], error_handler: Callable[[str], None]) -> List[str]: """Format *options* in a list or dictionary format into command line long form arguments. .. note:: The format of the output of this method is to prepare arguments for use in the ``initdb`` method of `self._postgres`. :Example: The *options* can be defined as a dictionary of key, values to be converted into arguments: >>> Bootstrap.process_user_options('foo', {'foo': 'bar'}, (), print) ['--foo=bar'] Or as a list of single string arguments >>> Bootstrap.process_user_options('foo', ['yes'], (), print) ['--yes'] Or as a list of key, value options >>> Bootstrap.process_user_options('foo', [{'foo': 'bar'}], (), print) ['--foo=bar'] Or a combination of single and key, values >>> Bootstrap.process_user_options('foo', ['yes', {'foo': 'bar'}], (), print) ['--yes', '--foo=bar'] Options that contain spaces are passed as is to ``subprocess.call`` >>> Bootstrap.process_user_options('foo', [{'foo': 'bar baz'}], (), print) ['--foo=bar baz'] Options that are quoted will be unquoted, so the quotes aren't interpreted literally by the postgres command >>> Bootstrap.process_user_options('foo', [{'foo': '"bar baz"'}], (), print) ['--foo=bar baz'] .. note:: The *error_handler* is called when any of these conditions are met: * Key, value dictionaries in the list form contains multiple keys. * If a key is listed in *not_allowed_options*. * If the options list is not in the required structure. :param tool: The name of the tool used in error reports to *error_handler* :param options: Options to parse as a list of key, values or single values, or a dictionary :param not_allowed_options: List of keys that cannot be used in the list of key, value formatted options :param error_handler: A function which will be called when an error condition is encountered :returns: List of long form arguments to pass to the named tool """ user_options: List[str] = [] def option_is_allowed(name: str) -> bool: ret = name not in not_allowed_options if not ret: error_handler('{0} option for {1} is not allowed'.format(name, tool)) return ret if isinstance(options, dict): for key, val in cast(Dict[str, str], options).items(): if key and val: user_options.append('--{0}={1}'.format(key, unquote(val))) elif isinstance(options, list): for opt in cast(List[Any], options): if isinstance(opt, str) and option_is_allowed(opt): user_options.append('--{0}'.format(opt)) elif isinstance(opt, dict): args = cast(Dict[str, Any], opt) keys = list(args.keys()) if len(keys) == 1 and isinstance(args[keys[0]], str) and option_is_allowed(keys[0]): user_options.append('--{0}={1}'.format(keys[0], unquote(args[keys[0]]))) else: error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed' ' and value should be a string'.format(tool, args[keys[0]])) else: error_handler('Error when parsing {0} option {1}: value should be string value' ' or a single key-value pair'.format(tool, opt)) else: error_handler('{0} options must be list or dict'.format(tool)) return user_options def _initdb(self, config: Any) -> bool: self._postgresql.set_state('initializing new cluster') not_allowed_options = ('pgdata', 'nosync', 'pwfile', 'sync-only', 'version') def error_handler(e: str) -> None: raise Exception(e) options = self.process_user_options('initdb', config or [], not_allowed_options, error_handler) pwfile = None if self._postgresql.config.superuser: if 'username' in self._postgresql.config.superuser: options.append('--username={0}'.format(self._postgresql.config.superuser['username'])) if 'password' in self._postgresql.config.superuser: (fd, pwfile) = tempfile.mkstemp() os.write(fd, self._postgresql.config.superuser['password'].encode('utf-8')) os.close(fd) options.append('--pwfile={0}'.format(pwfile)) ret = self._postgresql.initdb(*options) if pwfile: os.remove(pwfile) if ret: self._postgresql.configure_server_parameters() else: self._postgresql.set_state('initdb failed') return ret def _post_restore(self) -> None: self._postgresql.config.restore_configuration_files() self._postgresql.configure_server_parameters() # make sure there is no trigger file or postgres will be automatically promoted trigger_file = self._postgresql.config.triggerfile_good_name trigger_file = (self._postgresql.config.get('recovery_conf') or EMPTY_DICT).get(trigger_file) or 'promote' trigger_file = os.path.abspath(os.path.join(self._postgresql.data_dir, trigger_file)) if os.path.exists(trigger_file): os.unlink(trigger_file) def _custom_bootstrap(self, config: Any) -> bool: """Bootstrap a fresh Patroni cluster using a custom method provided by the user. :param config: configuration used for running a custom bootstrap method. It comes from the Patroni YAML file, so it is expected to be a :class:`dict`. .. note:: *config* must contain a ``command`` key, which value is the command or script to perform the custom bootstrap procedure. The exit code of the ``command`` dictates if the bootstrap succeeded or failed. When calling ``command``, Patroni will pass the following arguments to the ``command`` call: * ``--scope``: contains the value of ``scope`` configuration; * ``--data_dir``: contains the value of the ``postgresql.data_dir`` configuration. You can avoid that behavior by filling the optional key ``no_params`` with the value ``False`` in the configuration file, which will instruct Patroni to not pass these parameters to the ``command`` call. Besides that, a couple more keys are supported in *config*, but optional: * ``keep_existing_recovery_conf``: if ``True``, instruct Patroni to not remove the existing ``recovery.conf`` (PostgreSQL <= 11), to not discard recovery parameters from the configuration (PostgreSQL >= 12), and to not remove the files ``recovery.signal`` or ``standby.signal`` (PostgreSQL >= 12). This is specially useful when you are restoring backups through tools like pgBackRest and Barman, in which case they generated the appropriate recovery settings for you; * ``recovery_conf``: a section containing a map, where each key is the name of a recovery related setting, and the value is the value of the corresponding setting. Any key/value other than the ones that were described above will be interpreted as additional arguments for the ``command`` call. They will all be added to the call in the format ``--key=value``. :returns: ``True`` if the bootstrap was successful, i.e. the execution of the custom ``command`` from *config* exited with code ``0``, ``False`` otherwise. """ self._postgresql.set_state('running custom bootstrap script') params = [] if config.get('no_params') else ['--scope=' + self._postgresql.scope, '--datadir=' + self._postgresql.data_dir] # Add custom parameters specified by the user reserved_args = {'command', 'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'} params += [f"--{arg}={val}" for arg, val in config.items() if arg not in reserved_args] try: logger.info('Running custom bootstrap script: %s', config['command']) if self._postgresql.cancellable.call(shlex.split(config['command']) + params) != 0: self._postgresql.set_state('custom bootstrap failed') return False except Exception: logger.exception('Exception during custom bootstrap') return False self._post_restore() if 'recovery_conf' in config: self._postgresql.config.write_recovery_conf(config['recovery_conf']) elif not self.keep_existing_recovery_conf: self._postgresql.config.remove_recovery_conf() return True def call_post_bootstrap(self, config: Dict[str, Any]) -> bool: """ runs a script after initdb or custom bootstrap script is called and waits until completion. """ cmd = config.get('post_bootstrap') or config.get('post_init') if cmd: r = self._postgresql.connection_pool.conn_kwargs # https://www.postgresql.org/docs/current/static/libpq-pgpass.html # A host name of localhost matches both TCP (host name localhost) and Unix domain socket # (pghost empty or the default socket directory) connections coming from the local machine. env = self._postgresql.config.write_pgpass({'host': 'localhost', **r}) env['PGOPTIONS'] = '-c synchronous_commit=local -c statement_timeout=0' connstring = self._postgresql.config.format_dsn({**r, 'password': None}) try: ret = self._postgresql.cancellable.call(shlex.split(cmd) + [connstring], env=env) except OSError: logger.error('post_init script %s failed', cmd) return False if ret != 0: logger.error('post_init script %s returned non-zero code %d', cmd, ret) return False return True def create_replica(self, clone_member: Union[Leader, Member, None]) -> Optional[int]: """ create the replica according to the replica_method defined by the user. this is a list, so we need to loop through all methods the user supplies """ self._postgresql.set_state('creating replica') self._postgresql.schedule_sanity_checks_after_pause() is_remote_member = isinstance(clone_member, RemoteMember) # get list of replica methods either from clone member or from # the config. If there is no configuration key, or no value is # specified, use basebackup replica_methods = (clone_member.create_replica_methods if is_remote_member else self._postgresql.create_replica_methods) or ['basebackup'] if clone_member and clone_member.conn_url: r = clone_member.conn_kwargs(self._postgresql.config.replication) # add the credentials to connect to the replica origin to pgpass. env = self._postgresql.config.write_pgpass(r) connstring = self._postgresql.config.format_dsn({**r, 'password': None}) else: connstring = '' env = os.environ.copy() # if we don't have any source, leave only replica methods that work without it replica_methods = [r for r in replica_methods if self._postgresql.replica_method_can_work_without_replication_connection(r)] # go through them in priority order ret = 1 for replica_method in replica_methods: if self._postgresql.cancellable.is_cancelled: break method_config = self._postgresql.replica_method_options(replica_method) # if the method is basebackup, then use the built-in if replica_method == "basebackup": ret = self.basebackup(connstring, env, method_config) if ret == 0: logger.info("replica has been created using basebackup") # if basebackup succeeds, exit with success break else: if not self._postgresql.data_directory_empty(): if method_config.get('keep_data', False): logger.info('Leaving data directory uncleaned') else: self._postgresql.remove_data_directory() cmd = replica_method # user-defined method; check for configuration # not required, actually if method_config: # look to see if the user has supplied a full command path # if not, use the method name as the command cmd = method_config.pop('command', cmd) # add the default parameters if not method_config.get('no_params', False): method_config.update({"scope": self._postgresql.scope, "role": "replica", "datadir": self._postgresql.data_dir, "connstring": connstring}) else: for param in ('no_params', 'no_leader', 'keep_data'): method_config.pop(param, None) params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()] try: # call script with the full set of parameters ret = self._postgresql.cancellable.call(shlex.split(cmd) + params, env=env) # if we succeeded, stop if ret == 0: logger.info('replica has been created using %s', replica_method) break else: logger.error('Error creating replica using method %s: %s exited with code=%s', replica_method, cmd, ret) except Exception: logger.exception('Error creating replica using method %s', replica_method) ret = 1 self._postgresql.set_state('stopped') return ret def basebackup(self, conn_url: str, env: Dict[str, str], options: Dict[str, Any]) -> Optional[int]: # creates a replica data dir using pg_basebackup. # this is the default, built-in create_replica_methods # tries twice, then returns failure (as 1) # uses "stream" as the xlog-method to avoid sync issues # supports additional user-supplied options, those are not validated maxfailures = 2 ret = 1 not_allowed_options = ('pgdata', 'format', 'wal-method', 'xlog-method', 'gzip', 'version', 'compress', 'dbname', 'host', 'port', 'username', 'password') user_options = self.process_user_options('basebackup', options, not_allowed_options, logger.error) cmd = [ self._postgresql.pgcommand("pg_basebackup"), "--pgdata=" + self._postgresql.data_dir, "-X", "stream", "--dbname=" + conn_url, ] + user_options for bbfailures in range(0, maxfailures): if self._postgresql.cancellable.is_cancelled: break if not self._postgresql.data_directory_empty(): self._postgresql.remove_data_directory() try: logger.debug('calling: %r', cmd) ret = self._postgresql.cancellable.call(cmd, env=env) if ret == 0: break else: logger.error('Error when fetching backup: pg_basebackup exited with code=%s', ret) except Exception as e: logger.error('Error when fetching backup with pg_basebackup: %s', e) if bbfailures < maxfailures - 1: logger.warning('Trying again in 5 seconds') time.sleep(5) return ret def clone(self, clone_member: Union[Leader, Member, None]) -> bool: """ - initialize the replica from an existing member (primary or replica) - initialize the replica using the replica creation method that works without the replication connection (i.e. restore from on-disk base backup) """ ret = self.create_replica(clone_member) == 0 if ret: self._post_restore() return ret def bootstrap(self, config: Dict[str, Any]) -> bool: """ Initialize a new node from scratch and start it. """ pg_hba = config.get('pg_hba', []) method = config.get('method') or 'initdb' if method != 'initdb' and method in config and 'command' in config[method]: self._keep_existing_recovery_conf = config[method].get('keep_existing_recovery_conf') self._running_custom_bootstrap = True do_initialize = self._custom_bootstrap else: method = 'initdb' do_initialize = self._initdb return do_initialize(config.get(method)) and self._postgresql.config.append_pg_hba(pg_hba) \ and self._postgresql.config.save_configuration_files() and bool(self._postgresql.start()) def create_or_update_role(self, name: str, password: Optional[str], options: List[str]) -> None: options = list(map(str.upper, options)) if 'NOLOGIN' not in options and 'LOGIN' not in options: options.append('LOGIN') if password: options.extend(['PASSWORD', quote_literal(password)]) sql = """DO $$ BEGIN SET local synchronous_commit = 'local'; PERFORM * FROM pg_catalog.pg_authid WHERE rolname = {0}; IF FOUND THEN ALTER ROLE {1} WITH {2}; ELSE CREATE ROLE {1} WITH {2}; END IF; END;$$""".format(quote_literal(name), quote_ident(name, self._postgresql.connection()), ' '.join(options)) self._postgresql.query('SET log_statement TO none') self._postgresql.query('SET log_min_duration_statement TO -1') self._postgresql.query("SET log_min_error_statement TO 'log'") self._postgresql.query("SET pg_stat_statements.track_utility to 'off'") self._postgresql.query("SET pgaudit.log TO none") try: self._postgresql.query(sql) finally: self._postgresql.query('RESET log_min_error_statement') self._postgresql.query('RESET log_min_duration_statement') self._postgresql.query('RESET log_statement') self._postgresql.query('RESET pg_stat_statements.track_utility') self._postgresql.query('RESET pgaudit.log') def post_bootstrap(self, config: Dict[str, Any], task: CriticalTask) -> Optional[bool]: try: postgresql = self._postgresql superuser = postgresql.config.superuser if 'username' in superuser and 'password' in superuser: self.create_or_update_role(superuser['username'], superuser['password'], ['SUPERUSER']) task.complete(self.call_post_bootstrap(config)) if task.result: replication = postgresql.config.replication self.create_or_update_role(replication['username'], replication.get('password'), ['REPLICATION']) rewind = postgresql.config.rewind_credentials if not deep_compare(rewind, superuser): self.create_or_update_role(rewind['username'], rewind.get('password'), []) for f in ('pg_ls_dir(text, boolean, boolean)', 'pg_stat_file(text, boolean)', 'pg_read_binary_file(text)', 'pg_read_binary_file(text, bigint, bigint, boolean)'): sql = """DO $$ BEGIN SET local synchronous_commit = 'local'; GRANT EXECUTE ON function pg_catalog.{0} TO {1}; END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection())) postgresql.query(sql) if config.get('users'): logger.error('User creation is not be supported starting from v4.0.0. ' 'Please use "bootstrap.post_bootstrap" script to create users.') # We were doing a custom bootstrap instead of running initdb, therefore we opened trust # access from certain addresses to be able to reach cluster and change password if self._running_custom_bootstrap: self._running_custom_bootstrap = False # If we don't have custom configuration for pg_hba.conf we need to restore original file if not postgresql.config.get('pg_hba'): if os.path.exists(postgresql.config.pg_hba_conf): os.unlink(postgresql.config.pg_hba_conf) postgresql.config.restore_configuration_files() postgresql.config.write_postgresql_conf() postgresql.config.replace_pg_ident() # at this point there should be no recovery.conf postgresql.config.remove_recovery_conf() if postgresql.config.hba_file: postgresql.restart() else: postgresql.config.replace_pg_hba() if postgresql.pending_restart_reason: postgresql.restart() else: postgresql.reload() time.sleep(1) # give a time to postgres to "reload" configuration files postgresql.connection().close() # close connection to reconnect with a new password else: # initdb # We may want create database and extension for some MPP clusters self._postgresql.mpp_handler.bootstrap() except Exception: logger.exception('post_bootstrap') task.complete(False) return task.result patroni-4.0.4/patroni/postgresql/callback_executor.py000066400000000000000000000046271472010352700231100ustar00rootroot00000000000000import logging import sys from enum import Enum from threading import Condition, Thread from typing import Any, Dict, List from .cancellable import CancellableExecutor, CancellableSubprocess logger = logging.getLogger(__name__) class CallbackAction(str, Enum): NOOP = "noop" ON_START = "on_start" ON_STOP = "on_stop" ON_RESTART = "on_restart" ON_RELOAD = "on_reload" ON_ROLE_CHANGE = "on_role_change" def __repr__(self) -> str: return self.value class OnReloadExecutor(CancellableSubprocess): def call_nowait(self, cmd: List[str]) -> None: """Run one `on_reload` callback at most. To achieve it we always kill already running command including child processes.""" self.cancel(kill=True) self._kill_children() with self._lock: started = self._start_process(cmd, close_fds=True) if started and self._process is not None: Thread(target=self._process.wait).start() class CallbackExecutor(CancellableExecutor, Thread): def __init__(self): CancellableExecutor.__init__(self) Thread.__init__(self) self.daemon = True self._on_reload_executor = OnReloadExecutor() self._cmd = None self._condition = Condition() self.start() def call(self, cmd: List[str]) -> None: """Executes one callback at a time. Already running command is killed (including child processes). If it couldn't be killed we wait until it finishes. :param cmd: command to be executed""" kwargs: Dict[str, Any] = {'stacklevel': 3} if sys.version_info >= (3, 8) else {} logger.debug('CallbackExecutor.call(%s)', cmd, **kwargs) if cmd[-3] == CallbackAction.ON_RELOAD: return self._on_reload_executor.call_nowait(cmd) self._kill_process() with self._condition: self._cmd = cmd self._condition.notify() def run(self) -> None: while True: with self._condition: if self._cmd is None: self._condition.wait() cmd, self._cmd = self._cmd, None if cmd is not None: with self._lock: if not self._start_process(cmd, close_fds=True): continue if self._process: self._process.wait() self._kill_children() patroni-4.0.4/patroni/postgresql/cancellable.py000066400000000000000000000111251472010352700216520ustar00rootroot00000000000000import logging import subprocess from threading import Lock from typing import Any, Dict, List, Optional import psutil from patroni.exceptions import PostgresException from patroni.utils import polling_loop logger = logging.getLogger(__name__) class CancellableExecutor(object): """ There must be only one such process so that AsyncExecutor can easily cancel it. """ def __init__(self) -> None: self._process = None self._process_cmd = None self._process_children: List[psutil.Process] = [] self._lock = Lock() def _start_process(self, cmd: List[str], *args: Any, **kwargs: Any) -> Optional[bool]: """This method must be executed only when the `_lock` is acquired""" try: self._process_children = [] self._process_cmd = cmd self._process = psutil.Popen(cmd, *args, **kwargs) except Exception: return logger.exception('Failed to execute %s', cmd) return True def _kill_process(self) -> None: with self._lock: if self._process is not None and self._process.is_running() and not self._process_children: try: self._process.suspend() # Suspend the process before getting list of children except psutil.Error as e: logger.info('Failed to suspend the process: %s', e.msg) try: self._process_children = self._process.children(recursive=True) except psutil.Error: pass try: self._process.kill() logger.warning('Killed %s because it was still running', self._process_cmd) except psutil.NoSuchProcess: pass except psutil.AccessDenied as e: logger.warning('Failed to kill the process: %s', e.msg) def _kill_children(self) -> None: waitlist: List[psutil.Process] = [] with self._lock: for child in self._process_children: try: child.kill() except psutil.NoSuchProcess: continue except psutil.AccessDenied as e: logger.info('Failed to kill child process: %s', e.msg) waitlist.append(child) psutil.wait_procs(waitlist) class CancellableSubprocess(CancellableExecutor): def __init__(self) -> None: super(CancellableSubprocess, self).__init__() self._is_cancelled = False def call(self, *args: Any, **kwargs: Any) -> Optional[int]: for s in ('stdin', 'stdout', 'stderr'): kwargs.pop(s, None) communicate: Optional[Dict[str, str]] = kwargs.pop('communicate', None) input_data = None if isinstance(communicate, dict): input_data = communicate.get('input') if input_data: if input_data[-1] != '\n': input_data += '\n' input_data = input_data.encode('utf-8') kwargs['stdin'] = subprocess.PIPE kwargs['stdout'] = subprocess.PIPE kwargs['stderr'] = subprocess.PIPE try: with self._lock: if self._is_cancelled: raise PostgresException('cancelled') self._is_cancelled = False started = self._start_process(*args, **kwargs) if started and self._process is not None: if isinstance(communicate, dict): communicate['stdout'], communicate['stderr'] = \ self._process.communicate(input_data) # pyright: ignore [reportGeneralTypeIssues] return self._process.wait() finally: with self._lock: self._process = None self._kill_children() def reset_is_cancelled(self) -> None: with self._lock: self._is_cancelled = False @property def is_cancelled(self) -> bool: with self._lock: return self._is_cancelled def cancel(self, kill: bool = False) -> None: with self._lock: self._is_cancelled = True if self._process is None or not self._process.is_running(): return logger.info('Terminating %s', self._process_cmd) self._process.terminate() for _ in polling_loop(10): with self._lock: if self._process is None or not self._process.is_running(): return if kill: break self._kill_process() patroni-4.0.4/patroni/postgresql/config.py000066400000000000000000002116421472010352700207000ustar00rootroot00000000000000import logging import os import re import shutil import socket import stat import time from contextlib import contextmanager from types import TracebackType from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union from urllib.parse import parse_qsl, unquote, urlparse from .. import global_config from ..collections import CaseInsensitiveDict, CaseInsensitiveSet, EMPTY_DICT from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name from ..exceptions import PatroniFatalException, PostgresConnectionException from ..file_perm import pg_perm from ..postgresql.misc import get_major_from_minor_version, postgres_version_to_int from ..utils import compare_values, get_postgres_version, is_subpath, \ maybe_convert_from_base_unit, parse_bool, parse_int, split_host_port, uri, validate_directory from ..validator import EnumValidator, IntValidator from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value if TYPE_CHECKING: # pragma: no cover from . import Postgresql logger = logging.getLogger(__name__) PARAMETER_RE = re.compile(r'([a-z_]+)\s*=\s*') def conninfo_uri_parse(dsn: str) -> Dict[str, str]: ret: Dict[str, str] = {} r = urlparse(dsn) if r.username: ret['user'] = r.username if r.password: ret['password'] = r.password if r.path[1:]: ret['dbname'] = r.path[1:] hosts: List[str] = [] ports: List[str] = [] for netloc in r.netloc.split('@')[-1].split(','): host = None if '[' in netloc and ']' in netloc: tmp = netloc.split(']') + [''] host = tmp[0][1:] netloc = ':'.join(tmp[:2]) tmp = netloc.rsplit(':', 1) if host is None: host = tmp[0] hosts.append(host) ports.append(tmp[1] if len(tmp) == 2 else '') if hosts: ret['host'] = ','.join(hosts) if ports: ret['port'] = ','.join(ports) ret = {name: unquote(value) for name, value in ret.items()} ret.update({name: value for name, value in parse_qsl(r.query)}) if ret.get('ssl') == 'true': del ret['ssl'] ret['sslmode'] = 'require' return ret def read_param_value(value: str) -> Union[Tuple[None, None], Tuple[str, int]]: length = len(value) ret = '' is_quoted = value[0] == "'" i = int(is_quoted) while i < length: if is_quoted: if value[i] == "'": return ret, i + 1 elif value[i].isspace(): break if value[i] == '\\': i += 1 if i >= length: break ret += value[i] i += 1 return (None, None) if is_quoted else (ret, i) def conninfo_parse(dsn: str) -> Optional[Dict[str, str]]: ret: Dict[str, str] = {} length = len(dsn) i = 0 while i < length: if dsn[i].isspace(): i += 1 continue param_match = PARAMETER_RE.match(dsn[i:]) if not param_match: return param = param_match.group(1) i += param_match.end() if i >= length: return value, end = read_param_value(dsn[i:]) if value is None or end is None: return i += end ret[param] = value return ret def parse_dsn(value: str) -> Optional[Dict[str, str]]: """ Very simple equivalent of `psycopg2.extensions.parse_dsn` introduced in 2.7.0. We are not using psycopg2 function in order to remain compatible with 2.5.4+. There are a few minor differences though, this function sets the `sslmode`, 'gssencmode', and `channel_binding` to `prefer` if they are not present in the connection string. This is necessary to simplify comparison of the old and the new values. >>> r = parse_dsn('postgresql://u%2Fse:pass@:%2f123,[::1]/db%2Fsdf?application_name=mya%2Fpp&ssl=true') >>> r == {'application_name': 'mya/pp', 'dbname': 'db/sdf', 'host': ',::1', 'sslmode': 'require',\ 'password': 'pass', 'port': '/123,', 'user': 'u/se', 'gssencmode': 'prefer',\ 'channel_binding': 'prefer', 'sslnegotiation': 'postgres'} True >>> r = parse_dsn(" host = 'host' dbname = db\\\\ name requiressl=1 ") >>> r == {'dbname': 'db name', 'host': 'host', 'sslmode': 'require',\ 'gssencmode': 'prefer', 'channel_binding': 'prefer', 'sslnegotiation': 'postgres'} True >>> parse_dsn('requiressl = 0\\\\') == {'sslmode': 'prefer', 'gssencmode': 'prefer',\ 'channel_binding': 'prefer', 'sslnegotiation': 'postgres'} True >>> parse_dsn("host=a foo = '") is None True >>> parse_dsn("host=a foo = ") is None True >>> parse_dsn("1") is None True """ if value.startswith('postgres://') or value.startswith('postgresql://'): ret = conninfo_uri_parse(value) else: ret = conninfo_parse(value) if ret: if 'sslmode' not in ret: # allow sslmode to take precedence over requiressl requiressl = ret.pop('requiressl', None) if requiressl == '1': ret['sslmode'] = 'require' elif requiressl is not None: ret['sslmode'] = 'prefer' ret.setdefault('sslmode', 'prefer') ret.setdefault('gssencmode', 'prefer') ret.setdefault('channel_binding', 'prefer') ret.setdefault('sslnegotiation', 'postgres') return ret def strip_comment(value: str) -> str: i = value.find('#') if i > -1: value = value[:i].strip() return value def read_recovery_param_value(value: str) -> Optional[str]: """ >>> read_recovery_param_value('') is None True >>> read_recovery_param_value("'") is None True >>> read_recovery_param_value("''a") is None True >>> read_recovery_param_value('a b') is None True >>> read_recovery_param_value("'''") is None True >>> read_recovery_param_value("'\\\\") is None True >>> read_recovery_param_value("'a' s#") is None True >>> read_recovery_param_value("'\\\\'''' #a") "''" >>> read_recovery_param_value('asd') 'asd' """ value = value.strip() length = len(value) if length == 0: return None elif value[0] == "'": if length == 1: return None ret = '' i = 1 while i < length: if value[i] == '\\': i += 1 if i >= length: return None elif value[i] == "'": i += 1 if i >= length: break if value[i] in ('#', ' '): if strip_comment(value[i:]): return None break if value[i] != "'": return None ret += value[i] i += 1 else: return None return ret else: value = strip_comment(value) if not value or ' ' in value or '\\' in value: return None return value def mtime(filename: str) -> Optional[float]: try: return os.stat(filename).st_mtime except OSError: return None class ConfigWriter(object): def __init__(self, filename: str) -> None: self._filename = filename self._fd = None def __enter__(self) -> 'ConfigWriter': self._fd = open(self._filename, 'w') self.writeline('# Do not edit this file manually!\n# It will be overwritten by Patroni!') return self def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: if self._fd: self._fd.close() def writeline(self, line: str) -> None: if self._fd: self._fd.write(line) self._fd.write('\n') def writelines(self, lines: List[Optional[str]]) -> None: for line in lines: if isinstance(line, str): self.writeline(line) @staticmethod def escape(value: Any) -> str: # Escape (by doubling) any single quotes or backslashes in given string return re.sub(r'([\'\\])', r'\1\1', str(value)) def write_param(self, param: str, value: Any) -> None: self.writeline("{0} = '{1}'".format(param, self.escape(value))) def _false_validator(value: Any) -> bool: return False def _bool_validator(value: Any) -> bool: return parse_bool(value) is not None def _bool_is_true_validator(value: Any) -> bool: return parse_bool(value) is True def get_param_diff(old_value: Any, new_value: Any, vartype: Optional[str] = None, unit: Optional[str] = None) -> Dict[str, str]: """Get a dictionary representing a single PG parameter's value diff. :param old_value: current :class:`str` parameter value. :param new_value: :class:`str` value of the parameter after a restart. :param vartype: the target type to parse old/new_value. See ``vartype`` argument of :func:`~patroni.utils.maybe_convert_from_base_unit`. :param unit: unit of *old/new_value*. See ``base_unit`` argument of :func:`~patroni.utils.maybe_convert_from_base_unit`. :returns: a :class:`dict` object that contains two keys: ``old_value`` and ``new_value`` with their values casted to :class:`str` and converted from base units (if possible). """ str_value: Callable[[Any], str] = lambda x: '' if x is None else str(x) return { 'old_value': (maybe_convert_from_base_unit(str_value(old_value), vartype, unit) if vartype else str_value(old_value)), 'new_value': (maybe_convert_from_base_unit(str_value(new_value), vartype, unit) if vartype else str_value(new_value)) } class ConfigHandler(object): # List of parameters which must be always passed to postmaster as command line options # to make it not possible to change them with 'ALTER SYSTEM'. # Some of these parameters have sane default value assigned and Patroni doesn't allow # to decrease this value. E.g. 'wal_level' can't be lower then 'hot_standby' and so on. # These parameters could be changed only globally, i.e. via DCS. # P.S. 'listen_addresses' and 'port' are added here just for convenience, to mark them # as a parameters which should always be passed through command line. # # Format: # key - parameter name # value - tuple(default_value, check_function, min_version) # default_value -- some sane default value # check_function -- if the new value is not correct must return `!False` # min_version -- major version of PostgreSQL when parameter was introduced CMDLINE_OPTIONS = CaseInsensitiveDict({ 'listen_addresses': (None, _false_validator, 90100), 'port': (None, _false_validator, 90100), 'cluster_name': (None, _false_validator, 90500), 'wal_level': ('hot_standby', EnumValidator(('hot_standby', 'replica', 'logical')), 90100), 'hot_standby': ('on', _bool_is_true_validator, 90100), 'max_connections': (100, IntValidator(min=25), 90100), 'max_wal_senders': (10, IntValidator(min=3), 90100), 'wal_keep_segments': (8, IntValidator(min=1), 90100), 'wal_keep_size': ('128MB', IntValidator(min=16, base_unit='MB'), 130000), 'max_prepared_transactions': (0, IntValidator(min=0), 90100), 'max_locks_per_transaction': (64, IntValidator(min=32), 90100), 'track_commit_timestamp': ('off', _bool_validator, 90500), 'max_replication_slots': (10, IntValidator(min=4), 90400), 'max_worker_processes': (8, IntValidator(min=2), 90400), 'wal_log_hints': ('on', _bool_validator, 90400) }) _RECOVERY_PARAMETERS = CaseInsensitiveSet(recovery_parameters.keys()) def __init__(self, postgresql: 'Postgresql', config: Dict[str, Any]) -> None: self._postgresql = postgresql self._config_dir = os.path.abspath(config.get('config_dir', '') or postgresql.data_dir) config_base_name = config.get('config_base_name', 'postgresql') self._postgresql_conf = os.path.join(self._config_dir, config_base_name + '.conf') self._postgresql_conf_mtime = None self._postgresql_base_conf_name = config_base_name + '.base.conf' self._postgresql_base_conf = os.path.join(self._config_dir, self._postgresql_base_conf_name) self._pg_hba_conf = os.path.join(self._config_dir, 'pg_hba.conf') self._pg_ident_conf = os.path.join(self._config_dir, 'pg_ident.conf') self._recovery_conf = os.path.join(postgresql.data_dir, 'recovery.conf') self._recovery_conf_mtime = None self._recovery_signal = os.path.join(postgresql.data_dir, 'recovery.signal') self._standby_signal = os.path.join(postgresql.data_dir, 'standby.signal') self._auto_conf = os.path.join(postgresql.data_dir, 'postgresql.auto.conf') self._auto_conf_mtime = None self._pgpass = os.path.abspath(config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')) if os.path.exists(self._pgpass) and not os.path.isfile(self._pgpass): raise PatroniFatalException("'{0}' exists and it's not a file, check your `postgresql.pgpass` configuration" .format(self._pgpass)) self._passfile = None self._passfile_mtime = None self._postmaster_ctime = None self._current_recovery_params: Optional[CaseInsensitiveDict] = None self._config = {} self._recovery_params = CaseInsensitiveDict() self._server_parameters: CaseInsensitiveDict = CaseInsensitiveDict() self.reload_config(config) def load_current_server_parameters(self) -> None: """Read GUC's values from ``pg_settings`` when Patroni is joining the the postgres that is already running.""" exclude = [name.lower() for name, value in self.CMDLINE_OPTIONS.items() if value[1] == _false_validator] keep_values = {k: self._server_parameters[k] for k in exclude} server_parameters = CaseInsensitiveDict({r[0]: r[1] for r in self._postgresql.query( "SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings" " WHERE (source IN ('command line', 'environment variable') OR sourcefile = %s" " OR pg_catalog.lower(name) = ANY(%s)) AND pg_catalog.lower(name) != ALL(%s)", self._postgresql_conf, [n.lower() for n in self.CMDLINE_OPTIONS.keys()], exclude)}) recovery_params = CaseInsensitiveDict({k: server_parameters.pop(k) for k in self._RECOVERY_PARAMETERS if k in server_parameters}) # We also want to load current settings of recovery parameters, including primary_conninfo # and primary_slot_name, otherwise patronictl restart will update postgresql.conf # and remove them, what in the worst case will cause another restart. # We are doing it only for PostgreSQL v12 onwards, because older version still have recovery.conf if not self._postgresql.is_primary() and self._postgresql.major_version >= 120000: # primary_conninfo is expected to be a dict, therefore we need to parse it recovery_params['primary_conninfo'] = parse_dsn(recovery_params.pop('primary_conninfo', '')) or {} self._recovery_params = recovery_params self._server_parameters = CaseInsensitiveDict({**server_parameters, **keep_values}) def setup_server_parameters(self) -> None: self._server_parameters = self.get_server_parameters(self._config) self._adjust_recovery_parameters() def try_to_create_dir(self, d: str, msg: str) -> None: d = os.path.join(self._postgresql.data_dir, d) if (not is_subpath(self._postgresql.data_dir, d) or not self._postgresql.data_directory_empty()): validate_directory(d, msg) def check_directories(self) -> None: if "unix_socket_directories" in self._server_parameters: for d in self._server_parameters["unix_socket_directories"].split(","): self.try_to_create_dir(d.strip(), "'{}' is defined in unix_socket_directories, {}") if "stats_temp_directory" in self._server_parameters: self.try_to_create_dir(self._server_parameters["stats_temp_directory"], "'{}' is defined in stats_temp_directory, {}") if not self._krbsrvname: self.try_to_create_dir(os.path.dirname(self._pgpass), "'{}' is defined in `postgresql.pgpass`, {}") @property def config_dir(self) -> str: return self._config_dir @property def pg_version(self) -> int: """Current full postgres version if instance is running, major version otherwise. We can only use ``postgres --version`` output if major version there equals to the one in data directory. If it is not the case, we should use major version from the ``PG_VERSION`` file. """ if self._postgresql.state == 'running': try: return self._postgresql.server_version except AttributeError: pass bin_minor = postgres_version_to_int(get_postgres_version(bin_name=self._postgresql.pgcommand('postgres'))) bin_major = get_major_from_minor_version(bin_minor) datadir_major = self._postgresql.major_version return datadir_major if bin_major != datadir_major else bin_minor @property def _configuration_to_save(self) -> List[str]: configuration = [os.path.basename(self._postgresql_conf)] if 'custom_conf' not in self._config: configuration.append(os.path.basename(self._postgresql_base_conf_name)) if not self.hba_file: configuration.append('pg_hba.conf') if not self.ident_file: configuration.append('pg_ident.conf') return configuration def set_file_permissions(self, filename: str) -> None: """Set permissions of file *filename* according to the expected permissions if it resides under PGDATA. .. note:: Do nothing if the file is not under PGDATA. :param filename: path to a file which permissions might need to be adjusted. """ if is_subpath(self._postgresql.data_dir, filename): pg_perm.set_permissions_from_data_directory(self._postgresql.data_dir) os.chmod(filename, pg_perm.file_create_mode) @contextmanager def config_writer(self, filename: str) -> Iterator[ConfigWriter]: """Create :class:`ConfigWriter` object and set permissions on a *filename*. :param filename: path to a config file. :yields: :class:`ConfigWriter` object. """ with ConfigWriter(filename) as writer: yield writer self.set_file_permissions(filename) def save_configuration_files(self, check_custom_bootstrap: bool = False) -> bool: """ copy postgresql.conf to postgresql.conf.backup to be able to retrieve configuration files - originally stored as symlinks, those are normally skipped by pg_basebackup - in case of WAL-E basebackup (see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239) """ if not (check_custom_bootstrap and self._postgresql.bootstrap.running_custom_bootstrap): try: for f in self._configuration_to_save: config_file = os.path.join(self._config_dir, f) backup_file = os.path.join(self._postgresql.data_dir, f + '.backup') if os.path.isfile(config_file): shutil.copy(config_file, backup_file) self.set_file_permissions(backup_file) except IOError: logger.exception('unable to create backup copies of configuration files') return True def restore_configuration_files(self) -> None: """ restore a previously saved postgresql.conf """ try: for f in self._configuration_to_save: config_file = os.path.join(self._config_dir, f) backup_file = os.path.join(self._postgresql.data_dir, f + '.backup') if not os.path.isfile(config_file): if os.path.isfile(backup_file): shutil.copy(backup_file, config_file) self.set_file_permissions(config_file) # Previously we didn't backup pg_ident.conf, if file is missing just create empty elif f == 'pg_ident.conf': open(config_file, 'w').close() self.set_file_permissions(config_file) except IOError: logger.exception('unable to restore configuration files from backup') def write_postgresql_conf(self, configuration: Optional[CaseInsensitiveDict] = None) -> None: # rename the original configuration if it is necessary if 'custom_conf' not in self._config and not os.path.exists(self._postgresql_base_conf): os.rename(self._postgresql_conf, self._postgresql_base_conf) configuration = configuration or self._server_parameters.copy() # Due to the permanent logical replication slots configured we have to enable hot_standby_feedback if self._postgresql.enforce_hot_standby_feedback: configuration['hot_standby_feedback'] = 'on' with self.config_writer(self._postgresql_conf) as f: include = self._config.get('custom_conf') or self._postgresql_base_conf_name f.writeline("include '{0}'\n".format(ConfigWriter.escape(include))) version = self.pg_version for name, value in sorted((configuration).items()): value = transform_postgresql_parameter_value(version, name, value, self._postgresql.available_gucs) if value is not None and\ (name != 'hba_file' or not self._postgresql.bootstrap.running_custom_bootstrap): f.write_param(name, value) # when we are doing custom bootstrap we assume that we don't know superuser password # and in order to be able to change it, we are opening trust access from a certain address # therefore we need to make sure that hba_file is not overridden # after changing superuser password we will "revert" all these "changes" if self._postgresql.bootstrap.running_custom_bootstrap or 'hba_file' not in self._server_parameters: f.write_param('hba_file', self._pg_hba_conf) if 'ident_file' not in self._server_parameters: f.write_param('ident_file', self._pg_ident_conf) if self._postgresql.major_version >= 120000: if self._recovery_params: f.writeline('\n# recovery.conf') self._write_recovery_params(f, self._recovery_params) if not self._postgresql.bootstrap.keep_existing_recovery_conf: self._sanitize_auto_conf() def append_pg_hba(self, config: List[str]) -> bool: if not self.hba_file and not self._config.get('pg_hba'): with open(self._pg_hba_conf, 'a') as f: f.write('\n{}\n'.format('\n'.join(config))) self.set_file_permissions(self._pg_hba_conf) return True def replace_pg_hba(self) -> Optional[bool]: """ Replace pg_hba.conf content in the PGDATA if hba_file is not defined in the `postgresql.parameters` and pg_hba is defined in `postgresql` configuration section. :returns: True if pg_hba.conf was rewritten. """ # when we are doing custom bootstrap we assume that we don't know superuser password # and in order to be able to change it, we are opening trust access from a certain address if self._postgresql.bootstrap.running_custom_bootstrap: addresses = {} if os.name == 'nt' else {'': 'local'} # windows doesn't yet support unix-domain sockets if 'host' in self.local_replication_address and not self.local_replication_address['host'].startswith('/'): addresses.update({sa[0] + '/32': 'host' for _, _, _, _, sa in socket.getaddrinfo( self.local_replication_address['host'], self.local_replication_address['port'], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)}) with self.config_writer(self._pg_hba_conf) as f: for address, t in addresses.items(): f.writeline(( '{0}\treplication\t{1}\t{3}\ttrust\n' '{0}\tall\t{2}\t{3}\ttrust' ).format(t, self.replication['username'], self._superuser.get('username') or 'all', address)) elif not self.hba_file and self._config.get('pg_hba'): with self.config_writer(self._pg_hba_conf) as f: f.writelines(self._config['pg_hba']) return True def replace_pg_ident(self) -> Optional[bool]: """ Replace pg_ident.conf content in the PGDATA if ident_file is not defined in the `postgresql.parameters` and pg_ident is defined in the `postgresql` section. :returns: True if pg_ident.conf was rewritten. """ if not self.ident_file and self._config.get('pg_ident'): with self.config_writer(self._pg_ident_conf) as f: f.writelines(self._config['pg_ident']) return True def primary_conninfo_params(self, member: Union[Leader, Member, None]) -> Optional[Dict[str, Any]]: if not member or not member.conn_url or member.name == self._postgresql.name: return None ret = member.conn_kwargs(self.replication) ret['application_name'] = self._postgresql.name ret.setdefault('sslmode', 'prefer') if self._postgresql.major_version >= 120000: ret.setdefault('gssencmode', 'prefer') if self._postgresql.major_version >= 130000: ret.setdefault('channel_binding', 'prefer') if self._postgresql.major_version >= 170000: ret.setdefault('sslnegotiation', 'postgres') if self._krbsrvname: ret['krbsrvname'] = self._krbsrvname if not ret.get('dbname'): ret['dbname'] = self._postgresql.database return ret def format_dsn(self, params: Dict[str, Any]) -> str: """Format connection string from connection parameters. .. note:: only parameters from the below list are considered and values are escaped. :param params: :class:`dict` object with connection parameters. :returns: a connection string in a format "key1=value2 key2=value2" """ # A list of keywords that can be found in a conninfo string. Follows what is acceptable by libpq keywords = ('dbname', 'user', 'passfile' if params.get('passfile') else 'password', 'host', 'port', 'sslmode', 'sslcompression', 'sslcert', 'sslkey', 'sslpassword', 'sslrootcert', 'sslcrl', 'sslcrldir', 'application_name', 'krbsrvname', 'gssencmode', 'channel_binding', 'target_session_attrs', 'sslnegotiation') def escape(value: Any) -> str: return re.sub(r'([\'\\ ])', r'\\\1', str(value)) key_ver = {'target_session_attrs': 100000, 'gssencmode': 120000, 'channel_binding': 130000, 'sslpassword': 130000, 'sslcrldir': 140000, 'sslnegotiation': 170000} return ' '.join('{0}={1}'.format(kw, escape(params[kw])) for kw in keywords if params.get(kw) is not None and self._postgresql.major_version >= key_ver.get(kw, 0)) def _write_recovery_params(self, fd: ConfigWriter, recovery_params: CaseInsensitiveDict) -> None: if self._postgresql.major_version >= 90500: pause_at_recovery_target = parse_bool(recovery_params.pop('pause_at_recovery_target', None)) if pause_at_recovery_target is not None: recovery_params.setdefault('recovery_target_action', 'pause' if pause_at_recovery_target else 'promote') else: if str(recovery_params.pop('recovery_target_action', None)).lower() == 'promote': recovery_params.setdefault('pause_at_recovery_target', 'false') for name, value in sorted(recovery_params.items()): if name == 'primary_conninfo': if self._postgresql.major_version >= 100000 and 'PGPASSFILE' in self.write_pgpass(value): value['passfile'] = self._passfile = self._pgpass self._passfile_mtime = mtime(self._pgpass) value = self.format_dsn(value) else: value = transform_recovery_parameter_value(self._postgresql.major_version, name, value, self._postgresql.available_gucs) if value is None: continue fd.write_param(name, value) def build_recovery_params(self, member: Union[Leader, Member, None]) -> CaseInsensitiveDict: default: Dict[str, Any] = {} recovery_params = CaseInsensitiveDict({p: v for p, v in (self.get('recovery_conf') or default).items() if not p.lower().startswith('recovery_target') and p.lower() not in ('primary_conninfo', 'primary_slot_name')}) recovery_params.update({'standby_mode': 'on', 'recovery_target_timeline': 'latest'}) if self._postgresql.major_version >= 120000: # on pg12 we want to protect from following params being set in one of included files # not doing so might result in a standby being paused, promoted or shutted down. recovery_params.update({'recovery_target': '', 'recovery_target_name': '', 'recovery_target_time': '', 'recovery_target_xid': '', 'recovery_target_lsn': ''}) is_remote_member = isinstance(member, RemoteMember) primary_conninfo = self.primary_conninfo_params(member) if primary_conninfo: use_slots = global_config.use_slots and self._postgresql.major_version >= 90400 if use_slots and not (is_remote_member and member.no_replication_slot): primary_slot_name = member.primary_slot_name if is_remote_member else self._postgresql.name recovery_params['primary_slot_name'] = slot_name_from_member_name(primary_slot_name) # We are a standby leader and are using a replication slot. Make sure we connect to # the leader of the main cluster (in case more than one host is specified in the # connstr) by adding 'target_session_attrs=read-write' to primary_conninfo. if is_remote_member and ',' in primary_conninfo['host'] and self._postgresql.major_version >= 100000: primary_conninfo['target_session_attrs'] = 'read-write' recovery_params['primary_conninfo'] = primary_conninfo # standby_cluster config might have different parameters, we want to override them standby_cluster_params = ['restore_command', 'archive_cleanup_command']\ + (['recovery_min_apply_delay'] if is_remote_member else []) recovery_params.update({p: member.data.get(p) for p in standby_cluster_params if member and member.data.get(p)}) return recovery_params def recovery_conf_exists(self) -> bool: if self._postgresql.major_version >= 120000: return os.path.exists(self._standby_signal) or os.path.exists(self._recovery_signal) return os.path.exists(self._recovery_conf) @property def triggerfile_good_name(self) -> str: return 'trigger_file' if self._postgresql.major_version < 120000 else 'promote_trigger_file' @property def _triggerfile_wrong_name(self) -> str: return 'trigger_file' if self._postgresql.major_version >= 120000 else 'promote_trigger_file' @property def _recovery_parameters_to_compare(self) -> CaseInsensitiveSet: skip_params = CaseInsensitiveSet({'pause_at_recovery_target', 'recovery_target_inclusive', 'recovery_target_action', 'standby_mode', self._triggerfile_wrong_name}) return CaseInsensitiveSet(self._RECOVERY_PARAMETERS - skip_params) def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], bool]: """Read current recovery parameters values. .. note:: We query Postgres only if we detected that Postgresql was restarted or when at least one of the following files was updated: * ``postgresql.conf``; * ``postgresql.auto.conf``; * ``passfile`` that is used in the ``primary_conninfo``. :returns: a tuple with two elements: * :class:`CaseInsensitiveDict` object with current values of recovery parameters, or ``None`` if no configuration files were updated; * ``True`` if new values of recovery parameters were queried, ``False`` otherwise. """ if self._postgresql.is_starting(): return None, False pg_conf_mtime = mtime(self._postgresql_conf) auto_conf_mtime = mtime(self._auto_conf) passfile_mtime = mtime(self._passfile) if self._passfile else False postmaster_ctime = self._postgresql.is_running() if postmaster_ctime: postmaster_ctime = postmaster_ctime.create_time() if self._postgresql_conf_mtime == pg_conf_mtime and self._auto_conf_mtime == auto_conf_mtime \ and self._passfile_mtime == passfile_mtime and self._postmaster_ctime == postmaster_ctime: return None, False try: values = self._get_pg_settings(self._recovery_parameters_to_compare).values() values = CaseInsensitiveDict({p[0]: [p[1], p[4] == 'postmaster', p[5]] for p in values}) self._postgresql_conf_mtime = pg_conf_mtime self._auto_conf_mtime = auto_conf_mtime self._postmaster_ctime = postmaster_ctime except Exception as exc: if all((isinstance(exc, PostgresConnectionException), self._postgresql_conf_mtime == pg_conf_mtime, self._auto_conf_mtime == auto_conf_mtime, self._passfile_mtime == passfile_mtime, self._postmaster_ctime != postmaster_ctime)): # We detected that the connection to postgres fails, but the process creation time of the postmaster # doesn't match the old value. It is an indicator that Postgres crashed and either doing crash # recovery or down. In this case we return values like nothing changed in the config. return None, False values = None return values, True def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], bool]: recovery_conf_mtime = mtime(self._recovery_conf) passfile_mtime = mtime(self._passfile) if self._passfile else False if recovery_conf_mtime == self._recovery_conf_mtime and passfile_mtime == self._passfile_mtime: return None, False values = CaseInsensitiveDict() with open(self._recovery_conf, 'r') as f: for line in f: line = line.strip() if not line or line.startswith('#'): continue value = None match = PARAMETER_RE.match(line) if match: value = read_recovery_param_value(line[match.end():]) if match is None or value is None: return None, True values[match.group(1)] = [value, True] self._recovery_conf_mtime = recovery_conf_mtime values.setdefault('recovery_min_apply_delay', ['0', True]) values['recovery_min_apply_delay'][0] = parse_int(values['recovery_min_apply_delay'][0], 'ms') values.update({param: ['', True] for param in self._recovery_parameters_to_compare if param not in values}) return values, True def _check_passfile(self, passfile: str, wanted_primary_conninfo: Dict[str, Any]) -> bool: # If there is a passfile in the primary_conninfo try to figure out that # the passfile contains the line(s) allowing connection to the given node. # We assume that the passfile was created by Patroni and therefore doing # the full match and not covering cases when host, port or user are set to '*' passfile_mtime = mtime(passfile) if passfile_mtime: try: with open(passfile) as f: wanted_lines = (self._pgpass_content(wanted_primary_conninfo) or '').splitlines() file_lines = f.read().splitlines() if set(wanted_lines) == set(file_lines): self._passfile = passfile self._passfile_mtime = passfile_mtime return True except Exception: logger.info('Failed to read %s', passfile) return False def _check_primary_conninfo(self, primary_conninfo: Dict[str, Any], wanted_primary_conninfo: Dict[str, Any]) -> bool: # first we will cover corner cases, when we are replicating from somewhere while shouldn't # or there is no primary_conninfo but we should replicate from some specific node. if not wanted_primary_conninfo: return not primary_conninfo elif not primary_conninfo: return False if self._postgresql.major_version < 170000: # we want to compare dbname in primary_conninfo only for v17 onwards wanted_primary_conninfo.pop('dbname', None) if not self._postgresql.is_starting(): wal_receiver_primary_conninfo = self._postgresql.primary_conninfo() if wal_receiver_primary_conninfo: wal_receiver_primary_conninfo = parse_dsn(wal_receiver_primary_conninfo) # when wal receiver is alive use primary_conninfo from pg_stat_wal_receiver for comparison if wal_receiver_primary_conninfo: # dbname in pg_stat_wal_receiver is always `replication`, we need to use a "real" one wal_receiver_primary_conninfo.pop('dbname', None) dbname = primary_conninfo.get('dbname') if dbname: wal_receiver_primary_conninfo['dbname'] = dbname primary_conninfo = wal_receiver_primary_conninfo # There could be no password in the primary_conninfo or it is masked. # Just copy the "desired" value in order to make comparison succeed. if 'password' in wanted_primary_conninfo: primary_conninfo['password'] = wanted_primary_conninfo['password'] if 'passfile' in primary_conninfo and 'password' not in primary_conninfo \ and 'password' in wanted_primary_conninfo: if self._check_passfile(primary_conninfo['passfile'], wanted_primary_conninfo): primary_conninfo['password'] = wanted_primary_conninfo['password'] else: return False return all(str(primary_conninfo.get(p)) == str(v) for p, v in wanted_primary_conninfo.items() if v is not None) def check_recovery_conf(self, member: Union[Leader, Member, None]) -> Tuple[bool, bool]: """Returns a tuple. The first boolean element indicates that recovery params don't match and the second is set to `True` if the restart is required in order to apply new values""" # TODO: recovery.conf could be stale, would be nice to detect that. if self._postgresql.major_version >= 120000: if not os.path.exists(self._standby_signal): return True, True _read_recovery_params = self._read_recovery_params else: if not self.recovery_conf_exists(): return True, True _read_recovery_params = self._read_recovery_params_pre_v12 params, updated = _read_recovery_params() # updated indicates that mtime of postgresql.conf, postgresql.auto.conf, or recovery.conf # was changed and params were read either from the config or from the database connection. if updated: if params is None: # exception or unparsable config return True, True # We will cache parsed value until the next config change. self._current_recovery_params = params primary_conninfo = params['primary_conninfo'] if primary_conninfo[0]: primary_conninfo[0] = parse_dsn(params['primary_conninfo'][0]) # If we failed to parse non-empty connection string this indicates that config if broken. if not primary_conninfo[0]: return True, True else: # empty string, primary_conninfo is not in the config primary_conninfo[0] = {} if not self._postgresql.is_starting() and self._current_recovery_params: # when wal receiver is alive take primary_slot_name from pg_stat_wal_receiver wal_receiver_primary_slot_name = self._postgresql.primary_slot_name() if not wal_receiver_primary_slot_name and self._postgresql.primary_conninfo(): wal_receiver_primary_slot_name = '' if wal_receiver_primary_slot_name is not None: self._current_recovery_params['primary_slot_name'][0] = wal_receiver_primary_slot_name # Increment the 'reload' to enforce write of postgresql.conf when joining the running postgres required = {'restart': 0, 'reload': int(self._postgresql.major_version >= 120000 and not self._postgresql.cb_called and not self._postgresql.is_starting())} def record_mismatch(mtype: bool) -> None: required['restart' if mtype else 'reload'] += 1 wanted_recovery_params = self.build_recovery_params(member) for param, value in (self._current_recovery_params or EMPTY_DICT).items(): # Skip certain parameters defined in the included postgres config files # if we know that they are not specified in the patroni configuration. if len(value) > 2 and value[2] not in (self._postgresql_conf, self._auto_conf) and \ param in ('archive_cleanup_command', 'promote_trigger_file', 'recovery_end_command', 'recovery_min_apply_delay', 'restore_command') and param not in wanted_recovery_params: continue if param == 'recovery_min_apply_delay': if not compare_values('integer', 'ms', value[0], wanted_recovery_params.get(param, 0)): record_mismatch(value[1]) elif param == 'standby_mode': if not compare_values('bool', None, value[0], wanted_recovery_params.get(param, 'on')): record_mismatch(value[1]) elif param == 'primary_conninfo': if not self._check_primary_conninfo(value[0], wanted_recovery_params.get('primary_conninfo', {})): record_mismatch(value[1]) elif (param != 'primary_slot_name' or wanted_recovery_params.get('primary_conninfo')) \ and str(value[0]) != str(wanted_recovery_params.get(param, '')): record_mismatch(value[1]) return required['restart'] + required['reload'] > 0, required['restart'] > 0 @staticmethod def _remove_file_if_exists(name: str) -> None: if os.path.isfile(name) or os.path.islink(name): os.unlink(name) @staticmethod def _pgpass_content(record: Dict[str, Any]) -> Optional[str]: """Generate content of `pgpassfile` based on connection parameters. .. note:: In case if ``host`` is a comma separated string we generate one line per host. :param record: :class:`dict` object with connection parameters. :returns: a string with generated content of pgpassfile or ``None`` if there is no ``password``. """ if 'password' in record: def escape(value: Any) -> str: return re.sub(r'([:\\])', r'\\\1', str(value)) # 'host' could be several comma-separated hostnames, in this case we need to write on pgpass line per host hosts = [escape(host) for host in filter(None, map(str.strip, (record.get('host', '') or '*').split(',')))] # pyright: ignore [reportUnknownArgumentType] if any(host.startswith('/') for host in hosts) and 'localhost' not in hosts: hosts.append('localhost') record = {n: escape(record.get(n) or '*') for n in ('port', 'user', 'password')} return ''.join('{host}:{port}:*:{user}:{password}\n'.format(**record, host=host) for host in hosts) def write_pgpass(self, record: Dict[str, Any]) -> Dict[str, str]: """Maybe creates :attr:`_passfile` based on connection parameters. :param record: :class:`dict` object with connection parameters. :returns: a copy of environment variables, that will include ``PGPASSFILE`` in case if the file was written. """ content = self._pgpass_content(record) if not content: return os.environ.copy() with open(self._pgpass, 'w') as f: os.chmod(self._pgpass, stat.S_IWRITE | stat.S_IREAD) f.write(content) return {**os.environ, 'PGPASSFILE': self._pgpass} def write_recovery_conf(self, recovery_params: CaseInsensitiveDict) -> None: self._recovery_params = recovery_params if self._postgresql.major_version >= 120000: if parse_bool(recovery_params.pop('standby_mode', None)): open(self._standby_signal, 'w').close() self.set_file_permissions(self._standby_signal) else: self._remove_file_if_exists(self._standby_signal) open(self._recovery_signal, 'w').close() self.set_file_permissions(self._recovery_signal) def restart_required(name: str) -> bool: if self._postgresql.major_version >= 140000: return False return name == 'restore_command' or (self._postgresql.major_version < 130000 and name in ('primary_conninfo', 'primary_slot_name')) self._current_recovery_params = CaseInsensitiveDict({n: [v, restart_required(n), self._postgresql_conf] for n, v in recovery_params.items()}) else: with self.config_writer(self._recovery_conf) as f: self._write_recovery_params(f, recovery_params) def remove_recovery_conf(self) -> None: for name in (self._recovery_conf, self._standby_signal, self._recovery_signal): self._remove_file_if_exists(name) self._recovery_params = CaseInsensitiveDict() self._current_recovery_params = None def _sanitize_auto_conf(self) -> None: overwrite = False lines: List[str] = [] if os.path.exists(self._auto_conf): try: with open(self._auto_conf) as f: for raw_line in f: line = raw_line.strip() match = PARAMETER_RE.match(line) if match and match.group(1).lower() in self._RECOVERY_PARAMETERS: overwrite = True else: lines.append(raw_line) except Exception: logger.info('Failed to read %s', self._auto_conf) if overwrite: try: with open(self._auto_conf, 'w') as f: self.set_file_permissions(self._auto_conf) for raw_line in lines: f.write(raw_line) except Exception: logger.exception('Failed to remove some unwanted parameters from %s', self._auto_conf) def _adjust_recovery_parameters(self) -> None: # It is not strictly necessary, but we can make patroni configs crossi-compatible with all postgres versions. recovery_conf = {n: v for n, v in self._server_parameters.items() if n.lower() in self._RECOVERY_PARAMETERS} if recovery_conf: self._config['recovery_conf'] = recovery_conf if self.get('recovery_conf'): value = self._config['recovery_conf'].pop(self._triggerfile_wrong_name, None) if self.triggerfile_good_name not in self._config['recovery_conf'] and value: self._config['recovery_conf'][self.triggerfile_good_name] = value def get_server_parameters(self, config: Dict[str, Any]) -> CaseInsensitiveDict: parameters = config['parameters'].copy() listen_addresses, port = split_host_port(config['listen'], 5432) parameters.update(cluster_name=self._postgresql.scope, listen_addresses=listen_addresses, port=str(port)) if global_config.is_synchronous_mode: synchronous_standby_names = self._server_parameters.get('synchronous_standby_names') if synchronous_standby_names is None: if global_config.is_synchronous_mode_strict\ and self._postgresql.role in ('primary', 'promoted'): parameters['synchronous_standby_names'] = '*' else: parameters.pop('synchronous_standby_names', None) else: parameters['synchronous_standby_names'] = synchronous_standby_names # Handle hot_standby <-> replica rename if parameters.get('wal_level') == ('hot_standby' if self._postgresql.major_version >= 90600 else 'replica'): parameters['wal_level'] = 'replica' if self._postgresql.major_version >= 90600 else 'hot_standby' # Try to recalculate wal_keep_segments <-> wal_keep_size assuming that typical wal_segment_size is 16MB. # The real segment size could be estimated from pg_control, but we don't really care, because the only goal of # this exercise is improving cross version compatibility and user must set the correct parameter in the config. if self._postgresql.major_version >= 130000: wal_keep_segments = parameters.pop('wal_keep_segments', self.CMDLINE_OPTIONS['wal_keep_segments'][0]) parameters.setdefault('wal_keep_size', str(int(wal_keep_segments) * 16) + 'MB') elif self._postgresql.major_version: wal_keep_size = parse_int(parameters.pop('wal_keep_size', self.CMDLINE_OPTIONS['wal_keep_size'][0]), 'MB') parameters.setdefault('wal_keep_segments', int(((wal_keep_size or 0) + 8) / 16)) self._postgresql.mpp_handler.adjust_postgres_gucs(parameters) ret = CaseInsensitiveDict({k: v for k, v in parameters.items() if not self._postgresql.major_version or self._postgresql.major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]}) ret.update({k: os.path.join(self._config_dir, ret[k]) for k in ('hba_file', 'ident_file') if k in ret}) return ret @staticmethod def _get_unix_local_address(unix_socket_directories: str) -> str: for d in unix_socket_directories.split(','): d = d.strip() if d.startswith('/'): # Only absolute path can be used to connect via unix-socket return d return '' def _get_tcp_local_address(self) -> str: listen_addresses = self._server_parameters['listen_addresses'].split(',') for la in listen_addresses: if la.strip().lower() in ('*', '0.0.0.0', '127.0.0.1', 'localhost'): # we are listening on '*' or localhost return 'localhost' # connection via localhost is preferred return listen_addresses[0].strip() # can't use localhost, take first address from listen_addresses def resolve_connection_addresses(self) -> None: """Calculates and sets local and remote connection urls and options. This method sets: * :attr:`Postgresql.connection_string ` attribute, which is later written to the member key in DCS as ``conn_url``. * :attr:`ConfigHandler.local_replication_address` attribute, which is used for replication connections to local postgres. * :attr:`ConnectionPool.conn_kwargs ` attribute, which is used for superuser connections to local postgres. .. note:: If there is a valid directory in ``postgresql.parameters.unix_socket_directories`` in the Patroni configuration and ``postgresql.use_unix_socket`` and/or ``postgresql.use_unix_socket_repl`` are set to ``True``, we respectively use unix sockets for superuser and replication connections to local postgres. If there is a requirement to use unix sockets, but nothing is set in the ``postgresql.parameters.unix_socket_directories``, we omit a ``host`` in connection parameters relying on the ability of ``libpq`` to connect via some default unix socket directory. If unix sockets are not requested we "switch" to TCP, preferring to use ``localhost`` if it is possible to deduce that Postgres is listening on a local interface address. Otherwise we just used the first address specified in the ``listen_addresses`` GUC. """ port = self._server_parameters['port'] tcp_local_address = self._get_tcp_local_address() netloc = self._config.get('connect_address') or tcp_local_address + ':' + port unix_local_address = {'port': port} unix_socket_directories = self._server_parameters.get('unix_socket_directories') if unix_socket_directories is not None: # fallback to tcp if unix_socket_directories is set, but there are no suitable values unix_local_address['host'] = self._get_unix_local_address(unix_socket_directories) or tcp_local_address tcp_local_address = {'host': tcp_local_address, 'port': port} self.local_replication_address = unix_local_address\ if self._config.get('use_unix_socket_repl') else tcp_local_address self._postgresql.connection_string = uri('postgres', netloc, self._postgresql.database) local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address local_conn_kwargs = { **local_address, **self._superuser, 'dbname': self._postgresql.database, 'fallback_application_name': 'Patroni', 'connect_timeout': 3, 'options': '-c statement_timeout=2000' } # if the "username" parameter is present, it actually needs to be "user" for connecting to PostgreSQL if 'username' in local_conn_kwargs: local_conn_kwargs['user'] = local_conn_kwargs.pop('username') # "notify" connection_pool about the "new" local connection address self._postgresql.connection_pool.conn_kwargs = local_conn_kwargs def _get_pg_settings(self, names: Collection[str]) -> Dict[Any, Tuple[Any, ...]]: return {r[0]: r for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context, sourcefile' + ' FROM pg_catalog.pg_settings ' + ' WHERE pg_catalog.lower(name) = ANY(%s)'), [n.lower() for n in names])} @staticmethod def _handle_wal_buffers(old_values: Dict[Any, Tuple[Any, ...]], changes: CaseInsensitiveDict) -> None: wal_block_size = parse_int(old_values['wal_block_size'][1]) or 8192 wal_segment_size = old_values['wal_segment_size'] wal_segment_unit = parse_int(wal_segment_size[2], 'B') or 8192 \ if wal_segment_size[2] is not None and wal_segment_size[2][0].isdigit() else 1 wal_segment_size = parse_int(wal_segment_size[1]) or (16777216 if wal_segment_size[2] is None else 2048) wal_segment_size *= wal_segment_unit / wal_block_size default_wal_buffers = min(max((parse_int(old_values['shared_buffers'][1]) or 16384) / 32, 8), wal_segment_size) wal_buffers = old_values['wal_buffers'] new_value = str(changes['wal_buffers'] or -1) new_value = default_wal_buffers if new_value == '-1' else parse_int(new_value, wal_buffers[2]) old_value = default_wal_buffers if wal_buffers[1] == '-1' else parse_int(*wal_buffers[1:3]) if new_value == old_value: del changes['wal_buffers'] def reload_config(self, config: Dict[str, Any], sighup: bool = False) -> None: self._superuser = config['authentication'].get('superuser', {}) server_parameters = self.get_server_parameters(config) params_skip_changes = CaseInsensitiveSet((*self._RECOVERY_PARAMETERS, 'hot_standby')) conf_changed = hba_changed = ident_changed = local_connection_address_changed = False param_diff = CaseInsensitiveDict() if self._postgresql.state == 'running': changes = CaseInsensitiveDict({p: v for p, v in server_parameters.items() if p not in params_skip_changes}) changes.update({p: None for p in self._server_parameters.keys() if not (p in changes or p in params_skip_changes)}) if changes: undef = [] if 'wal_buffers' in changes: # we need to calculate the default value of wal_buffers undef = [p for p in ('shared_buffers', 'wal_segment_size', 'wal_block_size') if p not in changes] changes.update({p: None for p in undef}) # XXX: query can raise an exception old_values = self._get_pg_settings(changes.keys()) if 'wal_buffers' in changes: self._handle_wal_buffers(old_values, changes) for p in undef: del changes[p] for r in old_values.values(): if r[4] != 'internal' and r[0] in changes: new_value = changes.pop(r[0]) if new_value is None or not compare_values(r[3], r[2], r[1], new_value): conf_changed = True if r[4] == 'postmaster': param_diff[r[0]] = get_param_diff(r[1], new_value, r[3], r[2]) logger.info("Changed %s from '%s' to '%s' (restart might be required)", r[0], param_diff[r[0]]['old_value'], new_value) if config.get('use_unix_socket') and r[0] == 'unix_socket_directories'\ or r[0] in ('listen_addresses', 'port'): local_connection_address_changed = True else: logger.info("Changed %s from '%s' to '%s'", r[0], maybe_convert_from_base_unit(r[1], r[3], r[2]), new_value) elif r[0] in self._server_parameters \ and not compare_values(r[3], r[2], r[1], self._server_parameters[r[0]]): # Check if any parameter was set back to the current pg_settings value # We can use pg_settings value here, as it is proved to be equal to new_value logger.info("Changed %s from '%s' to '%s'", r[0], self._server_parameters[r[0]], new_value) conf_changed = True for param, value in changes.items(): if '.' in param: # Check that user-defined-parameters have changed (parameters with period in name) if value is None or param not in self._server_parameters \ or str(value) != str(self._server_parameters[param]): logger.info("Changed %s from '%s' to '%s'", param, self._server_parameters.get(param), value) conf_changed = True elif param in server_parameters: logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param) server_parameters.pop(param) if (not server_parameters.get('hba_file') or server_parameters['hba_file'] == self._pg_hba_conf) \ and config.get('pg_hba'): hba_changed = self._config.get('pg_hba', []) != config['pg_hba'] if (not server_parameters.get('ident_file') or server_parameters['ident_file'] == self._pg_hba_conf) \ and config.get('pg_ident'): ident_changed = self._config.get('pg_ident', []) != config['pg_ident'] self._config = config self._server_parameters = server_parameters self._adjust_recovery_parameters() self._krbsrvname = config.get('krbsrvname') # for not so obvious connection attempts that may happen outside of pyscopg2 if self._krbsrvname: os.environ['PGKRBSRVNAME'] = self._krbsrvname if not local_connection_address_changed: self.resolve_connection_addresses() proxy_addr = config.get('proxy_address') self._postgresql.proxy_url = uri('postgres', proxy_addr, self._postgresql.database) if proxy_addr else None if conf_changed: self.write_postgresql_conf() if hba_changed: self.replace_pg_hba() if ident_changed: self.replace_pg_ident() if sighup or conf_changed or hba_changed or ident_changed: logger.info('Reloading PostgreSQL configuration.') self._postgresql.reload() if self._postgresql.major_version >= 90500: time.sleep(1) try: settings_diff: CaseInsensitiveDict = CaseInsensitiveDict() for param, value, unit, vartype in self._postgresql.query( 'SELECT name, pg_catalog.current_setting(name), unit, vartype FROM pg_catalog.pg_settings' ' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart', [n.lower() for n in params_skip_changes]): new_value = self._postgresql.get_guc_value(param) new_value = '?' if new_value is None else new_value settings_diff[param] = get_param_diff(value, new_value, vartype, unit) external_change = {param: value for param, value in settings_diff.items() if param not in param_diff or value != param_diff[param]} if external_change: logger.info("PostgreSQL configuration parameters requiring restart" " (%s) seem to be changed bypassing Patroni config." " Setting 'Pending restart' flag", ', '.join(external_change)) param_diff = settings_diff except Exception as e: logger.warning('Exception %r when running query', e) else: logger.info('No PostgreSQL configuration items changed, nothing to reload.') self._postgresql.set_pending_restart_reason(param_diff) def set_synchronous_standby_names(self, value: Optional[str]) -> Optional[bool]: """Updates synchronous_standby_names and reloads if necessary. :returns: True if value was updated.""" if value != self._server_parameters.get('synchronous_standby_names'): if value is None: self._server_parameters.pop('synchronous_standby_names', None) else: self._server_parameters['synchronous_standby_names'] = value if self._postgresql.state == 'running': self.write_postgresql_conf() self._postgresql.reload() return True @property def effective_configuration(self) -> CaseInsensitiveDict: """It might happen that the current value of one (or more) below parameters stored in the controldata is higher than the value stored in the global cluster configuration. Example: max_connections in global configuration is 100, but in controldata `Current max_connections setting: 200`. If we try to start postgres with max_connections=100, it will immediately exit. As a workaround we will start it with the values from controldata and set `pending_restart` to true as an indicator that current values of parameters are not matching expectations.""" if self._postgresql.role == 'primary': return self._server_parameters options_mapping = { 'max_connections': 'max_connections setting', 'max_prepared_transactions': 'max_prepared_xacts setting', 'max_locks_per_transaction': 'max_locks_per_xact setting' } if self._postgresql.major_version >= 90400: options_mapping['max_worker_processes'] = 'max_worker_processes setting' if self._postgresql.major_version >= 120000: options_mapping['max_wal_senders'] = 'max_wal_senders setting' data = self._postgresql.controldata() effective_configuration = self._server_parameters.copy() param_diff = CaseInsensitiveDict() for name, cname in options_mapping.items(): value = parse_int(effective_configuration[name]) if cname not in data: logger.warning('%s is missing from pg_controldata output', cname) continue cvalue = parse_int(data[cname]) if cvalue is not None and value is not None and cvalue > value: effective_configuration[name] = cvalue logger.info("%s value in pg_controldata: %d, in the global configuration: %d." " pg_controldata value will be used. Setting 'Pending restart' flag", name, cvalue, value) param_diff[name] = get_param_diff(cvalue, value) self._postgresql.set_pending_restart_reason(param_diff) # If we are using custom bootstrap with PITR it could fail when values like max_connections # are increased, therefore we disable hot_standby if recovery_target_action == 'promote'. if self._postgresql.bootstrap.running_custom_bootstrap: disable_hot_standby = False if self._postgresql.bootstrap.keep_existing_recovery_conf: disable_hot_standby = True # trust that pgBackRest does the right thing # `pause_at_recovery_target` has no effect if hot_standby is not enabled, therefore we consider only 9.5+ elif self._postgresql.major_version >= 90500 and self._recovery_params: pause_at_recovery_target = parse_bool(self._recovery_params.get('pause_at_recovery_target')) recovery_target_action = self._recovery_params.get( 'recovery_target_action', 'promote' if pause_at_recovery_target is False else 'pause') disable_hot_standby = recovery_target_action == 'promote' if disable_hot_standby: effective_configuration['hot_standby'] = 'off' return effective_configuration @property def replication(self) -> Dict[str, Any]: return self._config['authentication']['replication'] @property def superuser(self) -> Dict[str, Any]: return self._superuser @property def rewind_credentials(self) -> Dict[str, Any]: return self._config['authentication'].get('rewind', self._superuser) \ if self._postgresql.major_version >= 110000 else self._superuser @property def ident_file(self) -> Optional[str]: ident_file = self._server_parameters.get('ident_file') return None if ident_file == self._pg_ident_conf else ident_file @property def hba_file(self) -> Optional[str]: hba_file = self._server_parameters.get('hba_file') return None if hba_file == self._pg_hba_conf else hba_file @property def pg_hba_conf(self) -> str: return self._pg_hba_conf @property def postgresql_conf(self) -> str: return self._postgresql_conf def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]: return self._config.get(key, default) def restore_command(self) -> Optional[str]: return (self.get('recovery_conf') or EMPTY_DICT).get('restore_command') @property def synchronous_standby_names(self) -> Optional[str]: """Get ``synchronous_standby_names`` value configured by the user. :returns: value of ``synchronous_standby_names`` in the Patroni configuration, if any, otherwise ``None``. """ return (self.get('parameters') or EMPTY_DICT).get('synchronous_standby_names') patroni-4.0.4/patroni/postgresql/connection.py000066400000000000000000000146161472010352700215740ustar00rootroot00000000000000import logging from contextlib import contextmanager from threading import Lock from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: # pragma: no cover from psycopg import Connection, Cursor from psycopg2 import connection, cursor from .. import psycopg from ..exceptions import PostgresConnectionException logger = logging.getLogger(__name__) class NamedConnection: """Helper class to manage ``psycopg`` connections from Patroni to PostgreSQL. :ivar server_version: PostgreSQL version in integer format where we are connected to. """ server_version: int def __init__(self, pool: 'ConnectionPool', name: str, kwargs_override: Optional[Dict[str, Any]]) -> None: """Create an instance of :class:`NamedConnection` class. :param pool: reference to a :class:`ConnectionPool` object. :param name: name of the connection. :param kwargs_override: :class:`dict` object with connection parameters that should be different from default values provided by connection *pool*. """ self._pool = pool self._name = name self._kwargs_override = kwargs_override or {} self._lock = Lock() # used to make sure that only one connection to postgres is established self._connection = None @property def _conn_kwargs(self) -> Dict[str, Any]: """Connection parameters for this :class:`NamedConnection`.""" return {**self._pool.conn_kwargs, **self._kwargs_override, 'application_name': f'Patroni {self._name}'} def get(self) -> Union['connection', 'Connection[Any]']: """Get ``psycopg``/``psycopg2`` connection object. .. note:: Opens a new connection if necessary. :returns: ``psycopg`` or ``psycopg2`` connection object. """ with self._lock: if not self._connection or self._connection.closed != 0: logger.info("establishing a new patroni %s connection to postgres", self._name) self._connection = psycopg.connect(**self._conn_kwargs) self.server_version = getattr(self._connection, 'server_version', 0) return self._connection def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]: """Execute a query with parameters and optionally returns a response. :param sql: SQL statement to execute. :param params: parameters to pass. :returns: a query response as a list of tuples if there is any. :raises: :exc:`~psycopg.Error` if had issues while executing *sql*. :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database. """ cursor = None try: with self.get().cursor() as cursor: cursor.execute(sql.encode('utf-8'), params or None) return cursor.fetchall() if cursor.rowcount and cursor.rowcount > 0 else [] except psycopg.Error as exc: if cursor and cursor.connection.closed == 0: # When connected via unix socket, psycopg2 can't recognize 'connection lost' and leaves # `self._connection.closed == 0`, but the generic exception is raised. It doesn't make # sense to continue with existing connection and we will close it, to avoid its reuse. if type(exc) in (psycopg.DatabaseError, psycopg.OperationalError): self.close() else: raise exc raise PostgresConnectionException('connection problems') from exc def close(self, silent: bool = False) -> bool: """Close the psycopg connection to postgres. :param silent: whether the method should not write logs. :returns: ``True`` if ``psycopg`` connection was closed, ``False`` otherwise.`` """ ret = False if self._connection and self._connection.closed == 0: self._connection.close() if not silent: logger.info("closed patroni %s connection to postgres", self._name) ret = True self._connection = None return ret class ConnectionPool: """Helper class to manage named connections from Patroni to PostgreSQL. The instance keeps named :class:`NamedConnection` objects and parameters that must be used for new connections. """ def __init__(self) -> None: """Create an instance of :class:`ConnectionPool` class.""" self._lock = Lock() self._connections: Dict[str, NamedConnection] = {} self._conn_kwargs: Dict[str, Any] = {} @property def conn_kwargs(self) -> Dict[str, Any]: """Connection parameters that must be used for new ``psycopg`` connections.""" with self._lock: return self._conn_kwargs.copy() @conn_kwargs.setter def conn_kwargs(self, value: Dict[str, Any]) -> None: """Set new connection parameters. :param value: :class:`dict` object with connection parameters. """ with self._lock: self._conn_kwargs = value def get(self, name: str, kwargs_override: Optional[Dict[str, Any]] = None) -> NamedConnection: """Get a new named :class:`NamedConnection` object from the pool. .. note:: Creates a new :class:`NamedConnection` object if it doesn't yet exist in the pool. :param name: name of the connection. :param kwargs_override: :class:`dict` object with connection parameters that should be different from default values provided by :attr:`conn_kwargs`. :returns: :class:`NamedConnection` object. """ with self._lock: if name not in self._connections: self._connections[name] = NamedConnection(self, name, kwargs_override) return self._connections[name] def close(self) -> None: """Close all named connections from Patroni to PostgreSQL registered in the pool.""" with self._lock: closed_connections = [conn.close(True) for conn in self._connections.values()] if any(closed_connections): logger.info("closed patroni connections to postgres") @contextmanager def get_connection_cursor(**kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]: conn = psycopg.connect(**kwargs) with conn.cursor() as cur: yield cur conn.close() patroni-4.0.4/patroni/postgresql/misc.py000066400000000000000000000065071472010352700203700ustar00rootroot00000000000000import errno import logging import os from typing import Iterable, Tuple from ..exceptions import PostgresException logger = logging.getLogger(__name__) def postgres_version_to_int(pg_version: str) -> int: """Convert the server_version to integer >>> postgres_version_to_int('9.5.3') 90503 >>> postgres_version_to_int('9.3.13') 90313 >>> postgres_version_to_int('10.1') 100001 >>> postgres_version_to_int('10') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 10' >>> postgres_version_to_int('9.6') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 9.6' >>> postgres_version_to_int('a.b.c') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... PostgresException: 'Invalid PostgreSQL version: a.b.c' """ try: components = list(map(int, pg_version.split('.'))) except ValueError: raise PostgresException('Invalid PostgreSQL version: {0}'.format(pg_version)) if len(components) < 2 or len(components) == 2 and components[0] < 10 or len(components) > 3: raise PostgresException('Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: {0}'.format(pg_version)) if len(components) == 2: # new style version numbers, i.e. 10.1 becomes 100001 components.insert(1, 0) return int(''.join('{0:02d}'.format(c) for c in components)) def postgres_major_version_to_int(pg_version: str) -> int: """ >>> postgres_major_version_to_int('10') 100000 >>> postgres_major_version_to_int('9.6') 90600 """ return postgres_version_to_int(pg_version + '.0') def get_major_from_minor_version(version: int) -> int: """Extract major PostgreSQL version from the provided full version. :param version: integer representation of PostgreSQL full version (major + minor). :returns: integer representation of the PostgreSQL major version. :Example: >>> get_major_from_minor_version(100012) 100000 >>> get_major_from_minor_version(90313) 90300 """ return version // 100 * 100 def parse_lsn(lsn: str) -> int: t = lsn.split('/') return int(t[0], 16) * 0x100000000 + int(t[1], 16) def parse_history(data: str) -> Iterable[Tuple[int, int, str]]: for line in data.split('\n'): values = line.strip().split('\t') if len(values) == 3: try: yield int(values[0]), parse_lsn(values[1]), values[2] except (IndexError, ValueError): logger.exception('Exception when parsing timeline history line "%s"', values) def format_lsn(lsn: int, full: bool = False) -> str: template = '{0:X}/{1:08X}' if full else '{0:X}/{1:X}' return template.format(lsn >> 32, lsn & 0xFFFFFFFF) def fsync_dir(path: str) -> None: if os.name != 'nt': fd = os.open(path, os.O_DIRECTORY) try: os.fsync(fd) except OSError as e: # Some filesystems don't like fsyncing directories and raise EINVAL. Ignoring it is usually safe. if e.errno != errno.EINVAL: raise finally: os.close(fd) patroni-4.0.4/patroni/postgresql/mpp/000077500000000000000000000000001472010352700176475ustar00rootroot00000000000000patroni-4.0.4/patroni/postgresql/mpp/__init__.py000066400000000000000000000257751472010352700220000ustar00rootroot00000000000000"""Abstract classes for MPP handler. MPP stands for Massively Parallel Processing, and Citus belongs to this architecture. Currently, Citus is the only supported MPP cluster. However, we may consider adapting other databases such as TimescaleDB, GPDB, etc. into Patroni. """ import abc from typing import Any, Dict, Iterator, Optional, Tuple, Type, TYPE_CHECKING, Union from ...dcs import Cluster from ...dynamic_loader import iter_classes from ...exceptions import PatroniException if TYPE_CHECKING: # pragma: no cover from ...config import Config from .. import Postgresql class AbstractMPP(abc.ABC): """An abstract class which should be passed to :class:`AbstractDCS`. .. note:: We create :class:`AbstractMPP` and :class:`AbstractMPPHandler` to solve the chicken-egg initialization problem. When initializing DCS, we dynamically create an object implementing :class:`AbstractMPP`, later this object is used to instantiate an object implementing :class:`AbstractMPPHandler`. """ group_re: Any # re.Pattern[str] def __init__(self, config: Dict[str, Union[str, int]]) -> None: """Init method for :class:`AbstractMPP`. :param config: configuration of MPP section. """ self._config = config def is_enabled(self) -> bool: """Check if MPP is enabled for a given MPP. .. note:: We just check that the :attr:`_config` object isn't empty and expect it to be empty only in case of :class:`Null`. :returns: ``True`` if MPP is enabled, otherwise ``False``. """ return bool(self._config) @staticmethod @abc.abstractmethod def validate_config(config: Any) -> bool: """Check whether provided config is good for a given MPP. :param config: configuration of MPP section. :returns: ``True`` is config passes validation, otherwise ``False``. """ @property @abc.abstractmethod def group(self) -> Any: """The group for a given MPP implementation.""" @property @abc.abstractmethod def coordinator_group_id(self) -> Any: """The group id of the coordinator PostgreSQL cluster.""" @property def type(self) -> str: """The type of the MPP cluster. :returns: A string representation of the type of a given MPP implementation. """ for base in self.__class__.__bases__: if not base.__name__.startswith('Abstract'): return base.__name__ return self.__class__.__name__ @property def k8s_group_label(self): """Group label used for kubernetes DCS of the MPP cluster. :returns: A string representation of the k8s group label of a given MPP implementation. """ return self.type.lower() + '-group' def is_coordinator(self) -> bool: """Check whether this node is running in the coordinator PostgreSQL cluster. :returns: ``True`` if MPP is enabled and the group id of this node matches with the :attr:`coordinator_group_id`, otherwise ``False``. """ return self.is_enabled() and self.group == self.coordinator_group_id def is_worker(self) -> bool: """Check whether this node is running as a MPP worker PostgreSQL cluster. :returns: ``True`` if MPP is enabled and this node is known to be not running as the coordinator PostgreSQL cluster, otherwise ``False``. """ return self.is_enabled() and not self.is_coordinator() def _get_handler_cls(self) -> Iterator[Type['AbstractMPPHandler']]: """Find Handler classes inherited from a class type of this object. :yields: handler classes for this object. """ for cls in self.__class__.__subclasses__(): if issubclass(cls, AbstractMPPHandler) and cls.__name__.startswith(self.__class__.__name__): yield cls def get_handler_impl(self, postgresql: 'Postgresql') -> 'AbstractMPPHandler': """Find and instantiate Handler implementation of this object. :param postgresql: a reference to :class:`Postgresql` object. :raises: :exc:`PatroniException`: if the Handler class haven't been found. :returns: an instantiated class that implements Handler for this object. """ for cls in self._get_handler_cls(): return cls(postgresql, self._config) raise PatroniException(f'Failed to initialize {self.__class__.__name__}Handler object') class AbstractMPPHandler(AbstractMPP): """An abstract class which defines interfaces that should be implemented by real handlers.""" def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None: """Init method for :class:`AbstractMPPHandler`. :param postgresql: a reference to :class:`Postgresql` object. :param config: configuration of MPP section. """ super().__init__(config) self._postgresql = postgresql @abc.abstractmethod def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None: """Handle an event sent from a worker node. :param cluster: the currently known cluster state from DCS. :param event: the event to be handled. """ @abc.abstractmethod def sync_meta_data(self, cluster: Cluster) -> None: """Sync meta data on the coordinator. :param cluster: the currently known cluster state from DCS. """ @abc.abstractmethod def on_demote(self) -> None: """On demote handler. Is called when the primary was demoted. """ @abc.abstractmethod def schedule_cache_rebuild(self) -> None: """Cache rebuild handler. Is called to notify handler that it has to refresh its metadata cache from the database. """ @abc.abstractmethod def bootstrap(self) -> None: """Bootstrap handler. Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method). """ @abc.abstractmethod def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None: """Adjust GUCs in the current PostgreSQL configuration. :param parameters: dictionary of GUCs, with key as GUC name and the corresponding value as current GUC value. """ @abc.abstractmethod def ignore_replication_slot(self, slot: Dict[str, str]) -> bool: """Check whether provided replication *slot* existing in the database should not be removed. .. note:: MPP database may create replication slots for its own use, for example to migrate data between workers using logical replication, and we don't want to suddenly drop them. :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and ``plugin``. :returns: ``True`` if the replication slots should not be removed, otherwise ``False``. """ class Null(AbstractMPP): """Dummy implementation of :class:`AbstractMPP`.""" def __init__(self) -> None: """Init method for :class:`Null`.""" super().__init__({}) @staticmethod def validate_config(config: Any) -> bool: """Check whether provided config is good for :class:`Null`. :returns: always ``True``. """ return True @property def group(self) -> None: """The group for :class:`Null`. :returns: always ``None``. """ return None @property def coordinator_group_id(self) -> None: """The group id of the coordinator PostgreSQL cluster. :returns: always ``None``. """ return None class NullHandler(Null, AbstractMPPHandler): """Dummy implementation of :class:`AbstractMPPHandler`.""" def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None: """Init method for :class:`NullHandler`. :param postgresql: a reference to :class:`Postgresql` object. :param config: configuration of MPP section. """ AbstractMPPHandler.__init__(self, postgresql, config) def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None: """Handle an event sent from a worker node. :param cluster: the currently known cluster state from DCS. :param event: the event to be handled. """ def sync_meta_data(self, cluster: Cluster) -> None: """Sync meta data on the coordinator. :param cluster: the currently known cluster state from DCS. """ def on_demote(self) -> None: """On demote handler. Is called when the primary was demoted. """ def schedule_cache_rebuild(self) -> None: """Cache rebuild handler. Is called to notify handler that it has to refresh its metadata cache from the database. """ def bootstrap(self) -> None: """Bootstrap handler. Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method). """ def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None: """Adjust GUCs in the current PostgreSQL configuration. :param parameters: dictionary of GUCs, with key as GUC name and corresponding value as current GUC value. """ def ignore_replication_slot(self, slot: Dict[str, str]) -> bool: """Check whether provided replication *slot* existing in the database should not be removed. .. note:: MPP database may create replication slots for its own use, for example to migrate data between workers using logical replication, and we don't want to suddenly drop them. :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and ``plugin``. :returns: always ``False``. """ return False def iter_mpp_classes( config: Optional[Union['Config', Dict[str, Any]]] = None ) -> Iterator[Tuple[str, Type[AbstractMPP]]]: """Attempt to import MPP modules that are present in the given configuration. :param config: configuration information with possible MPP names as keys. If given, only attempt to import MPP modules defined in the configuration. Else, if ``None``, attempt to import any supported MPP module. :yields: tuples, each containing the module ``name`` and the imported MPP class object. """ if TYPE_CHECKING: # pragma: no cover assert isinstance(__package__, str) yield from iter_classes(__package__, AbstractMPP, config) def get_mpp(config: Union['Config', Dict[str, Any]]) -> AbstractMPP: """Attempt to load and instantiate a MPP module from known available implementations. :param config: object or dictionary with Patroni configuration. :returns: The successfully loaded MPP or fallback to :class:`Null`. """ for name, mpp_class in iter_mpp_classes(config): if mpp_class.validate_config(config[name]): return mpp_class(config[name]) return Null() patroni-4.0.4/patroni/postgresql/mpp/citus.py000066400000000000000000001066071472010352700213620ustar00rootroot00000000000000import logging import re import time from threading import Condition, Event, Thread from typing import Any, cast, Collection, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, Union from urllib.parse import urlparse from ...dcs import Cluster from ...psycopg import connect, ProgrammingError, quote_ident from ...utils import parse_int from . import AbstractMPP, AbstractMPPHandler if TYPE_CHECKING: # pragma: no cover from .. import Postgresql CITUS_COORDINATOR_GROUP_ID = 0 CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$') logger = logging.getLogger(__name__) class PgDistNode: """Represents a single row in "pg_dist_node" table. .. note:: Unlike "noderole" possible values of ``role`` are ``primary``, ``secondary``, and ``demoted``. The last one is used to pause client connections on the coordinator to the worker by appending ``-demoted`` suffix to the "nodename". The actual "noderole" in DB remains ``primary``. :ivar host: "nodename" value :ivar port: "nodeport" value :ivar role: "noderole" value :ivar nodeid: "nodeid" value """ def __init__(self, host: str, port: int, role: str, nodeid: Optional[int] = None) -> None: """Create a :class:`PgDistNode` object based on given arguments. :param host: "nodename" of the Citus coordinator or worker. :param port: "nodeport" of the Citus coordinator or worker. :param role: "noderole" value. :param nodeid: id of the row in the "pg_dist_node". """ self.host = host self.port = port self.role = role self.nodeid = nodeid def __hash__(self) -> int: """Defines a hash function to put :class:`PgDistNode` objects to :class:`PgDistGroup` set-like object. .. note:: We use (:attr:`host`, :attr:`port`) tuple here because it is one of the UNIQUE constraints on the "pg_dist_node" table. The :attr:`role` value is irrelevant here because nodes may change their roles. """ return hash((self.host, self.port)) def __eq__(self, other: Any) -> bool: """Defines a comparison function. :returns: ``True`` if :attr:`host` and :attr:`port` between two instances are the same. """ return isinstance(other, PgDistNode) and self.host == other.host and self.port == other.port def __str__(self) -> str: return ('PgDistNode(nodeid={0},host={1},port={2},role={3})' .format(self.nodeid, self.host, self.port, self.role)) def __repr__(self) -> str: return str(self) def is_primary(self) -> bool: """Checks whether this object represents "primary" in a corresponding group. :returns: ``True`` if this object represents the ``primary``. """ return self.role in ('primary', 'demoted') def as_tuple(self, include_nodeid: bool = False) -> Tuple[str, int, str, Optional[int]]: """Helper method to compare two :class:`PgDistGroup` objects. .. note:: *include_nodeid* is set to ``True`` only in unit-tests. :param include_nodeid: whether :attr:`nodeid` should be taken into account when comparison is performed. :returns: :class:`tuple` object with :attr:`host`, :attr:`port`, :attr:`role`, and optionally :attr:`nodeid`. """ return self.host, self.port, self.role, (self.nodeid if include_nodeid else None) class PgDistGroup(Set[PgDistNode]): """A :class:`set`-like object that represents a Citus group in "pg_dist_node" table. This class implements a set of methods to compare topology and if it is necessary to transition from the old to the new topology in a "safe" manner: * register new primary/secondaries * replace gone secondaries with added secondaries * failover and switchover Typically there will be at least one :class:`PgDistNode` object registered (``primary``). In addition to that there could be one or more ``secondary`` nodes. :ivar failover: whether the ``primary`` row should be updated as a result of :func:`transition` method call. :ivar groupid: the "groupid" from "pg_dist_node". """ def __init__(self, groupid: int, nodes: Optional[Collection[PgDistNode]] = None) -> None: """Creates a :class:`PgDistGroup` object based on given arguments. :param groupid: the groupid from "pg_dist_node". :param nodes: a collection of :class:`PgDistNode` objects that belong to a *groupid*. """ self.failover = False self.groupid = groupid if nodes: self.update(nodes) def equals(self, other: 'PgDistGroup', check_nodeid: bool = False) -> bool: """Compares two :class:`PgDistGroup` objects. :param other: what we want to compare with. :param check_nodeid: whether :attr:`PgDistNode.nodeid` should be compared in addition to :attr:`PgDistNode.host`, :attr:`PgDistNode.port`, and :attr:`PgDistNode.role`. :returns: ``True`` if two :class:`PgDistGroup` objects are fully identical. """ return self.groupid == other.groupid\ and set(v.as_tuple(check_nodeid) for v in self) == set(v.as_tuple(check_nodeid) for v in other) def primary(self) -> Optional[PgDistNode]: """Finds and returns :class:`PgDistNode` object that represents the "primary". :returns: :class:`PgDistNode` object which represents the "primary" or ``None`` if not found. """ return next(iter(v for v in self if v.is_primary()), None) def get(self, value: PgDistNode) -> Optional[PgDistNode]: """Performs a lookup of the actual value in a set. .. note:: It is necessary because :func:`__hash__` and :func:`__eq__` methods in :class:`PgDistNode` are redefined and effectively they check only :attr:`PgDistNode.host` and :attr:`PgDistNode.port` attributes. :param value: the key we search for. :returns: the actual :class:`PgDistNode` value from this :class:`PgDistGroup` object or ``None`` if not found. """ return next(iter(v for v in self if v == value), None) def transition(self, old: 'PgDistGroup') -> Iterator[PgDistNode]: """Compares this topology with the old one and yields transitions that transform the old to the new one. .. note:: The actual yielded object is :class:`PgDistNode` that will be passed to the :meth:`CitusHandler.update_node` to execute all transitions in a transaction. In addition to the yielding transactions this method fills up :attr:`PgDistNode.nodeid` attribute for nodes that are presented in the old and in the new topology. There are a few simple rules/constraints that are imposed by Citus and must be followed: - adding/removing nodes is only possible when metadata is synced to all registered "priorities". - the "primary" row in "pg_dist_node" always keeps the nodeid (unless it is removed, but it is not supported by Patroni). - "nodename", "nodeport" must be unique across all rows in the "pg_dist_node". This means that every time we want to change the nodeid of an existing node (i.e. to change it from secondary to primary), we should first write some other "nodename"/"nodeport" to the row it's currently in. - updating "broken" nodes always works and metadata is synced asynchnonously after the commit. Following these rules below is an example of the switchover between node1 (primary, nodeid=4) and node2 (secondary, nodeid=5). .. code-block:: SQL BEGIN; SELECT citus_update_node(4, 'node1-demoted', 5432); SELECT citus_update_node(5, 'node1', 5432); SELECT citus_update_node(4, 'node2', 5432); COMMIT; :param old: the last known topology registered in "pg_dist_node" for a given :attr:`groupid`. :yields: :class:`PgDistNode` objects that must be updated/added/removed in "pg_dist_node". """ self.failover = old.failover new_primary = self.primary() assert new_primary is not None old_primary = old.primary() gone_nodes = old - self - {old_primary} added_nodes = self - old - {new_primary} if not old_primary: # We did not have any nodes in the group yet and we're adding one now yield new_primary elif old_primary == new_primary: new_primary.nodeid = old_primary.nodeid # Controlled switchover with pausing client connections. # Achieved by updating the primary row and putting hostname = '${host}-demoted' in a transaction. if old_primary.role != new_primary.role: self.failover = True yield new_primary elif old_primary != new_primary: self.failover = True new_primary_old_node = old.get(new_primary) old_primary_new_node = self.get(old_primary) # The new primary was registered as a secondary before failover if new_primary_old_node: new_node = None # Old primary is gone and some new secondaries were added. # We can use the row of promoted secondary to add the new secondary. if not old_primary_new_node and added_nodes: new_node = added_nodes.pop() new_node.nodeid = new_primary_old_node.nodeid yield new_node # notify _maybe_register_old_primary_as_secondary that the old primary should not be re-registered old_primary.role = 'secondary' # In opposite case we need to change the primary record to '${host}-demoted:${port}' # before we can put its host:port to the row of promoted secondary. elif old_primary.role == 'primary': old_primary.role = 'demoted' yield old_primary # The old primary is gone and the promoted secondary row wasn't yet used. if not old_primary_new_node and not new_node: # We have to "add" the gone primary to the row of promoted secondary because # nodes could not be removed while the metadata isn't synced. old_primary_new_node = PgDistNode(old_primary.host, old_primary.port, new_primary_old_node.role) self.add(old_primary_new_node) # put the old primary instead of promoted secondary if old_primary_new_node: old_primary_new_node.nodeid = new_primary_old_node.nodeid yield old_primary_new_node # update the primary record with the new information new_primary.nodeid = old_primary.nodeid yield new_primary # The new primary was never registered as a standby and there are secondaries that have gone away. Since # nodes can't be removed while metadata isn't synced we have to temporarily "add" the old primary back. if not new_primary_old_node and gone_nodes: # We were in the middle of controlled switchover while the primary disappeared. # If there are any gone nodes that can't be reused for new secondaries we will # use one of them to temporarily "add" the old primary back as a secondary. if not old_primary_new_node and old_primary.role == 'demoted' and len(gone_nodes) > len(added_nodes): old_primary_new_node = PgDistNode(old_primary.host, old_primary.port, 'secondary') self.add(old_primary_new_node) # Use one of the gone secondaries to put host:port of the old primary there. if old_primary_new_node: old_primary_new_node.nodeid = gone_nodes.pop().nodeid yield old_primary_new_node # Fill nodeid for standbys in the new topology from the old ones old_replicas = {v: v for v in old if not v.is_primary()} for n in self: if not n.is_primary() and not n.nodeid and n in old_replicas: n.nodeid = old_replicas[n].nodeid # Reuse nodeid's of gone standbys to "add" new standbys while gone_nodes and added_nodes: a = added_nodes.pop() a.nodeid = gone_nodes.pop().nodeid yield a # Adding or removing nodes operations are executed on primaries in all Citus groups in 2PC. # If we know that the primary was updated (self.failover is True) that automatically means that # adding/removing nodes calls will fail and the whole transaction will be aborted. Therefore # we discard operations that add/remove secondaries if we know that the primary was just updated. # The inconsistency will be automatically resolved on the next Patroni heartbeat loop. # Remove remaining nodes that are gone, but only in case if metadata is in sync (self.failover is False). for g in gone_nodes: if not self.failover: # Remove the node if we expect metadata to be in sync yield PgDistNode(g.host, g.port, '') else: # Otherwise add these nodes to the new topology self.add(g) # Add new nodes to the metadata, but only in case if metadata is in sync (self.failover is False). for a in added_nodes: if not self.failover: # Add the node if we expect metadata to be in sync yield a else: # Otherwise remove them from the new topology self.discard(a) class PgDistTask(PgDistGroup): """A "task" that represents the current or desired state of "pg_dist_node" for a provided *groupid*. :ivar group: the "groupid" in "pg_dist_node". :ivar event: an "event" that resulted in creating this task. possible values: "before_demote", "before_promote", "after_promote". :ivar timeout: a transaction timeout if the task resulted in starting a transaction. :ivar cooldown: the cooldown value for ``citus_update_node()`` UDF call. :ivar deadline: the time in unix seconds when the transaction is allowed to be rolled back. """ def __init__(self, groupid: int, nodes: Optional[Collection[PgDistNode]], event: str, timeout: Optional[float] = None, cooldown: Optional[float] = None) -> None: """Create a :class:`PgDistTask` object based on given arguments. :param groupid: the groupid from "pg_dist_node". :param nodes: a collection of :class:`PgDistNode` objects that belong to a *groupid*. :param event: an "event" that resulted in creating this task. :param timeout: a transaction timeout if the task resulted in starting a transaction. :param cooldown: the cooldown value for ``citus_update_node()`` UDF call. """ super(PgDistTask, self).__init__(groupid, nodes) # Event that is trying to change or changed the given row. # Possible values: before_demote, before_promote, after_promote. self.event = event # If transaction was started, we need to COMMIT/ROLLBACK before the deadline self.timeout = timeout self.cooldown = cooldown or 10000 # 10s by default self.deadline: float = 0 # All changes in the pg_dist_node are serialized on the Patroni # side by performing them from a thread. The thread, that is # requested a change, sometimes needs to wait for a result. # For example, we want to pause client connections before demoting # the worker, and once it is done notify the calling thread. self._event = Event() def wait(self) -> None: """Wait until this task is processed by a dedicated thread.""" self._event.wait() def wakeup(self) -> None: """Notify a thread that created a task that it was processed.""" self._event.set() def __eq__(self, other: Any) -> bool: return isinstance(other, PgDistTask) and self.event == other.event\ and super(PgDistTask, self).equals(other) def __ne__(self, other: Any) -> bool: return not self == other class Citus(AbstractMPP): group_re = re.compile('^(0|[1-9][0-9]*)$') @staticmethod def validate_config(config: Any) -> bool: """Check whether provided config is good for a given MPP. :param config: configuration of ``citus`` MPP section. :returns: ``True`` is config passes validation, otherwise ``False``. """ return isinstance(config, dict) \ and isinstance(cast(Dict[str, Any], config).get('database'), str) \ and parse_int(cast(Dict[str, Any], config).get('group')) is not None @property def group(self) -> int: """The group of this Citus node.""" return int(self._config['group']) @property def coordinator_group_id(self) -> int: """The group id of the Citus coordinator PostgreSQL cluster.""" return CITUS_COORDINATOR_GROUP_ID class CitusHandler(Citus, AbstractMPPHandler, Thread): """Define the interfaces for handling an underlying Citus cluster.""" def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None: """"Initialize a new instance of :class:`CitusHandler`. :param postgresql: the Postgres node. :param config: the ``citus`` MPP config section. """ Thread.__init__(self) AbstractMPPHandler.__init__(self, postgresql, config) self.daemon = True if config: self._connection = postgresql.connection_pool.get( 'citus', {'dbname': config['database'], 'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'}) self._pg_dist_group: Dict[int, PgDistTask] = {} # Cache of pg_dist_node: {groupid: PgDistTask()} self._tasks: List[PgDistTask] = [] # Requests to change pg_dist_group, every task is a `PgDistTask` self._in_flight: Optional[PgDistTask] = None # Reference to the `PgDistTask` being changed in a transaction self._schedule_load_pg_dist_group = True # Flag that "pg_dist_group" should be queried from the database self._condition = Condition() # protects _pg_dist_group, _tasks, _in_flight, and _schedule_load_pg_dist_group self.schedule_cache_rebuild() def schedule_cache_rebuild(self) -> None: """Cache rebuild handler. Is called to notify handler that it has to refresh its metadata cache from the database. """ with self._condition: self._schedule_load_pg_dist_group = True def on_demote(self) -> None: with self._condition: self._pg_dist_group.clear() empty_tasks: List[PgDistTask] = [] self._tasks[:] = empty_tasks self._in_flight = None def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]: try: logger.debug('query(%s, %s)', sql, params) return self._connection.query(sql, *params) except Exception as e: logger.error('Exception when executing query "%s", (%s): %r', sql, params, e) self._connection.close() with self._condition: self._in_flight = None self.schedule_cache_rebuild() raise e def load_pg_dist_group(self) -> bool: """Read from the `pg_dist_node` table and put it into the local cache""" with self._condition: if not self._schedule_load_pg_dist_group: return True self._schedule_load_pg_dist_group = False try: rows = self.query('SELECT groupid, nodename, nodeport, noderole, nodeid FROM pg_catalog.pg_dist_node') except Exception: return False pg_dist_group: Dict[int, PgDistTask] = {} for row in rows: if row[0] not in pg_dist_group: pg_dist_group[row[0]] = PgDistTask(row[0], nodes=set(), event='after_promote') pg_dist_group[row[0]].add(PgDistNode(*row[1:])) with self._condition: self._pg_dist_group = pg_dist_group return True def sync_meta_data(self, cluster: Cluster) -> None: """Maintain the ``pg_dist_node`` from the coordinator leader every heartbeat loop. We can't always rely on REST API calls from worker nodes in order to maintain `pg_dist_node`, therefore at least once per heartbeat loop we make sure that works registered in `self._pg_dist_group` cache are matching the cluster view from DCS by creating tasks the same way as it is done from the REST API.""" if not self.is_coordinator(): return with self._condition: if not self.is_alive(): self.start() self.add_task('after_promote', CITUS_COORDINATOR_GROUP_ID, cluster, self._postgresql.name, self._postgresql.connection_string) for groupid, worker in cluster.workers.items(): leader = worker.leader if leader and leader.conn_url\ and leader.data.get('role') in ('master', 'primary') and leader.data.get('state') == 'running': self.add_task('after_promote', groupid, worker, leader.name, leader.conn_url) def find_task_by_groupid(self, groupid: int) -> Optional[int]: for i, task in enumerate(self._tasks): if task.groupid == groupid: return i def pick_task(self) -> Tuple[Optional[int], Optional[PgDistTask]]: """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list Tasks are picked by following priorities: 1. If there is already a transaction in progress, pick a task that that will change already affected worker primary. 2. If the coordinator address should be changed - pick a task with groupid=0 (coordinators are always in groupid 0). 3. Pick a task that is the oldest (first from the self._tasks) """ with self._condition: if self._in_flight: i = self.find_task_by_groupid(self._in_flight.groupid) else: while True: i = self.find_task_by_groupid(CITUS_COORDINATOR_GROUP_ID) # set_coordinator if i is None and self._tasks: i = 0 if i is None: break task = self._tasks[i] if task == self._pg_dist_group.get(task.groupid): self._tasks.pop(i) # nothing to do because cached version of pg_dist_group already matches else: break task = self._tasks[i] if i is not None else None return i, task def update_node(self, groupid: int, node: PgDistNode, cooldown: float = 10000) -> None: if node.role not in ('primary', 'secondary', 'demoted'): self.query('SELECT pg_catalog.citus_remove_node(%s, %s)', node.host, node.port) elif node.nodeid is not None: host = node.host + ('-demoted' if node.role == 'demoted' else '') self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)', node.nodeid, host, node.port, cooldown) elif node.role != 'demoted': node.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default')", node.host, node.port, groupid, node.role)[0][0] def update_group(self, task: PgDistTask, transaction: bool) -> None: current_state = self._in_flight\ or self._pg_dist_group.get(task.groupid)\ or PgDistTask(task.groupid, set(), 'after_promote') transitions = list(task.transition(current_state)) if transitions: if not transaction and len(transitions) > 1: self.query('BEGIN') for node in transitions: self.update_node(task.groupid, node, task.cooldown) if not transaction and len(transitions) > 1: task.failover = False self.query('COMMIT') def process_task(self, task: PgDistTask) -> bool: """Updates a single row in `pg_dist_group` table, optionally in a transaction. The transaction is started if we do a demote of the worker node or before promoting the other worker if there is no transaction in progress. And, the transaction is committed when the switchover/failover completed. .. note: The maximum lifetime of the transaction in progress is controlled outside of this method. .. note: Read access to `self._in_flight` isn't protected because we know it can't be changed outside of our thread. :param task: reference to a :class:`PgDistTask` object that represents a row to be updated/created. :returns: ``True`` if the row was successfully created/updated or transaction in progress was committed as an indicator that the `self._pg_dist_group` cache should be updated, or, if the new transaction was opened, this method returns `False`. """ if task.event == 'after_promote': self.update_group(task, self._in_flight is not None) if self._in_flight: self.query('COMMIT') task.failover = False return True else: # before_demote, before_promote if task.timeout: task.deadline = time.time() + task.timeout if not self._in_flight: self.query('BEGIN') self.update_group(task, True) return False def process_tasks(self) -> None: while True: # Read access to `_in_flight` isn't protected because we know it can't be changed outside of our thread. if not self._in_flight and not self.load_pg_dist_group(): break i, task = self.pick_task() if not task or i is None: break try: update_cache = self.process_task(task) except Exception as e: logger.error('Exception when working with pg_dist_node: %r', e) update_cache = None with self._condition: if self._tasks: if update_cache: self._pg_dist_group[task.groupid] = task if update_cache is False: # an indicator that process_tasks has started a transaction self._in_flight = task else: self._in_flight = None if id(self._tasks[i]) == id(task): self._tasks.pop(i) task.wakeup() def run(self) -> None: while True: try: with self._condition: if self._schedule_load_pg_dist_group: timeout = -1 elif self._in_flight: timeout = self._in_flight.deadline - time.time() if self._tasks else None else: timeout = -1 if self._tasks else None if timeout is None or timeout > 0: self._condition.wait(timeout) elif self._in_flight: logger.warning('Rolling back transaction. Last known status: %s', self._in_flight) self.query('ROLLBACK') self._in_flight = None self.process_tasks() except Exception: logger.exception('run') def _add_task(self, task: PgDistTask) -> bool: with self._condition: i = self.find_task_by_groupid(task.groupid) # The `PgDistNode.timeout` == None is an indicator that it was scheduled from the sync_meta_data(). if task.timeout is None: # We don't want to override the already existing task created from REST API. if i is not None and self._tasks[i].timeout is not None: return False # There is a little race condition with tasks created from REST API - the call made "before" the member # key is updated in DCS. Therefore it is possible that :func:`sync_meta_data` will try to create a task # based on the outdated values of "state"/"role". To solve it we introduce an artificial timeout. # Only when the timeout is reached new tasks could be scheduled from sync_meta_data() if self._in_flight and self._in_flight.groupid == task.groupid and self._in_flight.timeout is not None\ and self._in_flight.deadline > time.time(): return False # Override already existing task for the same worker groupid if i is not None: if task != self._tasks[i]: logger.debug('Overriding existing task: %s != %s', self._tasks[i], task) self._tasks[i] = task self._condition.notify() return True # Add the task to the list if Worker node state is different from the cached `pg_dist_group` elif self._schedule_load_pg_dist_group or task != self._pg_dist_group.get(task.groupid)\ or self._in_flight and task.groupid == self._in_flight.groupid: logger.debug('Adding the new task: %s', task) self._tasks.append(task) self._condition.notify() return True return False @staticmethod def _pg_dist_node(role: str, conn_url: str) -> Optional[PgDistNode]: try: r = urlparse(conn_url) if r.hostname: return PgDistNode(r.hostname, r.port or 5432, role) except Exception as e: logger.error('Failed to parse connection url %s: %r', conn_url, e) def add_task(self, event: str, groupid: int, cluster: Cluster, leader_name: str, leader_url: str, timeout: Optional[float] = None, cooldown: Optional[float] = None) -> Optional[PgDistTask]: primary = self._pg_dist_node('demoted' if event == 'before_demote' else 'primary', leader_url) if not primary: return task = PgDistTask(groupid, {primary}, event=event, timeout=timeout, cooldown=cooldown) for member in cluster.members: secondary = self._pg_dist_node('secondary', member.conn_url)\ if member.name != leader_name and not member.noloadbalance and member.is_running and member.conn_url\ else None if secondary: task.add(secondary) return task if self._add_task(task) else None def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None: if not self.is_alive(): return worker = cluster.workers.get(event['group']) if not (worker and worker.leader and worker.leader.name == event['leader'] and worker.leader.conn_url): return logger.info('Discarding event %s', event) task = self.add_task(event['type'], event['group'], worker, worker.leader.name, worker.leader.conn_url, event['timeout'], event['cooldown'] * 1000) if task and event['type'] == 'before_demote': task.wait() def bootstrap(self) -> None: """Bootstrap handler. Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method). """ conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs, 'options': '-c synchronous_commit=local -c statement_timeout=0'} if self._config['database'] != self._postgresql.database: conn = connect(**conn_kwargs) try: with conn.cursor() as cur: cur.execute('CREATE DATABASE {0}'.format( quote_ident(self._config['database'], conn)).encode('utf-8')) except ProgrammingError as exc: if exc.diag.sqlstate == '42P04': # DuplicateDatabase logger.debug('Exception when creating database: %r', exc) else: raise exc finally: conn.close() conn_kwargs['dbname'] = self._config['database'] conn = connect(**conn_kwargs) try: with conn.cursor() as cur: cur.execute('CREATE EXTENSION IF NOT EXISTS citus') superuser = self._postgresql.config.superuser params = {k: superuser[k] for k in ('password', 'sslcert', 'sslkey') if k in superuser} if params: cur.execute("INSERT INTO pg_catalog.pg_dist_authinfo VALUES" "(0, pg_catalog.current_user(), %s)", (self._postgresql.config.format_dsn(params),)) if self.is_coordinator(): r = urlparse(self._postgresql.connection_string) cur.execute("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')", (r.hostname, r.port or 5432)) finally: conn.close() def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None: """Adjust GUCs in the current PostgreSQL configuration. :param parameters: dictionary of GUCs, with key as GUC name and the corresponding value as current GUC value. """ # citus extension must be on the first place in shared_preload_libraries shared_preload_libraries = list(filter( lambda el: el and el != 'citus', map(str.strip, parameters.get('shared_preload_libraries', '').split(','))) ) # pyright: ignore [reportUnknownArgumentType] parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries) # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2 if parameters['max_prepared_transactions'] == 0: parameters['max_prepared_transactions'] = parameters['max_connections'] * 2 # Resharding in Citus implemented using logical replication parameters['wal_level'] = 'logical' # Sometimes Citus needs to connect to the local postgres. We will do it the same way as Patroni does. parameters['citus.local_hostname'] = self._postgresql.connection_pool.conn_kwargs.get('host', 'localhost') def ignore_replication_slot(self, slot: Dict[str, str]) -> bool: """Check whether provided replication *slot* existing in the database should not be removed. .. note:: MPP database may create replication slots for its own use, for example to migrate data between workers using logical replication, and we don't want to suddenly drop them. :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and ``plugin``. :returns: ``True`` if the replication slots should not be removed, otherwise ``False``. """ if self._postgresql.is_primary() and slot['type'] == 'logical' and slot['database'] == self._config['database']: m = CITUS_SLOT_NAME_RE.match(slot['name']) return bool(m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin']) return False patroni-4.0.4/patroni/postgresql/postmaster.py000066400000000000000000000262741472010352700216410ustar00rootroot00000000000000import logging import multiprocessing import os import re import signal import subprocess import sys from multiprocessing.connection import Connection from typing import Dict, List, Optional import psutil from patroni import KUBERNETES_ENV_PREFIX, PATRONI_ENV_PREFIX # avoid spawning the resource tracker process if sys.version_info >= (3, 8): # pragma: no cover import multiprocessing.resource_tracker multiprocessing.resource_tracker.getfd = lambda: 0 elif sys.version_info >= (3, 4): # pragma: no cover import multiprocessing.semaphore_tracker multiprocessing.semaphore_tracker.getfd = lambda: 0 logger = logging.getLogger(__name__) STOP_SIGNALS = { 'smart': 'TERM', 'fast': 'INT', 'immediate': 'QUIT', } def pg_ctl_start(conn: Connection, cmdline: List[str], env: Dict[str, str]) -> None: if os.name != 'nt': os.setsid() try: postmaster = subprocess.Popen(cmdline, close_fds=True, env=env) conn.send(postmaster.pid) except Exception: logger.exception('Failed to execute %s', cmdline) conn.send(None) conn.close() class PostmasterProcess(psutil.Process): def __init__(self, pid: int) -> None: self._postmaster_pid: Dict[str, str] self.is_single_user = False if pid < 0: pid = -pid self.is_single_user = True super(PostmasterProcess, self).__init__(pid) @staticmethod def _read_postmaster_pidfile(data_dir: str) -> Dict[str, str]: """Reads and parses postmaster.pid from the data directory :returns dictionary of values if successful, empty dictionary otherwise """ pid_line_names = ['pid', 'data_dir', 'start_time', 'port', 'socket_dir', 'listen_addr', 'shmem_key'] try: with open(os.path.join(data_dir, 'postmaster.pid')) as f: return {name: line.rstrip('\n') for name, line in zip(pid_line_names, f)} except IOError: return {} def _is_postmaster_process(self) -> bool: try: start_time = int(self._postmaster_pid.get('start_time', 0)) if start_time and abs(self.create_time() - start_time) > 3: logger.info('Process %s is not postmaster, too much difference between PID file start time %s and ' 'process start time %s', self.pid, start_time, self.create_time()) return False except ValueError: logger.warning('Garbage start time value in pid file: %r', self._postmaster_pid.get('start_time')) # Extra safety check. The process can't be ourselves, our parent or our direct child. if self.pid == os.getpid() or self.pid == os.getppid() or self.ppid() == os.getpid(): logger.info('Patroni (pid=%s, ppid=%s), "fake postmaster" (pid=%s, ppid=%s)', os.getpid(), os.getppid(), self.pid, self.ppid()) return False return True @classmethod def _from_pidfile(cls, data_dir: str) -> Optional['PostmasterProcess']: postmaster_pid = PostmasterProcess._read_postmaster_pidfile(data_dir) try: pid = int(postmaster_pid.get('pid', 0)) if pid: proc = cls(pid) proc._postmaster_pid = postmaster_pid return proc except ValueError: return None @staticmethod def from_pidfile(data_dir: str) -> Optional['PostmasterProcess']: try: proc = PostmasterProcess._from_pidfile(data_dir) return proc if proc and proc._is_postmaster_process() else None except psutil.NoSuchProcess: return None @classmethod def from_pid(cls, pid: int) -> Optional['PostmasterProcess']: try: return cls(pid) except psutil.NoSuchProcess: return None def signal_kill(self) -> bool: """to suspend and kill postmaster and all children :returns True if postmaster and children are killed, False if error """ try: self.suspend() except psutil.NoSuchProcess: return True except psutil.Error as e: logger.warning('Failed to suspend postmaster: %s', e) try: children = self.children(recursive=True) except psutil.NoSuchProcess: return True except psutil.Error as e: logger.warning('Failed to get a list of postmaster children: %s', e) children = [] try: self.kill() except psutil.NoSuchProcess: return True except psutil.Error as e: logger.warning('Could not kill postmaster: %s', e) return False for child in children: try: child.kill() except psutil.Error: pass psutil.wait_procs(children + [self]) return True def signal_stop(self, mode: str, pg_ctl: str = 'pg_ctl') -> Optional[bool]: """Signal postmaster process to stop :returns None if signaled, True if process is already gone, False if error """ if self.is_single_user: logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(self.pid)) return False if os.name != 'posix': return self.pg_ctl_kill(mode, pg_ctl) try: self.send_signal(getattr(signal, 'SIG' + STOP_SIGNALS[mode])) except psutil.NoSuchProcess: return True except psutil.AccessDenied as e: logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e)) return False return None def pg_ctl_kill(self, mode: str, pg_ctl: str) -> Optional[bool]: try: status = subprocess.call([pg_ctl, "kill", STOP_SIGNALS[mode], str(self.pid)]) except OSError: return False if status == 0: return None else: return not self.is_running() def wait_for_user_backends_to_close(self, stop_timeout: Optional[float]) -> None: # These regexps are cross checked against versions PostgreSQL 9.1 .. 17 aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:(?:archiver|startup|autovacuum launcher|autovacuum worker|" "checkpointer|logger|stats collector|wal receiver|wal writer|writer)(?: process )?|" "walreceiver|wal sender process|walsender|walwriter|background writer|" "logical replication launcher|logical replication worker for subscription|" "logical replication tablesync worker for subscription|" "logical replication parallel apply worker for subscription|" "logical replication apply worker for subscription|" "slotsync worker|walsummarizer|bgworker:) ") try: children = self.children() except psutil.Error: return logger.debug('Failed to get list of postmaster children') user_backends: List[psutil.Process] = [] user_backends_cmdlines: Dict[int, str] = {} for child in children: try: cmdline = child.cmdline() if cmdline and not aux_proc_re.match(cmdline[0]): user_backends.append(child) user_backends_cmdlines[child.pid] = cmdline[0] except psutil.NoSuchProcess: pass if user_backends: logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines.values())) _, live = psutil.wait_procs(user_backends, stop_timeout) if stop_timeout and live: live = [user_backends_cmdlines[b.pid] for b in live] logger.warning('Backends still alive after %s: %s', stop_timeout, ', '.join(live)) else: logger.debug("Backends closed") @staticmethod def start(pgcommand: str, data_dir: str, conf: str, options: List[str]) -> Optional['PostmasterProcess']: # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start # in python. It will start postgres, wait for port to be open and wait until postgres will start # accepting connections. # Important!!! We can't just start postgres using subprocess.Popen, because in this case it # will be our child for the rest of our live and we will have to take care of it (`waitpid`). # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres. # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility # of init process to take care about postmaster. # In order to make everything portable we can't use fork&exec approach here, so we will call # ourselves and pass list of arguments which must be used to start postgres. # On Windows, in order to run a side-by-side assembly the specified env must include a valid SYSTEMROOT. env = {p: os.environ[p] for p in os.environ if not p.startswith( PATRONI_ENV_PREFIX) and not p.startswith(KUBERNETES_ENV_PREFIX)} try: proc = PostmasterProcess._from_pidfile(data_dir) if proc and not proc._is_postmaster_process(): # Upon start postmaster process performs various safety checks if there is a postmaster.pid # file in the data directory. Although Patroni already detected that the running process # corresponding to the postmaster.pid is not a postmaster, the new postmaster might fail # to start, because it thinks that postmaster.pid is already locked. # Important!!! Unlink of postmaster.pid isn't an option, because it has a lot of nasty race conditions. # Luckily there is a workaround to this problem, we can pass the pid from postmaster.pid # in the `PG_GRANDPARENT_PID` environment variable and postmaster will ignore it. logger.info("Telling pg_ctl that it is safe to ignore postmaster.pid for process %s", proc.pid) env['PG_GRANDPARENT_PID'] = str(proc.pid) except psutil.NoSuchProcess: pass cmdline = [pgcommand, '-D', data_dir, '--config-file={}'.format(conf)] + options logger.debug("Starting postgres: %s", " ".join(cmdline)) ctx = multiprocessing.get_context('spawn') parent_conn, child_conn = ctx.Pipe(False) proc = ctx.Process(target=pg_ctl_start, args=(child_conn, cmdline, env)) proc.start() pid = parent_conn.recv() proc.join() if pid is None: return logger.info('postmaster pid=%s', pid) # TODO: In an extremely unlikely case, the process could have exited and the pid reassigned. The start # initiation time is not accurate enough to compare to create time as start time would also likely # be relatively close. We need the subprocess extract pid+start_time in a race free manner. return PostmasterProcess.from_pid(pid) patroni-4.0.4/patroni/postgresql/rewind.py000066400000000000000000000702271472010352700207250ustar00rootroot00000000000000import logging import os import re import shlex import shutil import subprocess from enum import IntEnum from threading import Lock, Thread from typing import Any, Callable, Dict, List, Optional, Tuple, Union from ..async_executor import CriticalTask from ..collections import EMPTY_DICT from ..dcs import Leader, RemoteMember from . import Postgresql from .connection import get_connection_cursor from .misc import format_lsn, fsync_dir, parse_history, parse_lsn logger = logging.getLogger(__name__) class REWIND_STATUS(IntEnum): INITIAL = 0 CHECKPOINT = 1 CHECK = 2 NEED = 3 NOT_NEED = 4 SUCCESS = 5 FAILED = 6 class Rewind(object): def __init__(self, postgresql: Postgresql) -> None: self._postgresql = postgresql self._checkpoint_task_lock = Lock() self.reset_state() @staticmethod def configuration_allows_rewind(data: Dict[str, str]) -> bool: return data.get('wal_log_hints setting', 'off') == 'on' or data.get('Data page checksum version', '0') != '0' @property def enabled(self) -> bool: return bool(self._postgresql.config.get('use_pg_rewind')) @property def can_rewind(self) -> bool: """ check if pg_rewind executable is there and that pg_controldata indicates we have either wal_log_hints or checksums turned on """ # low-hanging fruit: check if pg_rewind configuration is there if not self.enabled: return False cmd = [self._postgresql.pgcommand('pg_rewind'), '--help'] try: ret = subprocess.call(cmd, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) if ret != 0: # pg_rewind is not there, close up the shop and go home return False except OSError: return False return self.configuration_allows_rewind(self._postgresql.controldata()) @property def should_remove_data_directory_on_diverged_timelines(self) -> bool: return bool(self._postgresql.config.get('remove_data_directory_on_diverged_timelines')) @property def can_rewind_or_reinitialize_allowed(self) -> bool: return self.should_remove_data_directory_on_diverged_timelines or self.can_rewind def trigger_check_diverged_lsn(self) -> None: if self.can_rewind_or_reinitialize_allowed and self._state != REWIND_STATUS.NEED: self._state = REWIND_STATUS.CHECK @staticmethod def check_leader_is_not_in_recovery(conn_kwargs: Dict[str, Any]) -> Optional[bool]: try: with get_connection_cursor(connect_timeout=3, options='-c statement_timeout=2000', **conn_kwargs) as cur: cur.execute('SELECT pg_catalog.pg_is_in_recovery()') row = cur.fetchone() if not row or not row[0]: return True logger.info('Leader is still in_recovery and therefore can\'t be used for rewind') except Exception: return logger.exception('Exception when working with leader') @staticmethod def check_leader_has_run_checkpoint(conn_kwargs: Dict[str, Any]) -> Optional[str]: try: with get_connection_cursor(connect_timeout=3, options='-c statement_timeout=2000', **conn_kwargs) as cur: cur.execute("SELECT NOT pg_catalog.pg_is_in_recovery()" " AND ('x' || pg_catalog.substr(pg_catalog.pg_walfile_name(" " pg_catalog.pg_current_wal_lsn()), 1, 8))::bit(32)::int = timeline_id" " FROM pg_catalog.pg_control_checkpoint()") row = cur.fetchone() if not row or not row[0]: return 'leader has not run a checkpoint yet' except Exception: logger.exception('Exception when working with leader') return 'not accessible or not healthy' def _get_checkpoint_end(self, timeline: int, lsn: int) -> int: """Get the end of checkpoint record from WAL. .. note:: The checkpoint record size in WAL depends on postgres major version and platform (memory alignment). Hence, the only reliable way to figure out where it ends, is to read the record from file with the help of ``pg_waldump`` and parse the output. We are trying to read two records, and expect that it will fail to read the second record with message: fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0; or fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: expected at least 24, got 0 The error message contains information about LSN of the next record, which is exactly where checkpoint ends. :param timeline: the checkpoint *timeline* from ``pg_controldata``. :param lsn: the checkpoint *location* as :class:`int` from ``pg_controldata``. :returns: the end of checkpoint record as :class:`int` or ``0`` if failed to parse ``pg_waldump`` output. """ lsn8 = format_lsn(lsn, True) lsn_str = format_lsn(lsn) out, err = self._postgresql.waldump(timeline, lsn_str, 2) if out is not None and err is not None: out = out.decode('utf-8').rstrip().split('\n') err = err.decode('utf-8').rstrip().split('\n') pattern = 'error in WAL record at {0}: invalid record length at '.format(lsn_str) if len(out) == 1 and len(err) == 1 and ', lsn: {0}, prev '.format(lsn8) in out[0] and pattern in err[0]: i = err[0].find(pattern) + len(pattern) # Message format depends on the major version: # * expected at least -- starting from v16 # * wanted -- before v16 # We will simply check all possible combinations. for pattern in (': expected at least ', ': wanted '): j = err[0].find(pattern, i) if j > -1: try: return parse_lsn(err[0][i:j]) except Exception as e: logger.error('Failed to parse lsn %s: %r', err[0][i:j], e) logger.error('Failed to parse pg_%sdump output', self._postgresql.wal_name) logger.error(' stdout=%s', '\n'.join(out)) logger.error(' stderr=%s', '\n'.join(err)) return 0 def _get_local_timeline_lsn_from_controldata(self) -> Tuple[Optional[bool], Optional[int], Optional[int]]: in_recovery = timeline = lsn = None data = self._postgresql.controldata() try: if data.get('Database cluster state') in ('shut down in recovery', 'in archive recovery'): in_recovery = True lsn = data.get('Minimum recovery ending location') timeline = int(data.get("Min recovery ending loc's timeline", "")) if lsn == '0/0' or timeline == 0: # it was a primary when it crashed data['Database cluster state'] = 'shut down' if data.get('Database cluster state') == 'shut down': in_recovery = False lsn = data.get('Latest checkpoint location') timeline = int(data.get("Latest checkpoint's TimeLineID", "")) except (TypeError, ValueError): logger.exception('Failed to get local timeline and lsn from pg_controldata output') if lsn is not None: try: lsn = parse_lsn(lsn) except (IndexError, ValueError) as e: logger.error('Exception when parsing lsn %s: %r', lsn, e) lsn = None return in_recovery, timeline, lsn def _get_local_timeline_lsn(self) -> Tuple[Optional[bool], Optional[int], Optional[int]]: if self._postgresql.is_running(): # if postgres is running - get timeline from replication connection in_recovery = True timeline = self._postgresql.get_replica_timeline() lsn = self._postgresql.replayed_location() else: # otherwise analyze pg_controldata output in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata() log_lsn = format_lsn(lsn) if isinstance(lsn, int) else lsn logger.info('Local timeline=%s lsn=%s', timeline, log_lsn) return in_recovery, timeline, lsn @staticmethod def _log_primary_history(history: List[Tuple[int, int, str]], i: int) -> None: start = max(0, i - 3) end = None if i + 4 >= len(history) else i + 2 history_show: List[str] = [] def format_history_line(line: Tuple[int, int, str]) -> str: return '{0}\t{1}\t{2}'.format(line[0], format_lsn(line[1]), line[2]) line = None for line in history[start:end]: history_show.append(format_history_line(line)) if line != history[-1]: history_show.append('...') history_show.append(format_history_line(history[-1])) logger.info('primary: history=%s', '\n'.join(history_show)) def _conn_kwargs(self, member: Union[Leader, RemoteMember], auth: Dict[str, Any]) -> Dict[str, Any]: ret = member.conn_kwargs(auth) if not ret.get('dbname'): ret['dbname'] = self._postgresql.database # Add target_session_attrs to make sure we hit the primary. # It is not strictly necessary for starting from PostgreSQL v14, which made it possible # to rewind from standby, but doing it from the real primary is always safer. if self._postgresql.major_version >= 100000: ret['target_session_attrs'] = 'read-write' return ret def _check_timeline_and_lsn(self, leader: Union[Leader, RemoteMember]) -> None: in_recovery, local_timeline, local_lsn = self._get_local_timeline_lsn() if local_timeline is None or local_lsn is None: return if isinstance(leader, Leader) and leader.member.data.get('role') not in ('master', 'primary'): return # We want to use replication credentials when connecting to the "postgres" database in case if # `use_pg_rewind` isn't enabled and only `remove_data_directory_on_diverged_timelines` is set # for Postgresql older than v11 (where Patroni can't use a dedicated user for rewind). # In all other cases we will use rewind or superuser credentials. check_credentials = self._postgresql.config.replication if not self.enabled and\ self.should_remove_data_directory_on_diverged_timelines and\ self._postgresql.major_version < 110000 else self._postgresql.config.rewind_credentials if not self.check_leader_is_not_in_recovery(self._conn_kwargs(leader, check_credentials)): return history = need_rewind = None try: with self._postgresql.get_replication_connection_cursor(**leader.conn_kwargs()) as cur: cur.execute('IDENTIFY_SYSTEM') row = cur.fetchone() if row: primary_timeline = row[1] logger.info('primary_timeline=%s', primary_timeline) if local_timeline > primary_timeline: # Not always supported by pg_rewind need_rewind = True elif local_timeline == primary_timeline: need_rewind = False elif primary_timeline > 1: cur.execute('TIMELINE_HISTORY {0}'.format(primary_timeline).encode('utf-8')) row = cur.fetchone() if row: history = row[1] if not isinstance(history, str): history = bytes(history).decode('utf-8') logger.debug('primary: history=%s', history) except Exception: return logger.exception('Exception when working with primary via replication connection') if history is not None: history = list(parse_history(history)) i = len(history) for i, (parent_timeline, switchpoint, _) in enumerate(history): if parent_timeline == local_timeline: # We don't need to rewind when: # 1. for replica: replayed location is not ahead of switchpoint # 2. for the former primary: end of checkpoint record is the same as switchpoint if in_recovery: need_rewind = local_lsn > switchpoint elif local_lsn >= switchpoint: need_rewind = True else: need_rewind = switchpoint != self._get_checkpoint_end(local_timeline, local_lsn) break elif parent_timeline > local_timeline: need_rewind = True break else: need_rewind = True self._log_primary_history(history, i) self._state = need_rewind and REWIND_STATUS.NEED or REWIND_STATUS.NOT_NEED def rewind_or_reinitialize_needed_and_possible(self, leader: Union[Leader, RemoteMember, None]) -> bool: if leader and leader.name != self._postgresql.name and leader.conn_url and self._state == REWIND_STATUS.CHECK: self._check_timeline_and_lsn(leader) return bool(leader and leader.conn_url) and self._state == REWIND_STATUS.NEED def __checkpoint(self, task: CriticalTask, wakeup: Callable[..., Any]) -> None: try: result = self._postgresql.checkpoint() except Exception as e: result = 'Exception: ' + str(e) with task: task.complete(not bool(result)) if task.result: wakeup() def ensure_checkpoint_after_promote(self, wakeup: Callable[..., Any]) -> None: """After promote issue a CHECKPOINT from a new thread and asynchronously check the result. In case if CHECKPOINT failed, just check that timeline in pg_control was updated.""" if self._state != REWIND_STATUS.CHECKPOINT and self._postgresql.is_primary(): with self._checkpoint_task_lock: if self._checkpoint_task: with self._checkpoint_task: if self._checkpoint_task.result is not None: self._state = REWIND_STATUS.CHECKPOINT self._checkpoint_task = None elif self._postgresql.get_primary_timeline() == self._postgresql.pg_control_timeline(): self._state = REWIND_STATUS.CHECKPOINT else: self._checkpoint_task = CriticalTask() Thread(target=self.__checkpoint, args=(self._checkpoint_task, wakeup)).start() def checkpoint_after_promote(self) -> bool: return self._state == REWIND_STATUS.CHECKPOINT def _build_archiver_command(self, command: str, wal_filename: str) -> str: """Replace placeholders in the given archiver command's template. Applicable for archive_command and restore_command. Can also be used for archive_cleanup_command and recovery_end_command, however %r value is always set to 000000010000000000000001.""" cmd = '' length = len(command) i = 0 while i < length: if command[i] == '%' and i + 1 < length: i += 1 if command[i] == 'p': cmd += os.path.join(self._postgresql.wal_dir, wal_filename) elif command[i] == 'f': cmd += wal_filename elif command[i] == 'r': cmd += '000000010000000000000001' elif command[i] == '%': cmd += '%' else: cmd += '%' i -= 1 else: cmd += command[i] i += 1 return cmd def _fetch_missing_wal(self, restore_command: str, wal_filename: str) -> bool: cmd = self._build_archiver_command(restore_command, wal_filename) logger.info('Trying to fetch the missing wal: %s', cmd) return self._postgresql.cancellable.call(shlex.split(cmd)) == 0 def _find_missing_wal(self, data: bytes) -> Optional[str]: # could not open file "$PGDATA/pg_wal/0000000A00006AA100000068": No such file or directory pattern = 'could not open file "' for line in data.decode('utf-8').split('\n'): b = line.find(pattern) if b > -1: b += len(pattern) e = line.find('": ', b) if e > -1 and '/' in line[b:e]: waldir, wal_filename = line[b:e].rsplit('/', 1) if waldir.endswith('/pg_' + self._postgresql.wal_name) and len(wal_filename) == 24: return wal_filename def _archive_ready_wals(self) -> None: """Try to archive WALs that have .ready files just in case archive_mode was not set to 'always' before promote, while after it the WALs were recycled on the promoted replica. With this we prevent the entire loss of such WALs and the consequent old leader's start failure.""" archive_mode = self._postgresql.get_guc_value('archive_mode') archive_cmd = self._postgresql.get_guc_value('archive_command') if archive_mode not in ('on', 'always') or not archive_cmd: return walseg_regex = re.compile(r'^[0-9A-F]{24}(\.partial){0,1}\.ready$') status_dir = os.path.join(self._postgresql.wal_dir, 'archive_status') try: wals_to_archive = [f[:-6] for f in os.listdir(status_dir) if walseg_regex.match(f)] except OSError as e: return logger.error('Unable to list %s: %r', status_dir, e) # skip fsync, as postgres --single or pg_rewind will anyway run it for wal in sorted(wals_to_archive): old_name = os.path.join(status_dir, wal + '.ready') # wal file might have already been archived if os.path.isfile(old_name) and os.path.isfile(os.path.join(self._postgresql.wal_dir, wal)): cmd = self._build_archiver_command(archive_cmd, wal) # it is the author of archive_command, who is responsible # for not overriding the WALs already present in archive logger.info('Trying to archive %s: %s', wal, cmd) if self._postgresql.cancellable.call([cmd], shell=True) == 0: new_name = os.path.join(status_dir, wal + '.done') try: shutil.move(old_name, new_name) except Exception as e: logger.error('Unable to rename %s to %s: %r', old_name, new_name, e) else: logger.info('Failed to archive WAL segment %s', wal) def _maybe_clean_pg_replslot(self) -> None: """Clean pg_replslot directory if pg version is less then 11 (pg_rewind deletes $PGDATA/pg_replslot content only since pg11).""" if self._postgresql.major_version < 110000: replslot_dir = self._postgresql.slots_handler.pg_replslot_dir try: for f in os.listdir(replslot_dir): shutil.rmtree(os.path.join(replslot_dir, f)) fsync_dir(replslot_dir) except Exception as e: logger.warning('Unable to clean %s: %r', replslot_dir, e) def pg_rewind(self, conn_kwargs: Dict[str, Any]) -> bool: """Do pg_rewind. .. note:: If ``pg_rewind`` doesn't support ``--restore-target-wal`` parameter and exited with non zero code, Patroni will parse stderr/stdout to figure out if it failed due to a missing WAL file and will repeat an attempt after downloading the missing file using ``restore_command``. :param conn_kwargs: :class:`dict` object with connection parameters. :returns: ``True`` if ``pg_rewind`` finished successfully, ``False`` otherwise. """ # prepare pg_rewind connection string env = self._postgresql.config.write_pgpass(conn_kwargs) env.update(LANG='C', LC_ALL='C', PGOPTIONS='-c statement_timeout=0') dsn = self._postgresql.config.format_dsn({**conn_kwargs, 'password': None}) logger.info('running pg_rewind from %s', dsn) restore_command = (self._postgresql.config.get('recovery_conf') or EMPTY_DICT).get('restore_command') \ if self._postgresql.major_version < 120000 else self._postgresql.get_guc_value('restore_command') # Until v15 pg_rewind expected postgresql.conf to be inside $PGDATA, which is not the case on e.g. Debian pg_rewind_can_restore = restore_command and (self._postgresql.major_version >= 150000 or (self._postgresql.major_version >= 130000 and self._postgresql.config.config_dir == self._postgresql.data_dir)) cmd = [self._postgresql.pgcommand('pg_rewind')] if pg_rewind_can_restore: cmd.append('--restore-target-wal') if self._postgresql.major_version >= 150000 and\ self._postgresql.config.config_dir != self._postgresql.data_dir: cmd.append('--config-file={0}'.format(self._postgresql.config.postgresql_conf)) cmd.extend(['-D', self._postgresql.data_dir, '--source-server', dsn]) while True: results: Dict[str, bytes] = {} ret = self._postgresql.cancellable.call(cmd, env=env, communicate=results) logger.info('pg_rewind exit code=%s', ret) if ret is None: return False logger.info(' stdout=%s', results['stdout'].decode('utf-8')) logger.info(' stderr=%s', results['stderr'].decode('utf-8')) if ret == 0: return True if not restore_command or pg_rewind_can_restore: return False missing_wal = self._find_missing_wal(results['stderr']) or self._find_missing_wal(results['stdout']) if not missing_wal: return False if not self._fetch_missing_wal(restore_command, missing_wal): logger.info('Failed to fetch WAL segment %s required for pg_rewind', missing_wal) return False def execute(self, leader: Union[Leader, RemoteMember]) -> Optional[bool]: if self._postgresql.is_running() and not self._postgresql.stop(checkpoint=False): return logger.warning('Can not run pg_rewind because postgres is still running') self._archive_ready_wals() # prepare pg_rewind connection r = self._conn_kwargs(leader, self._postgresql.config.rewind_credentials) # 1. make sure that we are really trying to rewind from the primary # 2. make sure that pg_control contains the new timeline by: # running a checkpoint or # waiting until Patroni on the primary will expose checkpoint_after_promote=True checkpoint_status = leader.checkpoint_after_promote if isinstance(leader, Leader) else None if checkpoint_status is None: # we are the standby-cluster leader or primary still runs the old Patroni # superuser credentials match rewind_credentials if the latter are not provided or we run 10 or older if self._postgresql.config.superuser == self._postgresql.config.rewind_credentials: leader_status = self._postgresql.checkpoint( self._conn_kwargs(leader, self._postgresql.config.superuser)) else: # we run 11+ and have a dedicated pg_rewind user leader_status = self.check_leader_has_run_checkpoint(r) if leader_status: # we tried to run/check for a checkpoint on the remote leader, but it failed return logger.warning('Can not use %s for rewind: %s', leader.name, leader_status) elif not checkpoint_status: return logger.info('Waiting for checkpoint on %s before rewind', leader.name) elif not self.check_leader_is_not_in_recovery(r): return if self.pg_rewind(r): self._maybe_clean_pg_replslot() self._state = REWIND_STATUS.SUCCESS else: if not self.check_leader_is_not_in_recovery(r): logger.warning('Failed to rewind because primary %s become unreachable', leader.name) if not self.can_rewind: # It is possible that the previous attempt damaged pg_control file! self._state = REWIND_STATUS.FAILED else: logger.error('Failed to rewind from healthy primary: %s', leader.name) self._state = REWIND_STATUS.FAILED if self.failed: for name in ('remove_data_directory_on_rewind_failure', 'remove_data_directory_on_diverged_timelines'): if self._postgresql.config.get(name): logger.warning('%s is set. removing...', name) self._postgresql.remove_data_directory() self._state = REWIND_STATUS.INITIAL break return False def reset_state(self) -> None: self._state = REWIND_STATUS.INITIAL with self._checkpoint_task_lock: self._checkpoint_task = None @property def is_needed(self) -> bool: return self._state in (REWIND_STATUS.CHECK, REWIND_STATUS.NEED) @property def executed(self) -> bool: return self._state > REWIND_STATUS.NOT_NEED @property def failed(self) -> bool: return self._state == REWIND_STATUS.FAILED def read_postmaster_opts(self) -> Dict[str, str]: """returns the list of option names/values from postgres.opts, Empty dict if read failed or no file""" result: Dict[str, str] = {} try: with open(os.path.join(self._postgresql.data_dir, 'postmaster.opts')) as f: data = f.read() for opt in data.split('" "'): if '=' in opt and opt.startswith('--'): name, val = opt.split('=', 1) result[name.strip('-')] = val.rstrip('"\n') except IOError: logger.exception('Error when reading postmaster.opts') return result def single_user_mode(self, communicate: Optional[Dict[str, Any]] = None, options: Optional[Dict[str, str]] = None) -> Optional[int]: """run a given command in a single-user mode. If the command is empty - then just start and stop""" cmd = [self._postgresql.pgcommand('postgres'), '--single', '-D', self._postgresql.data_dir] for opt, val in sorted((options or {}).items()): cmd.extend(['-c', '{0}={1}'.format(opt, val)]) # need a database name to connect cmd.append('template1') return self._postgresql.cancellable.call(cmd, communicate=communicate) def cleanup_archive_status(self) -> None: status_dir = os.path.join(self._postgresql.wal_dir, 'archive_status') try: for f in os.listdir(status_dir): path = os.path.join(status_dir, f) try: if os.path.islink(path): os.unlink(path) elif os.path.isfile(path): os.remove(path) except OSError: logger.exception('Unable to remove %s', path) except OSError: logger.exception('Unable to list %s', status_dir) def ensure_clean_shutdown(self) -> Optional[bool]: self._archive_ready_wals() self.cleanup_archive_status() # Start in a single user mode and stop to produce a clean shutdown opts = self.read_postmaster_opts() opts.update({'archive_mode': 'on', 'archive_command': 'false'}) self._postgresql.config.remove_recovery_conf() output: Dict[str, bytes] = {} ret = self.single_user_mode(communicate=output, options=opts) if ret != 0: logger.error('Crash recovery finished with code=%s', ret) logger.info(' stdout=%s', output['stdout'].decode('utf-8')) logger.info(' stderr=%s', output['stderr'].decode('utf-8')) return ret == 0 or None patroni-4.0.4/patroni/postgresql/slots.py000066400000000000000000001150241472010352700205740ustar00rootroot00000000000000"""Replication slot handling. Provides classes for the creation, monitoring, management and synchronisation of PostgreSQL replication slots. """ import logging import os import shutil from collections import defaultdict from contextlib import contextmanager from threading import Condition, Thread from typing import Any, Collection, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union from .. import global_config from ..dcs import Cluster, Leader from ..file_perm import pg_perm from ..psycopg import OperationalError from ..tags import Tags from .connection import get_connection_cursor from .misc import format_lsn, fsync_dir if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor from . import Postgresql logger = logging.getLogger(__name__) def compare_slots(s1: Dict[str, Any], s2: Dict[str, Any], dbid: str = 'database') -> bool: """Compare 2 replication slot objects for equality. ..note :: If the first argument is a ``physical`` replication slot then only the `type` of the second slot is compared. If the first argument is another ``type`` (e.g. ``logical``) then *dbid* and ``plugin`` are compared. :param s1: First slot dictionary to be compared. :param s2: Second slot dictionary to be compared. :param dbid: Optional attribute to be compared when comparing ``logical`` replication slots. :return: ``True`` if the slot ``type`` of *s1* and *s2* is matches, and the ``type`` of *s1* is ``physical``, OR the ``types`` match AND the *dbid* and ``plugin`` attributes are equal. """ return (s1['type'] == s2['type'] and (s1['type'] == 'physical' or s1.get(dbid) == s2.get(dbid) and s1['plugin'] == s2['plugin'])) class SlotsAdvanceThread(Thread): """Daemon process :class:``Thread`` object for advancing logical replication slots on replicas. This ensures that slot advancing queries sent to postgres do not block the main loop. """ def __init__(self, slots_handler: 'SlotsHandler') -> None: """Create and start a new thread for handling slot advance queries. :param slots_handler: The calling class instance for reference to slot information attributes. """ super().__init__() self.daemon = True self._slots_handler = slots_handler # _copy_slots and _failed are used to asynchronously give some feedback to the main thread self._copy_slots: List[str] = [] self._failed = False # {'dbname1': {'slot1': 100, 'slot2': 100}, 'dbname2': {'slot3': 100}} self._scheduled: Dict[str, Dict[str, int]] = defaultdict(dict) self._condition = Condition() # protect self._scheduled from concurrent access and to wakeup the run() method self.start() def sync_slot(self, cur: Union['cursor', 'Cursor[Any]'], database: str, slot: str, lsn: int) -> None: """Execute a ``pg_replication_slot_advance`` query and store success for scheduled synchronisation task. :param cur: database connection cursor. :param database: name of the database associated with the slot. :param slot: name of the slot to be synchronised. :param lsn: last known LSN position """ failed = copy = False try: cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", (slot, format_lsn(lsn))) except Exception as e: logger.error("Failed to advance logical replication slot '%s': %r", slot, e) failed = True # WAL file is gone or slot is invalidated copy = isinstance(e, OperationalError) and e.diag.sqlstate in ('58P01', '55000') with self._condition: if self._scheduled and failed: if copy and slot not in self._copy_slots: self._copy_slots.append(slot) self._failed = True new_lsn = self._scheduled.get(database, {}).get(slot, 0) # remove slot from the self._scheduled structure if it is to be copied or if it wasn't changed if copy or (new_lsn == lsn and database in self._scheduled): self._scheduled[database].pop(slot) if not self._scheduled[database]: self._scheduled.pop(database) def sync_slots_in_database(self, database: str, slots: List[str]) -> None: """Synchronise slots for a single database. :param database: name of the database. :param slots: list of slot names to synchronise. """ with self._slots_handler.get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur: for slot in slots: with self._condition: lsn = self._scheduled.get(database, {}).get(slot, 0) if lsn: self.sync_slot(cur, database, slot, lsn) def sync_slots(self) -> None: """Synchronise slots for all scheduled databases.""" with self._condition: databases = list(self._scheduled.keys()) for database in databases: with self._condition: slots = list(self._scheduled.get(database, {}).keys()) if slots: try: self.sync_slots_in_database(database, slots) except Exception as e: logger.error('Failed to advance replication slots in database %s: %r', database, e) def run(self) -> None: """Thread main loop entrypoint. .. note:: Thread will wait until a sync is scheduled from outside, normally triggered during the HA loop or a wakeup call. """ while True: with self._condition: if not self._scheduled: self._condition.wait() self.sync_slots() def schedule(self, advance_slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]: """Trigger a synchronisation of slots. This is the main entrypoint for Patroni HA loop wakeup call. :param advance_slots: dictionary containing slots that need to be advanced :return: tuple of failure status and a list of slots to be copied """ with self._condition: for database, values in advance_slots.items(): for name, value in values.items(): # Don't schedule sync for slots that just failed to be advanced and scheduled to be copied if name not in self._copy_slots: self._scheduled[database][name] = value ret = (self._failed, self._copy_slots) self._copy_slots = [] self._failed = False self._condition.notify() return ret def clean(self) -> None: """Reset state of the daemon.""" with self._condition: self._scheduled.clear() self._failed = False self._copy_slots = [] class SlotsHandler: """Handler for managing and storing information on replication slots in PostgreSQL. :ivar pg_replslot_dir: system location path of the PostgreSQL replication slots. :ivar _logical_slots_processing_queue: yet to be processed logical replication slots on the primary """ def __init__(self, postgresql: 'Postgresql') -> None: """Create an instance with storage attributes for replication slots and schedule the first synchronisation. :param postgresql: Calling class instance providing interface to PostgreSQL. """ self._force_readiness_check = False self._schedule_load_slots = False self._postgresql = postgresql self._advance = None self._replication_slots: Dict[str, Dict[str, Any]] = {} # already existing replication slots self._logical_slots_processing_queue: Dict[str, Optional[int]] = {} self.pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot') self.schedule() def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]: """Helper method for :meth:`Postgresql.query`. :param sql: SQL statement to execute. :param params: parameters to pass through to :meth:`Postgresql.query`. :returns: query response. """ return self._postgresql.query(sql, *params, retry=False) @staticmethod def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[Collection[str]] = None) -> None: """Select values from *src* dictionary to update in *dst* dictionary for optional supplied *keys*. :param src: source dictionary that *keys* will be looked up from. :param dst: destination dictionary to be updated. :param keys: optional list of keys to be looked up in the source dictionary. """ dst.update({key: src[key] for key in keys or ('datoid', 'catalog_xmin', 'confirmed_flush_lsn')}) def process_permanent_slots(self, slots: List[Dict[str, Any]]) -> Dict[str, int]: """Process replication slot information from the host and prepare information used in subsequent cluster tasks. .. note:: This methods solves three problems. The ``cluster_info_query`` from :class:``Postgresql`` is executed every HA loop and returns information about all replication slots that exists on the current host. Based on this information perform the following actions: 1. For the primary we want to expose to DCS permanent logical slots, therefore build (and return) a dict that maps permanent logical slot names to ``confirmed_flush_lsn``. 2. detect if one of the previously known permanent slots is missing and schedule resync. 3. Update the local cache with the fresh ``catalog_xmin`` and ``confirmed_flush_lsn`` for every known slot. This info is used when performing the check of logical slot readiness on standbys. :param slots: replication slot information that exists on the current host. :return: dictionary of logical slot names to ``confirmed_flush_lsn``. """ ret: Dict[str, int] = {} slots_dict: Dict[str, Dict[str, Any]] = {slot['slot_name']: slot for slot in slots or []} for name, value in slots_dict.items(): if name in self._replication_slots: if compare_slots(value, self._replication_slots[name], 'datoid'): if value['type'] == 'logical': ret[name] = value['confirmed_flush_lsn'] self._copy_items(value, self._replication_slots[name]) else: ret[name] = value['restart_lsn'] self._copy_items(value, self._replication_slots[name], ('restart_lsn', 'xmin')) else: self._schedule_load_slots = True # It could happen that the slot was deleted in the background, we want to detect this case if any(name not in slots_dict for name in self._replication_slots.keys()): self._schedule_load_slots = True return ret def load_replication_slots(self) -> None: """Query replication slot information from the database and store it for processing by other tasks. .. note:: Only supported from PostgreSQL version 9.4 onwards. Store replication slot ``name``, ``type``, ``plugin``, ``database`` and ``datoid``. If PostgreSQL version is 10 or newer also store ``catalog_xmin`` and ``confirmed_flush_lsn``. When using logical slots, store information separately for slot synchronisation on replica nodes. """ if self._postgresql.major_version >= 90400 and self._schedule_load_slots: replication_slots: Dict[str, Dict[str, Any]] = {} pg_wal_lsn_diff = f"pg_catalog.pg_{self._postgresql.wal_name}_{self._postgresql.lsn_name}_diff" extra = f", catalog_xmin, {pg_wal_lsn_diff}(confirmed_flush_lsn, '0/0')::bigint" \ if self._postgresql.major_version >= 100000 else "" skip_temp_slots = ' WHERE NOT temporary' if self._postgresql.major_version >= 100000 else '' for r in self._query("SELECT slot_name, slot_type, xmin, " f"{pg_wal_lsn_diff}(restart_lsn, '0/0')::bigint, plugin, database, datoid{extra}" f" FROM pg_catalog.pg_replication_slots{skip_temp_slots}"): value = {'type': r[1]} if r[1] == 'logical': value.update(plugin=r[4], database=r[5], datoid=r[6]) if self._postgresql.major_version >= 100000: value.update(catalog_xmin=r[7], confirmed_flush_lsn=r[8]) else: value.update(xmin=r[2], restart_lsn=r[3]) replication_slots[r[0]] = value self._replication_slots = replication_slots self._schedule_load_slots = False if self._force_readiness_check: self._logical_slots_processing_queue = {n: None for n, v in replication_slots.items() if v['type'] == 'logical'} self._force_readiness_check = False def ignore_replication_slot(self, cluster: Cluster, name: str) -> bool: """Check if slot *name* should not be managed by Patroni. :param cluster: cluster state information object. :param name: name of the slot to ignore :returns: ``True`` if slot *name* matches any slot specified in ``ignore_slots`` configuration, otherwise will pass through and return result of :meth:`AbstractMPPHandler.ignore_replication_slot`. """ slot = self._replication_slots[name] if cluster.config: for matcher in global_config.ignore_slots_matchers: if ( (matcher.get("name") is None or matcher["name"] == name) and all(not matcher.get(a) or matcher[a] == slot.get(a) for a in ('database', 'plugin', 'type')) ): return True return self._postgresql.mpp_handler.ignore_replication_slot(slot) def drop_replication_slot(self, name: str) -> Tuple[bool, bool]: """Drop a named slot from Postgres. :param name: name of the slot to be dropped. :returns: a tuple of ``active`` and ``dropped``. ``active`` is ``True`` if the slot is active, ``dropped`` is ``True`` if the slot was successfully dropped. If the slot was not found return ``False`` for both. """ rows = self._query(('WITH slots AS (SELECT slot_name, active' ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),' ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),' ' true AS dropped FROM slots WHERE not active) ' 'SELECT active, COALESCE(dropped, false) FROM slots' ' FULL OUTER JOIN dropped ON true'), name) return (rows[0][0], rows[0][1]) if rows else (False, False) def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any]) -> None: """Compare required slots and configured as permanent slots with those found, dropping extraneous ones. .. note:: Slots that are not contained in *slots* will be dropped. Slots can be filtered out with ``ignore_slots`` configuration. Slots that have matching names but do not match attributes in *slots* will also be dropped. :param cluster: cluster state information object. :param slots: dictionary of desired slot names as keys with slot attributes as a dictionary value, if known. """ # drop old replication slots which are not presented in desired slots. for name in set(self._replication_slots) - set(slots): if not global_config.is_paused and not self.ignore_replication_slot(cluster, name): active, dropped = self.drop_replication_slot(name) if dropped: logger.info("Dropped unknown replication slot '%s'", name) else: self._schedule_load_slots = True if active: logger.debug("Unable to drop unknown replication slot '%s', slot is still active", name) else: logger.error("Failed to drop replication slot '%s'", name) # drop slots with matching names but attributes that do not match, e.g. `plugin` or `database`. for name, value in slots.items(): if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]): logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s", name, self._replication_slots[name], value) if self.drop_replication_slot(name) == (False, True): self._replication_slots.pop(name) else: logger.error("Failed to drop replication slot '%s'", name) self._schedule_load_slots = True def _ensure_physical_slots(self, slots: Dict[str, Any]) -> None: """Create or advance physical replication *slots*. Any failures are logged and do not interrupt creation of all *slots*. :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot if the value is a dictionary with the key ``type`` and a value of ``physical``. """ immediately_reserve = ', true' if self._postgresql.major_version >= 90600 else '' for name, value in slots.items(): if value['type'] != 'physical': continue # First we want to detect physical replication slots that are not # expected to be active but have NOT NULL xmin value and drop them. # As the slot is not expected to be active, nothing would be consuming this # slot, consequently no hot-standby feedback messages would be received # by Postgres regarding this slot. In that case, the `xmin` value would never # change, which would prevent Postgres from advancing the xmin horizon. if self._postgresql.can_advance_slots and name in self._replication_slots and\ self._replication_slots[name]['type'] == 'physical': self._copy_items(self._replication_slots[name], value, ('restart_lsn', 'xmin')) if value.get('expected_active') is False and value['xmin']: logger.warning('Dropping physical replication slot %s because of its xmin value %s', name, value['xmin']) active, dropped = self.drop_replication_slot(name) if dropped: self._replication_slots.pop(name) else: self._schedule_load_slots = True if active: logger.warning("Unable to drop replication slot '%s', slot is active", name) else: logger.error("Failed to drop replication slot '%s'", name) # Now we will create physical replication slots that are missing. if name not in self._replication_slots: try: self._query(f"SELECT pg_catalog.pg_create_physical_replication_slot(%s{immediately_reserve})" f" WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots" f" WHERE slot_type = 'physical' AND slot_name = %s)", name, name) except Exception: logger.exception("Failed to create physical replication slot '%s'", name) self._schedule_load_slots = True # And advance restart_lsn on physical replication slots that are not expected to be active. elif self._postgresql.can_advance_slots and self._replication_slots[name]['type'] == 'physical' and\ value.get('expected_active') is not True and not value['xmin']: lsn = value.get('lsn') if lsn and lsn > value['restart_lsn']: # The slot has feedback in DCS and needs to be advanced try: lsn = format_lsn(lsn) self._query("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", name, lsn) except Exception as exc: logger.error("Error while advancing replication slot %s to position '%s': %r", name, lsn, exc) @contextmanager def get_local_connection_cursor(self, **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]: """Create a new database connection to local server. Create a non-blocking connection cursor to avoid the situation where an execution of the query of ``pg_replication_slot_advance`` takes longer than the timeout on a HA loop, which could cause a false failure state. :param kwargs: Any keyword arguments to pass to :func:`psycopg.connect`. :yields: connection cursor object, note implementation varies depending on version of :mod:`psycopg`. """ conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs, **kwargs} with get_connection_cursor(**conn_kwargs) as cur: yield cur def _ensure_logical_slots_primary(self, slots: Dict[str, Any]) -> None: """Create any missing logical replication *slots* on the primary. If the logical slot already exists, copy state information into the replication slots structure stored in the class instance. :param slots: Slots that should exist are supplied in a dictionary, mapping slot name to any attributes. The method will only consider slots that have a value that is a dictionary with a key ``type`` with a value that is ``logical``. """ # Group logical slots to be created by database name logical_slots: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(dict) for name, value in slots.items(): if value['type'] == 'logical': if self._replication_slots.get(name, {}).get('datoid'): self._copy_items(self._replication_slots[name], value) else: logical_slots[value['database']][name] = value # Create new logical slots for database, values in logical_slots.items(): with self.get_local_connection_cursor(dbname=database) as cur: for name, value in values.items(): try: cur.execute("SELECT pg_catalog.pg_create_logical_replication_slot(%s, %s)" " WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots" " WHERE slot_type = 'logical' AND slot_name = %s)", (name, value['plugin'], name)) except Exception as e: logger.error("Failed to create logical replication slot '%s' plugin='%s': %r", name, value['plugin'], e) slots.pop(name) self._schedule_load_slots = True def schedule_advance_slots(self, slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]: """Wrapper to ensure slots advance daemon thread is started if not already. :param slots: dictionary containing slot information. :return: tuple with the result of the scheduling of slot advancement: ``failed`` and list of slots to copy. """ if not self._advance: self._advance = SlotsAdvanceThread(self) return self._advance.schedule(slots) def _ensure_logical_slots_replica(self, slots: Dict[str, Any]) -> List[str]: """Update logical *slots* on replicas. If the logical slot already exists, copy state information into the replication slots structure stored in the class instance. Slots that exist are also advanced if their ``confirmed_flush_lsn`` is greater than the stored state of the slot. As logical slots can only be created when the primary is available, pass the list of slots that need to be copied back to the caller. They will be created on replicas with :meth:`SlotsHandler.copy_logical_slots`. :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot if the value is a dictionary with the key ``type`` and a value of ``logical``. :returns: list of slots to be copied from the primary. """ # Group logical slots to be advanced by database name advance_slots: Dict[str, Dict[str, int]] = defaultdict(dict) create_slots: List[str] = [] # Collect logical slots to be created on the replica for name, value in slots.items(): if value['type'] != 'logical': continue # If the logical already exists, copy some information about it into the original structure if name in self._replication_slots and compare_slots(value, self._replication_slots[name]): self._copy_items(self._replication_slots[name], value) if 'lsn' in value and value['confirmed_flush_lsn'] < value['lsn']: # The slot has feedback in DCS # Skip slots that don't need to be advanced advance_slots[value['database']][name] = value['lsn'] elif name not in self._replication_slots and 'lsn' in value: # We want to copy only slots with feedback in a DCS create_slots.append(name) # Slots to be copied from the primary should be removed from the *slots* structure, # otherwise Patroni falsely assumes that they already exist. for name in create_slots: slots.pop(name) error, copy_slots = self.schedule_advance_slots(advance_slots) if error: self._schedule_load_slots = True return create_slots + copy_slots def sync_replication_slots(self, cluster: Cluster, tags: Tags) -> List[str]: """During the HA loop read, check and alter replication slots found in the cluster. Read physical and logical slots from ``pg_replication_slots``, then compare to those configured in the DCS. Drop any slots that do not match those required by configuration and are not configured as permanent. Create any missing physical slots, or advance their position according to feedback stored in DCS. If we are the primary then create logical slots, otherwise if logical slots are known and active create them on replica nodes by copying slot files from the primary. :param cluster: object containing stateful information for the cluster. :param tags: reference to an object implementing :class:`Tags` interface. :returns: list of logical replication slots names that should be copied from the primary. """ ret = [] if self._postgresql.major_version >= 90400 and cluster.config: try: self.load_replication_slots() slots = cluster.get_replication_slots(self._postgresql, tags, show_error=True) self._drop_incorrect_slots(cluster, slots) self._ensure_physical_slots(slots) if self._postgresql.is_primary(): self._logical_slots_processing_queue.clear() self._ensure_logical_slots_primary(slots) else: self.check_logical_slots_readiness(cluster, tags) ret = self._ensure_logical_slots_replica(slots) self._replication_slots = slots except Exception: logger.exception('Exception when changing replication slots') self._schedule_load_slots = True return ret @contextmanager def _get_leader_connection_cursor(self, leader: Leader) -> Iterator[Union['cursor', 'Cursor[Any]']]: """Create a new database connection to the leader. .. note:: Uses rewind user credentials because it has enough permissions to read files from PGDATA. Sets the options ``connect_timeout`` to ``3`` and ``statement_timeout`` to ``2000``. :param leader: object with information on the leader :yields: connection cursor object, note implementation varies depending on version of ``psycopg``. """ conn_kwargs = leader.conn_kwargs(self._postgresql.config.rewind_credentials) conn_kwargs['dbname'] = self._postgresql.database with get_connection_cursor(connect_timeout=3, options="-c statement_timeout=2000", **conn_kwargs) as cur: yield cur def check_logical_slots_readiness(self, cluster: Cluster, tags: Tags) -> bool: """Determine whether all known logical slots are synchronised from the leader. 1) Retrieve the current ``catalog_xmin`` value for the physical slot from the cluster leader, and 2) using previously stored list of "unready" logical slots, those which have yet to be checked hence have no stored slot attributes, 3) store logical slot ``catalog_xmin`` when the physical slot ``catalog_xmin`` becomes valid. :param cluster: object containing stateful information for the cluster. :param tags: reference to an object implementing :class:`Tags` interface. :returns: ``False`` if any issue while checking logical slots readiness, ``True`` otherwise. """ catalog_xmin = None if self._logical_slots_processing_queue and cluster.leader: slot_name = cluster.get_slot_name_on_primary(self._postgresql.name, tags) try: with self._get_leader_connection_cursor(cluster.leader) as cur: cur.execute("SELECT slot_name, catalog_xmin FROM pg_catalog.pg_get_replication_slots()" " WHERE NOT pg_catalog.pg_is_in_recovery() AND slot_name = ANY(%s)", ([n for n, v in self._logical_slots_processing_queue.items() if v is None] + [slot_name],)) slots = {row[0]: row[1] for row in cur} if slot_name not in slots: logger.warning('Physical slot %s does not exist on the primary', slot_name) return False catalog_xmin = slots.pop(slot_name) except Exception as e: logger.error("Failed to check %s physical slot on the primary: %r", slot_name, e) return False if not self._update_pending_logical_slot_primary(slots, catalog_xmin): return False # since `catalog_xmin` isn't valid further checks don't make any sense self._ready_logical_slots(catalog_xmin) return True def _update_pending_logical_slot_primary(self, slots: Dict[str, Any], catalog_xmin: Optional[int] = None) -> bool: """Store pending logical slot information for ``catalog_xmin`` on the primary. Remember ``catalog_xmin`` of logical slots on the primary when ``catalog_xmin`` of the physical slot became valid. Logical slots on replica will be safe to use after promote when ``catalog_xmin`` of the physical slot overtakes these values. :param slots: dictionary of slot information from the primary :param catalog_xmin: ``catalog_xmin`` of the physical slot used by this replica to stream changes from primary. :returns: ``False`` if any issue was faced while processing, ``True`` otherwise. """ if catalog_xmin is not None: for name, value in slots.items(): self._logical_slots_processing_queue[name] = value return True # Replica isn't streaming or the hot_standby_feedback isn't enabled try: if not self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")[0][0]: logger.error('Logical slot failover requires "hot_standby_feedback". Please check postgresql.auto.conf') except Exception as e: logger.error('Failed to check the hot_standby_feedback setting: %r', e) return False def _ready_logical_slots(self, primary_physical_catalog_xmin: Optional[int] = None) -> None: """Ready logical slots by comparing primary physical slot ``catalog_xmin`` to logical ``catalog_xmin``. The logical slot on a replica is safe to use when the physical replica slot on the primary: 1. has a nonzero/non-null ``catalog_xmin`` represented by ``primary_physical_xmin``. 2. has a ``catalog_xmin`` that is not newer (greater) than the ``catalog_xmin`` of any slot on the standby 3. overtook the ``catalog_xmin`` of remembered values of logical slots on the primary. :param primary_physical_catalog_xmin: is the value retrieved from ``pg_catalog.pg_get_replication_slots()`` for the physical replication slot on the primary. """ # Make a copy of processing queue keys as a list as the queue dictionary is modified inside the loop. for name in list(self._logical_slots_processing_queue): primary_logical_catalog_xmin = self._logical_slots_processing_queue[name] standby_logical_slot = self._replication_slots.get(name, {}) standby_logical_catalog_xmin = standby_logical_slot.get('catalog_xmin', 0) if TYPE_CHECKING: # pragma: no cover assert primary_logical_catalog_xmin is not None if ( not standby_logical_slot or primary_physical_catalog_xmin is not None and primary_logical_catalog_xmin <= primary_physical_catalog_xmin <= standby_logical_catalog_xmin ): del self._logical_slots_processing_queue[name] if standby_logical_slot: logger.info('Logical slot %s is safe to be used after a failover', name) def copy_logical_slots(self, cluster: Cluster, tags: Tags, create_slots: List[str]) -> None: """Create logical replication slots on standby nodes. :param cluster: object containing stateful information for the cluster. :param tags: reference to an object implementing :class:`Tags` interface. :param create_slots: list of slot names to copy from the primary. """ leader = cluster.leader if not leader: return slots = cluster.get_replication_slots(self._postgresql, tags, role='replica') copy_slots: Dict[str, Dict[str, Any]] = {} with self._get_leader_connection_cursor(leader) as cur: try: cur.execute("SELECT slot_name, slot_type, datname, plugin, catalog_xmin, " "pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint, " "pg_catalog.pg_read_binary_file('pg_replslot/' || slot_name || '/state')" " FROM pg_catalog.pg_get_replication_slots() JOIN pg_catalog.pg_database ON datoid = oid" " WHERE NOT pg_catalog.pg_is_in_recovery() AND slot_name = ANY(%s)", (create_slots,)) for r in cur: if r[0] in slots: # slot_name is defined in the global configuration slot = {'type': r[1], 'database': r[2], 'plugin': r[3], 'catalog_xmin': r[4], 'confirmed_flush_lsn': r[5], 'data': r[6]} if compare_slots(slot, slots[r[0]]): copy_slots[r[0]] = slot else: logger.warning('Will not copy the logical slot "%s" due to the configuration mismatch: ' 'configuration=%s, slot on the primary=%s', r[0], slots[r[0]], slot) except Exception as e: logger.error("Failed to copy logical slots from the %s via postgresql connection: %r", leader.name, e) if copy_slots and self._postgresql.stop(): if self._advance: self._advance.clean() pg_perm.set_permissions_from_data_directory(self._postgresql.data_dir) for name, value in copy_slots.items(): slot_dir = os.path.join(self.pg_replslot_dir, name) slot_tmp_dir = slot_dir + '.tmp' if os.path.exists(slot_tmp_dir): shutil.rmtree(slot_tmp_dir) os.makedirs(slot_tmp_dir) os.chmod(slot_tmp_dir, pg_perm.dir_create_mode) fsync_dir(slot_tmp_dir) slot_filename = os.path.join(slot_tmp_dir, 'state') with open(slot_filename, 'wb') as f: os.chmod(slot_filename, pg_perm.file_create_mode) f.write(value['data']) f.flush() os.fsync(f.fileno()) if os.path.exists(slot_dir): shutil.rmtree(slot_dir) os.rename(slot_tmp_dir, slot_dir) os.chmod(slot_dir, pg_perm.dir_create_mode) fsync_dir(slot_dir) self._logical_slots_processing_queue[name] = None fsync_dir(self.pg_replslot_dir) self._postgresql.start() def schedule(self, value: Optional[bool] = None) -> None: """Schedule the loading of slot information from the database. :param value: the optional value can be used to unschedule if set to ``False`` or force it to be ``True``. If it is omitted the value will be ``True`` if this PostgreSQL node supports slot replication. """ if value is None: value = self._postgresql.major_version >= 90400 self._schedule_load_slots = self._force_readiness_check = value def on_promote(self) -> None: """Entry point from HA cycle used when a standby node is to be promoted to primary. .. note:: If logical replication slot synchronisation is enabled then slot advancement will be triggered. If any logical slots that were copied are yet to be confirmed as ready a warning message will be logged. """ if self._advance: self._advance.clean() if self._logical_slots_processing_queue: logger.warning('Logical replication slots that might be unsafe to use after promote: %s', set(self._logical_slots_processing_queue)) patroni-4.0.4/patroni/postgresql/sync.py000066400000000000000000000462701472010352700204120ustar00rootroot00000000000000import logging import re import time from copy import deepcopy from typing import Collection, List, NamedTuple, Optional, TYPE_CHECKING from .. import global_config from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..dcs import Cluster from ..psycopg import quote_ident if TYPE_CHECKING: # pragma: no cover from . import Postgresql logger = logging.getLogger(__name__) SYNC_STANDBY_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z_0-9\$]*$') SYNC_REP_PARSER_RE = re.compile(r""" (?P [fF][iI][rR][sS][tT] ) | (?P [aA][nN][yY] ) | (?P \s+ ) | (?P [A-Za-z_][A-Za-z_0-9\$]* ) | (?P " (?: [^"]+ | "" )* " ) | (?P [*] ) | (?P \d+ ) | (?P , ) | (?P \( ) | (?P \) ) | (?P . ) """, re.X) def quote_standby_name(value: str) -> str: """Quote provided *value* if it is nenecessary. :param value: name of a synchronous standby. :returns: a quoted value if it is required or the original one. """ return value if SYNC_STANDBY_NAME_RE.match(value) and value.lower() not in ('first', 'any') else quote_ident(value) class _SSN(NamedTuple): """class representing "synchronous_standby_names" value after parsing. :ivar sync_type: possible values: 'off', 'priority', 'quorum' :ivar has_star: is set to `True` if "synchronous_standby_names" contains '*' :ivar num: how many nodes are required to be synchronous :ivar members: collection of standby names listed in "synchronous_standby_names" """ sync_type: str has_star: bool num: int members: CaseInsensitiveSet _EMPTY_SSN = _SSN('off', False, 0, CaseInsensitiveSet()) def parse_sync_standby_names(value: str) -> _SSN: """Parse postgresql synchronous_standby_names to constituent parts. :param value: the value of `synchronous_standby_names` :returns: :class:`_SSN` object :raises `ValueError`: if the configuration value can not be parsed >>> parse_sync_standby_names('').sync_type 'off' >>> parse_sync_standby_names('FiRsT').sync_type 'priority' >>> 'first' in parse_sync_standby_names('FiRsT').members True >>> set(parse_sync_standby_names('"1"').members) {'1'} >>> parse_sync_standby_names(' a , b ').members == {'a', 'b'} True >>> parse_sync_standby_names(' a , b ').num 1 >>> parse_sync_standby_names('ANY 4("a",*,b)').has_star True >>> parse_sync_standby_names('ANY 4("a",*,b)').num 4 >>> parse_sync_standby_names('1') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError: Unparsable synchronous_standby_names value >>> parse_sync_standby_names('a,') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError: Unparsable synchronous_standby_names value >>> parse_sync_standby_names('ANY 4("a" b,"c c")') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError: Unparsable synchronous_standby_names value >>> parse_sync_standby_names('FIRST 4("a",)') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError: Unparsable synchronous_standby_names value >>> parse_sync_standby_names('2 (,)') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ValueError: Unparsable synchronous_standby_names value """ tokens = [(m.lastgroup, m.group(0), m.start()) for m in SYNC_REP_PARSER_RE.finditer(value) if m.lastgroup != 'space'] if not tokens: return deepcopy(_EMPTY_SSN) if [t[0] for t in tokens[0:3]] == ['any', 'num', 'parenstart'] and tokens[-1][0] == 'parenend': sync_type = 'quorum' num = int(tokens[1][1]) synclist = tokens[3:-1] elif [t[0] for t in tokens[0:3]] == ['first', 'num', 'parenstart'] and tokens[-1][0] == 'parenend': sync_type = 'priority' num = int(tokens[1][1]) synclist = tokens[3:-1] elif [t[0] for t in tokens[0:2]] == ['num', 'parenstart'] and tokens[-1][0] == 'parenend': sync_type = 'priority' num = int(tokens[0][1]) synclist = tokens[2:-1] else: sync_type = 'priority' num = 1 synclist = tokens has_star = False members = CaseInsensitiveSet() for i, (a_type, a_value, a_pos) in enumerate(synclist): if i % 2 == 1: # odd elements are supposed to be commas if len(synclist) == i + 1: # except the last token raise ValueError("Unparsable synchronous_standby_names value %r: Unexpected token %s %r at %d" % (value, a_type, a_value, a_pos)) if a_type != 'comma': raise ValueError("Unparsable synchronous_standby_names value %r: ""Got token %s %r while" " expecting comma at %d" % (value, a_type, a_value, a_pos)) elif a_type in {'ident', 'first', 'any'}: members.add(a_value) elif a_type == 'star': members.add(a_value) has_star = True elif a_type == 'dquot': members.add(a_value[1:-1].replace('""', '"')) else: raise ValueError("Unparsable synchronous_standby_names value %r: Unexpected token %s %r at %d" % (value, a_type, a_value, a_pos)) return _SSN(sync_type, has_star, num, members) class _SyncState(NamedTuple): """Class representing the current synchronous state. :ivar sync_type: possible values: ``off``, ``priority``, ``quorum`` :ivar numsync: how many nodes are required to be synchronous (according to ``synchronous_standby_names``). Is ``0`` if ``synchronous_standby_names`` value is invalid or contains ``*``. :ivar numsync_confirmed: how many nodes are known to be synchronous according to the ``pg_stat_replication`` view. Only nodes that caught up with the :attr:`SyncHandler._primary_flush_lsn` are counted. :ivar sync: collection of synchronous node names. In case of quorum commit all nodes listed in ``synchronous_standby_names``, otherwise nodes that are confirmed to be synchronous according to the ``pg_stat_replication`` view. :ivar active: collection of node names that are streaming and have no restrictions to become synchronous. """ sync_type: str numsync: int numsync_confirmed: int sync: CaseInsensitiveSet active: CaseInsensitiveSet class _Replica(NamedTuple): """Class representing a single replica that is eligible to be synchronous. Attributes are taken from ``pg_stat_replication`` view and respective ``Cluster.members``. :ivar pid: PID of walsender process. :ivar application_name: matches with the ``Member.name``. :ivar sync_state: possible values are: ``async``, ``potential``, ``quorum``, and ``sync``. :ivar lsn: ``write_lsn``, ``flush_lsn``, or ``replay_lsn``, depending on the value of ``synchronous_commit`` GUC. :ivar nofailover: whether the corresponding member has ``nofailover`` tag set to ``True``. """ pid: int application_name: str sync_state: str lsn: int nofailover: bool class _ReplicaList(List[_Replica]): """A collection of :class:``_Replica`` objects. Values are reverse ordered by ``_Replica.sync_state`` and ``_Replica.lsn``. That is, first there will be replicas that have ``sync_state`` == ``sync``, even if they are not the most up-to-date in term of write/flush/replay LSN. It helps to keep the result of choosing new synchronous nodes consistent in case if a synchronous standby member is slowed down OR async node is receiving changes faster than the sync member. Such cases would trigger sync standby member swapping, but only if lag on this member is exceeding a threshold (``maximum_lag_on_syncnode``). :ivar max_lsn: maximum value of ``_Replica.lsn`` among all values. In case if there is just one element in the list we take value of ``pg_current_wal_flush_lsn()``. """ def __init__(self, postgresql: 'Postgresql', cluster: Cluster) -> None: """Create :class:``_ReplicaList`` object. :param postgresql: reference to :class:``Postgresql`` object. :param cluster: currently known cluster state from DCS. """ super().__init__() # We want to prioritize candidates based on `write_lsn``, ``flush_lsn``, or ``replay_lsn``. # Which column exactly to pick depends on the values of ``synchronous_commit`` GUC. sort_col = { 'remote_apply': 'replay', 'remote_write': 'write' }.get(postgresql.synchronous_commit(), 'flush') + '_lsn' members = CaseInsensitiveDict({m.name: m for m in cluster.members if m.name.lower() != postgresql.name.lower()}) for row in postgresql.pg_stat_replication(): member = members.get(row['application_name']) # We want to consider only rows from ``pg_stat_replication` that: # 1. are known to be streaming (write/flush/replay LSN are not NULL). # 2. can be mapped to a ``Member`` of the ``Cluster``: # a. ``Member`` doesn't have ``nosync`` tag set; # b. PostgreSQL on the member is known to be running and accepting client connections. if member and row[sort_col] is not None and member.is_running and not member.nosync: self.append(_Replica(row['pid'], row['application_name'], row['sync_state'], row[sort_col], bool(member.nofailover))) # Prefer replicas that are in state ``sync`` and with higher values of ``write``/``flush``/``replay`` LSN. self.sort(key=lambda r: (r.sync_state, r.lsn), reverse=True) # When checking ``maximum_lag_on_syncnode`` we want to compare with the most # up-to-date replica otherwise with cluster LSN if there is only one replica. self.max_lsn = max(self, key=lambda x: x.lsn).lsn if len(self) > 1 else postgresql.last_operation() class SyncHandler(object): """Class responsible for working with the `synchronous_standby_names`. Sync standbys are chosen based on their state in `pg_stat_replication`. When `synchronous_standby_names` is changed we memorize the `_primary_flush_lsn` and the `current_state()` method will count newly added names as "sync" only when they reached memorized LSN and also reported as "sync" by `pg_stat_replication`""" def __init__(self, postgresql: 'Postgresql') -> None: self._postgresql = postgresql self._synchronous_standby_names = '' # last known value of synchronous_standby_names self._ssn_data = deepcopy(_EMPTY_SSN) self._primary_flush_lsn = 0 # "sync" replication connections, that were verified to reach self._primary_flush_lsn at some point self._ready_replicas = CaseInsensitiveDict({}) # keys: member names, values: connection pids def _handle_synchronous_standby_names_change(self) -> None: """Handles changes of "synchronous_standby_names" GUC. If "synchronous_standby_names" was changed, we need to check that newly added replicas have reached `self._primary_flush_lsn`. Only after that they could be counted as synchronous. """ synchronous_standby_names = self._postgresql.synchronous_standby_names() if synchronous_standby_names == self._synchronous_standby_names: return self._synchronous_standby_names = synchronous_standby_names try: self._ssn_data = parse_sync_standby_names(synchronous_standby_names) except ValueError as e: logger.warning('%s', e) self._ssn_data = deepcopy(_EMPTY_SSN) # Invalidate cache of "sync" connections for app_name in list(self._ready_replicas.keys()): if app_name not in self._ssn_data.members: del self._ready_replicas[app_name] # Newly connected replicas will be counted as sync only when reached self._primary_flush_lsn self._primary_flush_lsn = self._postgresql.last_operation() # Ensure some WAL traffic to move replication self._postgresql.query("""DO $$ BEGIN SET local synchronous_commit = 'off'; PERFORM * FROM pg_catalog.txid_current(); END;$$""") self._postgresql.reset_cluster_info_state(None) # Reset internal cache to query fresh values def _process_replica_readiness(self, cluster: Cluster, replica_list: _ReplicaList) -> None: """Flags replicas as truly "synchronous" when they have caught up with ``_primary_flush_lsn``. :param cluster: current cluster topology from DCS :param replica_list: collection of replicas that we want to evaluate. """ for replica in replica_list: # if standby name is listed in the /sync key we can count it as synchronous, otherwise # it becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up if replica.application_name not in self._ready_replicas\ and replica.application_name in self._ssn_data.members: if global_config.is_quorum_commit_mode: # When quorum commit is enabled we can't check against cluster.sync because nodes # are written there when at least one of them caught up with _primary_flush_lsn. if replica.lsn >= self._primary_flush_lsn\ and (replica.sync_state == 'quorum' or (not self._postgresql.supports_quorum_commit and replica.sync_state in ('sync', 'potential'))): self._ready_replicas[replica.application_name] = replica.pid elif cluster.sync.matches(replica.application_name)\ or replica.sync_state == 'sync' and replica.lsn >= self._primary_flush_lsn: # if standby name is listed in the /sync key we can count it as synchronous, otherwise it becomes # "really" synchronous when sync_state = 'sync' and we known that it managed to catch up self._ready_replicas[replica.application_name] = replica.pid def current_state(self, cluster: Cluster) -> _SyncState: """Find the best candidates to be the synchronous standbys. Current synchronous standby is always preferred, unless it has disconnected or does not want to be a synchronous standby any longer. Standbys are selected based on values from the global configuration: - ``maximum_lag_on_syncnode``: would help swapping unhealthy sync replica in case it stops responding (or hung). Please set the value high enough, so it won't unnecessarily swap sync standbys during high loads. Any value less or equal to ``0`` keeps the behavior backwards compatible. Please note that it will also not swap sync standbys when all replicas are hung. - ``synchronous_node_count``: controls how many nodes should be set as synchronous. :param cluster: current cluster topology from DCS :returns: current synchronous replication state as a :class:`_SyncState` object """ self._handle_synchronous_standby_names_change() replica_list = _ReplicaList(self._postgresql, cluster) self._process_replica_readiness(cluster, replica_list) active = CaseInsensitiveSet() sync_nodes = CaseInsensitiveSet() numsync_confirmed = 0 sync_node_count = global_config.synchronous_node_count if self._postgresql.supports_multiple_sync else 1 sync_node_maxlag = global_config.maximum_lag_on_syncnode # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable. for replica in sorted(replica_list, key=lambda x: x.nofailover): if sync_node_maxlag <= 0 or replica_list.max_lsn - replica.lsn <= sync_node_maxlag: if global_config.is_quorum_commit_mode: # We do not add nodes with `nofailover` enabled because that reduces availability. # We need to check LSN quorum only among nodes that are promotable because # there is a chance that a non-promotable node is ahead of a promotable one. if not replica.nofailover or len(active) < sync_node_count: if replica.application_name in self._ready_replicas: numsync_confirmed += 1 active.add(replica.application_name) else: active.add(replica.application_name) if replica.sync_state == 'sync' and replica.application_name in self._ready_replicas: sync_nodes.add(replica.application_name) numsync_confirmed += 1 if len(active) >= sync_node_count: break if global_config.is_quorum_commit_mode: sync_nodes = CaseInsensitiveSet() if self._ssn_data.has_star else self._ssn_data.members return _SyncState( self._ssn_data.sync_type, 0 if self._ssn_data.has_star else self._ssn_data.num, numsync_confirmed, sync_nodes, active) def set_synchronous_standby_names(self, sync: Collection[str], num: Optional[int] = None) -> None: """Constructs and sets ``synchronous_standby_names`` GUC value. .. note:: standbys in ``synchronous_standby_names`` will be sorted by name. :param sync: set of nodes to sync to :param num: specifies number of nodes to sync to. The *num* is set only in case if quorum commit is enabled """ # Special case. If sync nodes set is empty but requested num of sync nodes >= 1 # we want to set synchronous_standby_names to '*' has_asterisk = '*' in sync or num and num >= 1 and not sync if has_asterisk: sync = ['*'] else: sync = [quote_standby_name(x) for x in sorted(sync)] if self._postgresql.supports_multiple_sync and len(sync) > 1: if num is None: num = len(sync) sync_param = ','.join(sync) else: sync_param = next(iter(sync), None) if global_config.is_quorum_commit_mode and sync or self._postgresql.supports_multiple_sync and len(sync) > 1: prefix = 'ANY ' if global_config.is_quorum_commit_mode and self._postgresql.supports_quorum_commit else '' sync_param = f'{prefix}{num} ({sync_param})' if not (self._postgresql.config.set_synchronous_standby_names(sync_param) and self._postgresql.state == 'running' and self._postgresql.is_primary()) or has_asterisk: return time.sleep(0.1) # Usually it takes 1ms to reload postgresql.conf, but we will give it 100ms # Reset internal cache to query fresh values self._postgresql.reset_cluster_info_state(None) # timeline == 0 -- indicates that this is the replica if self._postgresql.get_primary_timeline() > 0: self._handle_synchronous_standby_names_change() patroni-4.0.4/patroni/postgresql/validator.py000066400000000000000000000476521472010352700214300ustar00rootroot00000000000000import abc import logging from copy import deepcopy from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union import yaml from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..exceptions import PatroniException from ..utils import parse_bool, parse_int, parse_real from .available_parameters import get_validator_files, PathLikeObj logger = logging.getLogger(__name__) class _Transformable(abc.ABC): def __init__(self, version_from: int, version_till: Optional[int] = None) -> None: self.__version_from = version_from self.__version_till = version_till @classmethod def get_subclasses(cls) -> Iterator[Type['_Transformable']]: """Recursively get all subclasses of :class:`_Transformable`. :yields: each subclass of :class:`_Transformable`. """ for subclass in cls.__subclasses__(): yield from subclass.get_subclasses() yield subclass @property def version_from(self) -> int: return self.__version_from @property def version_till(self) -> Optional[int]: return self.__version_till @abc.abstractmethod def transform(self, name: str, value: Any) -> Optional[Any]: """Verify that provided value is valid. :param name: GUC's name :param value: GUC's value :returns: the value (sometimes clamped) or ``None`` if the value isn't valid """ class Bool(_Transformable): def transform(self, name: str, value: Any) -> Optional[Any]: if parse_bool(value) is not None: return value logger.warning('Removing bool parameter=%s from the config due to the invalid value=%s', name, value) class Number(_Transformable): def __init__(self, *, version_from: int, version_till: Optional[int] = None, min_val: Union[int, float], max_val: Union[int, float], unit: Optional[str] = None) -> None: super(Number, self).__init__(version_from, version_till) self.__min_val = min_val self.__max_val = max_val self.__unit = unit @property def min_val(self) -> Union[int, float]: return self.__min_val @property def max_val(self) -> Union[int, float]: return self.__max_val @property def unit(self) -> Optional[str]: return self.__unit @staticmethod @abc.abstractmethod def parse(value: Any, unit: Optional[str]) -> Optional[Any]: """Convert provided value to unit.""" def transform(self, name: str, value: Any) -> Union[int, float, None]: num_value = self.parse(value, self.unit) if num_value is not None: if num_value < self.min_val: logger.warning('Value=%s of parameter=%s is too low, increasing to %s%s', value, name, self.min_val, self.unit or '') return self.min_val if num_value > self.max_val: logger.warning('Value=%s of parameter=%s is too big, decreasing to %s%s', value, name, self.max_val, self.unit or '') return self.max_val return value logger.warning('Removing %s parameter=%s from the config due to the invalid value=%s', self.__class__.__name__.lower(), name, value) class Integer(Number): @staticmethod def parse(value: Any, unit: Optional[str]) -> Optional[int]: return parse_int(value, unit) class Real(Number): @staticmethod def parse(value: Any, unit: Optional[str]) -> Optional[float]: return parse_real(value, unit) class Enum(_Transformable): def __init__(self, *, version_from: int, version_till: Optional[int] = None, possible_values: Tuple[str, ...]) -> None: super(Enum, self).__init__(version_from, version_till) self.__possible_values = possible_values @property def possible_values(self) -> Tuple[str, ...]: return self.__possible_values def transform(self, name: str, value: Optional[Any]) -> Optional[Any]: if str(value).lower() in self.possible_values: return value logger.warning('Removing enum parameter=%s from the config due to the invalid value=%s', name, value) class EnumBool(Enum): def transform(self, name: str, value: Optional[Any]) -> Optional[Any]: if parse_bool(value) is not None: return value return super(EnumBool, self).transform(name, value) class String(_Transformable): def transform(self, name: str, value: Optional[Any]) -> Optional[Any]: return value # Format: # key - parameter name # value - variable length tuple of `_Transformable` objects. Each object in the tuple represents a different # validation of the GUC across postgres versions. If a GUC validation has never changed over time, then it will # have a single object in the tuple. For example, `password_encryption` used to be a boolean GUC up to Postgres # 10, at which point it started being an enum. In that case the value of `password_encryption` would be a tuple # of 2 `_Transformable` objects (`Bool` and `Enum`, respectively), each one reprensenting a different # validation rule. parameters = CaseInsensitiveDict() recovery_parameters = CaseInsensitiveDict() class ValidatorFactoryNoType(PatroniException): """Raised when a validator spec misses a type.""" class ValidatorFactoryInvalidType(PatroniException): """Raised when a validator spec contains an invalid type.""" class ValidatorFactoryInvalidSpec(PatroniException): """Raised when a validator spec contains an invalid set of attributes.""" class ValidatorFactory: """Factory class used to build Patroni validator objects based on the given specs.""" TYPES: Dict[str, Type[_Transformable]] = {cls.__name__: cls for cls in _Transformable.get_subclasses()} def __new__(cls, validator: Dict[str, Any]) -> _Transformable: """Parse a given Postgres GUC *validator* into the corresponding Patroni validator object. :param validator: a validator spec for a given parameter. It usually comes from a parsed YAML file. :returns: the Patroni validator object that corresponds to the specification found in *validator*. :raises: :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key. :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value. :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the given ``type``. :Example: If a given validator was defined as follows in the YAML file: ```yaml - type: String version_from: 90300 version_till: null ``` Then this method would receive *validator* as: ```python { 'type': 'String', 'version_from': 90300, 'version_till': None } ``` And this method would return a :class:`String`: ```python String(90300, None) ``` """ validator = deepcopy(validator) try: type_ = validator.pop('type') except KeyError as exc: raise ValidatorFactoryNoType('Validator contains no type.') from exc if type_ not in cls.TYPES: raise ValidatorFactoryInvalidType(f'Unexpected validator type: `{type_}`.') for key, value in validator.items(): # :func:`_transform_parameter_value` expects :class:`tuple` instead of :class:`list` if isinstance(value, list): tmp_value: List[Any] = value validator[key] = tuple(tmp_value) try: return cls.TYPES[type_](**validator) except Exception as exc: raise ValidatorFactoryInvalidSpec( f'Failed to parse `{type_}` validator (`{validator}`): `{str(exc)}`.') from exc def _get_postgres_guc_validators(config: Dict[str, Any], parameter: str) -> Tuple[_Transformable, ...]: """Get all validators of *parameter* from *config*. Loop over all validators specs of *parameter* and return them parsed as Patroni validators. :param config: Python object corresponding to an YAML file, with values of either ``parameters`` or ``recovery_parameters`` key. :param parameter: name of the parameter found under *config* which validators should be parsed and returned. :rtype: yields any exception that is faced while parsing a validator spec into a Patroni validator object. """ validators: List[_Transformable] = [] for validator_spec in config.get(parameter, []): try: validator = ValidatorFactory(validator_spec) validators.append(validator) except (ValidatorFactoryNoType, ValidatorFactoryInvalidType, ValidatorFactoryInvalidSpec) as exc: logger.warning('Faced an issue while parsing a validator for parameter `%s`: `%r`', parameter, exc) return tuple(validators) class InvalidGucValidatorsFile(PatroniException): """Raised when reading or parsing of a YAML file faces an issue.""" def _read_postgres_gucs_validators_file(file: PathLikeObj) -> Dict[str, Any]: """Read an YAML file and return the corresponding Python object. :param file: path-like object to read from. It is expected to be encoded with ``UTF-8``, and to be a YAML document. :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then return ``None``. :raises: :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*. """ try: with file.open(encoding='UTF-8') as stream: return yaml.safe_load(stream) except Exception as exc: raise InvalidGucValidatorsFile( f'Unexpected issue while reading parameters file `{file}`: `{str(exc)}`.') from exc def _load_postgres_gucs_validators() -> None: """Load all Postgres GUC validators from YAML files. Recursively walk through ``available_parameters`` directory and load validators of each found YAML file into ``parameters`` and/or ``recovery_parameters`` variables. Walk through directories in top-down fashion and for each of them: * Sort files by name; * Load validators from YAML files that were found. Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the corresponding file or validator will be ignored. By default, Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom Postgres build, then they can create their custom YAML files under ``available_parameters`` directory. Each YAML file may contain either or both of these root attributes, here called sections: * ``parameters``: general GUCs that would be written to ``postgresql.conf``; * ``recovery_parameters``: recovery related GUCs that would be written to ``recovery.conf`` (Patroni later writes them to ``postgresql.conf`` if running PG 12 and above). Then, each of these sections, if specified, may contain one or more attributes with the following structure: * key: the name of a GUC; * value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among: * ``Bool``; or * ``Integer``; or * ``Real``; or * ``Enum``; or * ``EnumBool``; or * ``String``. Besides the ``type`` attribute, it should also contain all the required attributes as per the corresponding class in this module. .. seealso:: * :class:`Bool`; * :class:`Integer`; * :class:`Real`; * :class:`Enum`; * :class:`EnumBool`; * :class:`String`. :Example: This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and sections: .. code-block:: yaml parameters: archive_command: - type: String version_from: 90300 version_till: null archive_mode: - type: Bool version_from: 90300 version_till: 90500 - type: EnumBool version_from: 90500 version_till: null possible_values: - always archive_timeout: - type: Integer version_from: 90300 version_till: null min_val: 0 max_val: 1073741823 unit: s autovacuum_vacuum_cost_delay: - type: Integer version_from: 90300 version_till: 120000 min_val: -1 max_val: 100 unit: ms - type: Real version_from: 120000 version_till: null min_val: -1 max_val: 100 unit: ms client_min_messages: - type: Enum version_from: 90300 version_till: null possible_values: - debug5 - debug4 - debug3 - debug2 - debug1 - log - notice - warning - error recovery_parameters: archive_cleanup_command: - type: String version_from: 90300 version_till: null """ for file in get_validator_files(): try: config: Dict[str, Any] = _read_postgres_gucs_validators_file(file) except InvalidGucValidatorsFile as exc: logger.warning(str(exc)) continue logger.debug(f'Parsing validators from file `{file}`.') mapping = { 'parameters': parameters, 'recovery_parameters': recovery_parameters, } for section in ['parameters', 'recovery_parameters']: section_var = mapping[section] config_section = config.get(section, {}) for parameter in config_section.keys(): section_var[parameter] = _get_postgres_guc_validators(config_section, parameter) _load_postgres_gucs_validators() def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transformable, ...]], version: int, name: str, value: Any, available_gucs: CaseInsensitiveSet) -> Optional[Any]: """Validate *value* of GUC *name* for Postgres *version* using defined *validators* and *available_gucs*. :param validators: a dictionary of all GUCs across all Postgres versions. Each key is the name of a Postgres GUC, and the corresponding value is a variable length tuple of :class:`_Transformable`. Each item is a validation rule for the GUC for a given range of Postgres versions. Should either contain recovery GUCs or general GUCs, not both. :param version: Postgres version to validate the GUC against. :param name: name of the Postgres GUC. :param value: value of the Postgres GUC. :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres GUC. Used to avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres *version*. :returns: the return value may be one among: * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* has a validator in *validators* for the corresponding Postgres *version*; or * ``None`` if *name* does not have a validator in *validators* and is not present in *available_gucs*. """ for validator in validators.get(name, ()) or (): if version >= validator.version_from and\ (validator.version_till is None or version < validator.version_till): return validator.transform(name, value) # Ideally we should have a validator in *validators*. However, if none is available, we will not discard a # setting that exists in Postgres *version*, but rather allow the value with no validation. if name in available_gucs: return value logger.warning('Removing unexpected parameter=%s value=%s from the config', name, value) def transform_postgresql_parameter_value(version: int, name: str, value: Any, available_gucs: CaseInsensitiveSet) -> Optional[Any]: """Validate *value* of GUC *name* for Postgres *version* using ``parameters`` and *available_gucs*. :param version: Postgres version to validate the GUC against. :param name: name of the Postgres GUC. :param value: value of the Postgres GUC. :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres GUC. Used to avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres *version*. :returns: The return value may be one among: * The original *value* if *name* seems to be an extension GUC (contains a period '.'); or * ``None`` if **name** is a recovery GUC; or * *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in ``parameters``. Can also return ``None``. See :func:`_transform_parameter_value`. """ if '.' in name and name not in parameters: # likely an extension GUC, so just return as it is. Otherwise, if `name` is in `parameters`, it's likely a # namespaced GUC from a custom Postgres build, so we treat that over the usual validation means. return value if name in recovery_parameters: return None return _transform_parameter_value(parameters, version, name, value, available_gucs) def transform_recovery_parameter_value(version: int, name: str, value: Any, available_gucs: CaseInsensitiveSet) -> Optional[Any]: """Validate *value* of GUC *name* for Postgres *version* using ``recovery_parameters`` and *available_gucs*. :param version: Postgres version to validate the recovery GUC against. :param name: name of the Postgres recovery GUC. :param value: value of the Postgres recovery GUC. :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres GUC. Used to avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres *version*. :returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`. """ # Recovery settings are not present in ``postgres --describe-config`` output of Postgres <= 11. In that case we # just pass down the list of settings defined in Patroni validators so :func:`_transform_parameter_value` will not # discard the recovery GUCs when running Postgres <= 11. # NOTE: At the moment this change was done Postgres 11 was almost EOL, and had been likely extensively used with # Patroni, so we should be able to rely solely on Patroni validators as the source of truth. return _transform_parameter_value( recovery_parameters, version, name, value, available_gucs if version >= 120000 else CaseInsensitiveSet(recovery_parameters.keys())) patroni-4.0.4/patroni/psycopg.py000066400000000000000000000127611472010352700167150ustar00rootroot00000000000000"""Abstraction layer for :mod:`psycopg` module. This module is able to handle both :mod:`pyscopg2` and :mod:`psycopg`, and it exposes a common interface for both. :mod:`psycopg2` takes precedence. :mod:`psycopg` will only be used if :mod:`psycopg2` is either absent or older than ``2.5.4``. """ from typing import Any, Optional, TYPE_CHECKING, Union if TYPE_CHECKING: # pragma: no cover from psycopg import Connection from psycopg2 import connection, cursor __all__ = ['connect', 'quote_ident', 'quote_literal', 'DatabaseError', 'Error', 'OperationalError', 'ProgrammingError'] _legacy = False try: from psycopg2 import __version__ from . import MIN_PSYCOPG2, parse_version if parse_version(__version__) < MIN_PSYCOPG2: raise ImportError from psycopg2 import connect as _connect, DatabaseError, Error, OperationalError, ProgrammingError from psycopg2.extensions import adapt try: from psycopg2.extensions import quote_ident as _quote_ident except ImportError: _legacy = True def quote_literal(value: Any, conn: Optional[Any] = None) -> str: """Quote *value* as a SQL literal. .. note:: *value* is quoted through :mod:`psycopg2` adapters. :param value: value to be quoted. :param conn: if a connection is given then :func:`quote_literal` checks if any special handling based on server parameters needs to be applied to *value* before quoting it as a SQL literal. :returns: *value* quoted as a SQL literal. """ value = adapt(value) if conn: value.prepare(conn) return value.getquoted().decode('utf-8') except ImportError: import types from psycopg import DatabaseError, Error, OperationalError, ProgrammingError, sql # isort: off from psycopg import connect as __connect # pyright: ignore [reportUnknownVariableType] def __get_parameter_status(self: 'Connection[Any]', param_name: str) -> Optional[str]: """Helper function to be injected into :class:`Connection` object. :param param_name: the name of the connection parameter. :returns: the value for the *param_name* or ``None``. """ return self.info.parameter_status(param_name) def _connect(dsn: Optional[str] = None, **kwargs: Any) -> 'Connection[Any]': """Call :func:`psycopg.connect` with *dsn* and ``**kwargs``. .. note:: Will create following methods and attributes in the returning connection to keep compatibility with the object that would be returned by :func:`psycopg2.connect`: * ``server_version`` attribute. * ``get_parameter_status`` method. :param dsn: DSN to call :func:`psycopg.connect` with. :param kwargs: keyword arguments to call :func:`psycopg.connect` with. :returns: a connection to the database. """ ret: 'Connection[Any]' = __connect(dsn or "", **kwargs) # compatibility with psycopg2 setattr(ret, 'server_version', ret.pgconn.server_version) setattr(ret, 'get_parameter_status', types.MethodType(__get_parameter_status, ret)) return ret def _quote_ident(value: Any, scope: Any) -> str: """Quote *value* as a SQL identifier. :param value: value to be quoted. :param scope: connection to evaluate the returning string into. :returns: *value* quoted as a SQL identifier. """ return sql.Identifier(value).as_string(scope) def quote_literal(value: Any, conn: Optional[Any] = None) -> str: """Quote *value* as a SQL literal. :param value: value to be quoted. :param conn: connection to evaluate the returning string into. :returns: *value* quoted as a SQL literal. """ return sql.Literal(value).as_string(conn) def connect(*args: Any, **kwargs: Any) -> Union['connection', 'Connection[Any]']: """Get a connection to the database. .. note:: The connection will have ``autocommit`` enabled. It also enforces ``search_path=pg_catalog`` for non-replication connections to mitigate security issues as Patroni relies on superuser connections. :param args: positional arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module. :param kwargs: keyword arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module. :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using :mod:`psycopg`, or a :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`. """ if kwargs and 'replication' not in kwargs and kwargs.get('fallback_application_name') != 'Patroni ctl': options = [kwargs['options']] if 'options' in kwargs else [] options.append('-c search_path=pg_catalog') kwargs['options'] = ' '.join(options) ret = _connect(*args, **kwargs) ret.autocommit = True return ret def quote_ident(value: Any, conn: Optional[Union['cursor', 'connection', 'Connection[Any]']] = None) -> str: """Quote *value* as a SQL identifier. :param value: value to be quoted. :param conn: connection to evaluate the returning string into. Can be either a :class:`psycopg.Connection` if using :mod:`psycopg`, or a :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`. :returns: *value* quoted as a SQL identifier. """ if _legacy or conn is None: return '"{0}"'.format(value.replace('"', '""')) return _quote_ident(value, conn) patroni-4.0.4/patroni/quorum.py000066400000000000000000000567051472010352700165670ustar00rootroot00000000000000"""Implement state machine to manage ``synchronous_standby_names`` GUC and ``/sync`` key in DCS.""" import logging from typing import Collection, Iterator, NamedTuple, Optional from .collections import CaseInsensitiveSet from .exceptions import PatroniException logger = logging.getLogger(__name__) class Transition(NamedTuple): """Object describing transition of ``/sync`` or ``synchronous_standby_names`` to the new state. .. note:: Object attributes represent the new state. :ivar transition_type: possible values: * ``sync`` - indicates that we needed to update ``synchronous_standby_names``. * ``quorum`` - indicates that we need to update ``/sync`` key in DCS. * ``restart`` - caller should stop iterating over transitions and restart :class:`QuorumStateResolver`. :ivar leader: the new value of the ``leader`` field in the ``/sync`` key. :ivar num: the new value of the synchronous nodes count in ``synchronous_standby_names`` or value of the ``quorum`` field in the ``/sync`` key for :attr:`transition_type` values ``sync`` and ``quorum`` respectively. :ivar names: the new value of node names listed in ``synchronous_standby_names`` or value of ``voters`` field in the ``/sync`` key for :attr:`transition_type` values ``sync`` and ``quorum`` respectively. """ transition_type: str leader: str num: int names: CaseInsensitiveSet class QuorumError(PatroniException): """Exception indicating that the quorum state is broken.""" class QuorumStateResolver: """Calculates a list of state transitions and yields them as :class:`Transition` named tuples. Synchronous replication state is set in two places: * PostgreSQL configuration sets how many and which nodes are needed for a commit to succeed, abbreviated as ``numsync`` and ``sync`` set here; * DCS contains information about how many and which nodes need to be interrogated to be sure to see an wal position containing latest confirmed commit, abbreviated as ``quorum`` and ``voters`` set. .. note:: Both of above pairs have the meaning "ANY n OF set". The number of nodes needed for commit to succeed, ``numsync``, is also called the replication factor. To guarantee zero transaction loss on failover we need to keep the invariant that at all times any subset of nodes that can acknowledge a commit overlaps with any subset of nodes that can achieve quorum to promote a new leader. Given a desired replication factor and a set of nodes able to participate in sync replication there is one optimal state satisfying this condition. Given the node set ``active``, the optimal state is:: sync = voters = active numsync = min(sync_wanted, len(active)) quorum = len(active) - numsync We need to be able to produce a series of state changes that take the system to this desired state from any other arbitrary state given arbitrary changes is node availability, configuration and interrupted transitions. To keep the invariant the rule to follow is that when increasing ``numsync`` or ``quorum``, we need to perform the increasing operation first. When decreasing either, the decreasing operation needs to be performed later. In other words: * If a user increases ``synchronous_node_count`` configuration, first we increase ``synchronous_standby_names`` (``numsync``), then we decrease ``quorum`` field in the ``/sync`` key; * If a user decreases ``synchronous_node_count`` configuration, first we increase ``quorum`` field in the ``/sync`` key, then we decrease ``synchronous_standby_names`` (``numsync``). Order of adding or removing nodes from ``sync`` and ``voters`` depends on the state of ``synchronous_standby_names``. When adding new nodes:: if ``sync`` (``synchronous_standby_names``) is empty: add new nodes first to ``sync`` and then to ``voters`` when ``numsync_confirmed`` > ``0``. else: add new nodes first to ``voters`` and then to ``sync``. When removing nodes:: if ``sync`` (``synchronous_standby_names``) will become empty after removal: first remove nodes from ``voters`` and then from ``sync``. else: first remove nodes from ``sync`` and then from ``voters``. Make ``voters`` empty if ``numsync_confirmed`` == ``0``. :ivar leader: name of the leader, according to the ``/sync`` key. :ivar quorum: ``quorum`` value from the ``/sync`` key, the minimal number of nodes we need see when doing the leader race. :ivar voters: ``sync_standby`` value from the ``/sync`` key, set of node names we will be running the leader race against. :ivar numsync: the number of synchronous nodes from the ``synchronous_standby_names``. :ivar sync: set of node names listed in the ``synchronous_standby_names``. :ivar numsync_confirmed: the number of nodes that are confirmed to reach "safe" LSN after they were added to the ``synchronous_standby_names``. :ivar active: set of node names that are replicating from the primary (according to ``pg_stat_replication``) and are eligible to be listed in ``synchronous_standby_names``. :ivar sync_wanted: desired number of synchronous nodes (``synchronous_node_count`` from the global configuration). :ivar leader_wanted: the desired leader (could be different from the :attr:`leader` right after a failover). """ def __init__(self, leader: str, quorum: int, voters: Collection[str], numsync: int, sync: Collection[str], numsync_confirmed: int, active: Collection[str], sync_wanted: int, leader_wanted: str) -> None: """Instantiate :class:``QuorumStateResolver`` based on input parameters. :param leader: name of the leader, according to the ``/sync`` key. :param quorum: ``quorum`` value from the ``/sync`` key, the minimal number of nodes we need see when doing the leader race. :param voters: ``sync_standby`` value from the ``/sync`` key, set of node names we will be running the leader race against. :param numsync: the number of synchronous nodes from the ``synchronous_standby_names``. :param sync: Set of node names listed in the ``synchronous_standby_names``. :param numsync_confirmed: the number of nodes that are confirmed to reach "safe" LSN after they were added to the ``synchronous_standby_names``. :param active: set of node names that are replicating from the primary (according to ``pg_stat_replication``) and are eligible to be listed in ``synchronous_standby_names``. :param sync_wanted: desired number of synchronous nodes (``synchronous_node_count`` from the global configuration). :param leader_wanted: the desired leader (could be different from the *leader* right after a failover). """ self.leader = leader self.quorum = quorum self.voters = CaseInsensitiveSet(voters) self.numsync = min(numsync, len(sync)) # numsync can't be bigger than number of listed synchronous nodes. self.sync = CaseInsensitiveSet(sync) self.numsync_confirmed = numsync_confirmed self.active = CaseInsensitiveSet(active) self.sync_wanted = sync_wanted self.leader_wanted = leader_wanted def check_invariants(self) -> None: """Checks invariant of ``synchronous_standby_names`` and ``/sync`` key in DCS. .. seealso:: Check :class:`QuorumStateResolver`'s docstring for more information. :raises: :exc:`QuorumError`: in case of broken state""" voters = CaseInsensitiveSet(self.voters | CaseInsensitiveSet([self.leader])) sync = CaseInsensitiveSet(self.sync | CaseInsensitiveSet([self.leader_wanted])) # We need to verify that subset of nodes that can acknowledge a commit overlaps # with any subset of nodes that can achieve quorum to promote a new leader. # ``+ 1`` is required because the leader is included in the set. if self.voters and not (len(voters | sync) <= self.quorum + self.numsync + 1): len_nodes = len(voters | sync) raise QuorumError("Quorum and sync not guaranteed to overlap: " f"nodes {len_nodes} >= quorum {self.quorum} + sync {self.sync} + 1") # unstable cases, we are changing synchronous_standby_names and /sync key # one after another, hence one set is allowed to be a subset of another if not (voters.issubset(sync) or sync.issubset(voters)): voters_only = voters - sync sync_only = sync - voters raise QuorumError(f"Mismatched sets: voter only={voters_only} sync only={sync_only}") def quorum_update(self, quorum: int, voters: CaseInsensitiveSet, leader: Optional[str] = None, adjust_quorum: Optional[bool] = True) -> Iterator[Transition]: """Updates :attr:`quorum`, :attr:`voters` and optionally :attr:`leader` fields. :param quorum: the new value for :attr:`quorum`, could be adjusted depending on values of :attr:`numsync_confirmed` and *adjust_quorum*. :param voters: the new value for :attr:`voters`, could be adjusted if :attr:`numsync_confirmed` == ``0``. :param leader: the new value for :attr:`leader`, optional. :param adjust_quorum: if set to ``True`` the quorum requirement will be increased by the difference between :attr:`numsync` and :attr:`numsync_confirmed`. :yields: the new state of the ``/sync`` key as a :class:`Transition` object. :raises: :exc:`QuorumError` in case of invalid data or if the invariant after transition could not be satisfied. """ if quorum < 0: raise QuorumError(f'Quorum {quorum} < 0 of ({voters})') if quorum > 0 and quorum >= len(voters): raise QuorumError(f'Quorum {quorum} >= N of ({voters})') old_leader = self.leader if leader is not None: # Change of leader was requested self.leader = leader elif self.numsync_confirmed == 0: # If there are no nodes that known to caught up with the primary we want to reset quorum/voters in /sync key quorum = 0 voters = CaseInsensitiveSet() elif adjust_quorum: # It could be that the number of nodes that are known to catch up with the primary is below desired numsync. # We want to increase quorum to guarantee that the sync node will be found during the leader race. quorum += max(self.numsync - self.numsync_confirmed, 0) if (self.leader, quorum, voters) == (old_leader, self.quorum, self.voters): if self.voters: return # If transition produces no change of leader/quorum/voters we want to give a hint to # the caller to fetch the new state from the database and restart QuorumStateResolver. yield Transition('restart', self.leader, self.quorum, self.voters) self.quorum = quorum self.voters = voters self.check_invariants() logger.debug('quorum %s %s %s', self.leader, self.quorum, self.voters) yield Transition('quorum', self.leader, self.quorum, self.voters) def sync_update(self, numsync: int, sync: CaseInsensitiveSet) -> Iterator[Transition]: """Updates :attr:`numsync` and :attr:`sync` fields. :param numsync: the new value for :attr:`numsync`. :param sync: the new value for :attr:`sync`: :yields: the new state of ``synchronous_standby_names`` as a :class:`Transition` object. :raises: :exc:`QuorumError` in case of invalid data or if invariant after transition could not be satisfied """ if numsync < 0: raise QuorumError(f'Sync {numsync} < 0 of ({sync})') if numsync > len(sync): raise QuorumError(f'Sync {numsync} > N of ({sync})') self.numsync = numsync self.sync = sync self.check_invariants() logger.debug('sync %s %s %s', self.leader, self.numsync, self.sync) yield Transition('sync', self.leader, self.numsync, self.sync) def __iter__(self) -> Iterator[Transition]: """Iterate over the transitions produced by :meth:`_generate_transitions`. .. note:: Merge two transitions of the same type to a single one. This is always safe because skipping the first transition is equivalent to no one observing the intermediate state. :yields: transitions as :class:`Transition` objects. """ transitions = list(self._generate_transitions()) for cur_transition, next_transition in zip(transitions, transitions[1:] + [None]): if isinstance(next_transition, Transition) \ and cur_transition.transition_type == next_transition.transition_type: continue yield cur_transition if cur_transition.transition_type == 'restart': break def __handle_non_steady_cases(self) -> Iterator[Transition]: """Handle cases when set of transitions produced on previous run was interrupted. :yields: transitions as :class:`Transition` objects. """ if self.sync < self.voters: logger.debug("Case 1: synchronous_standby_names %s is a subset of DCS state %s", self.sync, self.voters) # Case 1: voters is superset of sync nodes. In the middle of changing voters (quorum). # Evict dead nodes from voters that are not being synced. remove_from_voters = self.voters - (self.sync | self.active) if remove_from_voters: yield from self.quorum_update( quorum=len(self.voters) - len(remove_from_voters) - self.numsync, voters=CaseInsensitiveSet(self.voters - remove_from_voters), adjust_quorum=not (self.sync - self.active)) # Start syncing to nodes that are in voters and alive add_to_sync = (self.voters & self.active) - self.sync if add_to_sync: yield from self.sync_update(self.numsync, CaseInsensitiveSet(self.sync | add_to_sync)) elif self.sync > self.voters: logger.debug("Case 2: synchronous_standby_names %s is a superset of DCS state %s", self.sync, self.voters) # Case 2: sync is superset of voters nodes. In the middle of changing replication factor (sync). # Add to voters nodes that are already synced and active add_to_voters = (self.sync - self.voters) & self.active if add_to_voters: voters = CaseInsensitiveSet(self.voters | add_to_voters) yield from self.quorum_update(len(voters) - self.numsync, voters) # Remove from sync nodes that are dead remove_from_sync = self.sync - self.voters if remove_from_sync: yield from self.sync_update( numsync=min(self.numsync, len(self.sync) - len(remove_from_sync)), sync=CaseInsensitiveSet(self.sync - remove_from_sync)) # After handling these two cases voters and sync must match. assert self.voters == self.sync safety_margin = self.quorum + min(self.numsync, self.numsync_confirmed) - len(self.voters | self.sync) if safety_margin > 0: # In the middle of changing replication factor. if self.numsync > self.sync_wanted: numsync = max(self.sync_wanted, len(self.voters) - self.quorum) logger.debug('Case 3: replication factor %d is bigger than needed %d', self.numsync, numsync) yield from self.sync_update(numsync, self.sync) else: quorum = len(self.sync) - self.numsync logger.debug('Case 4: quorum %d is bigger than needed %d', self.quorum, quorum) yield from self.quorum_update(quorum, self.voters) else: safety_margin = self.quorum + self.numsync - len(self.voters | self.sync) if self.numsync == self.sync_wanted and safety_margin > 0 and self.numsync > self.numsync_confirmed: yield from self.quorum_update(len(self.sync) - self.numsync, self.voters) def __remove_gone_nodes(self) -> Iterator[Transition]: """Remove inactive nodes from ``synchronous_standby_names`` and from ``/sync`` key. :yields: transitions as :class:`Transition` objects. """ to_remove = self.sync - self.active if to_remove and self.sync == to_remove: logger.debug("Removing nodes: %s", to_remove) yield from self.quorum_update(0, CaseInsensitiveSet(), adjust_quorum=False) yield from self.sync_update(0, CaseInsensitiveSet()) elif to_remove: logger.debug("Removing nodes: %s", to_remove) can_reduce_quorum_by = self.quorum # If we can reduce quorum size try to do so first if can_reduce_quorum_by: # Pick nodes to remove by sorted order to provide deterministic behavior for tests remove = CaseInsensitiveSet(sorted(to_remove, reverse=True)[:can_reduce_quorum_by]) sync = CaseInsensitiveSet(self.sync - remove) # when removing nodes from sync we can safely increase numsync if requested numsync = min(self.sync_wanted, len(sync)) if self.sync_wanted > self.numsync else self.numsync yield from self.sync_update(numsync, sync) voters = CaseInsensitiveSet(self.voters - remove) to_remove &= self.sync yield from self.quorum_update(len(voters) - self.numsync, voters, adjust_quorum=not to_remove) if to_remove: assert self.quorum == 0 numsync = self.numsync - len(to_remove) sync = CaseInsensitiveSet(self.sync - to_remove) voters = CaseInsensitiveSet(self.voters - to_remove) sync_decrease = numsync - min(self.sync_wanted, len(sync)) quorum = min(sync_decrease, len(voters) - 1) if sync_decrease else 0 yield from self.quorum_update(quorum, voters, adjust_quorum=False) yield from self.sync_update(numsync, sync) def __add_new_nodes(self) -> Iterator[Transition]: """Add new active nodes to ``synchronous_standby_names`` and to ``/sync`` key. :yields: transitions as :class:`Transition` objects. """ to_add = self.active - self.sync if to_add: # First get to requested replication factor logger.debug("Adding nodes: %s", to_add) sync_wanted = min(self.sync_wanted, len(self.sync | to_add)) increase_numsync_by = sync_wanted - self.numsync if increase_numsync_by > 0: if self.sync: add = CaseInsensitiveSet(sorted(to_add)[:increase_numsync_by]) increase_numsync_by = len(add) else: # there is only the leader add = to_add # and it is safe to add all nodes at once if sync is empty yield from self.sync_update(self.numsync + increase_numsync_by, CaseInsensitiveSet(self.sync | add)) voters = CaseInsensitiveSet(self.voters | add) yield from self.quorum_update(len(voters) - sync_wanted, voters) to_add -= self.sync if to_add: voters = CaseInsensitiveSet(self.voters | to_add) yield from self.quorum_update(len(voters) - sync_wanted, voters, adjust_quorum=sync_wanted > self.numsync_confirmed) yield from self.sync_update(sync_wanted, CaseInsensitiveSet(self.sync | to_add)) def __handle_replication_factor_change(self) -> Iterator[Transition]: """Handle change of the replication factor (:attr:`sync_wanted`, aka ``synchronous_node_count``). :yields: transitions as :class:`Transition` objects. """ # Apply requested replication factor change sync_increase = min(self.sync_wanted, len(self.sync)) - self.numsync if sync_increase > 0: # Increase replication factor logger.debug("Increasing replication factor to %s", self.numsync + sync_increase) yield from self.sync_update(self.numsync + sync_increase, self.sync) yield from self.quorum_update(len(self.voters) - self.numsync, self.voters) elif sync_increase < 0: # Reduce replication factor logger.debug("Reducing replication factor to %s", self.numsync + sync_increase) if self.quorum - sync_increase < len(self.voters): yield from self.quorum_update(len(self.voters) - self.numsync - sync_increase, self.voters, adjust_quorum=self.sync_wanted > self.numsync_confirmed) yield from self.sync_update(self.numsync + sync_increase, self.sync) def _generate_transitions(self) -> Iterator[Transition]: """Produce a set of changes to safely transition from the current state to the desired. :yields: transitions as :class:`Transition` objects. """ logger.debug("Quorum state: leader %s quorum %s, voters %s, numsync %s, sync %s, " "numsync_confirmed %s, active %s, sync_wanted %s leader_wanted %s", self.leader, self.quorum, self.voters, self.numsync, self.sync, self.numsync_confirmed, self.active, self.sync_wanted, self.leader_wanted) try: if self.leader_wanted != self.leader: # failover voters = (self.voters - CaseInsensitiveSet([self.leader_wanted])) | CaseInsensitiveSet([self.leader]) if not self.sync: # If sync is empty we need to update synchronous_standby_names first numsync = len(voters) - self.quorum yield from self.sync_update(numsync, CaseInsensitiveSet(voters)) # If leader changed we need to add the old leader to quorum (voters) yield from self.quorum_update(self.quorum, CaseInsensitiveSet(voters), self.leader_wanted) # right after promote there could be no replication connections yet if not self.sync & self.active: return # give another loop_wait seconds for replicas to reconnect before removing them from quorum else: self.check_invariants() except QuorumError as e: logger.warning('%s', e) yield from self.quorum_update(len(self.sync) - self.numsync, self.sync) assert self.leader == self.leader_wanted # numsync_confirmed could be 0 after restart/failover, we will calculate it from quorum if self.numsync_confirmed == 0 and self.sync & self.active: self.numsync_confirmed = min(len(self.sync & self.active), len(self.voters) - self.quorum) logger.debug('numsync_confirmed=0, adjusting it to %d', self.numsync_confirmed) yield from self.__handle_non_steady_cases() # We are in a steady state point. Find if desired state is different and act accordingly. yield from self.__remove_gone_nodes() yield from self.__add_new_nodes() yield from self.__handle_replication_factor_change() patroni-4.0.4/patroni/raft_controller.py000066400000000000000000000015621472010352700204250ustar00rootroot00000000000000import logging from .config import Config from .daemon import abstract_main, AbstractPatroniDaemon, get_base_arg_parser from .dcs.raft import KVStoreTTL logger = logging.getLogger(__name__) class RaftController(AbstractPatroniDaemon): def __init__(self, config: Config) -> None: super(RaftController, self).__init__(config) kvstore_config = self.config.get('raft') assert 'self_addr' in kvstore_config self._raft = KVStoreTTL(None, None, None, **kvstore_config) def _run_cycle(self) -> None: try: self._raft.doTick(self._raft.conf.autoTickPeriod) except Exception: logger.exception('doTick') def _shutdown(self) -> None: self._raft.destroy() def main() -> None: parser = get_base_arg_parser() args = parser.parse_args() abstract_main(RaftController, args.configfile) patroni-4.0.4/patroni/request.py000066400000000000000000000175631472010352700167260ustar00rootroot00000000000000"""Facilities for handling communication with Patroni's REST API.""" import json from typing import Any, Dict, Optional, Union import urllib3 from .config import Config from .dcs import Member from .utils import USER_AGENT class HTTPSConnectionPool(urllib3.HTTPSConnectionPool): def _validate_conn(self, *args: Any, **kwargs: Any) -> None: """Override parent method to silence warnings about requests without certificate verification enabled.""" class PatroniPoolManager(urllib3.PoolManager): def __init__(self, *args: Any, **kwargs: Any) -> None: super(PatroniPoolManager, self).__init__(*args, **kwargs) self.pool_classes_by_scheme = {'http': urllib3.HTTPConnectionPool, 'https': HTTPSConnectionPool} class PatroniRequest(object): """Wrapper for performing requests to Patroni's REST API. Prepares the request manager with the configured settings before performing the request. """ def __init__(self, config: Union[Config, Dict[str, Any]], insecure: Optional[bool] = None) -> None: """Create a new :class:`PatroniRequest` instance with given *config*. :param config: Patroni YAML configuration. :param insecure: how to deal with SSL certs verification: * If ``True`` it will perform REST API requests without verifying SSL certs; or * If ``False`` it will perform REST API requests and verify SSL certs; or * If ``None`` it will behave according to the value of ``ctl.insecure`` configuration; or * If none of the above applies, then it falls back to ``False``. """ self._insecure = insecure self._pool = PatroniPoolManager(num_pools=10, maxsize=10) self.reload_config(config) @staticmethod def _get_ctl_value(config: Union[Config, Dict[str, Any]], name: str, default: Any = None) -> Optional[Any]: """Get value of *name* setting from the ``ctl`` section of the *config*. :param config: Patroni YAML configuration. :param name: name of the setting value to be retrieved. :returns: value of ``ctl.*name*`` if present, ``None`` otherwise. """ return config.get('ctl', {}).get(name, default) @staticmethod def _get_restapi_value(config: Union[Config, Dict[str, Any]], name: str) -> Optional[Any]: """Get value of *name* setting from the ``restapi`` section of the *config*. :param config: Patroni YAML configuration. :param name: name of the setting value to be retrieved. :returns: value of ``restapi -> *name*`` if present, ``None`` otherwise. """ return config.get('restapi', {}).get(name) def _apply_pool_param(self, param: str, value: Any) -> None: """Configure *param* as *value* in the request manager. :param param: name of the setting to be changed. :param value: new value for *param*. If ``None``, ``0``, ``False``, and similar values, then explicit *param* declaration is removed, in which case it takes its default value, if any. """ if value: self._pool.connection_pool_kw[param] = value else: self._pool.connection_pool_kw.pop(param, None) def _apply_ssl_file_param(self, config: Union[Config, Dict[str, Any]], name: str) -> Union[str, None]: """Apply a given SSL related param to the request manager. :param config: Patroni YAML configuration. :param name: prefix of the Patroni SSL related setting name. Currently, supports these: * ``cert``: gets translated to ``certfile`` * ``key``: gets translated to ``keyfile`` Will attempt to fetch the requested key first from ``ctl`` section. :returns: value of ``ctl.*name*file`` if present, ``None`` otherwise. """ value = self._get_ctl_value(config, name + 'file') self._apply_pool_param(name + '_file', value) return value def reload_config(self, config: Union[Config, Dict[str, Any]]) -> None: """Apply *config* to request manager. Configure these HTTP headers for requests: * ``authorization``: based on Patroni' CTL or REST API authentication config; * ``user-agent``: based on ``patroni.utils.USER_AGENT``. Also configure SSL related settings for requests: * ``ca_certs`` is configured if ``ctl.cacert`` or ``restapi.cafile`` is available; * ``cert``, ``key`` and ``key_password`` are configured if ``ctl.certfile`` is available. :param config: Patroni YAML configuration. """ # ``ctl -> auth`` is equivalent to ``ctl -> authentication -> username`` + ``:`` + # ``ctl -> authentication -> password``. And the same for ``restapi -> auth`` basic_auth = self._get_ctl_value(config, 'auth') or self._get_restapi_value(config, 'auth') self._pool.headers = urllib3.make_headers(basic_auth=basic_auth, user_agent=USER_AGENT) self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED' insecure = self._insecure if isinstance(self._insecure, bool)\ else self._get_ctl_value(config, 'insecure', False) if self._apply_ssl_file_param(config, 'cert'): if insecure: # The assert_hostname = False helps to silence warnings self._pool.connection_pool_kw['assert_hostname'] = False self._apply_ssl_file_param(config, 'key') password = self._get_ctl_value(config, 'keyfile_password') self._apply_pool_param('key_password', password) else: if insecure: # Disable server certificate validation if requested self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE' self._pool.connection_pool_kw.pop('assert_hostname', None) self._pool.connection_pool_kw.pop('key_file', None) cacert = self._get_ctl_value(config, 'cacert') or self._get_restapi_value(config, 'cafile') self._apply_pool_param('ca_certs', cacert) def request(self, method: str, url: str, body: Optional[Any] = None, **kwargs: Any) -> urllib3.response.HTTPResponse: """Perform an HTTP request. :param method: the HTTP method to be used, e.g. ``GET``. :param url: the URL to be requested. :param body: anything to be used as the request body. :param kwargs: keyword arguments to be passed to :func:`urllib3.PoolManager.request`. :returns: the response returned upon request. """ if body is not None and not isinstance(body, str): body = json.dumps(body) return self._pool.request(method.upper(), url, body=body, **kwargs) def __call__(self, member: Member, method: str = 'GET', endpoint: Optional[str] = None, data: Optional[Any] = None, **kwargs: Any) -> urllib3.response.HTTPResponse: """Turn :class:`PatroniRequest` into a callable object. When called, perform a request through the manager. :param member: DCS member so we can fetch from it the configured base URL for the REST API. :param method: HTTP method to be used, e.g. ``GET``. :param endpoint: URL path of this request, e.g. ``switchover``. :param data: anything to be used as the request body. :returns: the response returned upon request. """ url = member.get_endpoint_url(endpoint) return self.request(method, url, data, **kwargs) def get(url: str, verify: bool = True, **kwargs: Any) -> urllib3.response.HTTPResponse: """Perform an HTTP GET request. .. note:: It uses :class:`PatroniRequest` so all relevant configuration is applied before processing the request. :param url: full URL for this GET request. :param verify: if it should verify SSL certificates when processing the request. :returns: the response returned from the request. """ http = PatroniRequest({}, not verify) return http.request('GET', url, **kwargs) patroni-4.0.4/patroni/scripts/000077500000000000000000000000001472010352700163375ustar00rootroot00000000000000patroni-4.0.4/patroni/scripts/__init__.py000066400000000000000000000000001472010352700204360ustar00rootroot00000000000000patroni-4.0.4/patroni/scripts/aws.py000077500000000000000000000062431472010352700175130ustar00rootroot00000000000000#!/usr/bin/env python import json import logging import sys from typing import Any, Optional import boto3 from botocore.exceptions import ClientError from botocore.utils import IMDSFetcher from ..utils import Retry, RetryFailedError logger = logging.getLogger(__name__) class AWSConnection(object): def __init__(self, cluster_name: Optional[str]) -> None: self.available = False self.cluster_name = cluster_name if cluster_name is not None else 'unknown' self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=ClientError) try: # get the instance id fetcher = IMDSFetcher(timeout=2.1) token = fetcher._fetch_metadata_token() r = fetcher._get_request("/latest/dynamic/instance-identity/document", None, token) except Exception: logger.error('cannot query AWS meta-data') return if r.status_code < 400: try: content = json.loads(r.text) self.instance_id = content['instanceId'] self.region = content['region'] except Exception: logger.exception('unable to fetch instance id and region from AWS meta-data') return self.available = True def retry(self, *args: Any, **kwargs: Any) -> Any: return self._retry.copy()(*args, **kwargs) def aws_available(self) -> bool: return self.available def _tag_ebs(self, conn: Any, role: str) -> None: """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """ tags = [{'Key': 'Name', 'Value': 'spilo_' + self.cluster_name}, {'Key': 'Role', 'Value': role}, {'Key': 'Instance', 'Value': self.instance_id}] volumes = conn.volumes.filter(Filters=[{'Name': 'attachment.instance-id', 'Values': [self.instance_id]}]) conn.create_tags(Resources=[v.id for v in volumes], Tags=tags) def _tag_ec2(self, conn: Any, role: str) -> None: """ tag the current EC2 instance with a cluster role """ tags = [{'Key': 'Role', 'Value': role}] conn.create_tags(Resources=[self.instance_id], Tags=tags) def on_role_change(self, new_role: str) -> bool: if not self.available: return False try: conn = boto3.resource('ec2', region_name=self.region) # type: ignore self.retry(self._tag_ec2, conn, new_role) self.retry(self._tag_ebs, conn, new_role) except RetryFailedError: logger.warning("Unable to communicate to AWS " "when setting tags for the EC2 instance {0} " "and attached EBS volumes".format(self.instance_id)) return False return True def main(): logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO) if len(sys.argv) == 4 and sys.argv[1] in ('on_start', 'on_stop', 'on_role_change'): AWSConnection(cluster_name=sys.argv[3]).on_role_change(sys.argv[2]) else: sys.exit("Usage: {0} action role name".format(sys.argv[0])) if __name__ == '__main__': main() patroni-4.0.4/patroni/scripts/barman/000077500000000000000000000000001472010352700175775ustar00rootroot00000000000000patroni-4.0.4/patroni/scripts/barman/__init__.py000066400000000000000000000000541472010352700217070ustar00rootroot00000000000000"""Create :mod:`patroni.scripts.barman`.""" patroni-4.0.4/patroni/scripts/barman/cli.py000066400000000000000000000167211472010352700207270ustar00rootroot00000000000000#!/usr/bin/env python """Perform operations on Barman through ``pg-backup-api``. The actual operations are implemented by separate modules. This module only builds the CLI that makes an interface with the actual commands. .. note:: See :class:ExitCode` for possible exit codes of this main script. """ import logging import sys from argparse import ArgumentParser from enum import IntEnum from .config_switch import run_barman_config_switch from .recover import run_barman_recover from .utils import ApiNotOk, PgBackupApi, set_up_logging class ExitCode(IntEnum): """Possible exit codes of this script. :cvar NO_COMMAND: if no sub-command of ``patroni_barman`` application has been selected by the user. :cvar API_NOT_OK: ``pg-backup-api`` status is not ``OK``. """ NO_COMMAND = -1 API_NOT_OK = -2 def main() -> None: """Entry point of ``patroni_barman`` application. Implements the parser for the application and for its sub-commands. The script exit code may be one of: * :attr:`ExitCode.NO_COMMAND`: if no sub-command was specified in the ``patroni_barman`` call; * :attr:`ExitCode.API_NOT_OK`: if ``pg-backup-api`` is not correctly up and running; * Value returned by :func:`~patroni.scripts.barman.config_switch.run_barman_config_switch`, if running ``patroni_barman config-switch``; * Value returned by :func:`~patroni.scripts.barman.recover.run_barman_recover`, if running ``patroni_barman recover``. The called sub-command is expected to exit execution once finished using its own set of exit codes. """ parser = ArgumentParser( description=( "Wrapper application for pg-backup-api. Communicate with the API " "running at the given URL to perform remote Barman operations." ), ) parser.add_argument( "--api-url", type=str, required=True, help="URL to reach the pg-backup-api, e.g. 'http://localhost:7480'", dest="api_url", ) parser.add_argument( "--cert-file", type=str, required=False, help="Certificate to authenticate against the API, if required.", dest="cert_file", ) parser.add_argument( "--key-file", type=str, required=False, help="Certificate key to authenticate against the API, if required.", dest="key_file", ) parser.add_argument( "--retry-wait", type=int, required=False, default=2, help="How long in seconds to wait before retrying a failed " "pg-backup-api request (default: '%(default)s')", dest="retry_wait", ) parser.add_argument( "--max-retries", type=int, required=False, default=5, help="Maximum number of retries when receiving malformed responses " "from the pg-backup-api (default: '%(default)s')", dest="max_retries", ) parser.add_argument( "--log-file", type=str, required=False, help="File where to log messages produced by this application, if any.", dest="log_file", ) subparsers = parser.add_subparsers(title="Sub-commands") recover_parser = subparsers.add_parser( "recover", help="Remote 'barman recover'", description="Restore a Barman backup of a given Barman server" ) recover_parser.add_argument( "--barman-server", type=str, required=True, help="Name of the Barman server from which to restore the backup.", dest="barman_server", ) recover_parser.add_argument( "--backup-id", type=str, required=False, default="latest", help="ID of the Barman backup to be restored. You can use any value " "supported by 'barman recover' command " "(default: '%(default)s')", dest="backup_id", ) recover_parser.add_argument( "--ssh-command", type=str, required=True, help="Value to be passed as '--remote-ssh-command' to 'barman recover'.", dest="ssh_command", ) recover_parser.add_argument( "--data-directory", "--datadir", type=str, required=True, help="Destination path where to restore the barman backup in the " "local host.", dest="data_directory", ) recover_parser.add_argument( "--loop-wait", type=int, required=False, default=10, help="How long to wait before checking again the status of the " "recovery process, in seconds. Use higher values if your " "recovery is expected to take long (default: '%(default)s')", dest="loop_wait", ) recover_parser.set_defaults(func=run_barman_recover) config_switch_parser = subparsers.add_parser( "config-switch", help="Remote 'barman config-switch'", description="Switch the configuration of a given Barman server. " "Intended to be used as a 'on_role_change' callback." ) config_switch_parser.add_argument( "action", type=str, choices=["on_role_change"], help="Name of the callback (automatically filled by Patroni)", ) config_switch_parser.add_argument( "role", type=str, choices=["primary", "promoted", "standby_leader", "replica", "demoted"], help="Name of the new role of this node (automatically filled by " "Patroni)", ) config_switch_parser.add_argument( "cluster", type=str, help="Name of the Patroni cluster involved in the callback " "(automatically filled by Patroni)", ) config_switch_parser.add_argument( "--barman-server", type=str, required=True, help="Name of the Barman server which config is to be switched.", dest="barman_server", ) group = config_switch_parser.add_mutually_exclusive_group(required=True) group.add_argument( "--barman-model", type=str, help="Name of the Barman config model to be applied to the server.", dest="barman_model", ) group.add_argument( "--reset", action="store_true", help="Unapply the currently active model for the server, if any.", dest="reset", ) config_switch_parser.add_argument( "--switch-when", type=str, required=True, default="promoted", choices=["promoted", "demoted", "always"], help="Controls under which circumstances the 'on_role_change' callback " "should actually switch config in Barman. 'promoted' means the " "'role' is either 'primary' or 'promoted'. 'demoted' " "means the 'role' is either 'replica' or 'demoted' " "(default: '%(default)s')", dest="switch_when", ) config_switch_parser.set_defaults(func=run_barman_config_switch) args, _ = parser.parse_known_args() set_up_logging(args.log_file) if not hasattr(args, "func"): parser.print_help() sys.exit(ExitCode.NO_COMMAND) api = None try: api = PgBackupApi(args.api_url, args.cert_file, args.key_file, args.retry_wait, args.max_retries) except ApiNotOk as exc: logging.error("pg-backup-api is not working: %r", exc) sys.exit(ExitCode.API_NOT_OK) sys.exit(args.func(api, args)) if __name__ == "__main__": main() patroni-4.0.4/patroni/scripts/barman/config_switch.py000066400000000000000000000120571472010352700230040ustar00rootroot00000000000000#!/usr/bin/env python """Implements ``patroni_barman config-switch`` sub-command. Apply a Barman configuration model through ``pg-backup-api``. This sub-command is specially useful as a ``on_role_change`` callback to change Barman configuration in response to failovers and switchovers. Check the output of ``--help`` to understand the parameters supported by the sub-command. It requires that you have previously configured a Barman server and Barman config models, and that you have ``pg-backup-api`` configured and running in the same host as Barman. Refer to :class:`ExitCode` for possible exit codes of this sub-command. """ import logging import time from argparse import Namespace from enum import IntEnum from typing import Optional, TYPE_CHECKING from .utils import OperationStatus, RetriesExceeded if TYPE_CHECKING: # pragma: no cover from .utils import PgBackupApi class ExitCode(IntEnum): """Possible exit codes of this script. :cvar CONFIG_SWITCH_DONE: config switch was successfully performed. :cvar CONFIG_SWITCH_SKIPPED: if the execution was skipped because of not matching user expectations. :cvar CONFIG_SWITCH_FAILED: config switch faced an issue. :cvar HTTP_ERROR: an error has occurred while communicating with ``pg-backup-api`` :cvar INVALID_ARGS: an invalid set of arguments has been given to the operation. """ CONFIG_SWITCH_DONE = 0 CONFIG_SWITCH_SKIPPED = 1 CONFIG_SWITCH_FAILED = 2 HTTP_ERROR = 3 INVALID_ARGS = 4 def _should_skip_switch(args: Namespace) -> bool: """Check if we should skip the config switch operation. :param args: arguments received from the command-line of ``patroni_barman config-switch`` command. :returns: if the operation should be skipped. """ if args.switch_when == "promoted": return args.role not in {"primary", "promoted"} if args.switch_when == "demoted": return args.role not in {"replica", "demoted"} return False def _switch_config(api: "PgBackupApi", barman_server: str, barman_model: Optional[str], reset: Optional[bool]) -> int: """Switch configuration of Barman server through ``pg-backup-api``. .. note:: If requests to ``pg-backup-api`` fail recurrently or we face HTTP errors, then exit with :attr:`ExitCode.HTTP_ERROR`. :param api: a :class:`PgBackupApi` instance to handle communication with the API. :param barman_server: name of the Barman server which config is to be switched. :param barman_model: name of the Barman model to be applied to the server, if any. :param reset: ``True`` if you would like to unapply the currently active model for the server, if any. :returns: the return code to be used when exiting the ``patroni_barman`` application. Refer to :class:`ExitCode`. """ operation_id = None try: operation_id = api.create_config_switch_operation( barman_server, barman_model, reset, ) except RetriesExceeded as exc: logging.error("An issue was faced while trying to create a config " "switch operation: %r", exc) return ExitCode.HTTP_ERROR logging.info("Created the config switch operation with ID %s", operation_id) status = None while True: try: status = api.get_operation_status(barman_server, operation_id) except RetriesExceeded: logging.error("Maximum number of retries exceeded, exiting.") return ExitCode.HTTP_ERROR if status != OperationStatus.IN_PROGRESS: break logging.info("Config switch operation %s is still in progress", operation_id) time.sleep(5) if status == OperationStatus.DONE: logging.info("Config switch operation finished successfully.") return ExitCode.CONFIG_SWITCH_DONE else: logging.error("Config switch operation failed.") return ExitCode.CONFIG_SWITCH_FAILED def run_barman_config_switch(api: "PgBackupApi", args: Namespace) -> int: """Run a remote ``barman config-switch`` through the ``pg-backup-api``. :param api: a :class:`PgBackupApi` instance to handle communication with the API. :param args: arguments received from the command-line of ``patroni_barman config-switch`` command. :returns: the return code to be used when exiting the ``patroni_barman`` application. Refer to :class:`ExitCode`. """ if _should_skip_switch(args): logging.info("Config switch operation was skipped (role=%s, " "switch_when=%s).", args.role, args.switch_when) return ExitCode.CONFIG_SWITCH_SKIPPED if not bool(args.barman_model) ^ bool(args.reset): logging.error("One, and only one among 'barman_model' ('%s') and " "'reset' ('%s') should be given", args.barman_model, args.reset) return ExitCode.INVALID_ARGS return _switch_config(api, args.barman_server, args.barman_model, args.reset) patroni-4.0.4/patroni/scripts/barman/recover.py000066400000000000000000000103201472010352700216120ustar00rootroot00000000000000#!/usr/bin/env python """Implements ``patroni_barman recover`` sub-command. Restore a Barman backup to the local node through ``pg-backup-api``. This sub-command can be used both as a custom bootstrap method, and as a custom create replica method. Check the output of ``--help`` to understand the parameters supported by the sub-command. ``--datadir`` is a special parameter and it is automatically filled by Patroni in both cases. It requires that you have previously configured a Barman server, and that you have ``pg-backup-api`` configured and running in the same host as Barman. Refer to :class:`ExitCode` for possible exit codes of this sub-command. """ import logging import time from argparse import Namespace from enum import IntEnum from typing import TYPE_CHECKING from .utils import OperationStatus, RetriesExceeded if TYPE_CHECKING: # pragma: no cover from .utils import PgBackupApi class ExitCode(IntEnum): """Possible exit codes of this script. :cvar RECOVERY_DONE: backup was successfully restored. :cvar RECOVERY_FAILED: recovery of the backup faced an issue. :cvar HTTP_ERROR: an error has occurred while communicating with ``pg-backup-api`` """ RECOVERY_DONE = 0 RECOVERY_FAILED = 1 HTTP_ERROR = 2 def _restore_backup(api: "PgBackupApi", barman_server: str, backup_id: str, ssh_command: str, data_directory: str, loop_wait: int) -> int: """Restore the configured Barman backup through ``pg-backup-api``. .. note:: If requests to ``pg-backup-api`` fail recurrently or we face HTTP errors, then exit with :attr:`ExitCode.HTTP_ERROR`. :param api: a :class:`PgBackupApi` instance to handle communication with the API. :param barman_server: name of the Barman server which backup is to be restored. :param backup_id: ID of the backup from the Barman server. :param ssh_command: SSH command to connect from the Barman host to the target host. :param data_directory: path to the Postgres data directory where to restore the backup in. :param loop_wait: how long in seconds to wait before checking again the status of the recovery process. Higher values are useful for backups that are expected to take longer to restore. :returns: the return code to be used when exiting the ``patroni_barman`` application. Refer to :class:`ExitCode`. """ operation_id = None try: operation_id = api.create_recovery_operation( barman_server, backup_id, ssh_command, data_directory, ) except RetriesExceeded as exc: logging.error("An issue was faced while trying to create a recovery " "operation: %r", exc) return ExitCode.HTTP_ERROR logging.info("Created the recovery operation with ID %s", operation_id) status = None while True: try: status = api.get_operation_status(barman_server, operation_id) except RetriesExceeded: logging.error("Maximum number of retries exceeded, exiting.") return ExitCode.HTTP_ERROR if status != OperationStatus.IN_PROGRESS: break logging.info("Recovery operation %s is still in progress", operation_id) time.sleep(loop_wait) if status == OperationStatus.DONE: logging.info("Recovery operation finished successfully.") return ExitCode.RECOVERY_DONE else: logging.error("Recovery operation failed.") return ExitCode.RECOVERY_FAILED def run_barman_recover(api: "PgBackupApi", args: Namespace) -> int: """Run a remote ``barman recover`` through the ``pg-backup-api``. :param api: a :class:`PgBackupApi` instance to handle communication with the API. :param args: arguments received from the command-line of ``patroni_barman recover`` command. :returns: the return code to be used when exiting the ``patroni_barman`` application. Refer to :class:`ExitCode`. """ return _restore_backup(api, args.barman_server, args.backup_id, args.ssh_command, args.data_directory, args.loop_wait) patroni-4.0.4/patroni/scripts/barman/utils.py000066400000000000000000000253301472010352700213140ustar00rootroot00000000000000#!/usr/bin/env python """Utilitary stuff to be used by Barman related scripts.""" import json import logging import time from enum import IntEnum from typing import Any, Callable, Dict, Optional, Tuple, Type, Union from urllib.parse import urljoin from urllib3 import PoolManager from urllib3.exceptions import MaxRetryError from urllib3.response import HTTPResponse class RetriesExceeded(Exception): """Maximum number of retries exceeded.""" def retry(exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]]) \ -> Any: """Retry an operation n times if expected *exceptions* are faced. .. note:: Should be used as a decorator of a class' method as it expects the first argument to be a class instance. The class which method is going to be decorated should contain a couple attributes: * ``max_retries``: maximum retry attempts before failing; * ``retry_wait``: how long in seconds to wait before retrying. :param exceptions: exceptions that could trigger a retry attempt. :raises: :exc:`RetriesExceeded`: if the maximum number of attempts has been exhausted. """ def decorator(func: Callable[..., Any]) -> Any: def inner_func(instance: object, *args: Any, **kwargs: Any) -> Any: times: int = getattr(instance, "max_retries") retry_wait: int = getattr(instance, "retry_wait") method_name = f"{instance.__class__.__name__}.{func.__name__}" attempt = 1 while attempt <= times: try: return func(instance, *args, **kwargs) except exceptions as exc: logging.warning("Attempt %d of %d on method %s failed " "with %r.", attempt, times, method_name, exc) attempt += 1 time.sleep(retry_wait) raise RetriesExceeded("Maximum number of retries exceeded for " f"method {method_name}.") return inner_func return decorator def set_up_logging(log_file: Optional[str] = None) -> None: """Set up logging to file, if *log_file* is given, otherwise to console. :param log_file: file where to log messages, if any. """ logging.basicConfig(filename=log_file, level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s") class OperationStatus(IntEnum): """Possible status of ``pg-backup-api`` operations. :cvar IN_PROGRESS: the operation is still ongoing. :cvar FAILED: the operation failed. :cvar DONE: the operation finished successfully. """ IN_PROGRESS = 0 FAILED = 1 DONE = 2 class ApiNotOk(Exception): """The ``pg-backup-api`` is not currently up and running.""" class PgBackupApi: """Facilities for communicating with the ``pg-backup-api``. :ivar api_url: base URL to reach the ``pg-backup-api``. :ivar cert_file: certificate to authenticate against the ``pg-backup-api``, if required. :ivar key_file: certificate key to authenticate against the ``pg-backup-api``, if required. :ivar retry_wait: how long in seconds to wait before retrying a failed request to the ``pg-backup-api``. :ivar max_retries: maximum number of retries when ``pg-backup-api`` returns malformed responses. :ivar http: a HTTP pool manager for performing web requests. """ def __init__(self, api_url: str, cert_file: Optional[str], key_file: Optional[str], retry_wait: int, max_retries: int) -> None: """Create a new instance of :class:`BarmanRecover`. Make sure the ``pg-backup-api`` is reachable and running fine. .. note:: When using any method which send requests to the API, be aware that they might raise :exc:`RetriesExceeded` upon HTTP request errors. Similarly, when instantiating this class you may face an :exc:`ApiNotOk`, if the API is down or returns a bogus status. :param api_url: base URL to reach the ``pg-backup-api``. :param cert_file: certificate to authenticate against the ``pg-backup-api``, if required. :param key_file: certificate key to authenticate against the ``pg-backup-api``, if required. :param retry_wait: how long in seconds to wait before retrying a failed request to the ``pg-backup-api``. :param max_retries: maximum number of retries when ``pg-backup-api`` returns malformed responses. """ self.api_url = api_url self.cert_file = cert_file self.key_file = key_file self.retry_wait = retry_wait self.max_retries = max_retries self._http = PoolManager(cert_file=cert_file, key_file=key_file) self._ensure_api_ok() def _build_full_url(self, url_path: str) -> str: """Build the full URL by concatenating *url_path* with the base URL. :param url_path: path to be accessed in the ``pg-backup-api``. :returns: the full URL after concatenating. """ return urljoin(self.api_url, url_path) @staticmethod def _deserialize_response(response: HTTPResponse) -> Any: """Retrieve body from *response* as a deserialized JSON object. :param response: response from which JSON body will be deserialized. :returns: the deserialized JSON body. """ return json.loads(response.data.decode("utf-8")) @staticmethod def _serialize_request(body: Any) -> Any: """Serialize a request body. :param body: content of the request body to be serialized. :returns: the serialized request body. """ return json.dumps(body).encode("utf-8") def _get_request(self, url_path: str) -> Any: """Perform a ``GET`` request to *url_path*. :param url_path: URL to perform the ``GET`` request against. :returns: the deserialized response body. :raises: :exc:`RetriesExceeded`: raised from the corresponding :mod:`urllib3` exception. """ url = self._build_full_url(url_path) response = None try: response = self._http.request("GET", url) except MaxRetryError as exc: msg = f"Failed to perform a GET request to {url}" raise RetriesExceeded(msg) from exc return self._deserialize_response(response) def _post_request(self, url_path: str, body: Any) -> Any: """Perform a ``POST`` request to *url_path* serializing *body* as JSON. :param url_path: URL to perform the ``POST`` request against. :param body: the body to be serialized as JSON and sent in the request. :returns: the deserialized response body. :raises: :exc:`RetriesExceeded`: raised from the corresponding :mod:`urllib3` exception. """ body = self._serialize_request(body) url = self._build_full_url(url_path) response = None try: response = self._http.request("POST", url, body=body, headers={ "Content-Type": "application/json" }) except MaxRetryError as exc: msg = f"Failed to perform a POST request to {url} with {body}" raise RetriesExceeded(msg) from exc return self._deserialize_response(response) def _ensure_api_ok(self) -> None: """Ensure ``pg-backup-api`` is reachable and ``OK``. :raises: :exc:`ApiNotOk`: if ``pg-backup-api`` status is not ``OK``. """ response = self._get_request("status") if response != "OK": msg = ( "pg-backup-api is currently not up and running at " f"{self.api_url}: {response}" ) raise ApiNotOk(msg) @retry(KeyError) def get_operation_status(self, barman_server: str, operation_id: str) -> OperationStatus: """Get status of the operation which ID is *operation_id*. :param barman_server: name of the Barman server related with the operation. :param operation_id: ID of the operation to be checked. :returns: the status of the operation. """ response = self._get_request( f"servers/{barman_server}/operations/{operation_id}", ) status = response["status"] return OperationStatus[status] @retry(KeyError) def create_recovery_operation(self, barman_server: str, backup_id: str, ssh_command: str, data_directory: str) -> str: """Create a recovery operation on the ``pg-backup-api``. :param barman_server: name of the Barman server which backup is to be restored. :param backup_id: ID of the backup from the Barman server. :param ssh_command: SSH command to connect from the Barman host to the target host. :param data_directory: path to the Postgres data directory where to restore the backup at. :returns: the ID of the recovery operation that has been created. """ response = self._post_request( f"servers/{barman_server}/operations", { "type": "recovery", "backup_id": backup_id, "remote_ssh_command": ssh_command, "destination_directory": data_directory, }, ) return response["operation_id"] @retry(KeyError) def create_config_switch_operation(self, barman_server: str, barman_model: Optional[str], reset: Optional[bool]) -> str: """Create a config switch operation on the ``pg-backup-api``. :param barman_server: name of the Barman server which config is to be switched. :param barman_model: name of the Barman model to be applied to the server, if any. :param reset: ``True`` if you would like to unapply the currently active model for the server, if any. :returns: the ID of the config switch operation that has been created. """ body: Dict[str, Any] = {"type": "config_switch"} if barman_model: body["model_name"] = barman_model elif reset: body["reset"] = reset response = self._post_request( f"servers/{barman_server}/operations", body, ) return response["operation_id"] patroni-4.0.4/patroni/scripts/wale_restore.py000077500000000000000000000347001472010352700214130ustar00rootroot00000000000000#!/usr/bin/env python # sample script to clone new replicas using WAL-E restore # falls back to pg_basebackup if WAL-E restore fails, or if # WAL-E backup is too far behind # note that pg_basebackup still expects to use restore from # WAL-E for transaction logs # theoretically should work with SWIFT, but not tested on it # arguments are: # - cluster scope # - cluster role # - leader connection string # - number of retries # - envdir for the WALE env # - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup # - WALE_BACKUP_THRESHOLD_PERCENTAGE if WAL size exceeds a certain percentage of the # this script depends on an envdir defining the S3 bucket (or SWIFT dir),and login # credentials per WALE Documentation. # currently also requires that you configure the restore_command to use wal_e, example: # recovery_conf: # restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 import argparse import csv import logging import os import subprocess import sys import time from enum import IntEnum from typing import Any, List, NamedTuple, Optional, Tuple, TYPE_CHECKING from .. import psycopg logger = logging.getLogger(__name__) RETRY_SLEEP_INTERVAL = 1 si_prefixes = ['K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'] # Meaningful names to the exit codes used by WALERestore class ExitCode(IntEnum): SUCCESS = 0 #: Succeeded RETRY_LATER = 1 #: External issue, retry later FAIL = 2 #: Don't try again unless configuration changes # We need to know the current PG version in order to figure out the correct WAL directory name def get_major_version(data_dir: str) -> float: version_file = os.path.join(data_dir, 'PG_VERSION') if os.path.isfile(version_file): # version file exists try: with open(version_file) as f: return float(f.read()) except Exception: logger.exception('Failed to read PG_VERSION from %s', data_dir) return 0.0 def repr_size(n_bytes: float) -> str: """ >>> repr_size(1000) '1000 Bytes' >>> repr_size(8257332324597) '7.5 TiB' """ if n_bytes < 1024: return '{0} Bytes'.format(n_bytes) i = -1 while n_bytes > 1023: n_bytes /= 1024.0 i += 1 return '{0} {1}iB'.format(round(n_bytes, 1), si_prefixes[i]) def size_as_bytes(size: float, prefix: str) -> int: """ >>> size_as_bytes(7.5, 'T') 8246337208320 """ prefix = prefix.upper() assert prefix in si_prefixes exponent = si_prefixes.index(prefix) + 1 return int(size * (1024.0 ** exponent)) class WALEConfig(NamedTuple): env_dir: str threshold_mb: int threshold_pct: int cmd: List[str] class WALERestore(object): def __init__(self, scope: str, datadir: str, connstring: str, env_dir: str, threshold_mb: int, threshold_pct: int, use_iam: int, no_leader: bool, retries: int) -> None: self.scope = scope self.leader_connection = connstring self.data_dir = datadir self.no_leader = no_leader wale_cmd = [ 'envdir', env_dir, 'wal-e', ] if use_iam == 1: wale_cmd += ['--aws-instance-profile'] self.wal_e = WALEConfig( env_dir=env_dir, threshold_mb=threshold_mb, threshold_pct=threshold_pct, cmd=wale_cmd, ) self.init_error = (not os.path.exists(self.wal_e.env_dir)) self.retries = retries def run(self) -> int: """ Creates a new replica using WAL-E Returns ------- ExitCode 0 = Success 1 = Error, try again 2 = Error, don't try again """ if self.init_error: logger.error('init error: %r did not exist at initialization time', self.wal_e.env_dir) return ExitCode.FAIL try: should_use_s3 = self.should_use_s3_to_create_replica() if should_use_s3 is None: # Need to retry return ExitCode.RETRY_LATER elif should_use_s3: return self.create_replica_with_s3() elif not should_use_s3: return ExitCode.FAIL except Exception: logger.exception("Unhandled exception when running WAL-E restore") return ExitCode.FAIL def should_use_s3_to_create_replica(self) -> Optional[bool]: """ determine whether it makes sense to use S3 and not pg_basebackup """ threshold_megabytes = self.wal_e.threshold_mb threshold_percent = self.wal_e.threshold_pct try: cmd = self.wal_e.cmd + ['backup-list', '--detail', 'LATEST'] logger.debug('calling %r', cmd) wale_output = subprocess.check_output(cmd) reader = csv.DictReader(wale_output.decode('utf-8').splitlines(), dialect='excel-tab') rows = list(reader) if not len(rows): logger.warning('wal-e did not find any backups') return False # This check might not add much, it was performed in the previous # version of this code. since the old version rolled CSV parsing the # check may have been part of the CSV parsing. if len(rows) > 1: logger.warning( 'wal-e returned more than one row of backups: %r', rows) return False backup_info = rows[0] except subprocess.CalledProcessError: logger.exception("could not query wal-e latest backup") return None try: backup_size = int(backup_info['expanded_size_bytes']) backup_start_segment = backup_info['wal_segment_backup_start'] backup_start_offset = backup_info['wal_segment_offset_backup_start'] except KeyError: logger.exception("unable to get some of WALE backup parameters") return None # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file, # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset, # that we have to convert to hex and 'prepend' to the high offset digits. lsn_segment = backup_start_segment[8:16] # first 2 characters of the result are 0x and the last one is L lsn_offset = hex((int(backup_start_segment[16:32], 16) << 24) + int(backup_start_offset))[2:-1] # construct the LSN from the segment and offset backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset) diff_in_bytes = backup_size attempts_no = 0 while True: if self.leader_connection: con = None try: # get the difference in bytes between the current WAL location and the backup start offset con = psycopg.connect(self.leader_connection) if getattr(con, 'server_version', 0) >= 100000: wal_name = 'wal' lsn_name = 'lsn' else: wal_name = 'xlog' lsn_name = 'location' with con.cursor() as cur: cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()" " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE(" "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, " "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)" " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint" " END").format(wal_name, lsn_name), (backup_start_lsn, backup_start_lsn, backup_start_lsn)) for row in cur: diff_in_bytes = int(row[0]) break except psycopg.Error: logger.exception('could not determine difference with the leader location') if attempts_no < self.retries: # retry in case of a temporarily connection issue attempts_no = attempts_no + 1 time.sleep(RETRY_SLEEP_INTERVAL) continue else: if not self.no_leader: return False # do no more retries on the outer level logger.info("continue with base backup from S3 since leader is not available") diff_in_bytes = 0 break finally: if con: con.close() else: # always try to use WAL-E if leader connection string is not available diff_in_bytes = 0 break # if the size of the accumulated WAL segments is more than a certain percentage of the backup size # or exceeds the pre-determined size - pg_basebackup is chosen instead. is_size_thresh_ok = diff_in_bytes < int(threshold_megabytes) * 1048576 threshold_pct_bytes = backup_size * threshold_percent / 100.0 is_percentage_thresh_ok = float(diff_in_bytes) < int(threshold_pct_bytes) are_thresholds_ok = is_size_thresh_ok and is_percentage_thresh_ok class Size(object): def __init__(self, n_bytes: float, prefix: Optional[str] = None) -> None: self.n_bytes = n_bytes self.prefix = prefix def __repr__(self) -> str: if self.prefix is not None: n_bytes = size_as_bytes(self.n_bytes, self.prefix) else: n_bytes = self.n_bytes return repr_size(n_bytes) class HumanContext(object): def __init__(self, items: List[Tuple[str, Any]]) -> None: self.items = items def __repr__(self) -> str: return ', '.join('{}={!r}'.format(key, value) for key, value in self.items) human_context = repr(HumanContext([ ('threshold_size', Size(threshold_megabytes, 'M')), ('threshold_percent', threshold_percent), ('threshold_percent_size', Size(threshold_pct_bytes)), ('backup_size', Size(backup_size)), ('backup_diff', Size(diff_in_bytes)), ('is_size_thresh_ok', is_size_thresh_ok), ('is_percentage_thresh_ok', is_percentage_thresh_ok), ])) if not are_thresholds_ok: logger.info('wal-e backup size diff is over threshold, falling back ' 'to other means of restore: %s', human_context) else: logger.info('Thresholds are OK, using wal-e basebackup: %s', human_context) return are_thresholds_ok def fix_subdirectory_path_if_broken(self, dirname: str) -> bool: # in case it is a symlink pointing to a non-existing location, remove it and create the actual directory path = os.path.join(self.data_dir, dirname) if not os.path.exists(path): if os.path.islink(path): # broken xlog symlink, to remove try: os.remove(path) except OSError: logger.exception("could not remove broken %s symlink pointing to %s", dirname, os.readlink(path)) return False try: os.mkdir(path) except OSError: logger.exception("could not create missing %s directory path", dirname) return False return True def create_replica_with_s3(self) -> int: # if we're set up, restore the replica using fetch latest try: cmd = self.wal_e.cmd + ['backup-fetch', '{}'.format(self.data_dir), 'LATEST'] logger.debug('calling: %r', cmd) exit_code = subprocess.call(cmd) except Exception as e: logger.error('Error when fetching backup with WAL-E: {0}'.format(e)) return ExitCode.RETRY_LATER if (exit_code == 0 and not self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10 else 'pg_wal')): return ExitCode.FAIL return exit_code def main() -> int: logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO) parser = argparse.ArgumentParser(description='Script to image replicas using WAL-E') parser.add_argument('--scope', required=True) parser.add_argument('--role', required=False) parser.add_argument('--datadir', required=True) parser.add_argument('--connstring', required=True) parser.add_argument('--retries', type=int, default=1) parser.add_argument('--envdir', required=True) parser.add_argument('--threshold_megabytes', type=int, default=10240) parser.add_argument('--threshold_backup_size_percentage', type=int, default=30) parser.add_argument('--use_iam', type=int, default=0) parser.add_argument('--no_leader', type=int, default=0) args = parser.parse_args() exit_code = None assert args.retries >= 0 # Retry cloning in a loop. We do separate retries for the leader # connection attempt inside should_use_s3_to_create_replica, # because we need to differentiate between the last attempt and # the rest and make a decision when the last attempt fails on # whether to use WAL-E or not depending on the no_leader flag. for _ in range(0, args.retries + 1): restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring, env_dir=args.envdir, threshold_mb=args.threshold_megabytes, threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam, no_leader=args.no_leader, retries=args.retries) exit_code = restore.run() if exit_code != ExitCode.RETRY_LATER: # only WAL-E failures lead to the retry logger.debug('exit_code is %r, not retrying', exit_code) break time.sleep(RETRY_SLEEP_INTERVAL) if TYPE_CHECKING: # pragma: no cover assert exit_code is not None return exit_code if __name__ == '__main__': sys.exit(main()) patroni-4.0.4/patroni/tags.py000066400000000000000000000076651472010352700161760ustar00rootroot00000000000000"""Tags handling.""" import abc from typing import Any, Dict, Optional from patroni.utils import parse_bool, parse_int class Tags(abc.ABC): """An abstract class that encapsulates all the ``tags`` logic. Child classes that want to use provided facilities must implement ``tags`` abstract property. .. note:: Due to backward-compatibility reasons, old tags may have a less strict type conversion than new ones. """ @staticmethod def _filter_tags(tags: Dict[str, Any]) -> Dict[str, Any]: """Get tags configured for this node, if any. Handle both predefined Patroni tags and custom defined tags. .. note:: A custom tag is any tag added to the configuration ``tags`` section that is not one of ``clonefrom``, ``nofailover``, ``noloadbalance``,``nosync`` or ``nostream``. For most of the Patroni predefined tags, the returning object will only contain them if they are enabled as they all are boolean values that default to disabled. However ``nofailover`` tag is always returned if ``failover_priority`` tag is defined. In this case, we need both values to see if they are contradictory and the ``nofailover`` value should be used. :returns: a dictionary of tags set for this node. The key is the tag name, and the value is the corresponding tag value. """ return {tag: value for tag, value in tags.items() if any((tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync', 'nostream'), value, tag == 'nofailover' and 'failover_priority' in tags))} @property @abc.abstractmethod def tags(self) -> Dict[str, Any]: """Configured tags. Must be implemented in a child class. """ raise NotImplementedError # pragma: no cover @property def clonefrom(self) -> bool: """``True`` if ``clonefrom`` tag is ``True``, else ``False``.""" return self.tags.get('clonefrom', False) @property def nofailover(self) -> bool: """Common logic for obtaining the value of ``nofailover`` from ``tags`` if defined. If ``nofailover`` is not defined, this methods returns ``True`` if ``failover_priority`` is non-positive, ``False`` otherwise. """ from_tags = self.tags.get('nofailover') if from_tags is not None: # Value of `nofailover` takes precedence over `failover_priority` return bool(from_tags) failover_priority = parse_int(self.tags.get('failover_priority')) return failover_priority is not None and failover_priority <= 0 @property def failover_priority(self) -> int: """Common logic for obtaining the value of ``failover_priority`` from ``tags`` if defined. If ``nofailover`` is defined as ``True``, this will return ``0``. Otherwise, it will return the value of ``failover_priority``, defaulting to ``1`` if it's not defined or invalid. """ from_tags = self.tags.get('nofailover') failover_priority = parse_int(self.tags.get('failover_priority')) failover_priority = 1 if failover_priority is None else failover_priority return 0 if from_tags else failover_priority @property def noloadbalance(self) -> bool: """``True`` if ``noloadbalance`` is ``True``, else ``False``.""" return bool(self.tags.get('noloadbalance', False)) @property def nosync(self) -> bool: """``True`` if ``nosync`` is ``True``, else ``False``.""" return bool(self.tags.get('nosync', False)) @property def replicatefrom(self) -> Optional[str]: """Value of ``replicatefrom`` tag, if any.""" return self.tags.get('replicatefrom') @property def nostream(self) -> bool: """``True`` if ``nostream`` is ``True``, else ``False``.""" return parse_bool(self.tags.get('nostream')) or False patroni-4.0.4/patroni/utils.py000066400000000000000000001366641472010352700164020ustar00rootroot00000000000000"""Utilitary objects and functions that can be used throughout Patroni code. :var tzutc: UTC time zone info object. :var logger: logger of this module. :var USER_AGENT: identifies the Patroni version, Python version, and the underlying platform. :var OCT_RE: regular expression to match octal numbers, signed or unsigned. :var DEC_RE: regular expression to match decimal numbers, signed or unsigned. :var HEX_RE: regular expression to match hex strings, signed or unsigned. :var DBL_RE: regular expression to match double precision numbers, signed or unsigned. Matches scientific notation too. :var WHITESPACE_RE: regular expression to match whitespace characters """ import errno import itertools import logging import os import platform import random import re import socket import subprocess import sys import tempfile import time from collections import OrderedDict from json import JSONDecoder from shlex import split from typing import Any, Callable, cast, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union from dateutil import tz from urllib3.response import HTTPResponse from .exceptions import PatroniException from .version import __version__ if TYPE_CHECKING: # pragma: no cover from .dcs import Cluster tzutc = tz.tzutc() logger = logging.getLogger(__name__) USER_AGENT = 'Patroni/{0} Python/{1} {2}'.format(__version__, platform.python_version(), platform.system()) OCT_RE = re.compile(r'^[-+]?0[0-7]*') DEC_RE = re.compile(r'^[-+]?(0|[1-9][0-9]*)') HEX_RE = re.compile(r'^[-+]?0x[0-9a-fA-F]+') DBL_RE = re.compile(r'^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?') WHITESPACE_RE = re.compile(r'[ \t\n\r]*', re.VERBOSE | re.MULTILINE | re.DOTALL) def get_conversion_table(base_unit: str) -> Dict[str, Dict[str, Union[int, float]]]: """Get conversion table for the specified base unit. If no conversion table exists for the passed unit, return an empty :class:`OrderedDict`. :param base_unit: unit to choose the conversion table for. :returns: :class:`OrderedDict` object. """ memory_unit_conversion_table: Dict[str, Dict[str, Union[int, float]]] = OrderedDict([ ('TB', {'B': 1024**4, 'kB': 1024**3, 'MB': 1024**2}), ('GB', {'B': 1024**3, 'kB': 1024**2, 'MB': 1024}), ('MB', {'B': 1024**2, 'kB': 1024, 'MB': 1}), ('kB', {'B': 1024, 'kB': 1, 'MB': 1024**-1}), ('B', {'B': 1, 'kB': 1024**-1, 'MB': 1024**-2}) ]) time_unit_conversion_table: Dict[str, Dict[str, Union[int, float]]] = OrderedDict([ ('d', {'ms': 1000 * 60**2 * 24, 's': 60**2 * 24, 'min': 60 * 24}), ('h', {'ms': 1000 * 60**2, 's': 60**2, 'min': 60}), ('min', {'ms': 1000 * 60, 's': 60, 'min': 1}), ('s', {'ms': 1000, 's': 1, 'min': 60**-1}), ('ms', {'ms': 1, 's': 1000**-1, 'min': 1 / (1000 * 60)}), ('us', {'ms': 1000**-1, 's': 1000**-2, 'min': 1 / (1000**2 * 60)}) ]) if base_unit in ('B', 'kB', 'MB'): return memory_unit_conversion_table elif base_unit in ('ms', 's', 'min'): return time_unit_conversion_table return OrderedDict() def deep_compare(obj1: Dict[Any, Any], obj2: Dict[Any, Any]) -> bool: """Recursively compare two dictionaries to check if they are equal in terms of keys and values. .. note:: Values are compared based on their string representation. :param obj1: dictionary to be compared with *obj2*. :param obj2: dictionary to be compared with *obj1*. :returns: ``True`` if all keys and values match between the two dictionaries. :Example: >>> deep_compare({'1': None}, {}) False >>> deep_compare({'1': {}}, {'1': None}) False >>> deep_compare({'1': [1]}, {'1': [2]}) False >>> deep_compare({'1': 2}, {'1': '2'}) True >>> deep_compare({'1': {'2': [3, 4]}}, {'1': {'2': [3, 4]}}) True """ if set(list(obj1.keys())) != set(list(obj2.keys())): # Objects have different sets of keys return False for key, value in obj1.items(): if isinstance(value, dict): if not (isinstance(obj2[key], dict) and deep_compare(cast(Dict[Any, Any], value), obj2[key])): return False elif str(value) != str(obj2[key]): return False return True def patch_config(config: Dict[Any, Any], data: Dict[Any, Any]) -> bool: """Update and append to dictionary *config* from overrides in *data*. .. note:: * If the value of a given key in *data* is ``None``, then the key is removed from *config*; * If a key is present in *data* but not in *config*, the key with the corresponding value is added to *config* * For keys that are present on both sides it will compare the string representation of the corresponding values, if the comparison doesn't match override the value :param config: configuration to be patched. :param data: new configuration values to patch *config* with. :returns: ``True`` if *config* was changed. """ is_changed = False for name, value in data.items(): if value is None: if config.pop(name, None) is not None: is_changed = True elif name in config: if isinstance(value, dict): if isinstance(config[name], dict): if patch_config(config[name], cast(Dict[Any, Any], value)): is_changed = True else: config[name] = value is_changed = True elif str(config[name]) != str(value): config[name] = value is_changed = True else: config[name] = value is_changed = True return is_changed def parse_bool(value: Any) -> Optional[bool]: """Parse a given value to a :class:`bool` object. .. note:: The parsing is case-insensitive, and takes into consideration these values: * ``on``, ``true``, ``yes``, and ``1`` as ``True``. * ``off``, ``false``, ``no``, and ``0`` as ``False``. :param value: value to be parsed to :class:`bool`. :returns: the parsed value. If not able to parse, returns ``None``. :Example: >>> parse_bool(1) True >>> parse_bool('off') False >>> parse_bool('foo') """ value = str(value).lower() if value in ('on', 'true', 'yes', '1'): return True if value in ('off', 'false', 'no', '0'): return False def strtol(value: Any, strict: Optional[bool] = True) -> Tuple[Optional[int], str]: """Extract the long integer part from the beginning of a string that represents a configuration value. As most as possible close equivalent of ``strtol(3)`` C function (with base=0), which is used by postgres to parse parameter values. Takes into consideration numbers represented either as hex, octal or decimal formats. :param value: any value from which we want to extract a long integer. :param strict: dictates how the first item in the returning tuple is set when :func:`strtol` is not able to find a long integer in *value*. If *strict* is ``True``, then the first item will be ``None``, else it will be ``1``. :returns: the first item is the extracted long integer from *value*, and the second item is the remaining string of *value*. If not able to match a long integer in *value*, then the first item will be either ``None`` or ``1`` (depending on *strict* argument), and the second item will be the original *value*. :Example: >>> strtol(0) == (0, '') True >>> strtol(1) == (1, '') True >>> strtol(9) == (9, '') True >>> strtol(' +0x400MB') == (1024, 'MB') True >>> strtol(' -070d') == (-56, 'd') True >>> strtol(' d ') == (None, 'd') True >>> strtol(' 1 d ') == (1, ' d') True >>> strtol('9s', False) == (9, 's') True >>> strtol(' s ', False) == (1, 's') True """ value = str(value).strip() for regex, base in ((HEX_RE, 16), (OCT_RE, 8), (DEC_RE, 10)): match = regex.match(value) if match: end = match.end() return int(value[:end], base), value[end:] return (None if strict else 1), value def strtod(value: Any) -> Tuple[Optional[float], str]: """Extract the double precision part from the beginning of a string that reprensents a configuration value. As most as possible close equivalent of ``strtod(3)`` C function, which is used by postgres to parse parameter values. :param value: any value from which we want to extract a double precision. :returns: the first item is the extracted double precision from *value*, and the second item is the remaining string of *value*. If not able to match a double precision in *value*, then the first item will be ``None``, and the second item will be the original *value*. :Example: >>> strtod(' A ') == (None, 'A') True >>> strtod('1 A ') == (1.0, ' A') True >>> strtod('1.5A') == (1.5, 'A') True >>> strtod('8.325e-10A B C') == (8.325e-10, 'A B C') True """ value = str(value).strip() match = DBL_RE.match(value) if match: end = match.end() return float(value[:end]), value[end:] return None, value def convert_to_base_unit(value: Union[int, float], unit: str, base_unit: Optional[str]) -> Union[int, float, None]: """Convert *value* as a *unit* of compute information or time to *base_unit*. :param value: value to be converted to the base unit. :param unit: unit of *value*. Accepts these units (case sensitive): * For space: ``B``, ``kB``, ``MB``, ``GB``, or ``TB``; * For time: ``d``, ``h``, ``min``, ``s``, ``ms``, or ``us``. :param base_unit: target unit in the conversion. May contain the target unit with an associated value, e.g ``512MB``. Accepts these units (case sensitive): * For space: ``B``, ``kB``, or ``MB``; * For time: ``ms``, ``s``, or ``min``. :returns: *value* in *unit* converted to *base_unit*. Returns ``None`` if *unit* or *base_unit* is invalid. :Example: >>> convert_to_base_unit(1, 'GB', '256MB') 4 >>> convert_to_base_unit(1, 'GB', 'MB') 1024 >>> convert_to_base_unit(1, 'gB', '512MB') is None True >>> convert_to_base_unit(1, 'GB', '512 MB') is None True """ base_value, base_unit = strtol(base_unit, False) if TYPE_CHECKING: # pragma: no cover assert isinstance(base_value, int) convert_tbl = get_conversion_table(base_unit) # {'TB': 'GB', 'GB': 'MB', ...} round_order = dict(zip(convert_tbl, itertools.islice(convert_tbl, 1, None))) if unit in convert_tbl and base_unit in convert_tbl[unit]: value *= convert_tbl[unit][base_unit] / float(base_value) if unit in round_order: multiplier = convert_tbl[round_order[unit]][base_unit] value = round(value / float(multiplier)) * multiplier return value def convert_int_from_base_unit(base_value: int, base_unit: Optional[str]) -> Optional[str]: """Convert an integer value in some base unit to a human-friendly unit. The output unit is chosen so that it's the greatest unit that can represent the value without loss. :param base_value: value to be converted from a base unit :param base_unit: unit of *value*. Should be one of the base units (case sensitive): * For space: ``B``, ``kB``, ``MB``; * For time: ``ms``, ``s``, ``min``. :returns: :class:`str` value representing *base_value* converted from *base_unit* to the greatest possible human-friendly unit, or ``None`` if conversion failed. :Example: >>> convert_int_from_base_unit(1024, 'kB') '1MB' >>> convert_int_from_base_unit(1025, 'kB') '1025kB' >>> convert_int_from_base_unit(4, '256MB') '1GB' >>> convert_int_from_base_unit(4, '256 MB') is None True >>> convert_int_from_base_unit(1024, 'KB') is None True """ base_value_mult, base_unit = strtol(base_unit, False) if TYPE_CHECKING: # pragma: no cover assert isinstance(base_value_mult, int) base_value *= base_value_mult convert_tbl = get_conversion_table(base_unit) for unit in convert_tbl: multiplier = convert_tbl[unit][base_unit] if multiplier <= 1.0 or base_value % multiplier == 0: return str(round(base_value / multiplier)) + unit def convert_real_from_base_unit(base_value: float, base_unit: Optional[str]) -> Optional[str]: """Convert an floating-point value in some base unit to a human-friendly unit. Same as :func:`convert_int_from_base_unit`, except we have to do the math a bit differently, and there's a possibility that we don't find any exact divisor. :param base_value: value to be converted from a base unit :param base_unit: unit of *value*. Should be one of the base units (case sensitive): * For space: ``B``, ``kB``, ``MB``; * For time: ``ms``, ``s``, ``min``. :returns: :class:`str` value representing *base_value* converted from *base_unit* to the greatest possible human-friendly unit, or ``None`` if conversion failed. :Example: >>> convert_real_from_base_unit(5, 'ms') '5ms' >>> convert_real_from_base_unit(2.5, 'ms') '2500us' >>> convert_real_from_base_unit(4.0, '256MB') '1GB' >>> convert_real_from_base_unit(4.0, '256 MB') is None True """ base_value_mult, base_unit = strtol(base_unit, False) if TYPE_CHECKING: # pragma: no cover assert isinstance(base_value_mult, int) base_value *= base_value_mult result = None convert_tbl = get_conversion_table(base_unit) for unit in convert_tbl: value = base_value / convert_tbl[unit][base_unit] result = f'{value:g}{unit}' if value > 0 and abs((round(value) / value) - 1.0) <= 1e-8: break return result def maybe_convert_from_base_unit(base_value: str, vartype: str, base_unit: Optional[str]) -> str: """Try to convert integer or real value in a base unit to a human-readable unit. Value is passed as a string. If parsing or subsequent conversion fails, the original value is returned. :param base_value: value to be converted from a base unit. :param vartype: the target type to parse *base_value* before converting (``integer`` or ``real`` is expected, any other type results in return value being equal to the *base_value* string). :param base_unit: unit of *value*. Should be one of the base units (case sensitive): * For space: ``B``, ``kB``, ``MB``; * For time: ``ms``, ``s``, ``min``. :returns: :class:`str` value representing *base_value* converted from *base_unit* to the greatest possible human-friendly unit, or *base_value* string if conversion failed. :Example: >>> maybe_convert_from_base_unit('5', 'integer', 'ms') '5ms' >>> maybe_convert_from_base_unit('4.2', 'real', 'ms') '4200us' >>> maybe_convert_from_base_unit('on', 'bool', None) 'on' >>> maybe_convert_from_base_unit('', 'integer', '256MB') '' """ converters: Dict[str, Tuple[Callable[[str, Optional[str]], Union[int, float, str, None]], Callable[[Any, Optional[str]], Optional[str]]]] = { 'integer': (parse_int, convert_int_from_base_unit), 'real': (parse_real, convert_real_from_base_unit), 'default': (lambda v, _: v, lambda v, _: v) } parser, converter = converters.get(vartype, converters['default']) parsed_value = parser(base_value, None) if parsed_value: return converter(parsed_value, base_unit) or base_value return base_value def parse_int(value: Any, base_unit: Optional[str] = None) -> Optional[int]: """Parse *value* as an :class:`int`. :param value: any value that can be handled either by :func:`strtol` or :func:`strtod`. If *value* contains a unit, then *base_unit* must be given. :param base_unit: an optional base unit to convert *value* through :func:`convert_to_base_unit`. Not used if *value* does not contain a unit. :returns: the parsed value, if able to parse. Otherwise returns ``None``. :Example: >>> parse_int('1') == 1 True >>> parse_int(' 0x400 MB ', '16384kB') == 64 True >>> parse_int('1MB', 'kB') == 1024 True >>> parse_int('1000 ms', 's') == 1 True >>> parse_int('1TB', 'GB') is None True >>> parse_int(50, None) == 50 True >>> parse_int("51", None) == 51 True >>> parse_int("nonsense", None) == None True >>> parse_int("nonsense", "kB") == None True >>> parse_int("nonsense") == None True >>> parse_int(0) == 0 True >>> parse_int('6GB', '16MB') == 384 True >>> parse_int('4097.4kB', 'kB') == 4097 True >>> parse_int('4097.5kB', 'kB') == 4098 True """ val, unit = strtol(value) if val is None and unit.startswith('.') or unit and unit[0] in ('.', 'e', 'E'): val, unit = strtod(value) if val is not None: unit = unit.strip() if not unit: return round(val) val = convert_to_base_unit(val, unit, base_unit) if val is not None: return round(val) def parse_real(value: Any, base_unit: Optional[str] = None) -> Optional[float]: """Parse *value* as a :class:`float`. :param value: any value that can be handled by :func:`strtod`. If *value* contains a unit, then *base_unit* must be given. :param base_unit: an optional base unit to convert *value* through :func:`convert_to_base_unit`. Not used if *value* does not contain a unit. :returns: the parsed value, if able to parse. Otherwise returns ``None``. :Example: >>> parse_real(' +0.0005 ') == 0.0005 True >>> parse_real('0.0005ms', 'ms') == 0.0 True >>> parse_real('0.00051ms', 'ms') == 0.001 True """ val, unit = strtod(value) if val is not None: unit = unit.strip() if not unit: return val return convert_to_base_unit(val, unit, base_unit) def compare_values(vartype: str, unit: Optional[str], settings_value: Any, config_value: Any) -> bool: """Check if the value from ``pg_settings`` and from Patroni config are equivalent after parsing them as *vartype*. :param vartype: the target type to parse *settings_value* and *config_value* before comparing them. Accepts any among of the following (case sensitive): * ``bool``: parse values using :func:`parse_bool`; or * ``integer``: parse values using :func:`parse_int`; or * ``real``: parse values using :func:`parse_real`; or * ``enum``: parse values as lowercase strings; or * ``string``: parse values as strings. This one is used by default if no valid value is passed as *vartype*. :param unit: base unit to be used as argument when calling :func:`parse_int` or :func:`parse_real` for *config_value*. :param settings_value: value to be compared with *config_value*. :param config_value: value to be compared with *settings_value*. :returns: ``True`` if *settings_value* is equivalent to *config_value* when both are parsed as *vartype*. :Example: >>> compare_values('enum', None, 'remote_write', 'REMOTE_WRITE') True >>> compare_values('string', None, 'remote_write', 'REMOTE_WRITE') False >>> compare_values('real', None, '1e-06', 0.000001) True >>> compare_values('integer', 'MB', '6GB', '6GB') False >>> compare_values('integer', None, '6GB', '6GB') False >>> compare_values('integer', '16384kB', '64', ' 0x400 MB ') True >>> compare_values('integer', '2MB', 524288, '1TB') True >>> compare_values('integer', 'MB', 1048576, '1TB') True >>> compare_values('integer', 'kB', 4098, '4097.5kB') True """ converters: Dict[str, Callable[[str, Optional[str]], Union[None, bool, int, float, str]]] = { 'bool': lambda v1, v2: parse_bool(v1), 'integer': parse_int, 'real': parse_real, 'enum': lambda v1, v2: str(v1).lower(), 'string': lambda v1, v2: str(v1) } converter = converters.get(vartype) or converters['string'] old_converted = converter(settings_value, None) new_converted = converter(config_value, unit) return old_converted is not None and new_converted is not None and old_converted == new_converted def _sleep(interval: Union[int, float]) -> None: """Wrap :func:`~time.sleep`. :param interval: Delay execution for a given number of seconds. The argument may be a floating point number for subsecond precision. """ time.sleep(interval) def read_stripped(file_path: str) -> Iterator[str]: """Iterate over stripped lines in the given file. :param file_path: path to the file to read from :yields: each line from the given file stripped """ with open(file_path) as f: for line in f: yield line.strip() class RetryFailedError(PatroniException): """Maximum number of attempts exhausted in retry operation.""" class Retry(object): """Helper for retrying a method in the face of retryable exceptions. :ivar max_tries: how many times to retry the command. :ivar delay: initial delay between retry attempts. :ivar backoff: backoff multiplier between retry attempts. :ivar max_jitter: additional max jitter period to wait between retry attempts to avoid slamming the server. :ivar max_delay: maximum delay in seconds, regardless of other backoff settings. :ivar sleep_func: function used to introduce artificial delays. :ivar deadline: timeout for operation retries. :ivar retry_exceptions: single exception or tuple """ def __init__(self, max_tries: Optional[int] = 1, delay: float = 0.1, backoff: int = 2, max_jitter: float = 0.8, max_delay: int = 3600, sleep_func: Callable[[Union[int, float]], None] = _sleep, deadline: Optional[Union[int, float]] = None, retry_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = PatroniException) -> None: """Create a :class:`Retry` instance for retrying function calls. :param max_tries: how many times to retry the command. ``-1`` means infinite tries. :param delay: initial delay between retry attempts. :param backoff: backoff multiplier between retry attempts. Defaults to ``2`` for exponential backoff. :param max_jitter: additional max jitter period to wait between retry attempts to avoid slamming the server. :param max_delay: maximum delay in seconds, regardless of other backoff settings. :param sleep_func: function used to introduce artificial delays. :param deadline: timeout for operation retries. :param retry_exceptions: single exception or tuple """ self.max_tries = max_tries self.delay = delay self.backoff = backoff self.max_jitter = int(max_jitter * 100) self.max_delay = float(max_delay) self._attempts = 0 self._cur_delay = delay self.deadline = deadline self._cur_stoptime = None self.sleep_func = sleep_func self.retry_exceptions = retry_exceptions def reset(self) -> None: """Reset the attempt counter, delay and stop time.""" self._attempts = 0 self._cur_delay = self.delay self._cur_stoptime = None def copy(self) -> 'Retry': """Return a clone of this retry manager.""" return Retry(max_tries=self.max_tries, delay=self.delay, backoff=self.backoff, max_jitter=self.max_jitter / 100.0, max_delay=int(self.max_delay), sleep_func=self.sleep_func, deadline=self.deadline, retry_exceptions=self.retry_exceptions) @property def sleeptime(self) -> float: """Get next cycle sleep time. It is based on the current delay plus a number up to ``max_jitter``. """ return self._cur_delay + (random.randint(0, self.max_jitter) / 100.0) def update_delay(self) -> None: """Set next cycle delay. It will be the minimum value between: * current delay with ``backoff``; or * ``max_delay``. """ self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay) @property def stoptime(self) -> float: """Get the current stop time.""" return self._cur_stoptime or 0 def ensure_deadline(self, timeout: float, raise_ex: Optional[Exception] = None) -> bool: """Calculates and checks the remaining deadline time. :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception. :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*. :returns: ``False`` if *deadline* is smaller than a provided *timeout* and *raise_ex* isn't set. Otherwise ``True``. :raises: :class:`Exception`: *raise_ex* if calculated deadline is smaller than provided *timeout*. """ if self.stoptime - time.time() < timeout: if raise_ex: raise raise_ex return False return True def __call__(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: """Call a function *func* with arguments ``*args`` and ``*kwargs`` in a loop. *func* will be called until one of the following conditions is met: * It completes without throwing one of the configured ``retry_exceptions``; or * ``max_retries`` is exceeded.; or * ``deadline`` is exceeded. .. note:: * It will set loop stop time based on ``deadline`` attribute. * It will adjust delay on each cycle. :param func: function to call. :param args: positional arguments to call *func* with. :params kwargs: keyword arguments to call *func* with. :raises: :class:`RetryFailedError`: * If ``max_tries`` is exceeded; or * If ``deadline`` is exceeded. """ self.reset() while True: try: if self.deadline is not None and self._cur_stoptime is None: self._cur_stoptime = time.time() + self.deadline return func(*args, **kwargs) except self.retry_exceptions as e: # Note: max_tries == -1 means infinite tries. if self._attempts == self.max_tries: logger.warning('Retry got exception: %s', e) raise RetryFailedError("Too many retry attempts") self._attempts += 1 sleeptime = getattr(e, 'sleeptime', None) if not isinstance(sleeptime, (int, float)): sleeptime = self.sleeptime if self._cur_stoptime is not None and time.time() + sleeptime >= self._cur_stoptime: logger.warning('Retry got exception: %s', e) raise RetryFailedError("Exceeded retry deadline") logger.debug('Retry got exception: %s', e) self.sleep_func(sleeptime) self.update_delay() def polling_loop(timeout: Union[int, float], interval: Union[int, float] = 1) -> Iterator[int]: """Return an iterator that returns values every *interval* seconds until *timeout* has passed. .. note:: Timeout is measured from start of iteration. :param timeout: for how long (in seconds) from now it should keep returning values. :param interval: for how long to sleep before returning a new value. :yields: current iteration counter, starting from ``0``. """ start_time = time.time() iteration = 0 end_time = start_time + timeout while time.time() < end_time: yield iteration iteration += 1 time.sleep(float(interval)) def split_host_port(value: str, default_port: Optional[int]) -> Tuple[str, int]: """Extract host(s) and port from *value*. :param value: string from where host(s) and port will be extracted. Accepts either of these formats: * ``host:port``; or * ``host1,host2,...,hostn:port``. Each ``host`` portion of *value* can be either: * A FQDN; or * An IPv4 address; or * An IPv6 address, with or without square brackets. :param default_port: if no port can be found in *param*, use *default_port* instead. :returns: the first item is composed of a CSV list of hosts from *value*, and the second item is either the port from *value* or *default_port*. :Example: >>> split_host_port('127.0.0.1', 5432) ('127.0.0.1', 5432) >>> split_host_port('127.0.0.1:5400', 5432) ('127.0.0.1', 5400) >>> split_host_port('127.0.0.1,192.168.0.101:5400', 5432) ('127.0.0.1,192.168.0.101', 5400) >>> split_host_port('127.0.0.1,www.mydomain.com,[fe80:0:0:0:213:72ff:fe3c:21bf], 0:0:0:0:0:0:0:0:5400', 5432) ('127.0.0.1,www.mydomain.com,fe80:0:0:0:213:72ff:fe3c:21bf,0:0:0:0:0:0:0:0', 5400) """ t = value.rsplit(':', 1) # If *value* contains ``:`` we consider it to be an IPv6 address, so we attempt to remove possible square brackets if ':' in t[0]: t[0] = ','.join([h.strip().strip('[]') for h in t[0].split(',')]) t.append(str(default_port)) return t[0], int(t[1]) def uri(proto: str, netloc: Union[List[str], Tuple[str, Union[int, str]], str], path: Optional[str] = '', user: Optional[str] = None) -> str: """Construct URI from given arguments. :param proto: the URI protocol. :param netloc: the URI host(s) and port. Can be specified in either way among * A :class:`list` or :class:`tuple`. The second item should be a port, and the first item should be composed of hosts in either of these formats: * ``host``; or. * ``host1,host2,...,hostn``. * A :class:`str` in either of these formats: * ``host:port``; or * ``host1,host2,...,hostn:port``. In all cases, each ``host`` portion of *netloc* can be either: * An FQDN; or * An IPv4 address; or * An IPv6 address, with or without square brackets. :param path: the URI path. :param user: the authenticating user, if any. :returns: constructed URI. """ host, port = netloc if isinstance(netloc, (list, tuple)) else split_host_port(netloc, 0) # If ``host`` contains ``:`` we consider it to be an IPv6 address, so we add square brackets if they are missing if host and ':' in host and host[0] != '[' and host[-1] != ']': host = '[{0}]'.format(host) port = ':{0}'.format(port) if port else '' path = '/{0}'.format(path) if path and not path.startswith('/') else path user = '{0}@'.format(user) if user else '' return '{0}://{1}{2}{3}{4}'.format(proto, user, host, port, path) def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]: """Iterate over the chunks of a :class:`~urllib3.response.HTTPResponse` and yield each JSON document that is found. :param response: the HTTP response from which JSON documents will be retrieved. :yields: current JSON document. """ prev = '' decoder = JSONDecoder() for chunk in response.read_chunked(decode_content=False): chunk = prev + chunk.decode('utf-8') length = len(chunk) # ``chunk`` is analyzed in parts. ``idx`` holds the position of the first character in the current part that is # neither space nor tab nor line-break, or in other words, the position in the ``chunk`` where it is likely # that a JSON document begins idx = WHITESPACE_RE.match(chunk, 0).end() # pyright: ignore [reportOptionalMemberAccess] while idx < length: try: # Get a JSON document from the chunk. ``message`` is a dictionary representing the JSON document, and # ``idx`` becomes the position in the ``chunk`` where the retrieved JSON document ends message, idx = decoder.raw_decode(chunk, idx) except ValueError: # malformed or incomplete JSON, unlikely to happen break else: yield message idx = WHITESPACE_RE.match(chunk, idx).end() # pyright: ignore [reportOptionalMemberAccess] # It is not usual that a ``chunk`` would contain more than one JSON document, but we handle that just in case prev = chunk[idx:] def cluster_as_json(cluster: 'Cluster') -> Dict[str, Any]: """Get a JSON representation of *cluster*. :param cluster: the :class:`~patroni.dcs.Cluster` object to be parsed as JSON. :returns: JSON representation of *cluster*. These are the possible keys in the returning object depending on the available information in *cluster*: * ``members``: list of members in the cluster. Each value is a :class:`dict` that may have the following keys: * ``name``: the name of the host (unique in the cluster). The ``members`` list is sorted by this key; * ``role``: ``leader``, ``standby_leader``, ``sync_standby``, ``quorum_standby``, or ``replica``; * ``state``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``, or ``creating replica``; * ``api_url``: REST API URL based on ``restapi->connect_address`` configuration; * ``host``: PostgreSQL host based on ``postgresql->connect_address``; * ``port``: PostgreSQL port based on ``postgresql->connect_address``; * ``timeline``: PostgreSQL current timeline; * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted; * ``scheduled_restart``: scheduled restart timestamp, if any; * ``tags``: any tags that were set for this member; * ``lag``: replication lag, if applicable; * ``pause``: ``True`` if cluster is in maintenance mode; * ``scheduled_switchover``: if a switchover has been scheduled, then it contains this entry with these keys: * ``at``: timestamp when switchover was scheduled to occur; * ``from``: name of the member to be demoted; * ``to``: name of the member to be promoted. """ from . import global_config config = global_config.from_cluster(cluster) leader_name = cluster.leader.name if cluster.leader else None cluster_lsn = cluster.status.last_lsn ret: Dict[str, Any] = {'members': []} sync_role = 'quorum_standby' if config.is_quorum_commit_mode else 'sync_standby' for m in cluster.members: if m.name == leader_name: role = 'standby_leader' if config.is_standby_cluster else 'leader' elif config.is_synchronous_mode and cluster.sync.matches(m.name): role = sync_role else: role = 'replica' state = (m.data.get('replication_state', '') if role != 'leader' else '') or m.data.get('state', '') member = {'name': m.name, 'role': role, 'state': state, 'api_url': m.api_url} conn_kwargs = m.conn_kwargs() if conn_kwargs.get('host'): member['host'] = conn_kwargs['host'] if conn_kwargs.get('port'): member['port'] = int(conn_kwargs['port']) optional_attributes = ('timeline', 'pending_restart', 'pending_restart_reason', 'scheduled_restart', 'tags') member.update({n: m.data[n] for n in optional_attributes if n in m.data}) if m.name != leader_name: lsn = m.lsn if lsn is None: member['lag'] = 'unknown' elif cluster_lsn >= lsn: member['lag'] = cluster_lsn - lsn else: member['lag'] = 0 ret['members'].append(member) # sort members by name for consistency cmp: Callable[[Dict[str, Any]], bool] = lambda m: m['name'] ret['members'].sort(key=cmp) if config.is_paused: ret['pause'] = True if cluster.failover and cluster.failover.scheduled_at: ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()} if cluster.failover.leader: ret['scheduled_switchover']['from'] = cluster.failover.leader if cluster.failover.candidate: ret['scheduled_switchover']['to'] = cluster.failover.candidate return ret def is_subpath(d1: str, d2: str) -> bool: """Check if the file system path *d2* is contained within *d1* after resolving symbolic links. .. note:: It will not check if the paths actually exist, it will only expand the paths and resolve any symbolic links that happen to be found. :param d1: path to a directory. :param d2: path to be checked if is within *d1*. :returns: ``True`` if *d1* is a subpath of *d2*. """ real_d1 = os.path.realpath(d1) + os.path.sep real_d2 = os.path.realpath(os.path.join(real_d1, d2)) return os.path.commonprefix([real_d1, real_d2 + os.path.sep]) == real_d1 def validate_directory(d: str, msg: str = "{} {}") -> None: """Ensure directory exists and is writable. .. note:: If the directory does not exist, :func:`validate_directory` will attempt to create it. :param d: the directory to be checked. :param msg: a message to be thrown when raising :class:`~patroni.exceptions.PatroniException`, if any issue is faced. It must contain 2 placeholders to be used by :func:`format`: * The first placeholder will be replaced with path *d*; * The second placeholder will be replaced with the error condition. :raises: :class:`~patroni.exceptions.PatroniException`: if any issue is observed while validating *d*. Can be thrown if: * *d* did not exist, and :func:`validate_directory` was not able to create it; or * *d* is an existing directory, but Patroni is not able to write to that directory; or * *d* is an existing file, not a directory. """ if not os.path.exists(d): try: os.makedirs(d) except OSError as e: logger.error(e) if e.errno != errno.EEXIST: raise PatroniException(msg.format(d, "couldn't create the directory")) elif os.path.isdir(d): try: fd, tmpfile = tempfile.mkstemp(dir=d) os.close(fd) os.remove(tmpfile) except OSError: raise PatroniException(msg.format(d, "the directory is not writable")) else: raise PatroniException(msg.format(d, "is not a directory")) def data_directory_is_empty(data_dir: str) -> bool: """Check if a PostgreSQL data directory is empty. .. note:: In non-Windows environments *data_dir* is also considered empty if it only contains hidden files and/or ``lost+found`` directory. :param data_dir: the PostgreSQL data directory to be checked. :returns: ``True`` if *data_dir* is empty. """ if not os.path.exists(data_dir): return True return all(os.name != 'nt' and (n.startswith('.') or n == 'lost+found') for n in os.listdir(data_dir)) def apply_keepalive_limit(option: str, value: int) -> int: """ Ensures provided *value* for keepalive *option* does not exceed the maximum allowed value for the current platform. :param option: The TCP keepalive option name. Possible values are: * ``TCP_USER_TIMEOUT``; * ``TCP_KEEPIDLE``; * ``TCP_KEEPINTVL``; * ``TCP_KEEPCNT``. :param value: The desired value for the keepalive option. :returns: maybe adjusted value. """ max_of_options = { 'linux': {'TCP_USER_TIMEOUT': 2147483647, 'TCP_KEEPIDLE': 32767, 'TCP_KEEPINTVL': 32767, 'TCP_KEEPCNT': 127}, 'darwin': {'TCP_KEEPIDLE': 4294967, 'TCP_KEEPINTVL': 4294967, 'TCP_KEEPCNT': 2147483647}, } platform = 'linux' if sys.platform.startswith('linux') else sys.platform max_possible_value = max_of_options.get(platform, {}).get(option) if max_possible_value is not None and value > max_possible_value: logger.debug('%s changed from %d to %d.', option, value, max_possible_value) value = max_possible_value return value def keepalive_intvl(timeout: int, idle: int, cnt: int = 3) -> int: """Calculate the value to be used as ``TCP_KEEPINTVL`` based on *timeout*, *idle*, and *cnt*. :param timeout: value for ``TCP_USER_TIMEOUT``. :param idle: value for ``TCP_KEEPIDLE``. :param cnt: value for ``TCP_KEEPCNT``. :returns: the value to be used as ``TCP_KEEPINTVL``. """ intvl = max(1, int(float(timeout - idle) / cnt)) return apply_keepalive_limit('TCP_KEEPINTVL', intvl) def keepalive_socket_options(timeout: int, idle: int, cnt: int = 3) -> Iterator[Tuple[int, int, int]]: """Get all keepalive related options to be set in a socket. :param timeout: value for ``TCP_USER_TIMEOUT``. :param idle: value for ``TCP_KEEPIDLE``. :param cnt: value for ``TCP_KEEPCNT``. :yields: all keepalive related socket options to be set. The first item in the tuple is the protocol, the second item is the option, and the third item is the value to be used. The return values depend on the platform: * ``Windows``: * ``SO_KEEPALIVE``. * ``Linux``: * ``SO_KEEPALIVE``; * ``TCP_USER_TIMEOUT``; * ``TCP_KEEPIDLE``; * ``TCP_KEEPINTVL``; * ``TCP_KEEPCNT``. * ``MacOS``: * ``SO_KEEPALIVE``; * ``TCP_KEEPIDLE``; * ``TCP_KEEPINTVL``; * ``TCP_KEEPCNT``. """ yield (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) if not (sys.platform.startswith('linux') or sys.platform.startswith('darwin')): return TCP_USER_TIMEOUT = getattr(socket, 'TCP_USER_TIMEOUT', None) if TCP_USER_TIMEOUT is not None: yield (socket.SOL_TCP, TCP_USER_TIMEOUT, apply_keepalive_limit('TCP_USER_TIMEOUT', int(timeout * 1000))) # The socket constants from MacOS netinet/tcp.h are not exported by python's # socket module, therefore we are using 0x10, 0x101, 0x102 constants. TCP_KEEPIDLE = getattr(socket, 'TCP_KEEPIDLE', 0x10 if sys.platform.startswith('darwin') else None) if TCP_KEEPIDLE is not None: idle = apply_keepalive_limit('TCP_KEEPIDLE', idle) yield (socket.IPPROTO_TCP, TCP_KEEPIDLE, idle) TCP_KEEPINTVL = getattr(socket, 'TCP_KEEPINTVL', 0x101 if sys.platform.startswith('darwin') else None) if TCP_KEEPINTVL is not None: intvl = keepalive_intvl(timeout, idle, cnt) yield (socket.IPPROTO_TCP, TCP_KEEPINTVL, intvl) TCP_KEEPCNT = getattr(socket, 'TCP_KEEPCNT', 0x102 if sys.platform.startswith('darwin') else None) if TCP_KEEPCNT is not None: cnt = apply_keepalive_limit('TCP_KEEPCNT', cnt) yield (socket.IPPROTO_TCP, TCP_KEEPCNT, cnt) def enable_keepalive(sock: socket.socket, timeout: int, idle: int, cnt: int = 3) -> None: """Enable keepalive for *sock*. Will set socket options depending on the platform, as per return of :func:`keepalive_socket_options`. .. note:: Value for ``TCP_KEEPINTVL`` will be calculated through :func:`keepalive_intvl` based on *timeout*, *idle*, and *cnt*. :param sock: the socket for which keepalive will be enabled. :param timeout: value for ``TCP_USER_TIMEOUT``. :param idle: value for ``TCP_KEEPIDLE``. :param cnt: value for ``TCP_KEEPCNT``. :returns: output of :func:`~socket.ioctl` if we are on Windows, nothing otherwise. """ SIO_KEEPALIVE_VALS = getattr(socket, 'SIO_KEEPALIVE_VALS', None) if SIO_KEEPALIVE_VALS is not None: # Windows intvl = keepalive_intvl(timeout, idle, cnt) sock.ioctl(SIO_KEEPALIVE_VALS, (1, idle * 1000, intvl * 1000)) for opt in keepalive_socket_options(timeout, idle, cnt): sock.setsockopt(*opt) def unquote(string: str) -> str: """Unquote a fully quoted *string*. :param string: The string to be checked for quoting. :returns: The string with quotes removed, if it is a fully quoted single string, or the original string if quoting is not detected, or unquoting was not possible. :Examples: A *string* with quotes will have those quotes removed >>> unquote('"a quoted string"') 'a quoted string' A *string* with multiple quotes will be returned as is >>> unquote('"a multi" "quoted string"') '"a multi" "quoted string"' So will a *string* with unbalanced quotes >>> unquote('unbalanced "quoted string') 'unbalanced "quoted string' """ try: ret = split(string) ret = ret[0] if len(ret) == 1 else string except ValueError: ret = string return ret def get_postgres_version(bin_dir: Optional[str] = None, bin_name: str = 'postgres') -> str: """Get full PostgreSQL version. It is based on the output of ``postgres --version``. :param bin_dir: path to the PostgreSQL binaries directory. If ``None`` or an empty string, it will use the first *bin_name* binary that is found by the subprocess in the ``PATH``. :param bin_name: name of the postgres binary to call (``postgres`` by default) :returns: the PostgreSQL version. :raises: :exc:`~patroni.exceptions.PatroniException`: if the postgres binary call failed due to :exc:`OSError`. :Example: * Returns `9.6.24` for PostgreSQL 9.6.24 * Returns `15.2` for PostgreSQL 15.2 """ if not bin_dir: binary = bin_name else: binary = os.path.join(bin_dir, bin_name) try: version = subprocess.check_output([binary, '--version']).decode() except OSError as e: raise PatroniException(f'Failed to get postgres version: {e}') version = re.match(r'^[^\s]+ [^\s]+ ((\d+)(\.\d+)*)', version) if TYPE_CHECKING: # pragma: no cover assert version is not None version = version.groups() # e.g., ('15.2', '15', '.2') major_version = int(version[1]) dot_count = version[0].count('.') if major_version < 10 and dot_count < 2 or major_version >= 10 and dot_count < 1: return '.'.join((version[0], '0')) return version[0] def get_major_version(bin_dir: Optional[str] = None, bin_name: str = 'postgres') -> str: """Get the major version of PostgreSQL. Like func:`get_postgres_version` but without minor version. :param bin_dir: path to the PostgreSQL binaries directory. If ``None`` or an empty string, it will use the first *bin_name* binary that is found by the subprocess in the ``PATH``. :param bin_name: name of the postgres binary to call (``postgres`` by default) :returns: the PostgreSQL major version. :raises: :exc:`~patroni.exceptions.PatroniException`: if the postgres binary call failed due to :exc:`OSError`. :Example: * Returns `9.6` for PostgreSQL 9.6.24 * Returns `15` for PostgreSQL 15.2 """ full_version = get_postgres_version(bin_dir, bin_name) return re.sub(r'\.\d+$', '', full_version) patroni-4.0.4/patroni/validator.py000066400000000000000000001506231472010352700172160ustar00rootroot00000000000000#!/usr/bin/env python3 """Patroni configuration validation helpers. This module contains facilities for validating configuration of Patroni processes. :var schema: configuration schema of the daemon launched by ``patroni`` command. """ import os import shutil import socket from typing import Any, cast, Dict, Iterator, List, Optional as OptionalType, Tuple, TYPE_CHECKING, Union from .collections import CaseInsensitiveSet, EMPTY_DICT from .dcs import dcs_modules from .exceptions import ConfigParseError from .log import type_logformat from .utils import data_directory_is_empty, get_major_version, parse_int, split_host_port # Additional parameters to fine-tune validation process _validation_params: Dict[str, Any] = {} def populate_validate_params(ignore_listen_port: bool = False) -> None: """Populate parameters used to fine-tune the validation of the Patroni config. :param ignore_listen_port: ignore the bind failures for the ports marked as `listen`. """ _validation_params['ignore_listen_port'] = ignore_listen_port def validate_log_field(field: Any) -> bool: """Checks if log field is valid. :param field: A log field to be validated. :returns: ``True`` if the field is either a string or a dictionary with exactly one key that has string value, ``False`` otherwise. """ if isinstance(field, str): return True elif isinstance(field, dict): field = cast(Dict[str, Any], field) return len(field) == 1 and isinstance(next(iter(field.values())), str) return False def validate_log_format(logformat: type_logformat) -> bool: """Checks if log format is valid. :param logformat: A log format to be validated. :returns: ``True`` if the log format is either a string or a list of valid log fields. :raises: :exc:`~patroni.exceptions.ConfigParseError`: * If the logformat is not a string or a list; or * If the logformat is an empty list; or * If the log format is a list and it with values that don't pass validation using :func:`validate_log_field`. """ if isinstance(logformat, str): return True elif isinstance(logformat, list): logformat = cast(List[Any], logformat) if len(logformat) == 0: raise ConfigParseError('should contain at least one item') if not all(map(validate_log_field, logformat)): raise ConfigParseError('each item should be a string or a dictionary with string values') return True else: raise ConfigParseError('Should be a string or a list') def data_directory_empty(data_dir: str) -> bool: """Check if PostgreSQL data directory is empty. :param data_dir: path to the PostgreSQL data directory to be checked. :returns: ``True`` if the data directory is empty. """ if os.path.isfile(os.path.join(data_dir, "global", "pg_control")): return False return data_directory_is_empty(data_dir) def validate_connect_address(address: str) -> bool: """Check if options related to connection address were properly configured. :param address: address to be validated in the format ``host:ip``. :returns: ``True`` if the address is valid. :raises: :class:`~patroni.exceptions.ConfigParseError`: * If the address is not in the expected format; or * If the host is set to not allowed values (``127.0.0.1``, ``0.0.0.0``, ``*``, ``::1``, or ``localhost``). """ try: host, _ = split_host_port(address, 1) except (AttributeError, TypeError, ValueError): raise ConfigParseError("contains a wrong value") if host in ["127.0.0.1", "0.0.0.0", "*", "::1", "localhost"]: raise ConfigParseError('must not contain "127.0.0.1", "0.0.0.0", "*", "::1", "localhost"') return True def validate_host_port(host_port: str, listen: bool = False, multiple_hosts: bool = False) -> bool: """Check if host(s) and port are valid and available for usage. :param host_port: the host(s) and port to be validated. It can be in either of these formats: * ``host:ip``, if *multiple_hosts* is ``False``; or * ``host_1,host_2,...,host_n:port``, if *multiple_hosts* is ``True``. :param listen: if the address is expected to be available for binding. ``False`` means it expects to connect to that address, and ``True`` that it expects to bind to that address. :param multiple_hosts: if *host_port* can contain multiple hosts. :returns: ``True`` if the host(s) and port are valid. :raises: :class:`~patroni.exceptions.ConfigParseError`: * If the *host_port* is not in the expected format; or * If ``*`` was specified along with more hosts in *host_port*; or * If we are expecting to bind to an address that is already in use; or * If we are not able to connect to an address that we are expecting to do so; or * If :class:`~socket.gaierror` is thrown by socket module when attempting to connect to the given address(es). """ try: hosts, port = split_host_port(host_port, 1) except (ValueError, TypeError): raise ConfigParseError("contains a wrong value") else: if multiple_hosts: hosts = hosts.split(",") else: hosts = [hosts] if "*" in hosts: if len(hosts) != 1: raise ConfigParseError("expecting '*' alone") # If host is set to "*" get all hostnames and/or IP addresses that the host would be able to listen to hosts = [p[-1][0] for p in socket.getaddrinfo(None, port, 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)] for host in hosts: # Check if "socket.IF_INET" or "socket.IF_INET6" is being used and instantiate a socket with the identified # protocol proto = socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE) s = socket.socket(proto[0][0], socket.SOCK_STREAM) try: if s.connect_ex((host, port)) == 0: # Do not raise an exception if ignore_listen_port is set to True. if listen and not _validation_params.get('ignore_listen_port', False): raise ConfigParseError("Port {} is already in use.".format(port)) elif not listen: raise ConfigParseError("{} is not reachable".format(host_port)) except socket.gaierror as e: raise ConfigParseError(e) finally: s.close() return True def validate_host_port_list(value: List[str]) -> bool: """Validate a list of host(s) and port items. Call :func:`validate_host_port` with each item in *value*. :param value: list of host(s) and port items to be validated. :returns: ``True`` if all items are valid. """ assert all([validate_host_port(v) for v in value]), "didn't pass the validation" return True def comma_separated_host_port(string: str) -> bool: """Validate a list of host and port items. Call :func:`validate_host_port_list` with a list represented by the CSV *string*. :param string: comma-separated list of host and port items. :returns: ``True`` if all items in the CSV string are valid. """ return validate_host_port_list([s.strip() for s in string.split(",")]) def validate_host_port_listen(host_port: str) -> bool: """Check if host and port are valid and available for binding. Call :func:`validate_host_port` with *listen* set to ``True``. :param host_port: the host and port to be validated. Must be in the format ``host:ip``. :returns: ``True`` if the host and port are valid and available for binding. """ return validate_host_port(host_port, listen=True) def validate_host_port_listen_multiple_hosts(host_port: str) -> bool: """Check if host(s) and port are valid and available for binding. Call :func:`validate_host_port` with both *listen* and *multiple_hosts* set to ``True``. :param host_port: the host(s) and port to be validated. It can be in either of these formats * ``host:ip``; or * ``host_1,host_2,...,host_n:port`` :returns: ``True`` if the host(s) and port are valid and available for binding. """ return validate_host_port(host_port, listen=True, multiple_hosts=True) def is_ipv4_address(ip: str) -> bool: """Check if *ip* is a valid IPv4 address. :param ip: the IP to be checked. :returns: ``True`` if the IP is an IPv4 address. :raises: :class:`~patroni.exceptions.ConfigParseError`: if *ip* is not a valid IPv4 address. """ try: socket.inet_aton(ip) except Exception: raise ConfigParseError("Is not a valid ipv4 address") return True def is_ipv6_address(ip: str) -> bool: """Check if *ip* is a valid IPv6 address. :param ip: the IP to be checked. :returns: ``True`` if the IP is an IPv6 address. :raises: :class:`~patroni.exceptions.ConfigParseError`: if *ip* is not a valid IPv6 address. """ try: socket.inet_pton(socket.AF_INET6, ip) except Exception: raise ConfigParseError("Is not a valid ipv6 address") return True def get_bin_name(bin_name: str) -> str: """Get the value of ``postgresql.bin_name[*bin_name*]`` configuration option. :param bin_name: a key to be retrieved from ``postgresql.bin_name`` configuration. :returns: value of ``postgresql.bin_name[*bin_name*]``, if present, otherwise *bin_name*. """ data = cast(Dict[Any, Any], schema.data) return (data.get('postgresql', {}).get('bin_name', {}) or EMPTY_DICT).get(bin_name, bin_name) def validate_data_dir(data_dir: str) -> bool: """Validate the value of ``postgresql.data_dir`` configuration option. It requires that ``postgresql.data_dir`` is set and match one of following conditions: * Point to a path that does not exist yet; or * Point to an empty directory; or * Point to a non-empty directory that seems to contain a valid PostgreSQL data directory. :param data_dir: the value of ``postgresql.data_dir`` configuration option. :returns: ``True`` if the PostgreSQL data directory is valid. :raises: :class:`~patroni.exceptions.ConfigParseError`: * If no *data_dir* was given; or * If *data_dir* is a file and not a directory; or * If *data_dir* is a non-empty directory and: * ``PG_VERSION`` file is not available in the directory * ``pg_wal``/``pg_xlog`` is not available in the directory * ``PG_VERSION`` content does not match the major version reported by ``postgres --version`` """ if not data_dir: raise ConfigParseError("is an empty string") elif os.path.exists(data_dir) and not os.path.isdir(data_dir): raise ConfigParseError("is not a directory") elif not data_directory_empty(data_dir): if not os.path.exists(os.path.join(data_dir, "PG_VERSION")): raise ConfigParseError("doesn't look like a valid data directory") else: with open(os.path.join(data_dir, "PG_VERSION"), "r") as version: pgversion = version.read().strip() waldir = ("pg_wal" if float(pgversion) >= 10 else "pg_xlog") if not os.path.isdir(os.path.join(data_dir, waldir)): raise ConfigParseError("data dir for the cluster is not empty, but doesn't contain" " \"{}\" directory".format(waldir)) data = cast(Dict[Any, Any], schema.data) bin_dir = data.get("postgresql", {}).get("bin_dir", None) major_version = get_major_version(bin_dir, get_bin_name('postgres')) if pgversion != major_version: raise ConfigParseError("data_dir directory postgresql version ({}) doesn't match with " "'postgres --version' output ({})".format(pgversion, major_version)) return True def validate_binary_name(bin_name: str) -> bool: """Validate the value of ``postgresql.binary_name[*bin_name*]`` configuration option. If ``postgresql.bin_dir`` is set and the value of the *bin_name* meets these conditions: * The path join of ``postgresql.bin_dir`` plus the *bin_name* value exists; and * The path join as above is executable If ``postgresql.bin_dir`` is not set, then validate that the value of *bin_name* meets this condition: * Is found in the system PATH using ``which`` :param bin_name: the value of the ``postgresql.bin_name[*bin_name*]`` :returns: ``True`` if the conditions are true :raises: :class:`~patroni.exceptions.ConfigParseError` if: * *bin_name* is not set; or * the path join of the ``postgresql.bin_dir`` plus *bin_name* does not exist; or * the path join as above is not executable; or * the *bin_name* cannot be found in the system PATH """ if not bin_name: raise ConfigParseError("is an empty string") data = cast(Dict[Any, Any], schema.data) bin_dir = data.get('postgresql', {}).get('bin_dir', None) if not shutil.which(bin_name, path=bin_dir): raise ConfigParseError(f"does not contain '{bin_name}' in '{bin_dir or '$PATH'}'") return True class Result(object): """Represent the result of a given validation that was performed. :ivar status: If the validation succeeded. :ivar path: YAML tree path of the configuration option. :ivar data: value of the configuration option. :ivar level: error level, in case of error. :ivar error: error message if the validation failed, otherwise ``None``. """ def __init__(self, status: bool, error: OptionalType[str] = "didn't pass validation", level: int = 0, path: str = "", data: Any = "") -> None: """Create a :class:`Result` object based on the given arguments. .. note:: ``error`` attribute is only set if *status* is failed. :param status: if the validation succeeded. :param error: error message related to the validation that was performed, if the validation failed. :param level: error level, in case of error. :param path: YAML tree path of the configuration option. :param data: value of the configuration option. """ self.status = status self.path = path self.data = data self.level = level self._error = error if not self.status: self.error = error else: self.error = None def __repr__(self) -> str: """Show configuration path and value. If the validation failed, also show the error message.""" return str(self.path) + (" " + str(self.data) + " " + str(self._error) if self.error else "") class Case(object): """Map how a list of available configuration options should be validated. .. note:: It should be used together with an :class:`Or` object. The :class:`Or` object will define the list of possible configuration options in a given context, and the :class:`Case` object will dictate how to validate each of them, if they are set. """ def __init__(self, schema: Dict[str, Any]) -> None: """Create a :class:`Case` object. :param schema: the schema for validating a set of attributes that may be available in the configuration. Each key is the configuration that is available in a given scope and that should be validated, and the related value is the validation function or expected type. :Example: .. code-block:: python Case({ "host": validate_host_port, "url": str, }) That will check that ``host`` configuration, if given, is valid based on :func:`validate_host_port`, and will also check that ``url`` configuration, if given, is a ``str`` instance. """ self._schema = schema class Or(object): """Represent the list of options that are available. It can represent either a list of configuration options that are available in a given scope, or a list of validation functions and/or expected types for a given configuration option. """ def __init__(self, *args: Any) -> None: """Create an :class:`Or` object. :param `*args`: any arguments that the caller wants to be stored in this :class:`Or` object. :Example: .. code-block:: python Or("host", "hosts"): Case({ "host": validate_host_port, "hosts": Or(comma_separated_host_port, [validate_host_port]), }) The outer :class:`Or` is used to define that ``host`` and ``hosts`` are possible options in this scope. The inner :class`Or` in the ``hosts`` key value is used to define that ``hosts`` option is valid if either of :func:`comma_separated_host_port` or :func:`validate_host_port` succeed to validate it. """ self.args = args class AtMostOne(object): """Mark that at most one option from a :class:`Case` can be supplied. Represents a list of possible configuration options in a given scope, where at most one can actually be provided. .. note:: It should be used together with a :class:`Case` object. """ def __init__(self, *args: str) -> None: """Create a :class`AtMostOne` object. :param `*args`: any arguments that the caller wants to be stored in this :class:`Or` object. :Example: .. code-block:: python AtMostOne("nofailover", "failover_priority"): Case({ "nofailover": bool, "failover_priority": IntValidator(min=0, raise_assert=True), }) The :class`AtMostOne` object is used to define that at most one of ``nofailover`` and ``failover_priority`` can be provided. """ self.args = args class Optional(object): """Mark a configuration option as optional. :ivar name: name of the configuration option. :ivar default: value to set if the configuration option is not explicitly provided """ def __init__(self, name: str, default: OptionalType[Any] = None) -> None: """Create an :class:`Optional` object. :param name: name of the configuration option. :param default: value to set if the configuration option is not explicitly provided """ self.name = name self.default = default class Directory(object): """Check if a directory contains the expected files. The attributes of objects of this class are used by their :func:`validate` method. :param contains: list of paths that should exist relative to a given directory. :param contains_executable: list of executable files that should exist directly under a given directory. """ def __init__(self, contains: OptionalType[List[str]] = None, contains_executable: OptionalType[List[str]] = None) -> None: """Create a :class:`Directory` object. :param contains: list of paths that should exist relative to a given directory. :param contains_executable: list of executable files that should exist directly under a given directory. """ self.contains = contains self.contains_executable = contains_executable def _check_executables(self, path: OptionalType[str] = None) -> Iterator[Result]: """Check that all executables from contains_executable list exist within the given directory or within ``PATH``. :param path: optional path to the base directory against which executables will be validated. If not provided, check within ``PATH``. :yields: objects with the error message containing the name of the executable, if any check fails. """ for program in self.contains_executable or []: if not shutil.which(program, path=path): yield Result(False, f"does not contain '{program}' in '{(path or '$PATH')}'") def validate(self, name: str) -> Iterator[Result]: """Check if the expected paths and executables can be found under *name* directory. :param name: path to the base directory against which paths and executables will be validated. Check against ``PATH`` if name is not provided. :yields: objects with the error message related to the failure, if any check fails. """ if not name: yield from self._check_executables() elif not os.path.exists(name): yield Result(False, "Directory '{}' does not exist.".format(name)) elif not os.path.isdir(name): yield Result(False, "'{}' is not a directory.".format(name)) else: if self.contains: for path in self.contains: if not os.path.exists(os.path.join(name, path)): yield Result(False, "'{}' does not contain '{}'".format(name, path)) yield from self._check_executables(path=name) class BinDirectory(Directory): """Check if a Postgres binary directory contains the expected files. It is a subclass of :class:`Directory` with an extended capability: translating ``BINARIES`` according to configured ``postgresql.bin_name``, if any. :cvar BINARIES: list of executable files that should exist directly under a given Postgres binary directory. """ # ``pg_rewind`` is not in the list because its usage by Patroni is optional. Also, it is not available by default on # Postgres 9.3 and 9.4, versions which Patroni supports. BINARIES = ["pg_ctl", "initdb", "pg_controldata", "pg_basebackup", "postgres", "pg_isready"] def validate(self, name: str) -> Iterator[Result]: """Check if the expected executables can be found under *name* binary directory. :param name: path to the base directory against which executables will be validated. Check against PATH if *name* is not provided. :yields: objects with the error message related to the failure, if any check fails. """ self.contains_executable: List[str] = [get_bin_name(binary) for binary in self.BINARIES] yield from super().validate(name) class Schema(object): """Define a configuration schema. It contains all the configuration options that are available in each scope, including the validation(s) that should be performed against each one of them. The validations will be performed whenever the :class:`Schema` object is called, or its :func:`validate` method is called. :ivar validator: validator of the configuration schema. Can be any of these: * :class:`str`: defines that a string value is required; or * :class:`type`: any subclass of :class:`type`, defines that a value of the given type is required; or * ``callable``: any callable object, defines that validation will follow the code defined in the callable object. If the callable object contains an ``expected_type`` attribute, then it will check if the configuration value is of the expected type before calling the code of the callable object; or * :class:`list`: list representing one or more values in the configuration; or * :class:`dict`: dictionary representing the YAML configuration tree. """ def __init__(self, validator: Union[Dict[Any, Any], List[Any], Any]) -> None: """Create a :class:`Schema` object. .. note:: This class is expected to be initially instantiated with a :class:`dict` based *validator* argument. The idea is that dict represents the full YAML tree of configuration options. The :func:`validate` method will then walk recursively through the configuration tree, creating new instances of :class:`Schema` with the new "base path", to validate the structure and the leaf values of the tree. The recursion stops on leaf nodes, when it performs checks of the actual setting values. :param validator: validator of the configuration schema. Can be any of these: * :class:`str`: defines that a string value is required; or * :class:`type`: any subclass of :class:`type`, defines that a value of the given type is required; or * ``callable``: Any callable object, defines that validation will follow the code defined in the callable object. If the callable object contains an ``expected_type`` attribute, then it will check if the configuration value is of the expected type before calling the code of the callable object; or * :class:`list`: list representing it expects to contain one or more values in the configuration; or * :class:`dict`: dictionary representing the YAML configuration tree. The first 3 items in the above list are here referenced as "base validators", which cause the recursion to stop. If *validator* is a :class:`dict`, then you should follow these rules: * For the keys it can be either: * A :class:`str` instance. It will be the name of the configuration option; or * An :class:`Optional` instance. The ``name`` attribute of that object will be the name of the configuration option, and that class makes this configuration option as optional to the user, allowing it to not be specified in the YAML; or * An :class:`Or` instance. The ``args`` attribute of that object will contain a tuple of configuration option names. At least one of them should be specified by the user in the YAML; * For the values it can be either: * A new :class:`dict` instance. It will represent a new level in the YAML configuration tree; or * A :class:`Case` instance. This is required if the key of this value is an :class:`Or` instance, and the :class:`Case` instance is used to map each of the ``args`` in :class:`Or` to their corresponding base validator in :class:`Case`; or * An :class:`Or` instance with one or more base validators; or * A :class:`list` instance with a single item which is the base validator; or * A base validator. :Example: .. code-block:: python Schema({ "application_name": str, "bind": { "host": validate_host, "port": int, }, "aliases": [str], Optional("data_directory"): "/var/lib/myapp", Or("log_to_file", "log_to_db"): Case({ "log_to_file": bool, "log_to_db": bool, }), "version": Or(int, float), }) This sample schema defines that your YAML configuration follows these rules: * It must contain an ``application_name`` entry which value should be a :class:`str` instance; * It must contain a ``bind.host`` entry which value should be valid as per function ``validate_host``; * It must contain a ``bind.port`` entry which value should be an :class:`int` instance; * It must contain a ``aliases`` entry which value should be a :class:`list` of :class:`str` instances; * It may optionally contain a ``data_directory`` entry, with a value which should be a string; * It must contain at least one of ``log_to_file`` or ``log_to_db``, with a value which should be a :class:`bool` instance; * It must contain a ``version`` entry which value should be either an :class:`int` or a :class:`float` instance. """ self.validator = validator def __call__(self, data: Any) -> List[str]: """Perform validation of data using the rules defined in this schema. :param data: configuration to be validated against ``validator``. :returns: list of errors identified while validating the *data*, if any. """ errors: List[str] = [] for i in self.validate(data): if not i.status: errors.append(str(i)) return errors def validate(self, data: Any) -> Iterator[Result]: """Perform all validations from the schema against the given configuration. It first checks that *data* argument type is compliant with the type of ``validator`` attribute. Additionally: * If ``validator`` attribute is a callable object, calls it to validate *data* argument. Before doing so, if `validator` contains an ``expected_type`` attribute, check if *data* argument is compliant with that expected type. * If ``validator`` attribute is an iterable object (:class:`dict`, :class:`list`, :class:`Directory` or :class:`Or`), then it iterates over it to validate each of the corresponding entries in *data* argument. :param data: configuration to be validated against ``validator``. :yields: objects with the error message related to the failure, if any check fails. """ self.data = data # New `Schema` objects can be created while validating a given `Schema`, depending on its structure. The first # 3 IF statements deal with the situation where we already reached a leaf node in the `Schema` structure, then # we are dealing with an actual value validation. The remaining logic in this method is used to iterate through # iterable objects in the structure, until we eventually reach a leaf node to validate its value. if isinstance(self.validator, str): yield Result(isinstance(self.data, str), "is not a string", level=1, data=self.data) elif isinstance(self.validator, type): yield Result(isinstance(self.data, self.validator), "is not {}".format(_get_type_name(self.validator)), level=1, data=self.data) elif callable(self.validator): if hasattr(self.validator, "expected_type"): expected_type = getattr(self.validator, 'expected_type') if not isinstance(data, expected_type): yield Result(False, "is not {}".format(_get_type_name(expected_type)), level=1, data=self.data) return try: self.validator(data) yield Result(True, data=self.data) except Exception as e: yield Result(False, "didn't pass validation: {}".format(e), data=self.data) elif isinstance(self.validator, dict): if not isinstance(self.data, dict): yield Result(isinstance(self.data, dict), "is not a dictionary", level=1, data=self.data) else: yield from self.iter_dict() elif isinstance(self.validator, list): if not isinstance(self.data, list): yield Result(isinstance(self.data, list), "is not a list", level=1, data=self.data) else: yield from self.iter_list() elif isinstance(self.validator, Or): yield from self.iter_or() elif isinstance(self.validator, Directory) and isinstance(self.data, str): yield from self.validator.validate(self.data) def iter_list(self) -> Iterator[Result]: """Iterate over a ``data`` object and perform validations using the first element of the ``validator``. :yields: objects with the error message related to the failure, if any check fails. """ data = cast(List[Any], self.data) if len(data) == 0: yield Result(False, "is an empty list", data=data) validators = cast(List[Any], self.validator) if len(validators): for key, value in enumerate(data): # Although the value in the configuration (`data`) is expected to contain 1 or more entries, only # the first validator defined in `validator` property list will be used. It is only defined as a # `list` in `validator` so this logic can understand that the value in `data` attribute should be a # `list`. For example: "pg_hba": [str] in `validator` attribute defines that "pg_hba" in `data` # attribute should contain a list with one or more `str` entries. for v in Schema(validators[0]).validate(value): yield Result(v.status, v.error, path=(str(key) + ("." + v.path if v.path else "")), level=v.level, data=value) def iter_dict(self) -> Iterator[Result]: """Iterate over a :class:`dict` based ``validator`` to validate the corresponding entries in ``data``. :yields: objects with the error message related to the failure, if any check fails. """ # One key in `validator` attribute (`key` variable) can be mapped to one or more keys in `data` attribute (`d` # variable), depending on the `key` type. data = cast(Dict[Any, Any], self.data) validators = cast(Dict[Any, Any], self.validator) for key in validators.keys(): if isinstance(key, AtMostOne) and len(list(self._data_key(key))) > 1: yield Result(False, f"Multiple of {key.args} provided") continue for d in self._data_key(key): if d not in data and not isinstance(key, Optional): yield Result(False, "is not defined.", path=d) elif d not in data and isinstance(key, Optional) and key.default is None: continue else: if d not in data and isinstance(key, Optional): data[d] = key.default validator = validators[key] if isinstance(key, (Or, AtMostOne)) and isinstance(validators[key], Case): validator = validators[key]._schema[d] # In this loop we may be calling a new `Schema` either over an intermediate node in the tree, or # over a leaf node. In the latter case the recursive calls in the given path will finish. for v in Schema(validator).validate(data[d]): yield Result(v.status, v.error, path=(d + ("." + v.path if v.path else "")), level=v.level, data=v.data) def iter_or(self) -> Iterator[Result]: """Perform all validations defined in an :class:`Or` object for a given configuration option. This method can be only called against leaf nodes in the configuration tree. :class:`Or` objects defined in the ``validator`` keys will be handled by :func:`iter_dict` method. :yields: objects with the error message related to the failure, if any check fails. """ if TYPE_CHECKING: # pragma: no cover assert isinstance(self.validator, Or) results: List[Result] = [] for a in self.validator.args: r: List[Result] = [] # Each of the `Or` validators can throw 0 to many `Result` instances. for v in Schema(a).validate(self.data): r.append(v) if any([x.status for x in r]) and not all([x.status for x in r]): results += [x for x in r if not x.status] else: results += r # None of the `Or` validators succeeded to validate `data`, so we report the issues back. if not any([x.status for x in results]): max_level = 3 for v in sorted(results, key=lambda x: x.level): if v.level > max_level: break max_level = v.level yield Result(v.status, v.error, path=v.path, level=v.level, data=v.data) def _data_key(self, key: Union[str, Optional, Or, AtMostOne]) -> Iterator[str]: """Map a key from the ``validator`` dictionary to the corresponding key(s) in the ``data`` dictionary. :param key: key from the ``validator`` attribute. :yields: keys that should be used to access corresponding value in the ``data`` attribute. """ data = cast(Dict[Any, Any], self.data) # If the key was defined as an `Optional` object in `validator` attribute, then its name is the key to access # the `data` dictionary. if isinstance(key, Optional): yield key.name # If the key was defined as a `str` object in `validator` attribute, then it is already the final key # to access the `data` dictionary. elif isinstance(key, str): yield key # If the key was defined as an `Or` object in `validator` attribute, then each of its values are # the keys to access the `data` dictionary. elif isinstance(key, Or): # At least one of the `Or` entries should be available in the `data` dictionary. If we find at least # one of them in `data`, then we return all found entries so the caller method can validate them all. if any([item in data for item in key.args]): for item in key.args: if item in data: yield item # If none of the `Or` entries is available in the `data` dictionary, then we return all entries so the # caller method will issue errors that they are all absent. else: for item in key.args: yield item # If the key was defined as a `AtMostOne` object in `validator` attribute, then each of its values # are the keys to access the `data` dictionary. elif isinstance(key, AtMostOne): # pyright: ignore [reportUnnecessaryIsInstance] # Yield back all of the entries from the `data` dictionary, each will be validated and then counted # to inform us if we've provided too many for item in key.args: if item in data: yield item def _get_type_name(python_type: Any) -> str: """Get a user-friendly name for a given Python type. :param python_type: Python type which user friendly name should be taken. :returns: User friendly name of the given Python type. """ types: Dict[Any, str] = {str: 'a string', int: 'an integer', float: 'a number', bool: 'a boolean', list: 'an array', dict: 'a dictionary'} return types.get(python_type, getattr(python_type, __name__, "unknown type")) def assert_(condition: bool, message: str = "Wrong value") -> None: """Assert that a given condition is ``True``. If the assertion fails, then throw a message. :param condition: result of a condition to be asserted. :param message: message to be thrown if the condition is ``False``. """ assert condition, message class IntValidator(object): """Validate an integer setting. :ivar min: minimum allowed value for the setting, if any. :ivar max: maximum allowed value for the setting, if any. :ivar base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range. :ivar expected_type: the expected Python type. :ivar raise_assert: if an ``assert`` test should be performed regarding expected type and valid range. """ def __init__(self, min: OptionalType[int] = None, max: OptionalType[int] = None, base_unit: OptionalType[str] = None, expected_type: Any = None, raise_assert: bool = False) -> None: """Create an :class:`IntValidator` object with the given rules. :param min: minimum allowed value for the setting, if any. :param max: maximum allowed value for the setting, if any. :param base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range. :param expected_type: the expected Python type. :param raise_assert: if an ``assert`` test should be performed regarding expected type and valid range. """ self.min = min self.max = max self.base_unit = base_unit if expected_type: self.expected_type = expected_type self.raise_assert = raise_assert def __call__(self, value: Any) -> bool: """Check if *value* is a valid integer and within the expected range. .. note:: If ``raise_assert`` is ``True`` and *value* is not valid, then an :class:`AssertionError` will be triggered. :param value: value to be checked against the rules defined for this :class:`IntValidator` instance. :returns: ``True`` if *value* is valid and within the expected range. """ value = parse_int(value, self.base_unit) ret = isinstance(value, int)\ and (self.min is None or value >= self.min)\ and (self.max is None or value <= self.max) if self.raise_assert: assert_(ret) return ret class EnumValidator(object): """Validate enum setting :ivar allowed_values: a ``set`` or ``CaseInsensitiveSet`` object with allowed enum values. :ivar raise_assert: if an ``assert`` call should be performed regarding expected type and valid range. """ def __init__(self, allowed_values: Tuple[str, ...], case_sensitive: bool = False, raise_assert: bool = False) -> None: """Create an :class:`EnumValidator` object with given allowed values. :param allowed_values: a tuple with allowed enum values :param case_sensitive: set to ``True`` to do case sensitive comparisons :param raise_assert: if an ``assert`` call should be performed regarding expected values. """ self.allowed_values = set(allowed_values) if case_sensitive else CaseInsensitiveSet(allowed_values) self.raise_assert = raise_assert def __call__(self, value: Any) -> bool: """Check if provided *value* could be found within *allowed_values*. .. note:: If ``raise_assert`` is ``True`` and *value* is not valid, then an ``AssertionError`` will be triggered. :param value: value to be checked. :returns: ``True`` if *value* could be found within *allowed_values*. """ ret = isinstance(value, str) and value in self.allowed_values if self.raise_assert: assert_(ret) return ret def validate_watchdog_mode(value: Any) -> None: """Validate ``watchdog.mode`` configuration option. :param value: value of ``watchdog.mode`` to be validated. """ assert_(isinstance(value, (str, bool)), "expected type is not a string") assert_(value in (False, "off", "automatic", "required")) userattributes = {"username": "", Optional("password"): ""} available_dcs = [m.split(".")[-1] for m in dcs_modules()] setattr(validate_host_port_list, 'expected_type', list) setattr(comma_separated_host_port, 'expected_type', str) setattr(validate_connect_address, 'expected_type', str) setattr(validate_host_port_listen, 'expected_type', str) setattr(validate_host_port_listen_multiple_hosts, 'expected_type', str) setattr(validate_data_dir, 'expected_type', str) setattr(validate_binary_name, 'expected_type', str) validate_etcd = { Or("host", "hosts", "srv", "srv_suffix", "url", "proxy"): Case({ "host": validate_host_port, "hosts": Or(comma_separated_host_port, [validate_host_port]), "srv": str, "srv_suffix": str, "url": str, "proxy": str }), Optional("protocol"): str, Optional("username"): str, Optional("password"): str, Optional("cacert"): str, Optional("cert"): str, Optional("key"): str } schema = Schema({ "name": str, "scope": str, Optional("log"): { Optional("type"): EnumValidator(('plain', 'json'), case_sensitive=True, raise_assert=True), Optional("level"): EnumValidator(('DEBUG', 'INFO', 'WARN', 'WARNING', 'ERROR', 'FATAL', 'CRITICAL'), case_sensitive=True, raise_assert=True), Optional("traceback_level"): EnumValidator(('DEBUG', 'ERROR'), raise_assert=True), Optional("format"): validate_log_format, Optional("dateformat"): str, Optional("static_fields"): dict, Optional("max_queue_size"): int, Optional("dir"): str, Optional("file_num"): int, Optional("file_size"): int, Optional("mode"): IntValidator(min=0, max=511, expected_type=int, raise_assert=True), Optional("loggers"): dict }, Optional("ctl"): { Optional("insecure"): bool, Optional("cacert"): str, Optional("certfile"): str, Optional("keyfile"): str, Optional("keyfile_password"): str }, "restapi": { "listen": validate_host_port_listen, "connect_address": validate_connect_address, Optional("authentication"): { "username": str, "password": str }, Optional("certfile"): str, Optional("keyfile"): str, Optional("keyfile_password"): str, Optional("cafile"): str, Optional("ciphers"): str, Optional("verify_client"): EnumValidator(("none", "optional", "required"), case_sensitive=True, raise_assert=True), Optional("allowlist"): [str], Optional("allowlist_include_members"): bool, Optional("http_extra_headers"): dict, Optional("https_extra_headers"): dict, Optional("request_queue_size"): IntValidator(min=0, max=4096, expected_type=int, raise_assert=True) }, Optional("bootstrap"): { "dcs": { Optional("ttl"): IntValidator(min=20, raise_assert=True), Optional("loop_wait"): IntValidator(min=1, raise_assert=True), Optional("retry_timeout"): IntValidator(min=3, raise_assert=True), Optional("maximum_lag_on_failover"): IntValidator(min=0, raise_assert=True), Optional("maximum_lag_on_syncnode"): IntValidator(min=-1, raise_assert=True), Optional('member_slots_ttl'): IntValidator(min=0, base_unit='s', raise_assert=True), Optional("postgresql"): { Optional("parameters"): { Optional("max_connections"): IntValidator(1, 262143, raise_assert=True), Optional("max_locks_per_transaction"): IntValidator(10, 2147483647, raise_assert=True), Optional("max_prepared_transactions"): IntValidator(0, 262143, raise_assert=True), Optional("max_replication_slots"): IntValidator(0, 262143, raise_assert=True), Optional("max_wal_senders"): IntValidator(0, 262143, raise_assert=True), Optional("max_worker_processes"): IntValidator(0, 262143, raise_assert=True), }, Optional("use_pg_rewind"): bool, Optional("pg_hba"): [str], Optional("pg_ident"): [str], Optional("pg_ctl_timeout"): IntValidator(min=0, raise_assert=True), Optional("use_slots"): bool, }, Optional("primary_start_timeout"): IntValidator(min=0, raise_assert=True), Optional("primary_stop_timeout"): IntValidator(min=0, raise_assert=True), Optional("standby_cluster"): { Or("host", "port", "restore_command"): Case({ "host": str, "port": IntValidator(max=65535, expected_type=int, raise_assert=True), "restore_command": str }), Optional("primary_slot_name"): str, Optional("create_replica_methods"): [str], Optional("archive_cleanup_command"): str, Optional("recovery_min_apply_delay"): str }, Optional("synchronous_mode"): bool, Optional("synchronous_mode_strict"): bool, Optional("synchronous_node_count"): IntValidator(min=1, raise_assert=True), }, Optional("initdb"): [Or(str, dict)], Optional("method"): str }, Or(*available_dcs): Case({ "consul": { Or("host", "url"): Case({ "host": validate_host_port, "url": str }), Optional("port"): IntValidator(max=65535, expected_type=int, raise_assert=True), Optional("scheme"): str, Optional("token"): str, Optional("verify"): bool, Optional("cacert"): str, Optional("cert"): str, Optional("key"): str, Optional("dc"): str, Optional("checks"): [str], Optional("register_service"): bool, Optional("service_tags"): [str], Optional("service_check_interval"): str, Optional("service_check_tls_server_name"): str, Optional("consistency"): EnumValidator(('default', 'consistent', 'stale'), case_sensitive=True, raise_assert=True) }, "etcd": validate_etcd, "etcd3": validate_etcd, "exhibitor": { "hosts": [str], "port": IntValidator(max=65535, expected_type=int, raise_assert=True), Optional("poll_interval"): IntValidator(min=1, expected_type=int, raise_assert=True), }, "raft": { "self_addr": validate_connect_address, Optional("bind_addr"): validate_host_port_listen, "partner_addrs": validate_host_port_list, Optional("data_dir"): str, Optional("password"): str }, "zookeeper": { "hosts": Or(comma_separated_host_port, [validate_host_port]), Optional("use_ssl"): bool, Optional("cacert"): str, Optional("cert"): str, Optional("key"): str, Optional("key_password"): str, Optional("verify"): bool, Optional("set_acls"): dict, Optional("auth_data"): dict, }, "kubernetes": { "labels": {}, Optional("bypass_api_service"): bool, Optional("namespace"): str, Optional("scope_label"): str, Optional("role_label"): str, Optional("leader_label_value"): str, Optional("follower_label_value"): str, Optional("standby_leader_label_value"): str, Optional("tmp_role_label"): str, Optional("use_endpoints"): bool, Optional("pod_ip"): Or(is_ipv4_address, is_ipv6_address), Optional("ports"): [{"name": str, "port": IntValidator(max=65535, expected_type=int, raise_assert=True)}], Optional("cacert"): str, Optional("retriable_http_codes"): Or(int, [int]), }, }), Optional("citus"): { "database": str, "group": IntValidator(min=0, expected_type=int, raise_assert=True), }, "postgresql": { "listen": validate_host_port_listen_multiple_hosts, "connect_address": validate_connect_address, Optional("proxy_address"): validate_connect_address, "authentication": { "replication": userattributes, "superuser": userattributes, Optional("rewind"): userattributes }, "data_dir": validate_data_dir, Optional("bin_name"): { Optional("pg_ctl"): validate_binary_name, Optional("initdb"): validate_binary_name, Optional("pg_controldata"): validate_binary_name, Optional("pg_basebackup"): validate_binary_name, Optional("postgres"): validate_binary_name, Optional("pg_isready"): validate_binary_name, Optional("pg_rewind"): validate_binary_name, }, Optional("bin_dir", ""): BinDirectory(), Optional("parameters"): { Optional("unix_socket_directories"): str }, Optional("pg_hba"): [str], Optional("pg_ident"): [str], Optional("pg_ctl_timeout"): IntValidator(min=0, raise_assert=True), Optional("use_pg_rewind"): bool }, Optional("watchdog"): { Optional("mode"): validate_watchdog_mode, Optional("device"): str, Optional("safety_margin"): IntValidator(min=-1, expected_type=int, raise_assert=True), }, Optional("tags"): { AtMostOne("nofailover", "failover_priority"): Case({ "nofailover": bool, "failover_priority": IntValidator(min=0, expected_type=int, raise_assert=True), }), Optional("clonefrom"): bool, Optional("noloadbalance"): bool, Optional("replicatefrom"): str, Optional("nosync"): bool, Optional("nostream"): bool } }) patroni-4.0.4/patroni/version.py000066400000000000000000000002001472010352700166770ustar00rootroot00000000000000"""This module specifies the current Patroni version. :var __version__: the current Patroni version. """ __version__ = '4.0.4' patroni-4.0.4/patroni/watchdog/000077500000000000000000000000001472010352700164505ustar00rootroot00000000000000patroni-4.0.4/patroni/watchdog/__init__.py000066400000000000000000000001431472010352700205570ustar00rootroot00000000000000from patroni.watchdog.base import Watchdog, WatchdogError __all__ = ['WatchdogError', 'Watchdog'] patroni-4.0.4/patroni/watchdog/base.py000066400000000000000000000305001472010352700177320ustar00rootroot00000000000000import abc import logging import platform import sys from threading import RLock from typing import Any, Callable, Dict, Optional, Union from ..config import Config from ..exceptions import WatchdogError __all__ = ['WatchdogError', 'Watchdog'] logger = logging.getLogger(__name__) MODE_REQUIRED = 'required' # Will not run if a watchdog is not available MODE_AUTOMATIC = 'automatic' # Will use a watchdog if one is available MODE_OFF = 'off' # Will not try to use a watchdog def parse_mode(mode: Union[bool, str]) -> str: if mode is False: return MODE_OFF mode = str(mode).lower() if mode in ['require', 'required']: return MODE_REQUIRED elif mode in ['auto', 'automatic']: return MODE_AUTOMATIC else: if mode not in ['off', 'disable', 'disabled']: logger.warning("Watchdog mode {0} not recognized, disabling watchdog".format(mode)) return MODE_OFF def synchronized(func: Callable[..., Any]) -> Callable[..., Any]: def wrapped(self: 'Watchdog', *args: Any, **kwargs: Any) -> Any: with self.lock: return func(self, *args, **kwargs) return wrapped class WatchdogConfig(object): """Helper to contain a snapshot of configuration""" def __init__(self, config: Config) -> None: watchdog_config = config.get("watchdog") or {'mode': 'automatic'} self.mode = parse_mode(watchdog_config.get('mode', 'automatic')) self.ttl = config['ttl'] self.loop_wait = config['loop_wait'] self.safety_margin = watchdog_config.get('safety_margin', 5) self.driver = watchdog_config.get('driver', 'default') self.driver_config = dict((k, v) for k, v in watchdog_config.items() if k not in ['mode', 'safety_margin', 'driver']) def __eq__(self, other: Any) -> bool: return isinstance(other, WatchdogConfig) and \ all(getattr(self, attr) == getattr(other, attr) for attr in ['mode', 'ttl', 'loop_wait', 'safety_margin', 'driver', 'driver_config']) def __ne__(self, other: Any) -> bool: return not self == other def get_impl(self) -> 'WatchdogBase': if self.driver == 'testing': # pragma: no cover from patroni.watchdog.linux import TestingWatchdogDevice return TestingWatchdogDevice.from_config(self.driver_config) elif platform.system() == 'Linux' and self.driver == 'default': from patroni.watchdog.linux import LinuxWatchdogDevice return LinuxWatchdogDevice.from_config(self.driver_config) else: return NullWatchdog() @property def timeout(self) -> int: if self.safety_margin == -1: return int(self.ttl // 2) else: return self.ttl - self.safety_margin @property def timing_slack(self) -> int: return self.timeout - self.loop_wait class Watchdog(object): """Facade to dynamically manage watchdog implementations and handle config changes. When activation fails underlying implementation will be switched to a Null implementation. To avoid log spam activation will only be retried when watchdog configuration is changed.""" def __init__(self, config: Config) -> None: self.config = WatchdogConfig(config) self.active_config: WatchdogConfig = self.config self.lock = RLock() self.active = False if self.config.mode == MODE_OFF: self.impl = NullWatchdog() else: self.impl = self.config.get_impl() if self.config.mode == MODE_REQUIRED and self.impl.is_null: logger.error("Configuration requires a watchdog, but watchdog is not supported on this platform.") sys.exit(1) @synchronized def reload_config(self, config: Config) -> None: self.config = WatchdogConfig(config) # Turning a watchdog off can always be done immediately if self.config.mode == MODE_OFF: if self.active: self._disable() self.active_config = self.config self.impl = NullWatchdog() # If watchdog is not active we can apply config immediately to show any warnings early. Otherwise we need to # delay until next time a keepalive is sent so timeout matches up with leader key update. if not self.active: if self.config.driver != self.active_config.driver or \ self.config.driver_config != self.active_config.driver_config: self.impl = self.config.get_impl() self.active_config = self.config @synchronized def activate(self) -> bool: """Activates the watchdog device with suitable timeouts. While watchdog is active keepalive needs to be called every time loop_wait expires. :returns False if a safe watchdog could not be configured, but is required. """ self.active = True return self._activate() def _activate(self) -> bool: self.active_config = self.config if self.config.timing_slack < 0: logger.warning('Watchdog not supported because leader TTL {0} is less than 2x loop_wait {1}' .format(self.config.ttl, self.config.loop_wait)) self.impl = NullWatchdog() try: self.impl.open() actual_timeout = self._set_timeout() except WatchdogError as e: logger.warning("Could not activate %s: %s", self.impl.describe(), e) self.impl = NullWatchdog() actual_timeout = self.impl.get_timeout() if self.impl.is_running and not self.impl.can_be_disabled: logger.warning("Watchdog implementation can't be disabled." " Watchdog will trigger after Patroni loses leader key.") if not self.impl.is_running or actual_timeout and actual_timeout > self.config.timeout: if self.config.mode == MODE_REQUIRED: if self.impl.is_null: logger.error("Configuration requires watchdog, but watchdog could not be configured.") else: logger.error("Configuration requires watchdog, but a safe watchdog timeout {0} could" " not be configured. Watchdog timeout is {1}.".format( self.config.timeout, actual_timeout)) return False else: if not self.impl.is_null: logger.warning("Watchdog timeout {0} seconds does not ensure safe termination within {1} seconds" .format(actual_timeout, self.config.timeout)) if self.is_running: logger.info("{0} activated with {1} second timeout, timing slack {2} seconds" .format(self.impl.describe(), actual_timeout, self.config.timing_slack)) else: if self.config.mode == MODE_REQUIRED: logger.error("Configuration requires watchdog, but watchdog could not be activated") return False return True def _set_timeout(self) -> Optional[int]: if self.impl.has_set_timeout(): self.impl.set_timeout(self.config.timeout) # Safety checks for watchdog implementations that don't support configurable timeouts actual_timeout = self.impl.get_timeout() if self.impl.is_running and actual_timeout < self.config.loop_wait: logger.error('loop_wait of {0} seconds is too long for watchdog {1} second timeout' .format(self.config.loop_wait, actual_timeout)) if self.impl.can_be_disabled: logger.info('Disabling watchdog due to unsafe timeout.') self.impl.close() self.impl = NullWatchdog() return None return actual_timeout @synchronized def disable(self) -> None: self._disable() self.active = False def _disable(self) -> None: try: if self.impl.is_running and not self.impl.can_be_disabled: # Give sysadmin some extra time to clean stuff up. self.impl.keepalive() logger.warning("Watchdog implementation can't be disabled. System will reboot after " "{0} seconds when watchdog times out.".format(self.impl.get_timeout())) self.impl.close() except WatchdogError as e: logger.error("Error while disabling watchdog: %s", e) @synchronized def keepalive(self) -> None: try: if self.active: self.impl.keepalive() # In case there are any pending configuration changes apply them now. if self.active and self.config != self.active_config: if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF: self.impl = self.config.get_impl() self._activate() if self.config.driver != self.active_config.driver \ or self.config.driver_config != self.active_config.driver_config: self._disable() self.impl = self.config.get_impl() self._activate() if self.config.timeout != self.active_config.timeout: self.impl.set_timeout(self.config.timeout) if self.is_running: logger.info("{0} updated with {1} second timeout, timing slack {2} seconds" .format(self.impl.describe(), self.impl.get_timeout(), self.config.timing_slack)) self.active_config = self.config except WatchdogError as e: logger.error("Error while sending keepalive: %s", e) @property @synchronized def is_running(self) -> bool: return self.impl.is_running @property @synchronized def is_healthy(self) -> bool: if self.config.mode != MODE_REQUIRED: return True return self.config.timing_slack >= 0 and self.impl.is_healthy class WatchdogBase(abc.ABC): """A watchdog object when opened requires periodic calls to keepalive. When keepalive is not called within a timeout the system will be terminated.""" is_null = False @property def is_running(self) -> bool: """Returns True when watchdog is activated and capable of performing it's task.""" return False @property def is_healthy(self) -> bool: """Returns False when calling open() is known to fail.""" return False @property def can_be_disabled(self) -> bool: """Returns True when watchdog will be disabled by calling close(). Some watchdog devices will keep running no matter what once activated. May raise WatchdogError if called without calling open() first.""" return True @abc.abstractmethod def open(self) -> None: """Open watchdog device. When watchdog is opened keepalive must be called. Returns nothing on success or raises WatchdogError if the device could not be opened.""" @abc.abstractmethod def close(self) -> None: """Gracefully close watchdog device.""" @abc.abstractmethod def keepalive(self) -> None: """Resets the watchdog timer. Watchdog must be open when keepalive is called.""" @abc.abstractmethod def get_timeout(self) -> int: """Returns the current keepalive timeout in effect.""" def has_set_timeout(self) -> bool: """Returns True if setting a timeout is supported.""" return False def set_timeout(self, timeout: int) -> None: """Set the watchdog timer timeout. :param timeout: watchdog timeout in seconds""" raise WatchdogError("Setting timeout is not supported on {0}".format(self.describe())) def describe(self) -> str: """Human readable name for this device""" return self.__class__.__name__ @classmethod def from_config(cls, config: Dict[str, Any]) -> 'WatchdogBase': return cls() class NullWatchdog(WatchdogBase): """Null implementation when watchdog is not supported.""" is_null = True def open(self) -> None: return def close(self) -> None: return def keepalive(self) -> None: return def get_timeout(self) -> int: # A big enough number to not matter return 1000000000 patroni-4.0.4/patroni/watchdog/linux.py000066400000000000000000000205021472010352700201600ustar00rootroot00000000000000# pyright: reportConstantRedefinition=false import ctypes import os import platform from typing import Any, Dict, NamedTuple from .base import WatchdogBase, WatchdogError # Pythonification of linux/ioctl.h IOC_NONE = 0 IOC_WRITE = 1 IOC_READ = 2 IOC_NRBITS = 8 IOC_TYPEBITS = 8 IOC_SIZEBITS = 14 IOC_DIRBITS = 2 # Non-generic platform special cases machine = platform.machine() if machine in ['mips', 'sparc', 'powerpc', 'ppc64', 'ppc64le']: # pragma: no cover IOC_SIZEBITS = 13 IOC_DIRBITS = 3 IOC_NONE, IOC_WRITE = 1, 4 elif machine == 'parisc': # pragma: no cover IOC_WRITE, IOC_READ = 2, 1 IOC_NRSHIFT = 0 IOC_TYPESHIFT = IOC_NRSHIFT + IOC_NRBITS IOC_SIZESHIFT = IOC_TYPESHIFT + IOC_TYPEBITS IOC_DIRSHIFT = IOC_SIZESHIFT + IOC_SIZEBITS def IOW(type_: str, nr: int, size: int) -> int: return IOC(IOC_WRITE, type_, nr, size) def IOR(type_: str, nr: int, size: int) -> int: return IOC(IOC_READ, type_, nr, size) def IOWR(type_: str, nr: int, size: int) -> int: return IOC(IOC_READ | IOC_WRITE, type_, nr, size) def IOC(dir_: int, type_: str, nr: int, size: int) -> int: return (dir_ << IOC_DIRSHIFT) \ | (ord(type_) << IOC_TYPESHIFT) \ | (nr << IOC_NRSHIFT) \ | (size << IOC_SIZESHIFT) # Pythonification of linux/watchdog.h WATCHDOG_IOCTL_BASE = 'W' class watchdog_info(ctypes.Structure): _fields_ = [ ('options', ctypes.c_uint32), # Options the card/driver supports ('firmware_version', ctypes.c_uint32), # Firmware version of the card ('identity', ctypes.c_uint8 * 32), # Identity of the board ] struct_watchdog_info_size = ctypes.sizeof(watchdog_info) int_size = ctypes.sizeof(ctypes.c_int) WDIOC_GETSUPPORT = IOR(WATCHDOG_IOCTL_BASE, 0, struct_watchdog_info_size) WDIOC_GETSTATUS = IOR(WATCHDOG_IOCTL_BASE, 1, int_size) WDIOC_GETBOOTSTATUS = IOR(WATCHDOG_IOCTL_BASE, 2, int_size) WDIOC_GETTEMP = IOR(WATCHDOG_IOCTL_BASE, 3, int_size) WDIOC_SETOPTIONS = IOR(WATCHDOG_IOCTL_BASE, 4, int_size) WDIOC_KEEPALIVE = IOR(WATCHDOG_IOCTL_BASE, 5, int_size) WDIOC_SETTIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 6, int_size) WDIOC_GETTIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 7, int_size) WDIOC_SETPRETIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 8, int_size) WDIOC_GETPRETIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 9, int_size) WDIOC_GETTIMELEFT = IOR(WATCHDOG_IOCTL_BASE, 10, int_size) WDIOF_UNKNOWN = -1 # Unknown flag error WDIOS_UNKNOWN = -1 # Unknown status error WDIOF = { "OVERHEAT": 0x0001, # Reset due to CPU overheat "FANFAULT": 0x0002, # Fan failed "EXTERN1": 0x0004, # External relay 1 "EXTERN2": 0x0008, # External relay 2 "POWERUNDER": 0x0010, # Power bad/power fault "CARDRESET": 0x0020, # Card previously reset the CPU "POWEROVER": 0x0040, # Power over voltage "SETTIMEOUT": 0x0080, # Set timeout (in seconds) "MAGICCLOSE": 0x0100, # Supports magic close char "PRETIMEOUT": 0x0200, # Pretimeout (in seconds), get/set "ALARMONLY": 0x0400, # Watchdog triggers a management or other external alarm not a reboot "KEEPALIVEPING": 0x8000, # Keep alive ping reply } WDIOS = { "DISABLECARD": 0x0001, # Turn off the watchdog timer "ENABLECARD": 0x0002, # Turn on the watchdog timer "TEMPPANIC": 0x0004, # Kernel panic on temperature trip } # Implementation class WatchdogInfo(NamedTuple): """Watchdog descriptor from the kernel""" options: int version: int identity: str def __getattr__(self, name: str) -> bool: """Convenience has_XYZ attributes for checking WDIOF bits in options""" if name.startswith('has_') and name[4:] in WDIOF: return bool(self.options & WDIOF[name[4:]]) raise AttributeError("WatchdogInfo instance has no attribute '{0}'".format(name)) class LinuxWatchdogDevice(WatchdogBase): DEFAULT_DEVICE = '/dev/watchdog' def __init__(self, device: str) -> None: self.device = device self._support_cache = None self._fd = None @classmethod def from_config(cls, config: Dict[str, Any]) -> 'LinuxWatchdogDevice': device = config.get('device', cls.DEFAULT_DEVICE) return cls(device) @property def is_running(self) -> bool: return self._fd is not None @property def is_healthy(self) -> bool: return os.path.exists(self.device) and os.access(self.device, os.W_OK) def open(self) -> None: try: self._fd = os.open(self.device, os.O_WRONLY) except OSError as e: raise WatchdogError("Can't open watchdog device: {0}".format(e)) def close(self) -> None: if self._fd is not None: # self.is_running try: os.write(self._fd, b'V') os.close(self._fd) self._fd = None except OSError as e: raise WatchdogError("Error while closing {0}: {1}".format(self.describe(), e)) @property def can_be_disabled(self) -> bool: return self.get_support().has_MAGICCLOSE def _ioctl(self, func: int, arg: Any) -> None: """Runs the specified ioctl on the underlying fd. Raises WatchdogError if the device is closed. Raises OSError or IOError (Python 2) when the ioctl fails.""" if self._fd is None: raise WatchdogError("Watchdog device is closed") if os.name != 'nt': import fcntl fcntl.ioctl(self._fd, func, arg, True) def get_support(self) -> WatchdogInfo: if self._support_cache is None: info = watchdog_info() try: self._ioctl(WDIOC_GETSUPPORT, info) except (WatchdogError, OSError, IOError) as e: raise WatchdogError("Could not get information about watchdog device: {}".format(e)) self._support_cache = WatchdogInfo(info.options, info.firmware_version, bytearray(info.identity).decode(errors='ignore').rstrip('\x00')) return self._support_cache def describe(self) -> str: dev_str = " at {0}".format(self.device) if self.device != self.DEFAULT_DEVICE else "" ver_str = "" identity = "Linux watchdog device" if self._fd: try: _, version, identity = self.get_support() ver_str = " (firmware {0})".format(version) if version else "" except WatchdogError: pass return identity + ver_str + dev_str def keepalive(self) -> None: if self._fd is None: raise WatchdogError("Watchdog device is closed") try: os.write(self._fd, b'1') except OSError as e: raise WatchdogError("Could not send watchdog keepalive: {0}".format(e)) def has_set_timeout(self) -> bool: """Returns True if setting a timeout is supported.""" return self.get_support().has_SETTIMEOUT def set_timeout(self, timeout: int) -> None: timeout = int(timeout) if not 0 < timeout < 0xFFFF: raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout)) try: self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout)) except (WatchdogError, OSError, IOError) as e: raise WatchdogError("Could not set timeout on watchdog device: {}".format(e)) def get_timeout(self) -> int: timeout = ctypes.c_int() try: self._ioctl(WDIOC_GETTIMEOUT, timeout) except (WatchdogError, OSError, IOError) as e: raise WatchdogError("Could not get timeout on watchdog device: {}".format(e)) return timeout.value class TestingWatchdogDevice(LinuxWatchdogDevice): # pragma: no cover """Converts timeout ioctls to regular writes that can be intercepted from a named pipe.""" timeout = 60 def get_support(self) -> WatchdogInfo: return WatchdogInfo(WDIOF['MAGICCLOSE'] | WDIOF['SETTIMEOUT'], 0, "Watchdog test harness") def set_timeout(self, timeout: int) -> None: if self._fd is None: raise WatchdogError("Watchdog device is closed") buf = "Ctimeout={0}\n".format(timeout).encode('utf8') while len(buf): buf = buf[os.write(self._fd, buf):] self.timeout = timeout def get_timeout(self) -> int: return self.timeout patroni-4.0.4/patroni_raft_controller.py000077500000000000000000000001471472010352700205060ustar00rootroot00000000000000#!/usr/bin/env python from patroni.raft_controller import main if __name__ == '__main__': main() patroni-4.0.4/patronictl.py000077500000000000000000000001341472010352700157260ustar00rootroot00000000000000#!/usr/bin/env python from patroni.ctl import ctl if __name__ == '__main__': ctl(None) patroni-4.0.4/postgres0.yml000066400000000000000000000106771472010352700156600ustar00rootroot00000000000000scope: batman #namespace: /service/ name: postgresql0 restapi: listen: 127.0.0.1:8008 connect_address: 127.0.0.1:8008 # cafile: /etc/ssl/certs/ssl-cacert-snakeoil.pem # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # authentication: # username: username # password: password #ctl: # insecure: false # Allow connections to Patroni REST API without verifying certificates # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem #citus: # database: citus # group: 0 # coordinator etcd: #Provide host to do the initial discovery of the cluster topology: host: 127.0.0.1:2379 #Or use "hosts" to provide multiple endpoints #Could be a comma separated string: #hosts: host1:port1,host2:port2 #or an actual yaml list: #hosts: #- host1:port1 #- host2:port2 #Once discovery is complete Patroni will use the list of advertised clientURLs #It is possible to change this behavior through by setting: #use_proxies: true #raft: # data_dir: . # self_addr: 127.0.0.1:2222 # partner_addrs: # - 127.0.0.1:2223 # - 127.0.0.1:2224 # The bootstrap configuration. Works only when the cluster is not yet initialized. # If the cluster is already initialized, all changes in the `bootstrap` section are ignored! bootstrap: # This section will be written into Etcd:///config after initializing new cluster # and all other cluster members will use it as a `global configuration`. # WARNING! If you want to change any of the parameters that were set up # via `bootstrap.dcs` section, please use `patronictl edit-config`! dcs: ttl: 30 loop_wait: 10 retry_timeout: 10 maximum_lag_on_failover: 1048576 # primary_start_timeout: 300 # synchronous_mode: false #standby_cluster: #host: 127.0.0.1 #port: 1111 #primary_slot_name: patroni postgresql: use_pg_rewind: true pg_hba: # For kerberos gss based connectivity (discard @.*$) #- host replication replicator 127.0.0.1/32 gss include_realm=0 #- host all all 0.0.0.0/0 gss include_realm=0 - host replication replicator 127.0.0.1/32 md5 - host all all 0.0.0.0/0 md5 # - hostssl all all 0.0.0.0/0 md5 # use_slots: true parameters: # wal_level: hot_standby # hot_standby: "on" # max_connections: 100 # max_worker_processes: 8 # wal_keep_segments: 8 # max_wal_senders: 10 # max_replication_slots: 10 # max_prepared_transactions: 0 # max_locks_per_transaction: 64 # wal_log_hints: "on" # track_commit_timestamp: "off" # archive_mode: "on" # archive_timeout: 1800s # archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f # recovery_conf: # restore_command: cp ../wal_archive/%f %p # some desired options for 'initdb' initdb: # Note: It needs to be a list (some options need values, others are switches) - encoding: UTF8 - data-checksums # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter) # post_init: /usr/local/bin/setup_cluster.sh postgresql: listen: 127.0.0.1:5432 connect_address: 127.0.0.1:5432 # proxy_address: 127.0.0.1:5433 # The address of connection pool (e.g., pgbouncer) running next to Patroni/Postgres. Only for service discovery. data_dir: data/postgresql0 # bin_dir: # config_dir: pgpass: /tmp/pgpass0 authentication: replication: username: replicator password: rep-pass superuser: username: postgres password: patroni rewind: # Has no effect on postgres 10 and lower username: rewind_user password: rewind_password # Server side kerberos spn # krbsrvname: postgres parameters: # Fully qualified kerberos ticket file for the running user # same as KRB5CCNAME used by the GSS # krb_server_keyfile: /var/spool/keytabs/postgres unix_socket_directories: '..' # parent directory of data_dir # Additional fencing script executed after acquiring the leader lock but before promoting the replica #pre_promote: /path/to/pre_promote.sh #watchdog: # mode: automatic # Allowed values: off, automatic, required # device: /dev/watchdog # safety_margin: 5 tags: # failover_priority: 1 noloadbalance: false clonefrom: false nosync: false nostream: false patroni-4.0.4/postgres1.yml000066400000000000000000000104411472010352700156460ustar00rootroot00000000000000scope: batman #namespace: /service/ name: postgresql1 restapi: listen: 127.0.0.1:8009 connect_address: 127.0.0.1:8009 # cafile: /etc/ssl/certs/ssl-cacert-snakeoil.pem # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # authentication: # username: username # password: password #ctl: # insecure: false # Allow connections to Patroni REST API without verifying certificates # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem #citus: # database: citus # group: 1 # worker etcd: #Provide host to do the initial discovery of the cluster topology: host: 127.0.0.1:2379 #Or use "hosts" to provide multiple endpoints #Could be a comma separated string: #hosts: host1:port1,host2:port2 #or an actual yaml list: #hosts: #- host1:port1 #- host2:port2 #Once discovery is complete Patroni will use the list of advertised clientURLs #It is possible to change this behavior through by setting: #use_proxies: true #raft: # data_dir: . # self_addr: 127.0.0.1:2223 # partner_addrs: # - 127.0.0.1:2222 # - 127.0.0.1:2224 # The bootstrap configuration. Works only when the cluster is not yet initialized. # If the cluster is already initialized, all changes in the `bootstrap` section are ignored! bootstrap: # This section will be written into Etcd:///config after initializing new cluster # and all other cluster members will use it as a `global configuration`. # WARNING! If you want to change any of the parameters that were set up # via `bootstrap.dcs` section, please use `patronictl edit-config`! dcs: ttl: 30 loop_wait: 10 retry_timeout: 10 maximum_lag_on_failover: 1048576 postgresql: use_pg_rewind: true pg_hba: # For kerberos gss based connectivity (discard @.*$) #- host replication replicator 127.0.0.1/32 gss include_realm=0 #- host all all 0.0.0.0/0 gss include_realm=0 - host replication replicator 127.0.0.1/32 md5 - host all all 0.0.0.0/0 md5 # - hostssl all all 0.0.0.0/0 md5 # use_slots: true parameters: # wal_level: hot_standby # hot_standby: "on" # max_connections: 100 # max_worker_processes: 8 # wal_keep_segments: 8 # max_wal_senders: 10 # max_replication_slots: 10 # max_prepared_transactions: 0 # max_locks_per_transaction: 64 # wal_log_hints: "on" # track_commit_timestamp: "off" # archive_mode: "on" # archive_timeout: 1800s # archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f # recovery_conf: # restore_command: cp ../wal_archive/%f %p # some desired options for 'initdb' initdb: # Note: It needs to be a list (some options need values, others are switches) - encoding: UTF8 - data-checksums # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter) # post_init: /usr/local/bin/setup_cluster.sh postgresql: listen: 127.0.0.1:5433 connect_address: 127.0.0.1:5433 # proxy_address: 127.0.0.1:5434 # The address of connection pool (e.g., pgbouncer) running next to Patroni/Postgres. Only for service discovery. data_dir: data/postgresql1 # bin_dir: # config_dir: pgpass: /tmp/pgpass1 authentication: replication: username: replicator password: rep-pass superuser: username: postgres password: patroni rewind: # Has no effect on postgres 10 and lower username: rewind_user password: rewind_password # Server side kerberos spn # krbsrvname: postgres parameters: # Fully qualified kerberos ticket file for the running user # same as KRB5CCNAME used by the GSS # krb_server_keyfile: /var/spool/keytabs/postgres unix_socket_directories: '..' # parent directory of data_dir basebackup: - verbose - max-rate: 100M # - waldir: /pg-wal-mount/external-waldir # only needed in case pg_wal is symlinked outside of data_dir # Additional fencing script executed after acquiring the leader lock but before promoting the replica #pre_promote: /path/to/pre_promote.sh tags: # failover_priority: 1 noloadbalance: false clonefrom: false patroni-4.0.4/postgres2.yml000066400000000000000000000075501472010352700156560ustar00rootroot00000000000000scope: batman #namespace: /service/ name: postgresql2 restapi: listen: 127.0.0.1:8010 connect_address: 127.0.0.1:8010 # cafile: /etc/ssl/certs/ssl-cacert-snakeoil.pem # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key authentication: username: username password: password #ctl: # insecure: false # Allow connections to Patroni REST API without verifying certificates # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem #citus: # database: citus # group: 1 # worker etcd: #Provide host to do the initial discovery of the cluster topology: host: 127.0.0.1:2379 #Or use "hosts" to provide multiple endpoints #Could be a comma separated string: #hosts: host1:port1,host2:port2 #or an actual yaml list: #hosts: #- host1:port1 #- host2:port2 #Once discovery is complete Patroni will use the list of advertised clientURLs #It is possible to change this behavior through by setting: #use_proxies: true #raft: # data_dir: . # self_addr: 127.0.0.1:2224 # partner_addrs: # - 127.0.0.1:2222 # - 127.0.0.1:2223 # The bootstrap configuration. Works only when the cluster is not yet initialized. # If the cluster is already initialized, all changes in the `bootstrap` section are ignored! bootstrap: # This section will be written into Etcd:///config after initializing new cluster # and all other cluster members will use it as a `global configuration`. # WARNING! If you want to change any of the parameters that were set up # via `bootstrap.dcs` section, please use `patronictl edit-config`! dcs: ttl: 30 loop_wait: 10 retry_timeout: 10 maximum_lag_on_failover: 1048576 postgresql: use_pg_rewind: true pg_hba: # For kerberos gss based connectivity (discard @.*$) #- host replication replicator 127.0.0.1/32 gss include_realm=0 #- host all all 0.0.0.0/0 gss include_realm=0 - host replication replicator 127.0.0.1/32 md5 - host all all 0.0.0.0/0 md5 # - hostssl all all 0.0.0.0/0 md5 # use_slots: true parameters: # wal_level: hot_standby # hot_standby: "on" # max_connections: 100 # max_worker_processes: 8 # wal_keep_segments: 8 # max_wal_senders: 10 # max_replication_slots: 10 # max_prepared_transactions: 0 # max_locks_per_transaction: 64 # wal_log_hints: "on" # track_commit_timestamp: "off" # archive_mode: "on" # archive_timeout: 1800s # archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f # recovery_conf: # restore_command: cp ../wal_archive/%f %p # some desired options for 'initdb' initdb: # Note: It needs to be a list (some options need values, others are switches) - encoding: UTF8 - data-checksums postgresql: listen: 127.0.0.1:5434 connect_address: 127.0.0.1:5434 # proxy_address: 127.0.0.1:5435 # The address of connection pool (e.g., pgbouncer) running next to Patroni/Postgres. Only for service discovery. data_dir: data/postgresql2 # bin_dir: # config_dir: pgpass: /tmp/pgpass2 authentication: replication: username: replicator password: rep-pass superuser: username: postgres password: patroni rewind: # Has no effect on postgres 10 and lower username: rewind_user password: rewind_password # Server side kerberos spn # krbsrvname: postgres parameters: # Fully qualified kerberos ticket file for the running user # same as KRB5CCNAME used by the GSS # krb_server_keyfile: /var/spool/keytabs/postgres unix_socket_directories: '..' # parent directory of data_dir tags: # failover_priority: 1 noloadbalance: false clonefrom: false # replicatefrom: postgresql1 patroni-4.0.4/pyrightconfig.json000066400000000000000000000005111472010352700167400ustar00rootroot00000000000000{ "include": [ "patroni" ], "exclude": [ "**/__pycache__" ], "ignore": [ ], "defineConstant": { "DEBUG": true }, "stubPath": "typings/", "reportMissingImports": true, "reportMissingTypeStubs": false, "pythonVersion": "3.12", "pythonPlatform": "All", "typeCheckingMode": "strict" } patroni-4.0.4/release.sh000077500000000000000000000015221472010352700151530ustar00rootroot00000000000000#!/bin/bash # Release process: # 1. Open a PR that updates release notes, Patroni version and pyright version in the tests workflow. # 2. Resolve possible typing issues. # 3. Merge the PR. # 4. Run release.sh # 5. After the new tag is pushed, the .github/workflows/release.yaml will run tests and upload the new package to test.pypi.org # 6. Once the release is created, the .github/workflows/release.yaml will run tests and upload the new package to pypi.org ## Bail out on any non-zero exitcode from the called processes set -xe if python3 --version &> /dev/null; then alias python=python3 shopt -s expand_aliases fi python --version git --version version=$(python -c 'from patroni.version import __version__; print(__version__)') python setup.py clean python setup.py test python setup.py flake8 git tag "v$version" git push --tags patroni-4.0.4/requirements.dev.txt000066400000000000000000000001741472010352700172370ustar00rootroot00000000000000psycopg2-binary==2.9.9; sys_platform == "darwin" psycopg2-binary behave coverage flake8>=3.0.0 pytest-cov pytest setuptools patroni-4.0.4/requirements.docs.txt000066400000000000000000000002351472010352700174070ustar00rootroot00000000000000sphinx>=4 sphinx_rtd_theme>1 sphinxcontrib-apidoc sphinx-github-style<1.0.3 psycopg[binary] psycopg2-binary==2.9.9; sys_platform == "darwin" psycopg2-binary patroni-4.0.4/requirements.txt000066400000000000000000000003631472010352700164620ustar00rootroot00000000000000urllib3>=1.19.1,!=1.21 boto3 PyYAML kazoo>=1.3.1 python-etcd>=0.4.3,<0.5 py-consul>=1.1.1 click>=4.1 prettytable>=0.7 python-dateutil pysyncobj>=0.3.8 cryptography>=1.4 psutil>=2.0.0 ydiff>=1.2.0,<1.5,!=1.4.0,!=1.4.1 python-json-logger>=2.0.2 patroni-4.0.4/setup.py000066400000000000000000000163721472010352700147170ustar00rootroot00000000000000#!/usr/bin/env python """ Setup file for patroni """ import glob import inspect import logging import os import sys from setuptools import Command, find_packages, setup __location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) NAME = 'patroni' MAIN_PACKAGE = NAME DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI' LICENSE = 'The MIT License' URL = 'https://github.com/patroni/patroni' AUTHOR = 'Alexander Kukushkin, Polina Bungina' AUTHOR_EMAIL = 'akukushkin@microsoft.com, polina.bungina@zalando.de' KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\ ' zookeeper exhibitor consul streaming replication kubernetes k8s' EXTRAS_REQUIRE = {'aws': ['boto3'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'], 'consul': ['py-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': [], 'raft': ['pysyncobj', 'cryptography'], 'jsonlogger': ['python-json-logger']} # Add here all kinds of additional classifiers as defined under # https://pypi.python.org/pypi?%3Aaction=list_classifiers CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: MIT License', 'Operating System :: MacOS', 'Operating System :: POSIX :: Linux', 'Operating System :: POSIX :: BSD :: FreeBSD', 'Operating System :: Microsoft :: Windows', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', ] CONSOLE_SCRIPTS = ['patroni = patroni.__main__:main', 'patronictl = patroni.ctl:ctl', 'patroni_raft_controller = patroni.raft_controller:main', "patroni_wale_restore = patroni.scripts.wale_restore:main", "patroni_aws = patroni.scripts.aws:main", "patroni_barman = patroni.scripts.barman.cli:main"] class _Command(Command): user_options = [] def initialize_options(self): pass def finalize_options(self): pass class _Lint(_Command): def package_modules(self): package_dirs = self.distribution.package_dir or {} for package in self.distribution.packages or []: if package in package_dirs: yield package_dirs[package] elif '' in package_dirs: yield os.path.join(package_dirs[''], package) else: yield package def package_directories(self): for module in self.package_modules(): yield module.replace('.', os.path.sep) def aux_directories(self): for dir_name in ('tests', 'features'): yield dir_name for root, dirs, files in os.walk(dir_name): for name in dirs: yield os.path.join(root, name) def dirs_to_check(self): yield from self.package_directories() yield from self.aux_directories() def files_to_check(self): for path in self.dirs_to_check(): for python_file in glob.iglob(os.path.join(path, '*.py')): yield python_file for filename in self.distribution.py_modules or []: yield f'{filename}.py' yield 'setup.py' class Flake8(_Lint): def run(self): from flake8.main.cli import main logging.getLogger().setLevel(logging.ERROR) raise SystemExit(main(list(self.files_to_check()))) class ISort(_Lint): def run(self): from isort import api wrong_sorted_files = False for python_file in self.files_to_check(): try: if not api.check_file(python_file, settings_path=__location__, show_diff=True): wrong_sorted_files = True except OSError as error: logging.warning('Unable to parse file %s due to %r', python_file, error) wrong_sorted_files = True if wrong_sorted_files: sys.exit(1) class PyTest(_Command): def run(self): try: import pytest except Exception: raise RuntimeError('py.test is not installed, run: pip install pytest') logging.getLogger().setLevel(logging.WARNING) args = ['--verbose', 'tests', '--doctest-modules', MAIN_PACKAGE] +\ ['-s' if logging.getLogger().getEffectiveLevel() < logging.WARNING else '--capture=fd'] +\ ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing', '--cov-report', 'xml'] errno = pytest.main(args=args) sys.exit(errno) def read(fname): with open(os.path.join(__location__, fname), encoding='utf-8') as fd: return fd.read() def get_versions(): old_modules = sys.modules.copy() try: from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3 from patroni.version import __version__ return __version__, MIN_PSYCOPG2, MIN_PSYCOPG3 finally: sys.modules.clear() sys.modules.update(old_modules) def main(): logging.basicConfig(format='%(message)s', level=os.getenv('LOGLEVEL', logging.WARNING)) install_requires = [] for r in read('requirements.txt').split('\n'): r = r.strip() if r == '': continue extra = False for e, deps in EXTRAS_REQUIRE.items(): for i, v in enumerate(deps): if r.startswith(v): deps[i] = r EXTRAS_REQUIRE[e] = deps extra = True if not extra: install_requires.append(r) # Just for convenience, if someone wants to install dependencies for all extras EXTRAS_REQUIRE['all'] = list({e for extras in EXTRAS_REQUIRE.values() for e in extras}) patroni_version, min_psycopg2, min_psycopg3 = get_versions() # Make it possible to specify psycopg dependency as extra for name, version in {'psycopg[binary]': min_psycopg3, 'psycopg2': min_psycopg2, 'psycopg2-binary': None}.items(): EXTRAS_REQUIRE[name] = [name + ('>=' + '.'.join(map(str, version)) if version else '')] EXTRAS_REQUIRE['psycopg3'] = EXTRAS_REQUIRE.pop('psycopg[binary]') setup( name=NAME, version=patroni_version, url=URL, author=AUTHOR, author_email=AUTHOR_EMAIL, description=DESCRIPTION, license=LICENSE, keywords=KEYWORDS, long_description=read('README.rst'), classifiers=CLASSIFIERS, packages=find_packages(exclude=['tests', 'tests.*']), package_data={MAIN_PACKAGE: [ "postgresql/available_parameters/*.yml", "postgresql/available_parameters/*.yaml", ]}, install_requires=install_requires, extras_require=EXTRAS_REQUIRE, cmdclass={'test': PyTest, 'flake8': Flake8, 'isort': ISort}, entry_points={'console_scripts': CONSOLE_SCRIPTS}, ) if __name__ == '__main__': main() patroni-4.0.4/tests/000077500000000000000000000000001472010352700143365ustar00rootroot00000000000000patroni-4.0.4/tests/__init__.py000066400000000000000000000317521472010352700164570ustar00rootroot00000000000000import datetime import os import shutil import unittest from unittest.mock import Mock, patch, PropertyMock import urllib3 import patroni.psycopg as psycopg from patroni.dcs import Leader, Member from patroni.postgresql import Postgresql from patroni.postgresql.config import ConfigHandler from patroni.postgresql.mpp import get_mpp from patroni.utils import RetryFailedError, tzutc class SleepException(Exception): pass mock_available_gucs = PropertyMock(return_value={ 'cluster_name', 'constraint_exclusion', 'force_parallel_mode', 'hot_standby', 'listen_addresses', 'max_connections', 'max_locks_per_transaction', 'max_prepared_transactions', 'max_replication_slots', 'max_stack_depth', 'max_wal_senders', 'max_worker_processes', 'port', 'search_path', 'shared_preload_libraries', 'stats_temp_directory', 'synchronous_standby_names', 'track_commit_timestamp', 'unix_socket_directories', 'vacuum_cost_delay', 'vacuum_cost_limit', 'wal_keep_size', 'wal_level', 'wal_log_hints', 'zero_damaged_pages', 'autovacuum', 'wal_segment_size', 'wal_block_size', 'shared_buffers', 'wal_buffers', 'fork_specific_param', }) GET_PG_SETTINGS_RESULT = [ ('wal_segment_size', '2048', '8kB', 'integer', 'internal'), ('wal_block_size', '8192', None, 'integer', 'internal'), ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'), ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'), ('max_connections', '100', None, 'integer', 'postmaster'), ('max_prepared_transactions', '200', None, 'integer', 'postmaster'), ('max_worker_processes', '8', None, 'integer', 'postmaster'), ('max_locks_per_transaction', '64', None, 'integer', 'postmaster'), ('max_wal_senders', '5', None, 'integer', 'postmaster'), ('search_path', 'public', None, 'string', 'user'), ('port', '5432', None, 'integer', 'postmaster'), ('listen_addresses', '127.0.0.2, 127.0.0.3', None, 'string', 'postmaster'), ('autovacuum', 'on', None, 'bool', 'sighup'), ('unix_socket_directories', '/tmp', None, 'string', 'postmaster'), ('shared_preload_libraries', 'citus', None, 'string', 'postmaster'), ('wal_keep_size', '128', 'MB', 'integer', 'sighup'), ('cluster_name', 'batman', None, 'string', 'postmaster'), ('vacuum_cost_delay', '200', 'ms', 'real', 'user'), ('vacuum_cost_limit', '-1', None, 'integer', 'user'), ('max_stack_depth', '2048', 'kB', 'integer', 'superuser'), ('constraint_exclusion', '', None, 'enum', 'user'), ('force_parallel_mode', '1', None, 'enum', 'user'), ('zero_damaged_pages', 'off', None, 'bool', 'superuser'), ('stats_temp_directory', '/tmp', None, 'string', 'sighup'), ('track_commit_timestamp', 'off', None, 'bool', 'postmaster'), ('wal_log_hints', 'on', None, 'bool', 'postmaster'), ('hot_standby', 'on', None, 'bool', 'postmaster'), ('max_replication_slots', '5', None, 'integer', 'postmaster'), ('wal_level', 'logical', None, 'enum', 'postmaster'), ] class MockResponse(object): def __init__(self, status_code=200): self.status_code = status_code self.headers = {'content-type': 'json', 'lsn': 100} self.content = '{}' self.reason = 'Not Found' @property def data(self): return self.content.encode('utf-8') @property def status(self): return self.status_code @staticmethod def getheader(*args): return '' def requests_get(url, method='GET', endpoint=None, data='', **kwargs): members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\ '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]' response = MockResponse() if endpoint == 'failsafe': response.content = 'Accepted' elif url.startswith('http://local'): raise urllib3.exceptions.HTTPError() elif ':8011/patroni' in url: response.content = '{"role": "replica", "wal": {"received_location": 0}, "tags": {}}' elif url.endswith('/members'): response.content = '[{}]' if url.startswith('http://error') else members elif url.startswith('http://exhibitor'): response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}' elif url.endswith(':8011/reinitialize'): if ' false}' in data: response.status_code = 503 response.content = 'restarting after failure already in progress' else: response.status_code = 404 return response class MockPostmaster(object): def __init__(self, pid=1): self.is_running = Mock(return_value=self) self.wait_for_user_backends_to_close = Mock() self.signal_stop = Mock(return_value=None) self.wait = Mock() self.signal_kill = Mock(return_value=False) class MockCursor(object): def __init__(self, connection): self.connection = connection self.closed = False self.rowcount = 0 self.results = [] self.description = [Mock()] def execute(self, sql, *params): if isinstance(sql, bytes): sql = sql.decode('utf-8') if sql.startswith('blabla'): raise psycopg.ProgrammingError() elif sql == 'CHECKPOINT' or sql.startswith('SELECT pg_catalog.pg_create_'): raise psycopg.OperationalError() elif sql.startswith('RetryFailedError'): raise RetryFailedError('retry') elif sql.startswith('SELECT slot_name, catalog_xmin'): self.results = [('postgresql0', 100), ('ls', 100)] elif sql.startswith('SELECT slot_name, slot_type, datname, plugin, catalog_xmin'): self.results = [('ls', 'logical', 'a', 'b', 100, 500, b'123456')] elif sql.startswith('SELECT slot_name'): self.results = [('blabla', 'physical', 1, 12345), ('foobar', 'physical', 1, 12345), ('ls', 'logical', 1, 499, 'b', 'a', 5, 100, 500)] elif sql.startswith('WITH slots AS (SELECT slot_name, active'): self.results = [(False, True)] if self.rowcount == 1 else [] elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'): self.results = [(1, 2, 1, 0, False, 1, 1, None, None, 'streaming', '', [{"slot_name": "ls", "confirmed_flush_lsn": 12345, "restart_lsn": 12344}], 'on', 'n1', None)] elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'): self.results = [(False, 2)] elif sql.startswith('SELECT pg_catalog.pg_postmaster_start_time'): self.results = [(datetime.datetime.now(tzutc),)] elif sql.endswith('AND pending_restart'): self.results = [] elif sql.startswith('SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings'): self.results = [('data_directory', 'data'), ('hba_file', os.path.join('data', 'pg_hba.conf')), ('ident_file', os.path.join('data', 'pg_ident.conf')), ('max_connections', 42), ('max_locks_per_transaction', 73), ('max_prepared_transactions', 0), ('max_replication_slots', 21), ('max_wal_senders', 37), ('track_commit_timestamp', 'off'), ('wal_level', 'replica'), ('listen_addresses', '6.6.6.6'), ('port', 1984), ('archive_command', 'my archive command'), ('cluster_name', 'my_cluster')] elif sql.startswith('SELECT name, setting'): self.results = GET_PG_SETTINGS_RESULT elif sql.startswith('IDENTIFY_SYSTEM'): self.results = [('1', 3, '0/402EEC0', '')] elif sql.startswith('TIMELINE_HISTORY '): self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' b'1\t0/40159C0\tno recovery target specified\n\n' b'2\t0/402DD98\tno recovery target specified\n\n' b'3\t0/403DD98\tno recovery target specified\n')] elif sql.startswith('SELECT pg_catalog.citus_add_node'): self.results = [(2,)] elif sql.startswith('SELECT groupid, nodename'): self.results = [(0, 'host1', 5432, 'primary', 1), (0, '127.0.0.1', 5436, 'secondary', 2), (1, 'host4', 5432, 'primary', 3), (1, '127.0.0.1', 5437, 'secondary', 4), (1, '127.0.0.1', 5438, 'secondary', 5)] else: self.results = [(None, None, None, None, None, None, None, None, None, None)] self.rowcount = len(self.results) def fetchone(self): return self.results[0] def fetchall(self): return self.results def __iter__(self): for i in self.results: yield i def __enter__(self): return self def __exit__(self, *args): pass class MockConnect(object): server_version = 99999 autocommit = False closed = 0 def get_parameter_status(self, param_name): if param_name == 'is_superuser': return 'on' return '0' def cursor(self): return MockCursor(self) def __enter__(self): return self def __exit__(self, *args): pass @staticmethod def close(): pass def psycopg_connect(*args, **kwargs): return MockConnect() class PostgresInit(unittest.TestCase): _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar', 'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5, 'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64, 'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 200, 'track_commit_timestamp': 'off', 'unix_socket_directories': '/tmp', 'trigger_file': 'bla', 'stats_temp_directory': '/tmp', 'zero_damaged_pages': 'off', 'force_parallel_mode': '1', 'constraint_exclusion': '', 'max_stack_depth': 2048, 'vacuum_cost_limit': -1, 'vacuum_cost_delay': 200} @patch('patroni.psycopg._connect', psycopg_connect) @patch('patroni.postgresql.CallbackExecutor', Mock()) @patch.object(ConfigHandler, 'write_postgresql_conf', Mock()) @patch.object(ConfigHandler, 'replace_pg_hba', Mock()) @patch.object(ConfigHandler, 'replace_pg_ident', Mock()) @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='primary')) def setUp(self): data_dir = os.path.join('data', 'test0') config = {'name': 'postgresql0', 'scope': 'batman', 'data_dir': data_dir, 'config_dir': data_dir, 'retry_timeout': 10, 'krbsrvname': 'postgres', 'pgpass': os.path.join(data_dir, 'pgpass0'), 'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432', 'proxy_address': '127.0.0.2:5433', 'authentication': {'superuser': {'username': 'foo', 'password': 'test'}, 'replication': {'username': '', 'password': 'rep-pass'}, 'rewind': {'username': 'rewind', 'password': 'test'}}, 'remove_data_directory_on_rewind_failure': True, 'use_pg_rewind': True, 'pg_ctl_timeout': 'bla', 'use_unix_socket': True, 'parameters': self._PARAMETERS, 'recovery_conf': {'foo': 'bar'}, 'pg_hba': ['host all all 0.0.0.0/0 md5'], 'pg_ident': ['krb realm postgres'], 'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true', 'on_restart': 'true', 'on_role_change': 'true'}, 'citus': {'group': 0, 'database': 'citus'}} self.p = Postgresql(config, get_mpp(config)) class BaseTestPostgresql(PostgresInit): @patch('time.sleep', Mock()) def setUp(self): super(BaseTestPostgresql, self).setUp() if not os.path.exists(self.p.data_dir): os.makedirs(self.p.data_dir) self.leadermem = Member(0, 'leader', 28, {'xlog_location': 100, 'state': 'running', 'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5435/postgres'}) self.leader = Leader(-1, 28, self.leadermem) self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5433/postgres', 'state': 'running', 'tags': {'replicatefrom': 'leader'}}) self.me = Member(0, 'test0', 28, { 'state': 'running', 'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5434/postgres'}) def tearDown(self): if os.path.exists(self.p.data_dir): shutil.rmtree(self.p.data_dir) patroni-4.0.4/tests/test_api.py000066400000000000000000001115011472010352700165170ustar00rootroot00000000000000import datetime import json import socket import unittest from http.server import HTTPServer from io import BytesIO as IO from socketserver import ThreadingMixIn from unittest.mock import Mock, patch, PropertyMock from patroni import global_config from patroni.api import RestApiHandler, RestApiServer from patroni.dcs import ClusterConfig, Member from patroni.exceptions import PostgresConnectionException from patroni.ha import _MemberStatus from patroni.postgresql.config import get_param_diff from patroni.psycopg import OperationalError from patroni.utils import RetryFailedError, tzutc from . import MockConnect, psycopg_connect from .test_ha import get_cluster_initialized_without_leader future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5) postmaster_start_time = datetime.datetime.now(tzutc) class MockConnection: @staticmethod def get(*args): return psycopg_connect() @staticmethod def query(sql, *params): return [(postmaster_start_time, 0, '', 0, '', False, postmaster_start_time, 'streaming', None, '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' + '"state":"streaming","sync_state":"async","sync_priority":0}]')] class MockConnectionPool: @staticmethod def get(*args): return MockConnection() class MockPostgresql: connection_pool = MockConnectionPool() name = 'test' state = 'running' role = 'primary' server_version = 90625 major_version = 90600 sysid = 'dummysysid' scope = 'dummy' pending_restart_reason = {} wal_name = 'wal' lsn_name = 'lsn' wal_flush = '_flush' POSTMASTER_START_TIME = 'pg_catalog.pg_postmaster_start_time()' TL_LSN = 'CASE WHEN pg_catalog.pg_is_in_recovery()' mpp_handler = Mock() @staticmethod def postmaster_start_time(): return postmaster_start_time @staticmethod def replica_cached_timeline(_): return 2 @staticmethod def is_running(): return True @staticmethod def replication_state_from_parameters(*args): return 'streaming' class MockWatchdog(object): is_healthy = False class MockHa(object): state_handler = MockPostgresql() watchdog = MockWatchdog() @staticmethod def update_failsafe(*args): return 'foo' @staticmethod def failsafe_is_active(*args): return True @staticmethod def is_leader(): return False @staticmethod def reinitialize(_): return 'reinitialize' @staticmethod def restart(*args, **kwargs): return (True, '') @staticmethod def restart_scheduled(): return False @staticmethod def delete_future_restart(): return True @staticmethod def fetch_nodes_statuses(members): return [_MemberStatus(None, True, None, 0, {})] @staticmethod def schedule_future_restart(data): return True @staticmethod def is_lagging(wal): return False @staticmethod def get_effective_tags(): return {'nosync': True} @staticmethod def wakeup(): pass @staticmethod def is_paused(): return True class MockLogger(object): NORMAL_LOG_QUEUE_SIZE = 2 queue_size = 3 records_lost = 1 class MockPatroni(object): ha = MockHa() postgresql = ha.state_handler dcs = Mock() logger = MockLogger() tags = {"key1": True, "key2": False, "key3": 1, "key4": 1.4, "key5": "RandomTag"} version = '0.00' noloadbalance = PropertyMock(return_value=False) scheduled_restart = {'schedule': future_restart_time, 'postmaster_start_time': postgresql.postmaster_start_time()} @staticmethod def sighup_handler(): pass @staticmethod def api_sigterm(): pass class MockRequest(object): def __init__(self, request): self.request = request.encode('utf-8') def makefile(self, *args, **kwargs): return IO(self.request) def sendall(self, *args, **kwargs): pass class MockRestApiServer(RestApiServer): def __init__(self, Handler, request, config=None): self.socket = 0 self.serve_forever = Mock() MockRestApiServer._BaseServer__is_shut_down = Mock() MockRestApiServer._BaseServer__shutdown_request = True config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb', 'verify_client': 'a', 'http_extra_headers': {'foo': 'bar'}, 'https_extra_headers': {'foo': 'sbar'}} super(MockRestApiServer, self).__init__(MockPatroni(), config) Handler(MockRequest(request), ('0.0.0.0', 8080), self) @patch('ssl.SSLContext.load_cert_chain', Mock()) @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0)) @patch.object(HTTPServer, '__init__', Mock()) class TestRestApiHandler(unittest.TestCase): _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0' def test_do_GET(self): MockPostgresql.pending_restart_reason = {'max_connections': get_param_diff('200', '100')} MockPatroni.dcs.cluster.status.last_lsn = 20 with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /replica') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M') MockRestApiServer(RestApiHandler, 'GET /replica?lag=10MB') MockRestApiServer(RestApiHandler, 'GET /replica?lag=10485760') MockRestApiServer(RestApiHandler, 'GET /read-only') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})): MockRestApiServer(RestApiHandler, 'GET /replica') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})): MockRestApiServer(RestApiHandler, 'GET /replica') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'state': 'running'})): MockRestApiServer(RestApiHandler, 'GET /health') MockRestApiServer(RestApiHandler, 'GET /leader') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica', 'sync_standby': True})): MockRestApiServer(RestApiHandler, 'GET /synchronous') MockRestApiServer(RestApiHandler, 'GET /read-only-sync') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica', 'quorum_standby': True})): MockRestApiServer(RestApiHandler, 'GET /quorum') MockRestApiServer(RestApiHandler, 'GET /read-only-quorum') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})): MockRestApiServer(RestApiHandler, 'GET /asynchronous') with patch.object(MockHa, 'is_leader', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /replica') MockRestApiServer(RestApiHandler, 'GET /read-only-sync') MockRestApiServer(RestApiHandler, 'GET /read-only-quorum') with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /standby_leader') MockPatroni.dcs.cluster = None with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})): MockRestApiServer(RestApiHandler, 'GET /primary') with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /primary') self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /primary')) with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, None, None, '')])): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni')) with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)), \ patch.object(global_config.__class__, 'is_paused', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /standby_leader') # test tags # MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=False&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2') # with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})): MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=False&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2') # with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'standby_leader'})): MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&' 'tag_key1=true&tag_key2=False&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2') # MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=False&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2') # with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})): MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=False&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2') # MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&' 'tag_key1=true&tag_key2=False&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag') MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&' 'tag_key1=true&tag_key2=false&' 'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2') def test_do_OPTIONS(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0')) def test_do_HEAD(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'HEAD / HTTP/1.0')) @patch.object(MockPatroni, 'dcs') def test_do_GET_liveness(self, mock_dcs): mock_dcs.ttl.return_value = PropertyMock(30) self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /liveness HTTP/1.0')) def test_do_GET_readiness(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /readiness HTTP/1.0')) with patch.object(MockHa, 'is_leader', Mock(return_value=True)): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /readiness HTTP/1.0')) with patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped')): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /readiness HTTP/1.0')) @patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped')) def test_do_GET_patroni(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni')) def test_basicauth(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0')) MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:') @patch.object(MockPatroni, 'dcs') def test_do_GET_cluster(self, mock_dcs): mock_dcs.get_cluster.return_value = get_cluster_initialized_without_leader() mock_dcs.get_cluster.return_value.members[1].data['xlog_location'] = 11 self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /cluster')) @patch.object(MockPatroni, 'dcs') def test_do_GET_history(self, mock_dcs): mock_dcs.cluster = get_cluster_initialized_without_leader() self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /history')) @patch.object(MockPatroni, 'dcs') def test_do_GET_config(self, mock_dcs): mock_dcs.cluster.config.data = {} self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config')) mock_dcs.cluster.config = None self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config')) @patch.object(MockPatroni, 'dcs') def test_do_GET_metrics(self, mock_dcs): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /metrics')) @patch.object(MockPatroni, 'dcs') def test_do_PATCH_config(self, mock_dcs): config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}} mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config)) request = 'PATCH /config HTTP/1.0' + self._authorization self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) request += '\nContent-Length: ' self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '34\n\n{"postgresql":{"use_slots":false}}')) config['ttl'] = 5 config['postgresql'].update({'use_slots': {'foo': True}, "parameters": None}) config = json.dumps(config) request += str(len(config)) + '\n\n' + config MockRestApiServer(RestApiHandler, request) mock_dcs.set_config_value.return_value = False MockRestApiServer(RestApiHandler, request) mock_dcs.get_cluster.return_value.config = None MockRestApiServer(RestApiHandler, request) @patch.object(MockPatroni, 'dcs') def test_do_PUT_config(self, mock_dcs): mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}') request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: ' self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2\n\n{}')) config = '{"foo": "bar"}' request += str(len(config)) + '\n\n' + config MockRestApiServer(RestApiHandler, request) mock_dcs.set_config_value.return_value = False MockRestApiServer(RestApiHandler, request) mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config) MockRestApiServer(RestApiHandler, request) @patch.object(MockPatroni, 'dcs') def test_do_GET_failsafe(self, mock_dcs): type(mock_dcs).failsafe = PropertyMock(return_value={'node1': 'http://foo:8080/patroni'}) self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /failsafe')) type(mock_dcs).failsafe = PropertyMock(return_value=None) self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /failsafe')) def test_do_POST_failsafe(self): with patch.object(MockHa, 'is_failsafe_mode', Mock(return_value=False), create=True): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /failsafe HTTP/1.0' + self._authorization)) with patch.object(MockHa, 'is_failsafe_mode', Mock(return_value=True), create=True): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /failsafe HTTP/1.0' + self._authorization + '\nContent-Length: 9\n\n{"a":"b"}')) @patch.object(MockPatroni, 'sighup_handler', Mock()) def test_do_POST_reload(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)) @patch('os.environ', {'BEHAVE_DEBUG': 'true'}) @patch('os.name', 'nt') def test_do_POST_sigterm(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /sigterm HTTP/1.0' + self._authorization)) def test_do_POST_restart(self): request = 'POST /restart HTTP/1.0' + self._authorization self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) with patch.object(MockHa, 'restart', Mock(side_effect=Exception)): MockRestApiServer(RestApiHandler, request) post = request + '\nContent-Length: ' def make_request(request=None, **kwargs): request = json.dumps(kwargs) if request is None else request return '{0}{1}\n\n{2}'.format(post, len(request), request) # empty request request = make_request('') MockRestApiServer(RestApiHandler, request) # invalid request request = make_request('foobar=baz') MockRestApiServer(RestApiHandler, request) # wrong role request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3') MockRestApiServer(RestApiHandler, request) # wrong version request = make_request(schedule=future_restart_time.isoformat(), role='primary', postgres_version='9.5.3.1') MockRestApiServer(RestApiHandler, request) # unknown filter request = make_request(schedule=future_restart_time.isoformat(), batman='lives') MockRestApiServer(RestApiHandler, request) # incorrect schedule request = make_request(schedule='2016-08-42 12:45TZ+1', role='primary') MockRestApiServer(RestApiHandler, request) # everything fine, but the schedule is missing request = make_request(role='primary', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) for retval in (True, False): with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)): request = make_request(schedule=future_restart_time.isoformat()) MockRestApiServer(RestApiHandler, request) with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))): request = make_request(role='primary', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='primary')) # Valid timeout MockRestApiServer(RestApiHandler, make_request(timeout='60s')) # Invalid timeout MockRestApiServer(RestApiHandler, make_request(timeout='42towels')) def test_do_DELETE_restart(self): for retval in (True, False): with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)): request = 'DELETE /restart HTTP/1.0' + self._authorization self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) @patch.object(MockPatroni, 'dcs') def test_do_DELETE_switchover(self, mock_dcs): request = 'DELETE /switchover HTTP/1.0' + self._authorization self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) mock_dcs.manual_failover.return_value = False self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) mock_dcs.get_cluster.return_value.failover = None self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) def test_do_POST_reinitialize(self): request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}' MockRestApiServer(RestApiHandler, request) with patch.object(MockHa, 'reinitialize', Mock(return_value=None)): MockRestApiServer(RestApiHandler, request) @patch('time.sleep', Mock()) def test_RestApiServer_query(self): with patch.object(MockConnection, 'query', Mock(side_effect=RetryFailedError('bla'))): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni')) @patch('time.sleep', Mock()) @patch.object(MockPatroni, 'dcs') def test_do_POST_switchover(self, dcs): dcs.loop_wait = 10 cluster = dcs.get_cluster.return_value post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: ' # Invalid content with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}') response_mock.assert_called_with(400, 'Switchover could be performed only from a specific leader') # Empty content request = post + '0\n\n' MockRestApiServer(RestApiHandler, request) # [Switchover without a candidate] # Cluster with only a leader with patch.object(RestApiHandler, 'write_response') as response_mock: cluster.leader.name = 'postgresql1' request = post + '25\n\n{"leader": "postgresql1"}' MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with( 412, 'switchover is not possible: cluster does not have members except leader') # Switchover in pause mode with patch.object(RestApiHandler, 'write_response') as response_mock, \ patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with( 400, 'Switchover is possible only to a specific candidate in a paused state') # No healthy nodes to promote in both sync and async mode for is_synchronous_mode, response in ( (True, 'switchover is not possible: can not find sync_standby'), (False, 'switchover is not possible: cluster does not have members except leader')): with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(412, response) # [Switchover to the candidate specified] # Candidate to promote is the same as the leader specified with patch.object(RestApiHandler, 'write_response') as response_mock: request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}' MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(400, 'Switchover target and source are the same') # Current leader is different from the one specified with patch.object(RestApiHandler, 'write_response') as response_mock: cluster.leader.name = 'postgresql2' request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(412, 'leader name does not match') # Candidate to promote is not a member of the cluster cluster.leader.name = 'postgresql1' cluster.sync.matches.return_value = False for is_synchronous_mode, response in ( (True, 'candidate name does not match with sync_standby'), (False, 'candidate does not exists')): with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(412, response) cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}), Member(0, 'postgresql2', 30, {'api_url': 'http'})] # Failover key is empty in DCS with patch.object(RestApiHandler, 'write_response') as response_mock: cluster.failover = None MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(503, 'Switchover failed') # Result polling failed with patch.object(RestApiHandler, 'write_response') as response_mock: dcs.get_cluster.side_effect = [cluster] MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(503, 'Switchover status unknown') # Switchover to a node different from the candidate specified with patch.object(RestApiHandler, 'write_response') as response_mock: cluster2 = cluster.copy() cluster2.leader.name = 'postgresql0' cluster2.is_unlocked.return_value = False dcs.get_cluster.side_effect = [cluster, cluster2] MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(200, 'Switched over to "postgresql0" instead of "postgresql2"') # Successful switchover to the candidate with patch.object(RestApiHandler, 'write_response') as response_mock: cluster2.leader.name = 'postgresql2' dcs.get_cluster.side_effect = [cluster, cluster2] MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(200, 'Successfully switched over to "postgresql2"') with patch.object(RestApiHandler, 'write_response') as response_mock: dcs.manual_failover.return_value = False dcs.get_cluster.side_effect = None MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(503, 'failed to write failover key into DCS') dcs.manual_failover.return_value = True # Candidate is not healthy to be promoted with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(412, 'switchover is not possible: no good candidates have been found') # [Scheduled switchover] # Valid future date with patch.object(RestApiHandler, 'write_response') as response_mock: request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' + \ ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(202, 'Switchover scheduled') # Schedule in paused mode with patch.object(RestApiHandler, 'write_response') as response_mock, \ patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): dcs.manual_failover.return_value = False MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(400, "Can't schedule switchover in the paused state") # No timezone specified with patch.object(RestApiHandler, 'write_response') as response_mock: request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \ ' "scheduled_at": "6016-02-15T18:13:30.568224"}' MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(400, 'Timezone information is mandatory for the scheduled switchover') request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "' # Scheduled in the past with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}') response_mock.assert_called_with(422, 'Cannot schedule switchover in the past') # Invalid date with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}') response_mock.assert_called_with( 422, 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601') def test_do_POST_failover(self): post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: ' with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') response_mock.assert_called_once_with(400, 'Failover could be performed only to a specific candidate') with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}') response_mock.assert_called_once_with(400, "Failover can't be scheduled") with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}') response_mock.assert_called_once_with(412, 'leader name does not match') @patch.object(MockHa, 'is_leader', Mock(return_value=True)) def test_do_POST_citus(self): post = 'POST /citus HTTP/1.0' + self._authorization + '\nContent-Length: ' MockRestApiServer(RestApiHandler, post + '0\n\n') MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') @patch.object(MockHa, 'is_leader', Mock(return_value=True)) def test_do_POST_mpp(self): post = 'POST /mpp HTTP/1.0' + self._authorization + '\nContent-Length: ' MockRestApiServer(RestApiHandler, post + '0\n\n') MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') class TestRestApiServer(unittest.TestCase): @patch('ssl.SSLContext.load_cert_chain', Mock()) @patch('ssl.SSLContext.set_ciphers', Mock()) @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0)) @patch.object(HTTPServer, '__init__', Mock()) def setUp(self): self.srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required', 'ciphers': '!SSLv1:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1', 'allowlist': ['127.0.0.1', '::1/128', '::1/zxc'], 'allowlist_include_members': True}) @patch.object(HTTPServer, '__init__', Mock()) def test_reload_config(self): bad_config = {'listen': 'foo'} self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config) self.assertRaises(ValueError, self.srv.reload_config, bad_config) self.assertRaises(ValueError, self.srv.reload_config, {}) with patch.object(socket.socket, 'setsockopt', Mock(side_effect=socket.error)), \ patch.object(MockRestApiServer, 'server_close', Mock()): self.srv.reload_config({'listen': ':8008'}) @patch.object(MockPatroni, 'dcs') def test_check_access(self, mock_dcs): mock_dcs.cluster = get_cluster_initialized_without_leader() mock_dcs.cluster.members[1].data['api_url'] = 'http://127.0.0.1z:8011/patroni' mock_dcs.cluster.members.append(Member(0, 'bad-api-url', 30, {'api_url': 123})) mock_rh = Mock() mock_rh.client_address = ('127.0.0.2',) self.assertIsNot(self.srv.check_access(mock_rh), True) mock_rh.client_address = ('127.0.0.1',) mock_rh.request.getpeercert.return_value = None self.assertIsNot(self.srv.check_access(mock_rh), True) def test_handle_error(self): try: raise Exception() except Exception: self.assertIsNone(self.srv.handle_error(None, ('127.0.0.1', 55555))) @patch.object(HTTPServer, '__init__', Mock(side_effect=socket.error)) def test_socket_error(self): self.assertRaises(socket.error, MockRestApiServer, Mock(), '', {'listen': '*:8008'}) def __create_socket(self): sock = socket.socket() try: import ssl ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) ctx.check_hostname = False sock = ctx.wrap_socket(sock=sock) sock.do_handshake = Mock() sock.unwrap = Mock(side_effect=Exception) except Exception: pass return sock @patch.object(ThreadingMixIn, 'process_request_thread', Mock()) def test_process_request_thread(self): self.srv.process_request_thread(self.__create_socket(), ('2', 54321)) @patch.object(MockRestApiServer, 'process_request', Mock(side_effect=RuntimeError)) @patch.object(MockRestApiServer, 'get_request') def test_process_request_error(self, mock_get_request): mock_get_request.return_value = (self.__create_socket(), ('127.0.0.1', 55555)) self.srv._handle_request_noblock() @patch('ssl._ssl._test_decode_cert', Mock()) def test_reload_local_certificate(self): self.assertTrue(self.srv.reload_local_certificate()) def test_get_certificate_serial_number(self): self.assertIsNone(self.srv.get_certificate_serial_number()) def test_query(self): with patch.object(MockConnection, 'get', Mock(side_effect=OperationalError)): self.assertRaises(PostgresConnectionException, self.srv.query, 'SELECT 1') with patch.object(MockConnection, 'get', Mock(side_effect=[MockConnect(), OperationalError])), \ patch.object(MockConnection, 'query') as mock_query: self.srv.query('SELECT 1') mock_query.assert_called_once_with('SELECT 1') patroni-4.0.4/tests/test_async_executor.py000066400000000000000000000013761472010352700210110ustar00rootroot00000000000000import unittest from threading import Thread from unittest.mock import Mock, patch from patroni.async_executor import AsyncExecutor, CriticalTask class TestAsyncExecutor(unittest.TestCase): def setUp(self): self.a = AsyncExecutor(Mock(), Mock()) @patch.object(Thread, 'start', Mock()) def test_run_async(self): self.a.run_async(Mock(return_value=True)) def test_run(self): self.a.run(Mock(side_effect=Exception())) def test_cancel(self): self.a.cancel() self.a.schedule('foo') self.a.cancel() self.a.run(Mock()) class TestCriticalTask(unittest.TestCase): def test_completed_task(self): ct = CriticalTask() ct.complete(1) self.assertFalse(ct.cancel()) patroni-4.0.4/tests/test_aws.py000066400000000000000000000046011472010352700165420ustar00rootroot00000000000000import sys import unittest from collections import namedtuple from unittest.mock import Mock, patch, PropertyMock import botocore import botocore.awsrequest from patroni.scripts.aws import AWSConnection, main as _main class MockVolumes(object): @staticmethod def filter(*args, **kwargs): oid = namedtuple('Volume', 'id') return [oid(id='a'), oid(id='b')] class MockEc2Connection(object): volumes = MockVolumes() @staticmethod def create_tags(Resources, **kwargs): if len(Resources) == 0: raise botocore.exceptions.ClientError({'Error': {'Code': 503, 'Message': 'Request limit exceeded'}}, 'create_tags') return True class MockIMDSFetcher(object): def __init__(self, timeout): pass @staticmethod def _fetch_metadata_token(): return '' @staticmethod def _get_request(*args): return botocore.awsrequest.AWSResponse(url='', status_code=200, headers={}, raw=None) @patch('boto3.resource', Mock(return_value=MockEc2Connection())) @patch('patroni.scripts.aws.IMDSFetcher', MockIMDSFetcher) class TestAWSConnection(unittest.TestCase): @patch.object(botocore.awsrequest.AWSResponse, 'text', PropertyMock(return_value='{"instanceId": "012345", "region": "eu-west-1"}')) def test_on_role_change(self): conn = AWSConnection('test') self.assertTrue(conn.on_role_change('primary')) with patch.object(MockVolumes, 'filter', Mock(return_value=[])): conn._retry.max_tries = 1 self.assertFalse(conn.on_role_change('primary')) @patch.object(MockIMDSFetcher, '_get_request', Mock(side_effect=Exception('foo'))) def test_non_aws(self): conn = AWSConnection('test') self.assertFalse(conn.on_role_change("primary")) @patch.object(botocore.awsrequest.AWSResponse, 'text', PropertyMock(return_value='boo')) def test_aws_bizarre_response(self): conn = AWSConnection('test') self.assertFalse(conn.aws_available()) @patch.object(MockIMDSFetcher, '_get_request', Mock(return_value=botocore.awsrequest.AWSResponse( url='', status_code=503, headers={}, raw=None))) @patch('sys.exit', Mock()) def test_main(self): self.assertIsNone(_main()) sys.argv = ['aws.py', 'on_start', 'replica', 'foo'] self.assertIsNone(_main()) patroni-4.0.4/tests/test_barman.py000066400000000000000000000732171472010352700172210ustar00rootroot00000000000000import logging import unittest from unittest import mock from unittest.mock import MagicMock, Mock, patch from urllib3.exceptions import MaxRetryError from patroni.scripts.barman.cli import main from patroni.scripts.barman.config_switch import _should_skip_switch, _switch_config, \ ExitCode as BarmanConfigSwitchExitCode, run_barman_config_switch from patroni.scripts.barman.recover import _restore_backup, ExitCode as BarmanRecoverExitCode, run_barman_recover from patroni.scripts.barman.utils import ApiNotOk, OperationStatus, PgBackupApi, RetriesExceeded, set_up_logging API_URL = "http://localhost:7480" BARMAN_SERVER = "my_server" BARMAN_MODEL = "my_model" BACKUP_ID = "backup_id" SSH_COMMAND = "ssh postgres@localhost" DATA_DIRECTORY = "/path/to/pgdata" LOOP_WAIT = 10 RETRY_WAIT = 2 MAX_RETRIES = 5 # stuff from patroni.scripts.barman.utils @patch("logging.basicConfig") def test_set_up_logging(mock_log_config): log_file = "/path/to/some/file.log" set_up_logging(log_file) mock_log_config.assert_called_once_with(filename=log_file, level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s") class TestPgBackupApi(unittest.TestCase): @patch.object(PgBackupApi, "_ensure_api_ok", Mock()) @patch("patroni.scripts.barman.utils.PoolManager", MagicMock()) def setUp(self): self.api = PgBackupApi(API_URL, None, None, RETRY_WAIT, MAX_RETRIES) # Reset the mock as the same instance is used across tests self.api._http.request.reset_mock() self.api._http.request.side_effect = None def test__build_full_url(self): self.assertEqual(self.api._build_full_url("/some/path"), f"{API_URL}/some/path") @patch("json.loads") def test__deserialize_response(self, mock_json_loads): mock_response = MagicMock() self.assertIsNotNone(self.api._deserialize_response(mock_response)) mock_json_loads.assert_called_once_with(mock_response.data.decode("utf-8")) @patch("json.dumps") def test__serialize_request(self, mock_json_dumps): body = "some_body" ret = self.api._serialize_request(body) self.assertIsNotNone(ret) mock_json_dumps.assert_called_once_with(body) mock_json_dumps.return_value.encode.assert_called_once_with("utf-8") @patch.object(PgBackupApi, "_deserialize_response", Mock(return_value="test")) def test__get_request(self): mock_request = self.api._http.request # with no error self.assertEqual(self.api._get_request("/some/path"), "test") mock_request.assert_called_once_with("GET", f"{API_URL}/some/path") # with MaxRetryError http_error = MaxRetryError(self.api._http, f"{API_URL}/some/path") mock_request.side_effect = http_error with self.assertRaises(RetriesExceeded) as exc: self.assertIsNone(self.api._get_request("/some/path")) self.assertEqual( str(exc.exception), "Failed to perform a GET request to http://localhost:7480/some/path" ) @patch.object(PgBackupApi, "_deserialize_response", Mock(return_value="test")) @patch.object(PgBackupApi, "_serialize_request") def test__post_request(self, mock_serialize): mock_request = self.api._http.request # with no error self.assertEqual(self.api._post_request("/some/path", "some body"), "test") mock_serialize.assert_called_once_with("some body") mock_request.assert_called_once_with("POST", f"{API_URL}/some/path", body=mock_serialize.return_value, headers={"Content-Type": "application/json"}) # with HTTPError http_error = MaxRetryError(self.api._http, f"{API_URL}/some/path") mock_request.side_effect = http_error with self.assertRaises(RetriesExceeded) as exc: self.assertIsNone(self.api._post_request("/some/path", "some body")) self.assertEqual( str(exc.exception), f"Failed to perform a POST request to http://localhost:7480/some/path with {mock_serialize.return_value}" ) @patch.object(PgBackupApi, "_get_request") def test__ensure_api_ok(self, mock_get_request): # API ok mock_get_request.return_value = "OK" self.assertIsNone(self.api._ensure_api_ok()) # API not ok mock_get_request.return_value = "random" with self.assertRaises(ApiNotOk) as exc: self.assertIsNone(self.api._ensure_api_ok()) self.assertEqual( str(exc.exception), "pg-backup-api is currently not up and running at http://localhost:7480: random", ) @patch("patroni.scripts.barman.utils.OperationStatus") @patch("logging.warning") @patch("time.sleep") @patch.object(PgBackupApi, "_get_request") def test_get_operation_status(self, mock_get_request, mock_sleep, mock_logging, mock_op_status): # well formed response mock_get_request.return_value = {"status": "some status"} mock_op_status.__getitem__.return_value = "SOME_STATUS" self.assertEqual(self.api.get_operation_status(BARMAN_SERVER, "some_id"), "SOME_STATUS") mock_get_request.assert_called_once_with(f"servers/{BARMAN_SERVER}/operations/some_id") mock_sleep.assert_not_called() mock_logging.assert_not_called() mock_op_status.__getitem__.assert_called_once_with("some status") # malformed response mock_get_request.return_value = {"statuss": "some status"} with self.assertRaises(RetriesExceeded) as exc: self.api.get_operation_status(BARMAN_SERVER, "some_id") self.assertEqual(str(exc.exception), "Maximum number of retries exceeded for method PgBackupApi.get_operation_status.") self.assertEqual(mock_sleep.call_count, self.api.max_retries) mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries) self.assertEqual(mock_logging.call_count, self.api.max_retries) for i in range(mock_logging.call_count): call_args = mock_logging.call_args_list[i][0] self.assertEqual(len(call_args), 5) self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") self.assertEqual(call_args[1], i + 1) self.assertEqual(call_args[2], self.api.max_retries) self.assertEqual(call_args[3], "PgBackupApi.get_operation_status") self.assertIsInstance(call_args[4], KeyError) self.assertEqual(call_args[4].args, ('status',)) @patch("logging.warning") @patch("time.sleep") @patch.object(PgBackupApi, "_post_request") def test_create_recovery_operation(self, mock_post_request, mock_sleep, mock_logging): # well formed response mock_post_request.return_value = {"operation_id": "some_id"} self.assertEqual( self.api.create_recovery_operation(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY), "some_id", ) mock_sleep.assert_not_called() mock_logging.assert_not_called() mock_post_request.assert_called_once_with( f"servers/{BARMAN_SERVER}/operations", { "type": "recovery", "backup_id": BACKUP_ID, "remote_ssh_command": SSH_COMMAND, "destination_directory": DATA_DIRECTORY, } ) # malformed response mock_post_request.return_value = {"operation_idd": "some_id"} with self.assertRaises(RetriesExceeded) as exc: self.api.create_recovery_operation(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY) self.assertEqual(str(exc.exception), "Maximum number of retries exceeded for method PgBackupApi.create_recovery_operation.") self.assertEqual(mock_sleep.call_count, self.api.max_retries) mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries) self.assertEqual(mock_logging.call_count, self.api.max_retries) for i in range(mock_logging.call_count): call_args = mock_logging.call_args_list[i][0] self.assertEqual(len(call_args), 5) self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") self.assertEqual(call_args[1], i + 1) self.assertEqual(call_args[2], self.api.max_retries) self.assertEqual(call_args[3], "PgBackupApi.create_recovery_operation") self.assertIsInstance(call_args[4], KeyError) self.assertEqual(call_args[4].args, ('operation_id',)) @patch("logging.warning") @patch("time.sleep") @patch.object(PgBackupApi, "_post_request") def test_create_config_switch_operation(self, mock_post_request, mock_sleep, mock_logging): # well formed response -- sample 1 mock_post_request.return_value = {"operation_id": "some_id"} self.assertEqual( self.api.create_config_switch_operation(BARMAN_SERVER, BARMAN_MODEL, None), "some_id", ) mock_sleep.assert_not_called() mock_logging.assert_not_called() mock_post_request.assert_called_once_with( f"servers/{BARMAN_SERVER}/operations", { "type": "config_switch", "model_name": BARMAN_MODEL, } ) # well formed response -- sample 2 mock_post_request.reset_mock() self.assertEqual( self.api.create_config_switch_operation(BARMAN_SERVER, None, True), "some_id", ) mock_sleep.assert_not_called() mock_logging.assert_not_called() mock_post_request.assert_called_once_with( f"servers/{BARMAN_SERVER}/operations", { "type": "config_switch", "reset": True, } ) # malformed response mock_post_request.return_value = {"operation_idd": "some_id"} with self.assertRaises(RetriesExceeded) as exc: self.api.create_config_switch_operation(BARMAN_SERVER, BARMAN_MODEL, None) self.assertEqual(str(exc.exception), "Maximum number of retries exceeded for method PgBackupApi.create_config_switch_operation.") self.assertEqual(mock_sleep.call_count, self.api.max_retries) mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries) self.assertEqual(mock_logging.call_count, self.api.max_retries) for i in range(mock_logging.call_count): call_args = mock_logging.call_args_list[i][0] self.assertEqual(len(call_args), 5) self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") self.assertEqual(call_args[1], i + 1) self.assertEqual(call_args[2], self.api.max_retries) self.assertEqual(call_args[3], "PgBackupApi.create_config_switch_operation") self.assertIsInstance(call_args[4], KeyError) self.assertEqual(call_args[4].args, ('operation_id',)) # stuff from patroni.scripts.barman.recover class TestBarmanRecover(unittest.TestCase): def setUp(self): self.api = MagicMock() # Reset the mock as the same instance is used across tests self.api._http.request.reset_mock() self.api._http.request.side_effect = None @patch("time.sleep") @patch("logging.info") @patch("logging.error") def test__restore_backup(self, mock_log_error, mock_log_info, mock_sleep): mock_create_op = self.api.create_recovery_operation mock_get_status = self.api.get_operation_status # successful fast restore mock_create_op.return_value = "some_id" mock_get_status.return_value = OperationStatus.DONE self.assertEqual( _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT), BarmanRecoverExitCode.RECOVERY_DONE, ) mock_create_op.assert_called_once_with(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY) mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id") mock_log_info.assert_has_calls([ mock.call("Created the recovery operation with ID %s", "some_id"), mock.call("Recovery operation finished successfully."), ]) mock_log_error.assert_not_called() mock_sleep.assert_not_called() # successful slow restore mock_create_op.reset_mock() mock_get_status.reset_mock() mock_log_info.reset_mock() mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.DONE] self.assertEqual( _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT), BarmanRecoverExitCode.RECOVERY_DONE, ) mock_create_op.assert_called_once() self.assertEqual(mock_get_status.call_count, 21) mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21) self.assertEqual(mock_log_info.call_count, 22) mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")] + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20 + [mock.call("Recovery operation finished successfully.")]) mock_log_error.assert_not_called() self.assertEqual(mock_sleep.call_count, 20) mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20) # failed fast restore mock_create_op.reset_mock() mock_get_status.reset_mock() mock_log_info.reset_mock() mock_sleep.reset_mock() mock_get_status.side_effect = None mock_get_status.return_value = OperationStatus.FAILED self.assertEqual( _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT), BarmanRecoverExitCode.RECOVERY_FAILED, ) mock_create_op.assert_called_once() mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id") mock_log_info.assert_has_calls([ mock.call("Created the recovery operation with ID %s", "some_id"), ]) mock_log_error.assert_has_calls([ mock.call("Recovery operation failed."), ]) mock_sleep.assert_not_called() # failed slow restore mock_create_op.reset_mock() mock_get_status.reset_mock() mock_log_info.reset_mock() mock_log_error.reset_mock() mock_sleep.reset_mock() mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.FAILED] self.assertEqual( _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT), BarmanRecoverExitCode.RECOVERY_FAILED, ) mock_create_op.assert_called_once() self.assertEqual(mock_get_status.call_count, 21) mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21) self.assertEqual(mock_log_info.call_count, 21) mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")] + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20) mock_log_error.assert_has_calls([ mock.call("Recovery operation failed."), ]) self.assertEqual(mock_sleep.call_count, 20) mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20) # create retries exceeded mock_log_info.reset_mock() mock_log_error.reset_mock() mock_sleep.reset_mock() mock_create_op.side_effect = RetriesExceeded() mock_get_status.side_effect = None self.assertEqual( _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT), BarmanRecoverExitCode.HTTP_ERROR, ) mock_log_info.assert_not_called() mock_log_error.assert_called_once_with("An issue was faced while trying to create a recovery operation: %r", mock_create_op.side_effect) mock_sleep.assert_not_called() # get status retries exceeded mock_create_op.reset_mock() mock_create_op.side_effect = None mock_log_error.reset_mock() mock_log_info.reset_mock() mock_get_status.side_effect = RetriesExceeded self.assertEqual( _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT), BarmanRecoverExitCode.HTTP_ERROR, ) mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id") mock_log_error.assert_called_once_with("Maximum number of retries exceeded, exiting.") mock_sleep.assert_not_called() class TestBarmanRecoverCli(unittest.TestCase): @patch("patroni.scripts.barman.recover._restore_backup") def test_run_barman_recover(self, mock_rb): api = MagicMock() args = MagicMock() # successful execution mock_rb.return_value = BarmanRecoverExitCode.RECOVERY_DONE self.assertEqual( run_barman_recover(api, args), BarmanRecoverExitCode.RECOVERY_DONE, ) mock_rb.assert_called_once_with(api, args.barman_server, args.backup_id, args.ssh_command, args.data_directory, args.loop_wait) # failed execution mock_rb.reset_mock() mock_rb.return_value = BarmanRecoverExitCode.RECOVERY_FAILED self.assertEqual( run_barman_recover(api, args), BarmanRecoverExitCode.RECOVERY_FAILED, ) mock_rb.assert_called_once_with(api, args.barman_server, args.backup_id, args.ssh_command, args.data_directory, args.loop_wait) # stuff from patroni.scripts.barman.config_switch class TestBarmanConfigSwitch(unittest.TestCase): def setUp(self): self.api = MagicMock() # Reset the mock as the same instance is used across tests self.api._http.request.reset_mock() self.api._http.request.side_effect = None @patch("time.sleep") @patch("logging.info") @patch("logging.error") def test__switch_config(self, mock_log_error, mock_log_info, mock_sleep): mock_create_op = self.api.create_config_switch_operation mock_get_status = self.api.get_operation_status # successful fast config-switch mock_create_op.return_value = "some_id" mock_get_status.return_value = OperationStatus.DONE self.assertEqual( _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None), BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE, ) mock_create_op.assert_called_once_with(BARMAN_SERVER, BARMAN_MODEL, None) mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id") mock_log_info.assert_has_calls([ mock.call("Created the config switch operation with ID %s", "some_id"), mock.call("Config switch operation finished successfully."), ]) mock_log_error.assert_not_called() mock_sleep.assert_not_called() # successful slow config-switch mock_create_op.reset_mock() mock_get_status.reset_mock() mock_log_info.reset_mock() mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.DONE] self.assertEqual( _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None), BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE, ) mock_create_op.assert_called_once_with(BARMAN_SERVER, BARMAN_MODEL, None) self.assertEqual(mock_get_status.call_count, 21) mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21) self.assertEqual(mock_log_info.call_count, 22) mock_log_info.assert_has_calls([mock.call("Created the config switch operation with ID %s", "some_id")] + [mock.call("Config switch operation %s is still in progress", "some_id")] * 20 + [mock.call("Config switch operation finished successfully.")]) mock_log_error.assert_not_called() self.assertEqual(mock_sleep.call_count, 20) mock_sleep.assert_has_calls([mock.call(5)] * 20) # failed fast config-switch mock_create_op.reset_mock() mock_get_status.reset_mock() mock_log_info.reset_mock() mock_sleep.reset_mock() mock_get_status.side_effect = None mock_get_status.return_value = OperationStatus.FAILED self.assertEqual( _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None), BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED, ) mock_create_op.assert_called_once() mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id") mock_log_info.assert_called_once_with("Created the config switch operation with ID %s", "some_id") mock_log_error.assert_called_once_with("Config switch operation failed.") mock_sleep.assert_not_called() # failed slow config-switch mock_create_op.reset_mock() mock_get_status.reset_mock() mock_log_info.reset_mock() mock_log_error.reset_mock() mock_sleep.reset_mock() mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.FAILED] self.assertEqual( _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None), BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED, ) mock_create_op.assert_called_once() self.assertEqual(mock_get_status.call_count, 21) mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21) self.assertEqual(mock_log_info.call_count, 21) mock_log_info.assert_has_calls([mock.call("Created the config switch operation with ID %s", "some_id")] + [mock.call("Config switch operation %s is still in progress", "some_id")] * 20) mock_log_error.assert_called_once_with("Config switch operation failed.") self.assertEqual(mock_sleep.call_count, 20) mock_sleep.assert_has_calls([mock.call(5)] * 20) # create retries exceeded mock_log_info.reset_mock() mock_log_error.reset_mock() mock_sleep.reset_mock() mock_create_op.side_effect = RetriesExceeded() mock_get_status.side_effect = None self.assertEqual( _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None), BarmanConfigSwitchExitCode.HTTP_ERROR, ) mock_log_info.assert_not_called() mock_log_error.assert_called_once_with("An issue was faced while trying to create a config switch operation: " "%r", mock_create_op.side_effect) mock_sleep.assert_not_called() # get status retries exceeded mock_create_op.reset_mock() mock_create_op.side_effect = None mock_log_error.reset_mock() mock_get_status.side_effect = RetriesExceeded self.assertEqual( _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None), BarmanConfigSwitchExitCode.HTTP_ERROR, ) mock_log_info.assert_called_once_with("Created the config switch operation with ID %s", "some_id") mock_log_error.assert_called_once_with("Maximum number of retries exceeded, exiting.") mock_sleep.assert_not_called() class TestBarmanConfigSwitchCli(unittest.TestCase): def test__should_skip_switch(self): args = MagicMock() for role, switch_when, expected in [ ("primary", "promoted", False), ("primary", "demoted", True), ("primary", "always", False), ("promoted", "promoted", False), ("promoted", "demoted", True), ("promoted", "always", False), ("standby_leader", "promoted", True), ("standby_leader", "demoted", True), ("standby_leader", "always", False), ("replica", "promoted", True), ("replica", "demoted", False), ("replica", "always", False), ("demoted", "promoted", True), ("demoted", "demoted", False), ("demoted", "always", False), ]: args.role = role args.switch_when = switch_when self.assertEqual(_should_skip_switch(args), expected) @patch("patroni.scripts.barman.config_switch._should_skip_switch") @patch("patroni.scripts.barman.config_switch._switch_config") @patch("logging.error") @patch("logging.info") def test_run_barman_config_switch(self, mock_log_info, mock_log_error, mock_sc, mock_skip): api = MagicMock() args = MagicMock() args.reset = None # successful execution mock_skip.return_value = False mock_sc.return_value = BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE self.assertEqual( run_barman_config_switch(api, args), BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE, ) mock_sc.assert_called_once_with(api, args.barman_server, args.barman_model, args.reset) # failed execution mock_sc.reset_mock() mock_sc.return_value = BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED self.assertEqual( run_barman_config_switch(api, args), BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED, ) mock_sc.assert_called_once_with(api, args.barman_server, args.barman_model, args.reset) # skipped execution mock_sc.reset_mock() mock_skip.return_value = True self.assertEqual( run_barman_config_switch(api, args), BarmanConfigSwitchExitCode.CONFIG_SWITCH_SKIPPED ) mock_sc.assert_not_called() mock_log_info.assert_called_once_with("Config switch operation was skipped (role=%s, " "switch_when=%s).", args.role, args.switch_when) mock_log_error.assert_not_called() # invalid args -- sample 1 mock_skip.return_value = False args = MagicMock() args.barman_server = BARMAN_SERVER args.barman_model = BARMAN_MODEL args.reset = True self.assertEqual( run_barman_config_switch(api, args), BarmanConfigSwitchExitCode.INVALID_ARGS, ) mock_log_error.assert_called_once_with("One, and only one among 'barman_model' ('%s') and 'reset' " "('%s') should be given", BARMAN_MODEL, True) api.assert_not_called() # invalid args -- sample 2 args = MagicMock() args.barman_server = BARMAN_SERVER args.barman_model = None args.reset = None mock_log_error.reset_mock() api.reset_mock() self.assertEqual( run_barman_config_switch(api, args), BarmanConfigSwitchExitCode.INVALID_ARGS, ) mock_log_error.assert_called_once_with("One, and only one among 'barman_model' ('%s') and 'reset' " "('%s') should be given", None, None) api.assert_not_called() # stuff from patroni.scripts.barman.cli class TestMain(unittest.TestCase): @patch("patroni.scripts.barman.cli.PgBackupApi") @patch("patroni.scripts.barman.cli.set_up_logging") @patch("patroni.scripts.barman.cli.ArgumentParser") def test_main(self, mock_arg_parse, mock_set_up_log, mock_api): # sub-command specified args = MagicMock() args.func.return_value = 0 mock_arg_parse.return_value.parse_known_args.return_value = (args, None) with self.assertRaises(SystemExit) as exc: main() mock_arg_parse.assert_called_once() mock_set_up_log.assert_called_once_with(args.log_file) mock_api.assert_called_once_with(args.api_url, args.cert_file, args.key_file, args.retry_wait, args.max_retries) mock_arg_parse.return_value.print_help.assert_not_called() args.func.assert_called_once_with(mock_api.return_value, args) self.assertEqual(exc.exception.code, 0) # Issue in the API mock_arg_parse.reset_mock() mock_set_up_log.reset_mock() mock_api.reset_mock() mock_api.side_effect = ApiNotOk() with self.assertRaises(SystemExit) as exc: main() mock_arg_parse.assert_called_once() mock_set_up_log.assert_called_once_with(args.log_file) mock_api.assert_called_once_with(args.api_url, args.cert_file, args.key_file, args.retry_wait, args.max_retries) mock_arg_parse.return_value.print_help.assert_not_called() self.assertEqual(exc.exception.code, -2) # sub-command not specified mock_arg_parse.reset_mock() mock_set_up_log.reset_mock() mock_api.reset_mock() delattr(args, "func") mock_api.side_effect = None with self.assertRaises(SystemExit) as exc: main() mock_arg_parse.assert_called_once() mock_set_up_log.assert_called_once_with(args.log_file) mock_api.assert_not_called() mock_arg_parse.return_value.print_help.assert_called_once_with() self.assertEqual(exc.exception.code, -1) patroni-4.0.4/tests/test_bootstrap.py000066400000000000000000000362441472010352700177750ustar00rootroot00000000000000import os import sys from unittest.mock import Mock, patch, PropertyMock from patroni.async_executor import CriticalTask from patroni.collections import CaseInsensitiveDict from patroni.postgresql import Postgresql from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.cancellable import CancellableSubprocess from patroni.postgresql.config import ConfigHandler, get_param_diff from . import BaseTestPostgresql, mock_available_gucs, psycopg_connect @patch('subprocess.call', Mock(return_value=0)) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1")) @patch('patroni.psycopg.connect', psycopg_connect) @patch('os.rename', Mock()) @patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestBootstrap(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) def setUp(self): super(TestBootstrap, self).setUp() self.b = self.p.bootstrap @patch('time.sleep', Mock()) @patch.object(CancellableSubprocess, 'call') @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True)) @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False)) @patch.object(Bootstrap, '_post_restore', Mock(side_effect=OSError)) def test_create_replica(self, mock_cancellable_subprocess_call): self.p.config._config['create_replica_methods'] = ['pgBackRest'] self.p.config._config['pgBackRest'] = {'command': 'pgBackRest', 'keep_data': True, 'no_params': True} mock_cancellable_subprocess_call.return_value = 0 self.assertEqual(self.b.create_replica(self.leader), 0) self.p.config._config['create_replica_methods'] = ['basebackup'] self.p.config._config['basebackup'] = [{'max_rate': '100M'}, 'no-sync'] self.assertEqual(self.b.create_replica(self.leader), 0) self.p.config._config['basebackup'] = [{'max_rate': '100M', 'compress': '9'}] with patch('patroni.postgresql.bootstrap.logger.error', new_callable=Mock()) as mock_logger: self.b.create_replica(self.leader) mock_logger.assert_called_once() self.assertTrue("only one key-value is allowed and value should be a string" in mock_logger.call_args[0][0], "not matching {0}".format(mock_logger.call_args[0][0])) self.p.config._config['basebackup'] = [42] with patch('patroni.postgresql.bootstrap.logger.error', new_callable=Mock()) as mock_logger: self.b.create_replica(self.leader) mock_logger.assert_called_once() self.assertTrue("value should be string value or a single key-value pair" in mock_logger.call_args[0][0], "not matching {0}".format(mock_logger.call_args[0][0])) self.p.config._config['basebackup'] = {"foo": "bar"} self.assertEqual(self.b.create_replica(self.leader), 0) self.p.config._config['create_replica_methods'] = ['wale', 'basebackup'] del self.p.config._config['basebackup'] mock_cancellable_subprocess_call.return_value = 1 self.assertEqual(self.b.create_replica(self.leader), 1) mock_cancellable_subprocess_call.side_effect = Exception('foo') self.assertEqual(self.b.create_replica(self.leader), 1) mock_cancellable_subprocess_call.side_effect = [1, 0] self.assertEqual(self.b.create_replica(self.leader), 0) mock_cancellable_subprocess_call.side_effect = [Exception(), 0] self.assertEqual(self.b.create_replica(self.leader), 0) self.p.cancellable.cancel() self.assertEqual(self.b.create_replica(self.leader), 1) @patch('time.sleep', Mock()) @patch.object(CancellableSubprocess, 'call') @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True)) @patch.object(Bootstrap, '_post_restore', Mock(side_effect=OSError)) def test_create_replica_old_format(self, mock_cancellable_subprocess_call): """ The same test as before but with old 'create_replica_method' to test backward compatibility """ self.p.config._config['create_replica_method'] = ['wale', 'basebackup'] self.p.config._config['wale'] = {'command': 'foo'} mock_cancellable_subprocess_call.return_value = 0 self.assertEqual(self.b.create_replica(self.leader), 0) del self.p.config._config['wale'] self.assertEqual(self.b.create_replica(self.leader), 0) self.p.config._config['create_replica_method'] = ['wale'] mock_cancellable_subprocess_call.return_value = 1 self.assertEqual(self.b.create_replica(self.leader), 1) @patch.object(CancellableSubprocess, 'call', Mock(return_value=0)) @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=True)) def test_basebackup(self): with patch('patroni.postgresql.bootstrap.logger.debug') as mock_debug: self.p.cancellable.cancel() self.b.basebackup("", None, {'foo': 'bar'}) mock_debug.assert_not_called() self.p.cancellable.reset_is_cancelled() self.b.basebackup("", None, {'foo': 'bar'}) mock_debug.assert_called_with( 'calling: %r', ['pg_basebackup', f'--pgdata={self.p.data_dir}', '-X', 'stream', '--dbname=', '--foo=bar'], ) def test__initdb(self): self.assertRaises(Exception, self.b.bootstrap, {'initdb': [{'pgdata': 'bar'}]}) self.assertRaises(Exception, self.b.bootstrap, {'initdb': [{'foo': 'bar', 1: 2}]}) self.assertRaises(Exception, self.b.bootstrap, {'initdb': [1]}) self.assertRaises(Exception, self.b.bootstrap, {'initdb': 1}) def test__process_user_options(self): def error_handler(msg): raise Exception(msg) self.assertEqual(self.b.process_user_options('initdb', ['string'], (), error_handler), ['--string']) self.assertEqual( self.b.process_user_options( 'initdb', [{'key': 'value'}], (), error_handler ), ['--key=value']) if sys.platform != 'win32': self.assertEqual( self.b.process_user_options( 'initdb', [{'key': 'value with spaces'}], (), error_handler ), ["--key=value with spaces"]) self.assertEqual( self.b.process_user_options( 'initdb', [{'key': "'value with spaces'"}], (), error_handler ), ["--key=value with spaces"]) self.assertEqual( self.b.process_user_options( 'initdb', {'key': 'value with spaces'}, (), error_handler ), ["--key=value with spaces"]) self.assertEqual( self.b.process_user_options( 'initdb', {'key': "'value with spaces'"}, (), error_handler ), ["--key=value with spaces"]) # not allowed options in list of dicts/strs are filtered out self.assertEqual( self.b.process_user_options( 'pg_basebackup', [{'checkpoint': 'fast'}, {'dbname': 'dbname=postgres'}, 'gzip', {'label': 'standby'}, 'verbose'], ('dbname', 'verbose'), print ), ['--checkpoint=fast', '--gzip', '--label=standby'], ) @patch.object(CancellableSubprocess, 'call', Mock()) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False)) @patch.object(Postgresql, 'controldata', Mock(return_value={'max_connections setting': 100, 'max_prepared_xacts setting': 0, 'max_locks_per_xact setting': 64})) def test_bootstrap(self): with patch('subprocess.call', Mock(return_value=1)): self.assertFalse(self.b.bootstrap({})) config = {} with patch.object(Postgresql, 'is_running', Mock(return_value=False)), \ patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)), \ patch('multiprocessing.Process', Mock(side_effect=Exception)), \ patch('multiprocessing.get_context', Mock(side_effect=Exception), create=True): self.assertRaises(Exception, self.b.bootstrap, config) with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f: lines = f.readlines() self.assertTrue('host all all 0.0.0.0/0 md5\n' in lines) self.p.config._config.pop('pg_hba') config.update({'post_init': '/bin/false', 'pg_hba': ['host replication replicator 127.0.0.1/32 md5', 'hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5']}) self.b.bootstrap(config) with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f: lines = f.readlines() self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines) @patch.object(CancellableSubprocess, 'call') @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600)) @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'in production'})) def test_custom_bootstrap(self, mock_cancellable_subprocess_call): self.p.config._config.pop('pg_hba') config = {'method': 'foo', 'foo': {'command': 'bar --arg1=val1'}} mock_cancellable_subprocess_call.return_value = 1 self.assertFalse(self.b.bootstrap(config)) self.assertEqual(mock_cancellable_subprocess_call.call_args_list[0][0][0], ['bar', '--arg1=val1', '--scope=batman', '--datadir=' + os.path.join('data', 'test0')]) mock_cancellable_subprocess_call.reset_mock() config['foo']['no_params'] = 1 self.assertFalse(self.b.bootstrap(config)) self.assertEqual(mock_cancellable_subprocess_call.call_args_list[0][0][0], ['bar', '--arg1=val1']) mock_cancellable_subprocess_call.return_value = 0 with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))), \ patch('multiprocessing.get_context', Mock(side_effect=Exception("42")), create=True), \ patch('os.path.isfile', Mock(return_value=True)), \ patch('os.unlink', Mock()), \ patch.object(ConfigHandler, 'save_configuration_files', Mock()), \ patch.object(ConfigHandler, 'restore_configuration_files', Mock()), \ patch.object(ConfigHandler, 'write_recovery_conf', Mock()): with self.assertRaises(Exception) as e: self.b.bootstrap(config) self.assertEqual(str(e.exception), '42') config['foo']['recovery_conf'] = {'foo': 'bar'} with self.assertRaises(Exception) as e: self.b.bootstrap(config) self.assertEqual(str(e.exception), '42') mock_cancellable_subprocess_call.side_effect = Exception self.assertFalse(self.b.bootstrap(config)) @patch('time.sleep', Mock()) @patch('os.unlink', Mock()) @patch('shutil.copy', Mock()) @patch('os.path.isfile', Mock(return_value=True)) @patch('patroni.postgresql.bootstrap.quote_ident', Mock()) @patch.object(Bootstrap, 'call_post_bootstrap', Mock(return_value=True)) @patch.object(Bootstrap, '_custom_bootstrap', Mock(return_value=True)) @patch.object(Postgresql, 'start', Mock(return_value=True)) @patch.object(Postgresql, 'get_major_version', Mock(return_value=110000)) def test_post_bootstrap(self): config = {'method': 'foo', 'foo': {'command': 'bar'}} self.b.bootstrap(config) task = CriticalTask() with patch.object(Bootstrap, 'create_or_update_role', Mock(side_effect=Exception)): self.b.post_bootstrap({}, task) self.assertFalse(task.result) self.p.config._config.pop('pg_hba') with patch('patroni.postgresql.bootstrap.logger.error', new_callable=Mock()) as mock_logger: self.b.post_bootstrap({'users': 1}, task) self.assertEqual(mock_logger.call_args_list[0][0][0], 'User creation is not be supported starting from v4.0.0. ' 'Please use "bootstrap.post_bootstrap" script to create users.') self.assertTrue(task.result) self.b.bootstrap(config) with patch.object(Postgresql, 'pending_restart_reason', PropertyMock(CaseInsensitiveDict({'max_connections': get_param_diff('200', '100')}))), \ patch.object(Postgresql, 'restart', Mock()) as mock_restart: self.b.post_bootstrap({}, task) mock_restart.assert_called_once() self.b.bootstrap(config) self.p.set_state('stopped') self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'}, 'replication': {'username': 'r', 'password': 'r'}, 'rewind': {'username': 'rw', 'password': 'rw'}}, 'listen': '*', 'retry_timeout': 10, 'parameters': {'wal_level': '', 'hba_file': 'foo', 'max_prepared_transactions': 10}}) with patch.object(Postgresql, 'major_version', PropertyMock(return_value=110000)), \ patch.object(Postgresql, 'restart', Mock()) as mock_restart: self.b.post_bootstrap({}, task) mock_restart.assert_called_once() @patch.object(CancellableSubprocess, 'call') def test_call_post_bootstrap(self, mock_cancellable_subprocess_call): mock_cancellable_subprocess_call.return_value = 1 self.assertFalse(self.b.call_post_bootstrap({'post_init': '/bin/false'})) mock_cancellable_subprocess_call.return_value = 0 self.p.connection_pool._conn_kwargs.pop('user') self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'})) mock_cancellable_subprocess_call.assert_called() args, kwargs = mock_cancellable_subprocess_call.call_args self.assertTrue('PGPASSFILE' in kwargs['env']) self.assertEqual(args[0], ['/bin/false', 'dbname=postgres host=/tmp port=5432']) mock_cancellable_subprocess_call.reset_mock() self.p.connection_pool._conn_kwargs.pop('host') self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'})) mock_cancellable_subprocess_call.assert_called() self.assertEqual(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'dbname=postgres port=5432']) mock_cancellable_subprocess_call.side_effect = OSError self.assertFalse(self.b.call_post_bootstrap({'post_init': '/bin/false'})) @patch('os.path.exists', Mock(return_value=True)) @patch('os.unlink', Mock()) @patch.object(Bootstrap, 'create_replica', Mock(return_value=0)) def test_clone(self): self.b.clone(self.leader) patroni-4.0.4/tests/test_callback_executor.py000066400000000000000000000025631472010352700214270ustar00rootroot00000000000000import unittest from unittest.mock import Mock, patch import psutil from patroni.postgresql.callback_executor import CallbackExecutor class TestCallbackExecutor(unittest.TestCase): @patch('psutil.Popen') def test_callback_executor(self, mock_popen): mock_popen.return_value.children.return_value = [] mock_popen.return_value.is_running.return_value = True callback = ['test.sh', 'on_start', 'replica', 'foo'] ce = CallbackExecutor() ce._kill_children = Mock(side_effect=Exception) ce._invoke_excepthook = Mock() self.assertIsNone(ce.call(callback)) ce.join() self.assertIsNone(ce.call(callback)) mock_popen.return_value.kill.side_effect = psutil.AccessDenied() self.assertIsNone(ce.call(callback)) ce._process_children = [] mock_popen.return_value.children.side_effect = psutil.Error() mock_popen.return_value.kill.side_effect = psutil.NoSuchProcess(123) self.assertIsNone(ce.call(callback)) mock_popen.side_effect = Exception ce = CallbackExecutor() ce._condition.wait = Mock(side_effect=[None, Exception]) ce._invoke_excepthook = Mock() self.assertIsNone(ce.call(callback)) mock_popen.side_effect = [Mock()] self.assertIsNone(ce.call(['test.sh', 'on_reload', 'replica', 'foo'])) ce.join() patroni-4.0.4/tests/test_cancellable.py000066400000000000000000000022551472010352700202000ustar00rootroot00000000000000import unittest from unittest.mock import Mock, patch import psutil from patroni.exceptions import PostgresException from patroni.postgresql.cancellable import CancellableSubprocess class TestCancellableSubprocess(unittest.TestCase): def setUp(self): self.c = CancellableSubprocess() def test_call(self): self.c.cancel() self.assertRaises(PostgresException, self.c.call) def test__kill_children(self): self.c._process_children = [Mock()] self.c._kill_children() self.c._process_children[0].kill.side_effect = psutil.AccessDenied() self.c._kill_children() self.c._process_children[0].kill.side_effect = psutil.NoSuchProcess(123) self.c._kill_children() @patch('patroni.postgresql.cancellable.polling_loop', Mock(return_value=[0, 0])) def test_cancel(self): self.c._process = Mock() self.c._process.is_running.return_value = True self.c._process.children.side_effect = psutil.NoSuchProcess(123) self.c._process.suspend.side_effect = psutil.AccessDenied() self.c.cancel() self.c._process.is_running.side_effect = [True, False] self.c.cancel() patroni-4.0.4/tests/test_citus.py000066400000000000000000000566221472010352700171110ustar00rootroot00000000000000import time import unittest from copy import deepcopy from typing import List from unittest.mock import Mock, patch, PropertyMock from patroni.postgresql.mpp.citus import CitusHandler, PgDistGroup, PgDistNode from patroni.psycopg import ProgrammingError from . import BaseTestPostgresql, MockCursor, psycopg_connect, SleepException from .test_ha import get_cluster_initialized_with_leader @patch('patroni.postgresql.mpp.citus.Thread', Mock()) @patch('patroni.psycopg.connect', psycopg_connect) class TestCitus(BaseTestPostgresql): def setUp(self): super(TestCitus, self).setUp() self.c = self.p.mpp_handler self.cluster = get_cluster_initialized_with_leader() self.cluster.workers[1] = self.cluster @patch('time.time', Mock(side_effect=[100, 130, 160, 190, 220, 250, 280, 310, 340, 370, 400, 430, 460, 490])) @patch('patroni.postgresql.mpp.citus.logger.exception', Mock(side_effect=SleepException)) @patch('patroni.postgresql.mpp.citus.logger.warning') @patch('patroni.postgresql.mpp.citus.PgDistTask.wait', Mock()) @patch.object(CitusHandler, 'is_alive', Mock(return_value=True)) def test_run(self, mock_logger_warning): # `before_demote` or `before_promote` REST API calls starting a # transaction. We want to make sure that it finishes during # certain timeout. In case if it is not, we want to roll it back # in order to not block other workers that want to update # `pg_dist_node`. self.c._condition.wait = Mock(side_effect=[Mock(), Mock(), Mock(), SleepException]) self.c.handle_event(self.cluster, {'type': 'before_demote', 'group': 1, 'leader': 'leader', 'timeout': 30, 'cooldown': 10}) self.c.add_task('after_promote', 2, self.cluster, self.cluster.leader_name, 'postgres://host3:5432/postgres') self.assertRaises(SleepException, self.c.run) mock_logger_warning.assert_called_once() self.assertTrue(mock_logger_warning.call_args[0][0].startswith('Rolling back transaction')) self.assertTrue(repr(mock_logger_warning.call_args[0][1]).startswith('PgDistTask')) @patch.object(CitusHandler, 'is_alive', Mock(return_value=False)) @patch.object(CitusHandler, 'start', Mock()) def test_sync_meta_data(self): with patch.object(CitusHandler, 'is_enabled', Mock(return_value=False)): self.c.sync_meta_data(self.cluster) self.c.sync_meta_data(self.cluster) def test_handle_event(self): self.c.handle_event(self.cluster, {}) with patch.object(CitusHandler, 'is_alive', Mock(return_value=True)): self.c.handle_event(self.cluster, {'type': 'after_promote', 'group': 2, 'leader': 'leader', 'timeout': 30, 'cooldown': 10}) def test_add_task(self): with patch('patroni.postgresql.mpp.citus.logger.error') as mock_logger, \ patch('patroni.postgresql.mpp.citus.urlparse', Mock(side_effect=Exception)): self.c.add_task('', 1, self.cluster, '', None) mock_logger.assert_called_once() with patch('patroni.postgresql.mpp.citus.logger.debug') as mock_logger: self.c.add_task('before_demote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres', 30) mock_logger.assert_called_once() self.assertTrue(mock_logger.call_args[0][0].startswith('Adding the new task:')) with patch('patroni.postgresql.mpp.citus.logger.debug') as mock_logger: self.c.add_task('before_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres', 30) mock_logger.assert_called_once() self.assertTrue(mock_logger.call_args[0][0].startswith('Overriding existing task:')) # add_task called from sync_pg_dist_node should not override already scheduled or in flight task until deadline self.assertIsNotNone(self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres', 30)) self.assertIsNone(self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres')) self.c._in_flight = self.c._tasks.pop() self.c._in_flight.deadline = self.c._in_flight.timeout + time.time() self.assertIsNone(self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres')) self.c._in_flight.deadline = 0 self.assertIsNotNone(self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres')) # If there is no transaction in progress and cached pg_dist_node matching desired state task should not be added self.c._schedule_load_pg_dist_node = False self.c._pg_dist_group[self.c._in_flight.groupid] = self.c._in_flight self.c._in_flight = None self.assertIsNone(self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host:5432/postgres')) def test_pick_task(self): self.c.add_task('after_promote', 0, self.cluster, self.cluster.leader_name, 'postgres://host1:5432/postgres') with patch.object(CitusHandler, 'update_node') as mock_update_node: self.c.process_tasks() # process_task() shouldn't be called because pick_task double checks with _pg_dist_group mock_update_node.assert_not_called() def test_process_task(self): self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host2:5432/postgres') task = self.c.add_task('before_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host4:5432/postgres', 30) self.c.process_tasks() self.assertTrue(task._event.is_set()) # the after_promote should result only in COMMIT task = self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host4:5432/postgres', 30) with patch.object(CitusHandler, 'query') as mock_query: self.c.process_tasks() mock_query.assert_called_once() self.assertEqual(mock_query.call_args[0][0], 'COMMIT') def test_process_tasks(self): self.c.add_task('after_promote', 0, self.cluster, self.cluster.leader_name, 'postgres://host2:5432/postgres') self.c.process_tasks() self.c.add_task('after_promote', 0, self.cluster, self.cluster.leader_name, 'postgres://host3:5432/postgres') with patch('patroni.postgresql.mpp.citus.logger.error') as mock_logger, \ patch.object(CitusHandler, 'query', Mock(side_effect=Exception)): self.c.process_tasks() mock_logger.assert_called_once() self.assertTrue(mock_logger.call_args[0][0].startswith('Exception when working with pg_dist_node: ')) def test_on_demote(self): self.c.on_demote() @patch('patroni.postgresql.mpp.citus.logger.error') @patch.object(MockCursor, 'execute', Mock(side_effect=Exception)) def test_load_pg_dist_group(self, mock_logger): # load_pg_dist_group) triggers, query fails and exception is property handled self.c.process_tasks() self.assertTrue(self.c._schedule_load_pg_dist_group) mock_logger.assert_called_once() self.assertTrue(mock_logger.call_args[0][0].startswith('Exception when executing query')) self.assertTrue(mock_logger.call_args[0][1].startswith('SELECT groupid, nodename, ')) def test_wait(self): task = self.c.add_task('before_demote', 1, self.cluster, self.cluster.leader_name, u'postgres://host:5432/postgres', 30) task._event.wait = Mock() task.wait() def test_adjust_postgres_gucs(self): parameters = {'max_connections': 101, 'max_prepared_transactions': 0, 'shared_preload_libraries': 'foo , citus, bar '} self.c.adjust_postgres_gucs(parameters) self.assertEqual(parameters['max_prepared_transactions'], 202) self.assertEqual(parameters['shared_preload_libraries'], 'citus,foo,bar') self.assertEqual(parameters['wal_level'], 'logical') self.assertEqual(parameters['citus.local_hostname'], '/tmp') def test_ignore_replication_slot(self): self.assertFalse(self.c.ignore_replication_slot({'name': 'foo', 'type': 'physical', 'database': 'bar', 'plugin': 'wal2json'})) self.assertFalse(self.c.ignore_replication_slot({'name': 'foo', 'type': 'logical', 'database': 'bar', 'plugin': 'wal2json'})) self.assertFalse(self.c.ignore_replication_slot({'name': 'foo', 'type': 'logical', 'database': 'bar', 'plugin': 'pgoutput'})) self.assertFalse(self.c.ignore_replication_slot({'name': 'foo', 'type': 'logical', 'database': 'citus', 'plugin': 'pgoutput'})) self.assertTrue(self.c.ignore_replication_slot({'name': 'citus_shard_move_slot_1_2_3', 'type': 'logical', 'database': 'citus', 'plugin': 'pgoutput'})) self.assertFalse(self.c.ignore_replication_slot({'name': 'citus_shard_move_slot_1_2_3', 'type': 'logical', 'database': 'citus', 'plugin': 'citus'})) self.assertFalse(self.c.ignore_replication_slot({'name': 'citus_shard_split_slot_1_2_3', 'type': 'logical', 'database': 'citus', 'plugin': 'pgoutput'})) self.assertTrue(self.c.ignore_replication_slot({'name': 'citus_shard_split_slot_1_2_3', 'type': 'logical', 'database': 'citus', 'plugin': 'citus'})) @patch('patroni.postgresql.mpp.citus.logger.debug') @patch('patroni.postgresql.mpp.citus.connect', psycopg_connect) @patch('patroni.postgresql.mpp.citus.quote_ident', Mock()) def test_bootstrap_duplicate_database(self, mock_logger): with patch.object(MockCursor, 'execute', Mock(side_effect=ProgrammingError)): self.assertRaises(ProgrammingError, self.c.bootstrap) with patch.object(MockCursor, 'execute', Mock(side_effect=[ProgrammingError, None, None, None])), \ patch.object(ProgrammingError, 'diag') as mock_diag: type(mock_diag).sqlstate = PropertyMock(return_value='42P04') self.c.bootstrap() mock_logger.assert_called_once() self.assertTrue(mock_logger.call_args[0][0].startswith('Exception when creating database')) class TestGroupTransition(unittest.TestCase): nodeid = 100 def map_to_sql(self, group: int, transition: PgDistNode) -> str: if transition.role not in ('primary', 'demoted', 'secondary'): return "citus_remove_node('{0}', {1})".format(transition.host, transition.port) elif transition.nodeid: host = transition.host + ('-demoted' if transition.role == 'demoted' else '') return "citus_update_node({0}, '{1}', {2})".format(transition.nodeid, host, transition.port) else: transition.nodeid = self.nodeid self.nodeid += 1 return "citus_add_node('{0}', {1}, {2}, '{3}')".format(transition.host, transition.port, group, transition.role) def check_transitions(self, old_topology: PgDistGroup, new_topology: PgDistGroup, expected_transitions: List[str]) -> None: check_topology = deepcopy(old_topology) transitions: List[str] = [] for node in new_topology.transition(old_topology): self.assertTrue(node not in check_topology or (check_topology.get(node) or node).role == 'demoted') old_node = node.nodeid and next(iter(v for v in check_topology if v.nodeid == node.nodeid), None) if old_node: check_topology.discard(old_node) transitions.append(self.map_to_sql(new_topology.groupid, node)) check_topology.add(node) self.assertEqual(transitions, expected_transitions) def test_new_topology(self): old = PgDistGroup(0) new = PgDistGroup(0, {PgDistNode('1', 5432, 'primary'), PgDistNode('2', 5432, 'secondary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=100), PgDistNode('2', 5432, 'secondary', nodeid=101)}) self.check_transitions(old, new, ["citus_add_node('1', 5432, 0, 'primary')", "citus_add_node('2', 5432, 0, 'secondary')"]) self.assertTrue(new.equals(expected, True)) def test_switchover(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'), PgDistNode('2', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1), PgDistNode('1', 5432, 'secondary', nodeid=2)}) self.check_transitions(old, new, ["citus_update_node(1, '1-demoted', 5432)", "citus_update_node(2, '1', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_failover(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1), PgDistNode('1', 5432, 'secondary', nodeid=2)}) self.check_transitions(old, new, ["citus_update_node(1, '1-demoted', 5432)", "citus_update_node(2, '1', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_failover_and_new_secondary(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary'), PgDistNode('3', 5432, 'secondary')}) expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1), PgDistNode('3', 5432, 'secondary', nodeid=2)}) # the secondary record is used to add the new standby and primary record is updated with the new hostname self.check_transitions(old, new, ["citus_update_node(2, '3', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_and_new_secondary_primary_gone(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary'), PgDistNode('3', 5432, 'secondary')}) expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1), PgDistNode('3', 5432, 'secondary', nodeid=2)}) # the secondary record is used to add the new standby and primary record is updated with the new hostname self.check_transitions(old, new, ["citus_update_node(2, '3', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_secondary_replaced(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'primary'), PgDistNode('3', 5432, 'secondary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('3', 5432, 'secondary', nodeid=2)}) self.check_transitions(old, new, ["citus_update_node(2, '3', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_secondary_repmoved(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2), PgDistNode('3', 5432, 'secondary', nodeid=3)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'primary'), PgDistNode('3', 5432, 'secondary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('3', 5432, 'secondary', nodeid=3)}) self.check_transitions(old, new, ["citus_remove_node('2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_and_secondary_removed(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2), PgDistNode('3', 5432, 'secondary', nodeid=3)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'), PgDistNode('2', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2), PgDistNode('2', 5432, 'primary', nodeid=1), PgDistNode('3', 5432, 'secondary', nodeid=3)}) self.check_transitions(old, new, ["citus_update_node(1, '1-demoted', 5432)", "citus_update_node(2, '1', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_and_new_secondary(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'), PgDistNode('2', 5432, 'primary'), PgDistNode('3', 5432, 'secondary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2), PgDistNode('2', 5432, 'primary', nodeid=1)}) self.check_transitions(old, new, ["citus_update_node(1, '1-demoted', 5432)", "citus_update_node(2, '1', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_failover_to_new_node_secondary_remains(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('2', 5432, 'secondary'), PgDistNode('3', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('3', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_failover_to_new_node_secondary_removed(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('3', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('3', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) # the secondary record needs to be removed before we update the primary record self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_to_new_node_and_secondary_removed(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'), PgDistNode('3', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('3', 5432, 'primary', nodeid=1), PgDistNode('1', 5432, 'secondary', nodeid=2)}) self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)", "citus_update_node(2, '1', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_with_pause(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) self.check_transitions(old, new, ["citus_update_node(1, '1-demoted', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_after_paused_connections(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2), PgDistNode('2', 5432, 'primary', nodeid=1)}) self.check_transitions(old, new, ["citus_update_node(2, '1', 5432)", "citus_update_node(1, '2', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_to_new_node_after_paused_connections(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('3', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2), PgDistNode('3', 5432, 'primary', nodeid=1)}) self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)", "citus_update_node(2, '1', 5432)"]) self.assertTrue(new.equals(expected, True)) def test_switchover_to_new_node_after_paused_connections_secondary_added(self): old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1), PgDistNode('2', 5432, 'secondary', nodeid=2)}) new = PgDistGroup(0, {PgDistNode('4', 5432, 'secondary'), PgDistNode('3', 5432, 'primary')}) expected = PgDistGroup(0, {PgDistNode('4', 5432, 'secondary', nodeid=2), PgDistNode('3', 5432, 'primary', nodeid=1)}) self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)", "citus_update_node(2, '4', 5432)"]) self.assertTrue(new.equals(expected, True)) patroni-4.0.4/tests/test_config.py000066400000000000000000000305001472010352700172120ustar00rootroot00000000000000import io import os import sys import unittest from copy import deepcopy from unittest.mock import MagicMock, Mock, patch from patroni import global_config from patroni.config import ClusterConfig, Config, ConfigParseError from .test_ha import get_cluster_initialized_with_only_leader class TestConfig(unittest.TestCase): @patch('os.path.isfile', Mock(return_value=True)) @patch('json.load', Mock(side_effect=Exception)) @patch('builtins.open', MagicMock()) def setUp(self): sys.argv = ['patroni.py'] os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}' self.config = Config(None) def test_set_dynamic_configuration(self): with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)): self.assertFalse(self.config.set_dynamic_configuration({'foo': 'bar'})) self.assertTrue(self.config.set_dynamic_configuration({'standby_cluster': {}, 'postgresql': { 'parameters': {'cluster_name': 1, 'hot_standby': 1, 'wal_keep_size': 1, 'track_commit_timestamp': 1, 'wal_level': 1, 'max_connections': '100'}}})) def test_reload_local_configuration(self): os.environ.update({ 'PATRONI_NAME': 'postgres0', 'PATRONI_NAMESPACE': '/patroni/', 'PATRONI_SCOPE': 'batman2', 'PATRONI_LOGLEVEL': 'ERROR', 'PATRONI_LOG_FORMAT': '["message", {"levelname": "level"}]', 'PATRONI_LOG_LOGGERS': 'patroni.postmaster: WARNING, urllib3: DEBUG', 'PATRONI_LOG_FILE_NUM': '5', 'PATRONI_LOG_MODE': '0123', 'PATRONI_CITUS_DATABASE': 'citus', 'PATRONI_CITUS_GROUP': '0', 'PATRONI_CITUS_HOST': '0', 'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password', 'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008', 'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008', 'PATRONI_RESTAPI_CERTFILE': '/certfile', 'PATRONI_RESTAPI_KEYFILE': '/keyfile', 'PATRONI_RESTAPI_ALLOWLIST_INCLUDE_MEMBERS': 'on', 'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432', 'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432', 'PATRONI_POSTGRESQL_PROXY_ADDRESS': '127.0.0.1:5433', 'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0', 'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0', 'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0', 'PATRONI_ETCD_HOST': '127.0.0.1:2379', 'PATRONI_ETCD_URL': 'https://127.0.0.1:2379', 'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379', 'PATRONI_ETCD_SRV': 'test', 'PATRONI_ETCD_CACERT': '/cacert', 'PATRONI_ETCD_CERT': '/cert', 'PATRONI_ETCD_KEY': '/key', 'PATRONI_CONSUL_HOST': '127.0.0.1:8500', 'PATRONI_CONSUL_REGISTER_SERVICE': 'on', 'PATRONI_KUBERNETES_LABELS': 'a: b: c', 'PATRONI_KUBERNETES_SCOPE_LABEL': 'a', 'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]', 'PATRONI_KUBERNETES_RETRIABLE_HTTP_CODES': '401', 'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'", 'PATRONI_EXHIBITOR_HOSTS': 'host1,host2', 'PATRONI_EXHIBITOR_PORT': '8181', 'PATRONI_RAFT_PARTNER_ADDRS': "'host1:1234','host2:1234'", 'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list 'PATRONI_SUPERUSER_USERNAME': 'postgres', 'PATRONI_SUPERUSER_PASSWORD': 'patroni', 'PATRONI_REPLICATION_USERNAME': 'replicator', 'PATRONI_REPLICATION_PASSWORD': 'rep-pass', 'PATRONI_admin_PASSWORD': 'admin', 'PATRONI_admin_OPTIONS': 'createrole,createdb', 'PATRONI_POSTGRESQL_BIN_POSTGRES': 'sergtsop' }) config = Config('postgres0.yml') self.assertEqual(config.local_configuration['log']['mode'], 0o123) with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})): with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)): config.reload_local_configuration() self.assertTrue(config.reload_local_configuration()) self.assertIsNone(config.reload_local_configuration()) @patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla'])) @patch('os.path.exists', Mock(return_value=True)) @patch('os.remove', Mock(side_effect=IOError)) @patch('os.close', Mock(side_effect=IOError)) @patch('os.chmod', Mock()) @patch('shutil.move', Mock(return_value=None)) @patch('json.dump', Mock()) def test_save_cache(self): self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}}) with patch('os.fdopen', Mock(side_effect=IOError)): self.config.save_cache() with patch('os.fdopen', MagicMock()): self.config.save_cache() def test_standby_cluster_parameters(self): dynamic_configuration = { 'standby_cluster': { 'create_replica_methods': ['wal_e', 'basebackup'], 'host': 'localhost', 'port': 5432 } } self.config.set_dynamic_configuration(dynamic_configuration) for name, value in dynamic_configuration['standby_cluster'].items(): self.assertEqual(self.config['standby_cluster'][name], value) @patch('os.path.exists', Mock(return_value=True)) @patch('os.path.isfile', Mock(side_effect=lambda fname: fname != 'postgres0')) @patch('os.path.isdir', Mock(return_value=True)) @patch('os.listdir', Mock(return_value=['01-specific.yml', '00-base.yml'])) def test_configuration_directory(self): def open_mock(fname, *args, **kwargs): if fname.endswith('00-base.yml'): return io.StringIO( u''' test: True test2: child-1: somestring child-2: 5 child-3: False test3: True test4: - abc: 3 - abc: 4 ''') elif fname.endswith('01-specific.yml'): return io.StringIO( u''' test: False test2: child-2: 10 child-3: !!null test4: - ab: 5 new-attr: True ''') with patch('builtins.open', MagicMock(side_effect=open_mock)): config = Config('postgres0') self.assertEqual(config._local_configuration, {'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10}, 'test3': True, 'test4': [{'ab': 5}], 'new-attr': True}) @patch('os.path.exists', Mock(return_value=True)) @patch('os.path.isfile', Mock(return_value=False)) @patch('os.path.isdir', Mock(return_value=False)) def test_invalid_path(self): self.assertRaises(ConfigParseError, Config, 'postgres0') @patch.object(Config, 'get') @patch('patroni.config.logger') def test__validate_failover_tags(self, mock_logger, mock_get): """Ensures that only one of `nofailover` or `failover_priority` can be provided""" # Providing one of `nofailover` or `failover_priority` is fine for single_param in ({"nofailover": True}, {"failover_priority": 1}, {"failover_priority": 0}): mock_get.side_effect = [single_param] * 2 self.assertIsNone(self.config._validate_failover_tags()) mock_logger.warning.assert_not_called() # Providing both `nofailover` and `failover_priority` is fine if consistent for consistent_state in ( {"nofailover": False, "failover_priority": 1}, {"nofailover": True, "failover_priority": 0}, {"nofailover": "False", "failover_priority": 0} ): mock_get.side_effect = [consistent_state] * 2 self.assertIsNone(self.config._validate_failover_tags()) mock_logger.warning.assert_not_called() # Providing both inconsistently should log a warning for inconsistent_state in ( {"nofailover": False, "failover_priority": 0}, {"nofailover": True, "failover_priority": 1}, {"nofailover": "False", "failover_priority": 1}, {"nofailover": "", "failover_priority": 0} ): mock_get.side_effect = [inconsistent_state] * 2 self.assertIsNone(self.config._validate_failover_tags()) mock_logger.warning.assert_called_once_with( 'Conflicting configuration between nofailover: %s and failover_priority: %s.' + ' Defaulting to nofailover: %s', inconsistent_state['nofailover'], inconsistent_state['failover_priority'], inconsistent_state['nofailover']) mock_logger.warning.reset_mock() def test__process_postgresql_parameters(self): expected_params = { 'f.oo': 'bar', # not in ConfigHandler.CMDLINE_OPTIONS 'max_connections': 100, # IntValidator 'wal_keep_size': '128MB', # IntValidator 'wal_level': 'hot_standby', # EnumValidator } input_params = deepcopy(expected_params) input_params['max_connections'] = '100' self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params) expected_params['f.oo'] = input_params['f.oo'] = '100' self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params) input_params['wal_level'] = 'cold_standby' expected_params.pop('wal_level') self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params) input_params['max_connections'] = 10 expected_params.pop('max_connections') self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params) def test__validate_and_adjust_timeouts(self): with patch('patroni.config.logger.warning') as mock_logger: self.config._validate_and_adjust_timeouts({'ttl': 15}) self.assertEqual(mock_logger.call_args_list[0][0], ("%s=%d can't be smaller than %d, adjusting...", 'ttl', 15, 20)) with patch('patroni.config.logger.warning') as mock_logger: self.config._validate_and_adjust_timeouts({'loop_wait': 0}) self.assertEqual(mock_logger.call_args_list[0][0], ("%s=%d can't be smaller than %d, adjusting...", 'loop_wait', 0, 1)) with patch('patroni.config.logger.warning') as mock_logger: self.config._validate_and_adjust_timeouts({'retry_timeout': 1}) self.assertEqual(mock_logger.call_args_list[0][0], ("%s=%d can't be smaller than %d, adjusting...", 'retry_timeout', 1, 3)) with patch('patroni.config.logger.warning') as mock_logger: self.config._validate_and_adjust_timeouts({'ttl': 20, 'loop_wait': 11, 'retry_timeout': 5}) self.assertEqual(mock_logger.call_args_list[0][0], ('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d ' 'and retry_timeout=%d. Adjusting loop_wait from %d to %d', 20, 5, 11, 10)) with patch('patroni.config.logger.warning') as mock_logger: self.config._validate_and_adjust_timeouts({'ttl': 20, 'loop_wait': 10, 'retry_timeout': 10}) self.assertEqual(mock_logger.call_args_list[0][0], ('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. Adjusting' ' loop_wait from %d to %d and retry_timeout from %d to %d', 20, 10, 1, 10, 9)) def test_global_config_is_synchronous_mode(self): # we should ignore synchronous_mode setting in a standby cluster config = {'standby_cluster': {'host': 'some_host'}, 'synchronous_mode': True} cluster = get_cluster_initialized_with_only_leader(cluster_config=ClusterConfig(1, config, 1)) test_config = global_config.from_cluster(cluster) self.assertFalse(test_config.is_synchronous_mode) patroni-4.0.4/tests/test_config_generator.py000066400000000000000000000405261472010352700212710ustar00rootroot00000000000000import os import unittest from copy import deepcopy from unittest.mock import MagicMock, Mock, mock_open as _mock_open, patch, PropertyMock import psutil import yaml from patroni.__main__ import main as _main from patroni.config import Config from patroni.config_generator import AbstractConfigGenerator, get_address, NO_VALUE_MSG from patroni.log import PatroniLogger from patroni.utils import parse_bool, patch_config from . import MockConnect, MockCursor, psycopg_connect HOSTNAME = 'test_hostname' IP = '1.9.8.4' def mock_open(*args, **kwargs): ret = _mock_open(*args, **kwargs) ret.return_value.__iter__ = lambda o: iter(o.readline, '') if not kwargs.get('read_data'): ret.return_value.readline = Mock(return_value=None) return ret @patch('patroni.psycopg.connect', psycopg_connect) @patch('builtins.open', MagicMock()) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 17.0")) @patch('psutil.Process.exe', Mock(return_value='/bin/dir/from/running/postgres')) @patch('psutil.Process.__init__', Mock(return_value=None)) @patch('patroni.config_generator.get_address', Mock(return_value=(HOSTNAME, IP))) class TestGenerateConfig(unittest.TestCase): def setUp(self): os.environ['PATRONI_SCOPE'] = 'scope_from_env' os.environ['PATRONI_POSTGRESQL_BIN_DIR'] = '/bin/from/env' os.environ['PATRONI_SUPERUSER_USERNAME'] = 'su_user_from_env' os.environ['PATRONI_SUPERUSER_PASSWORD'] = 'su_pwd_from_env' os.environ['PATRONI_REPLICATION_USERNAME'] = 'repl_user_from_env' os.environ['PATRONI_REPLICATION_PASSWORD'] = 'repl_pwd_from_env' os.environ['PATRONI_REWIND_USERNAME'] = 'rewind_user_from_env' os.environ['PGUSER'] = 'pguser_from_env' os.environ['PGPASSWORD'] = 'pguser_pwd_from_env' os.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'] = 'localhost:8080' os.environ['PATRONI_RESTAPI_LISTEN'] = 'localhost:8080' os.environ['PATRONI_POSTGRESQL_BIN_POSTGRES'] = 'custom_postgres_bin_from_env' self.environ = deepcopy(os.environ) dynamic_config = Config.get_default_config() dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters']) del dynamic_config['standby_cluster'] dynamic_config['postgresql']['parameters']['wal_keep_segments'] = 8 dynamic_config['postgresql']['use_pg_rewind'] = \ parse_bool(dynamic_config['postgresql']['parameters']['wal_log_hints']) is True self.config = { 'scope': self.environ['PATRONI_SCOPE'], 'name': HOSTNAME, 'log': { 'type': PatroniLogger.DEFAULT_TYPE, 'format': PatroniLogger.DEFAULT_FORMAT, 'level': PatroniLogger.DEFAULT_LEVEL, 'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL, 'max_queue_size': PatroniLogger.DEFAULT_MAX_QUEUE_SIZE }, 'restapi': { 'connect_address': self.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'], 'listen': self.environ['PATRONI_RESTAPI_LISTEN'] }, 'bootstrap': { 'dcs': dynamic_config }, 'postgresql': { 'connect_address': IP + ':5432', 'data_dir': NO_VALUE_MSG, 'listen': IP + ':5432', 'pg_hba': ['host all all all md5', f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all md5'], 'authentication': {'superuser': {'username': self.environ['PATRONI_SUPERUSER_USERNAME'], 'password': self.environ['PATRONI_SUPERUSER_PASSWORD']}, 'replication': {'username': self.environ['PATRONI_REPLICATION_USERNAME'], 'password': self.environ['PATRONI_REPLICATION_PASSWORD']}, 'rewind': {'username': self.environ['PATRONI_REWIND_USERNAME']}}, 'bin_dir': self.environ['PATRONI_POSTGRESQL_BIN_DIR'], 'bin_name': {'postgres': self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']}, 'parameters': {'password_encryption': 'md5'} } } def _set_running_instance_config_vals(self): # values are taken from tests/__init__.py conf = { 'scope': 'my_cluster', 'bootstrap': { 'dcs': { 'postgresql': { 'parameters': { 'max_connections': 42, 'max_locks_per_transaction': 73, 'max_replication_slots': 21, 'max_wal_senders': 37, 'wal_level': 'replica', 'wal_keep_segments': None }, 'use_pg_rewind': None } } }, 'postgresql': { 'connect_address': f'{IP}:bar', 'listen': '6.6.6.6:1984', 'data_dir': 'data', 'bin_dir': '/bin/dir/from/running', 'parameters': { 'archive_command': 'my archive command', 'hba_file': os.path.join('data', 'pg_hba.conf'), 'ident_file': os.path.join('data', 'pg_ident.conf'), 'password_encryption': None }, 'authentication': { 'superuser': { 'username': 'foobar', 'password': 'qwerty', 'channel_binding': 'prefer', 'gssencmode': 'prefer', 'sslmode': 'prefer', 'sslnegotiation': 'postgres' }, 'replication': { 'username': NO_VALUE_MSG, 'password': NO_VALUE_MSG }, 'rewind': None }, }, 'tags': { 'failover_priority': 1, 'noloadbalance': False, 'clonefrom': True, 'nosync': False, 'nostream': False } } patch_config(self.config, conf) def _get_running_instance_open_res(self): hba_content = '\n'.join(self.config['postgresql']['pg_hba'] + ['#host all all all md5', ' host all all all md5', '', 'hostall all all md5']) ident_content = '\n'.join(['# something very interesting', ' ']) self.config['postgresql']['pg_hba'] += ['host all all all md5'] return [ mock_open(read_data=hba_content)(), mock_open(read_data=ident_content)(), mock_open(read_data='1984')(), mock_open()() ] @patch('os.makedirs') def test_generate_sample_config_pre_13_dir_creation(self, mock_makedir): with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \ patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 9.4.3")) as pg_bin_mock, \ patch('builtins.open', _mock_open()) as mocked_file, \ self.assertRaises(SystemExit) as e: _main() self.assertEqual(self.config, yaml.safe_load(mocked_file().write.call_args_list[0][0][0])) self.assertEqual(e.exception.code, 0) mock_makedir.assert_called_once() pg_bin_mock.assert_called_once_with([os.path.join(self.environ['PATRONI_POSTGRESQL_BIN_DIR'], self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']), '--version']) @patch('os.makedirs', Mock()) def test_generate_sample_config_17(self): conf = { 'bootstrap': { 'dcs': { 'postgresql': { 'parameters': { 'wal_keep_size': '128MB', 'wal_keep_segments': None }, } } }, 'postgresql': { 'parameters': { 'password_encryption': 'scram-sha-256' }, 'pg_hba': ['host all all all scram-sha-256', f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all scram-sha-256'], 'authentication': { 'rewind': { 'username': self.environ['PATRONI_REWIND_USERNAME'], 'password': NO_VALUE_MSG} }, } } patch_config(self.config, conf) with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \ patch('builtins.open', _mock_open()) as mocked_file, \ self.assertRaises(SystemExit) as e: _main() self.assertEqual(self.config, yaml.safe_load(mocked_file().write.call_args_list[0][0][0])) self.assertEqual(e.exception.code, 0) @patch('os.makedirs', Mock()) @patch('sys.stdout') def test_generate_config_running_instance_17(self, mock_sys_stdout): self._set_running_instance_config_vals() with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \ patch('sys.argv', ['patroni.py', '--generate-config', '--dsn', 'host=foo port=bar user=foobar password=qwerty']), \ self.assertRaises(SystemExit) as e: _main() self.assertEqual(e.exception.code, 0) self.assertEqual(self.config, yaml.safe_load(mock_sys_stdout.write.call_args_list[0][0][0])) @patch('os.makedirs', Mock()) @patch('sys.stdout') def test_generate_config_running_instance_17_connect_from_env(self, mock_sys_stdout): self._set_running_instance_config_vals() # su auth params and connect host from env os.environ['PGCHANNELBINDING'] = \ self.config['postgresql']['authentication']['superuser']['channel_binding'] = 'disable' os.environ['PGSSLNEGOTIATION'] = \ self.config['postgresql']['authentication']['superuser']['sslnegotiation'] = 'direct' conf = { 'scope': 'my_cluster', 'bootstrap': { 'dcs': { 'postgresql': { 'parameters': { 'max_connections': 42, 'max_locks_per_transaction': 73, 'max_replication_slots': 21, 'max_wal_senders': 37, 'wal_level': 'replica', 'wal_keep_segments': None }, 'use_pg_rewind': None } } }, 'postgresql': { 'connect_address': f'{IP}:1984', 'authentication': { 'superuser': { 'username': self.environ['PGUSER'], 'password': self.environ['PGPASSWORD'], 'gssencmode': None, 'sslmode': None }, }, } } patch_config(self.config, conf) with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \ patch('sys.argv', ['patroni.py', '--generate-config']), \ patch.object(MockConnect, 'server_version', PropertyMock(return_value=170000)), \ self.assertRaises(SystemExit) as e: _main() self.assertEqual(e.exception.code, 0) self.assertEqual(self.config, yaml.safe_load(mock_sys_stdout.write.call_args_list[0][0][0])) def test_generate_config_running_instance_errors(self): # 1. Wrong DSN format with patch('sys.argv', ['patroni.py', '--generate-config', '--dsn', 'host:foo port:bar user:foobar']), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Failed to parse DSN string', e.exception.code) # 2. User is not a superuser with patch('sys.argv', ['patroni.py', '--generate-config', '--dsn', 'host=foo port=bar user=foobar password=pwd_from_dsn']), \ patch.object(MockCursor, 'rowcount', PropertyMock(return_value=0), create=True), \ patch.object(MockConnect, 'get_parameter_status', Mock(return_value='off')), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('The provided user does not have superuser privilege', e.exception.code) # 3. Error while calling postgres --version with patch('subprocess.check_output', Mock(side_effect=OSError)), \ patch('sys.argv', ['patroni.py', '--generate-sample-config']), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Failed to get postgres version:', e.exception.code) with patch('sys.argv', ['patroni.py', '--generate-config']): # 4. empty postmaster.pid with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), mock_open(read_data='ident_content')(), mock_open(read_data='')()])), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Failed to obtain postmaster pid from postmaster.pid file', e.exception.code) # 5. Failed to open postmaster.pid with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), mock_open(read_data='ident_content')(), OSError])), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Error while reading postmaster.pid file', e.exception.code) # 6. Invalid postmaster pid with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), mock_open(read_data='ident_content')(), mock_open(read_data='1984')()])), \ patch('psutil.Process.__init__', Mock(return_value=None)), \ patch('psutil.Process.exe', Mock(side_effect=psutil.NoSuchProcess(1984))), \ self.assertRaises(SystemExit) as e: _main() self.assertIn("Obtained postmaster pid doesn't exist", e.exception.code) # 7. Failed to open pg_hba with patch('builtins.open', Mock(side_effect=OSError)), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Failed to read pg_hba.conf', e.exception.code) # 8. Failed to open pg_ident with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), OSError])), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Failed to read pg_ident.conf', e.exception.code) # 9. Failed PG connection from . import psycopg with patch('patroni.psycopg.connect', side_effect=psycopg.Error), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Failed to establish PostgreSQL connection', e.exception.code) # 10. An unexpected error with patch.object(AbstractConfigGenerator, '__init__', side_effect=psycopg.Error), \ self.assertRaises(SystemExit) as e: _main() self.assertIn('Unexpected exception', e.exception.code) def test_get_address(self): with patch('socket.getaddrinfo', Mock(side_effect=Exception)), \ patch('logging.warning') as mock_warning: self.assertEqual(get_address(), (NO_VALUE_MSG, NO_VALUE_MSG)) self.assertIn('Failed to obtain address: %r', mock_warning.call_args_list[0][0]) patroni-4.0.4/tests/test_consul.py000066400000000000000000000375711472010352700172670ustar00rootroot00000000000000import unittest from unittest.mock import Mock, patch, PropertyMock import consul from consul import ConsulException, NotFound from patroni.dcs import get_dcs from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulClient, ConsulError, \ ConsulInternalError, HTTPClient, InvalidSession, InvalidSessionTTL, RetryFailedError from patroni.postgresql.mpp import get_mpp from . import SleepException def kv_get(self, key, **kwargs): if key == 'service/test/members/postgresql1': return '1', {'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7'} if key == 'service/test/': return None, None if key == 'service/good/leader': return '1', None if key == 'service/good/sync': return '1', {'ModifyIndex': 1, 'Value': b'{}'} good_cls = ('6429', [{'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'failover', 'LockIndex': 0, 'ModifyIndex': 1334, 'Value': b''}, {'CreateIndex': 1334, 'Flags': 0, 'Key': key + '1/initialize', 'LockIndex': 0, 'ModifyIndex': 1334, 'Value': b'postgresql0'}, {'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'initialize', 'LockIndex': 0, 'ModifyIndex': 1334, 'Value': b'postgresql0'}, {'CreateIndex': 2621, 'Flags': 0, 'Key': key + 'leader', 'LockIndex': 1, 'ModifyIndex': 2621, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7', 'Value': b'postgresql1'}, {'CreateIndex': 6156, 'Flags': 0, 'Key': key + 'members/postgresql0', 'LockIndex': 1, 'ModifyIndex': 6156, 'Session': '782e6da4-ed02-3aef-7963-99a90ed94b53', 'Value': ('postgres://replicator:rep-pass@127.0.0.1:5432/postgres' + '?application_name=http://127.0.0.1:8008/patroni').encode('utf-8')}, {'CreateIndex': 2630, 'Flags': 0, 'Key': key + 'members/postgresql1', 'LockIndex': 1, 'ModifyIndex': 2630, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7', 'Value': ('postgres://replicator:rep-pass@127.0.0.1:5433/postgres' + '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')}, {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0, 'ModifyIndex': 6429, 'Value': b'4496294792'}, {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0, 'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'}, {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'failsafe', 'LockIndex': 0, 'ModifyIndex': 6429, 'Value': b'{'}, {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'status', 'LockIndex': 0, 'ModifyIndex': 6429, 'Value': b'{"optime":4496294792,"slots":{"ls":12345},"retain_slots":["postgresql0","postgresql1"]}'}]) if key == 'service/good/': return good_cls if key == 'service/broken/': good_cls[1][-1]['Value'] = b'{' return good_cls if key == 'service/legacy/': good_cls[1].pop() return good_cls raise ConsulException class TestHTTPClient(unittest.TestCase): def setUp(self): c = ConsulClient() self.client = c.http self.client.http.request = Mock() def test_get(self): self.client.get(Mock(), '') self.client.get(Mock(), '', {'wait': '1s', 'index': 1, 'token': 'foo'}) self.client.http.request.return_value.status = 500 self.client.http.request.return_value.data = b'Foo' self.assertRaises(ConsulInternalError, self.client.get, Mock(), '') self.client.http.request.return_value.data = b"Invalid Session TTL '3000000000', must be between [10s=24h0m0s]" self.assertRaises(InvalidSessionTTL, self.client.get, Mock(), '') self.client.http.request.return_value.data = b"invalid session '16492f43-c2d6-5307-432f-e32d6f7bcbd0'" self.assertRaises(InvalidSession, self.client.get, Mock(), '') def test_unknown_method(self): try: self.client.bla(Mock(), '') self.assertFail() except Exception as e: self.assertTrue(isinstance(e, AttributeError)) def test_put(self): self.client.put(Mock(), '/v1/session/create') self.client.put(Mock(), '/v1/session/create', params=[], data='{"foo": "bar"}') KV = consul.Consul.KV if hasattr(consul.Consul, 'KV') else consul.api.kv.KV Session = consul.Consul.Session if hasattr(consul.Consul, 'Session') else consul.api.session.Session Agent = consul.Consul.Agent if hasattr(consul.Consul, 'Agent') else consul.api.agent.Agent @patch.object(KV, 'get', kv_get) class TestConsul(unittest.TestCase): @patch.object(Session, 'create', Mock(return_value='fd4f44fe-2cac-bba5-a60b-304b51ff39b7')) @patch.object(Session, 'renew', Mock(side_effect=NotFound)) @patch.object(KV, 'get', kv_get) @patch.object(KV, 'delete', Mock()) def setUp(self): self.assertIsInstance(get_dcs({'ttl': 30, 'scope': 't', 'name': 'p', 'retry_timeout': 10, 'consul': {'url': 'https://l:1', 'verify': 'on', 'key': 'foo', 'cert': 'bar', 'cacert': 'buz', 'token': 'asd', 'dc': 'dc1', 'register_service': True}}), Consul) self.assertIsInstance(get_dcs({'ttl': 30, 'scope': 't_', 'name': 'p', 'retry_timeout': 10, 'consul': {'url': 'https://l:1', 'verify': 'on', 'cert': 'bar', 'cacert': 'buz', 'register_service': True}}), Consul) self.c = get_dcs({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'retry_timeout': 10, 'consul': {'host': 'localhost:1', 'register_service': True, 'service_check_tls_server_name': True}}) self.assertIsInstance(self.c, Consul) self.c._base_path = 'service/good' self.c.get_cluster() @patch('time.sleep', Mock(side_effect=SleepException)) @patch.object(Session, 'create', Mock(side_effect=ConsulException)) def test_create_session(self): self.c._session = None self.assertRaises(SleepException, self.c.create_session) @patch.object(Session, 'renew', Mock(side_effect=NotFound)) @patch.object(Session, 'create', Mock(side_effect=[InvalidSessionTTL, ConsulException])) @patch.object(Agent, 'self', Mock(return_value={'Config': {'SessionTTLMin': 0}})) @patch.object(HTTPClient, 'set_ttl', Mock(side_effect=ValueError)) def test_referesh_session(self): self.c._session = '1' self.assertFalse(self.c.refresh_session()) self.c._last_session_refresh = 0 self.assertRaises(ConsulError, self.c.refresh_session) @patch.object(KV, 'delete', Mock()) def test_get_cluster(self): self.c._base_path = 'service/test' self.assertIsInstance(self.c.get_cluster(), Cluster) self.assertIsInstance(self.c.get_cluster(), Cluster) self.c._base_path = 'service/fail' self.assertRaises(ConsulError, self.c.get_cluster) self.c._base_path = 'service/broken' self.assertIsInstance(self.c.get_cluster(), Cluster) self.c._base_path = 'service/legacy' self.assertIsInstance(self.c.get_cluster(), Cluster) def test__get_citus_cluster(self): self.c._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) cluster = self.c.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsInstance(cluster.workers[1], Cluster) @patch.object(KV, 'delete', Mock(side_effect=[ConsulException, True, True, True])) @patch.object(KV, 'put', Mock(side_effect=[True, ConsulException, InvalidSession])) def test_touch_member(self): self.c.refresh_session = Mock(return_value=False) with patch.object(Consul, 'update_service', Mock(side_effect=Exception)): self.c.touch_member({'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5433/postgres', 'api_url': 'http://127.0.0.1:8009/patroni'}) self.c._register_service = True self.c.refresh_session = Mock(return_value=True) for _ in range(0, 4): self.c.touch_member({'balbla': 'blabla'}) self.c.refresh_session = Mock(side_effect=ConsulError('foo')) self.assertFalse(self.c.touch_member({'balbla': 'blabla'})) @patch.object(KV, 'put', Mock(side_effect=[InvalidSession, False, InvalidSession])) def test_take_leader(self): self.c.set_ttl(20) self.c._do_refresh_session = Mock() self.assertFalse(self.c.take_leader()) with patch('time.time', Mock(side_effect=[0, 0, 0, 100, 100, 100])): self.assertFalse(self.c.take_leader()) @patch.object(KV, 'put', Mock(return_value=True)) def test_set_failover_value(self): self.c.set_failover_value('') @patch.object(KV, 'put', Mock(return_value=True)) def test_set_config_value(self): self.c.set_config_value('') @patch.object(Cluster, 'min_version', PropertyMock(return_value=(2, 0))) @patch.object(KV, 'put', Mock(side_effect=ConsulException)) def test_write_leader_optime(self): self.c.get_cluster() self.c.write_leader_optime('1') @patch.object(Session, 'renew') @patch.object(KV, 'put', Mock(side_effect=ConsulException)) def test_update_leader(self, mock_renew): cluster = self.c.get_cluster() self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8' with patch.object(KV, 'delete', Mock(return_value=True)): with patch.object(KV, 'put', Mock(return_value=True)): self.assertTrue(self.c.update_leader(cluster, 12345, failsafe={'foo': 'bar'})) with patch.object(KV, 'put', Mock(side_effect=ConsulException)): self.assertFalse(self.c.update_leader(cluster, 12345)) with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 100, 200, 300])): self.assertRaises(ConsulError, self.c.update_leader, cluster, 12345) with patch('time.time', Mock(side_effect=[0, 100, 200, 300])): self.assertRaises(ConsulError, self.c.update_leader, cluster, 12345) with patch.object(KV, 'delete', Mock(side_effect=ConsulException)): self.assertFalse(self.c.update_leader(cluster, 12347)) mock_renew.side_effect = RetryFailedError('') self.c._last_session_refresh = 0 self.assertRaises(ConsulError, self.c.update_leader, cluster, 12346) mock_renew.side_effect = ConsulException self.assertFalse(self.c.update_leader(cluster, 12347)) @patch.object(KV, 'delete', Mock(return_value=True)) def test_delete_leader(self): leader = self.c.get_cluster().leader self.c.delete_leader(leader) self.c._name = 'other' self.c.delete_leader(leader) @patch.object(KV, 'put', Mock(return_value=True)) def test_initialize(self): self.c.initialize() @patch.object(KV, 'delete', Mock(return_value=True)) def test_cancel_initialization(self): self.c.cancel_initialization() @patch.object(KV, 'delete', Mock(return_value=True)) def test_delete_cluster(self): self.c.delete_cluster() @patch.object(AbstractDCS, 'watch', Mock()) def test_watch(self): self.c.watch(None, 1) self.c._name = '' self.c.watch(6429, 1) with patch.object(KV, 'get', Mock(side_effect=ConsulException)): self.c.watch(6429, 1) def test_set_retry_timeout(self): self.c.set_retry_timeout(10) @patch.object(KV, 'delete', Mock(return_value=True)) @patch.object(KV, 'put', Mock(return_value=True)) def test_sync_state(self): self.assertEqual(self.c.set_sync_state_value('{}'), 1) with patch('time.time', Mock(side_effect=[1, 100, 1000])): self.assertFalse(self.c.set_sync_state_value('{}')) with patch.object(KV, 'put', Mock(return_value=False)): self.assertFalse(self.c.set_sync_state_value('{}')) self.assertTrue(self.c.delete_sync_state()) @patch.object(KV, 'put', Mock(return_value=True)) def test_set_history_value(self): self.assertTrue(self.c.set_history_value('{}')) @patch.object(Agent.Service, 'register', Mock(side_effect=(False, True, True, True))) @patch.object(Agent.Service, 'deregister', Mock(return_value=True)) def test_update_service(self): d = {'role': 'replica', 'api_url': 'http://a/t', 'conn_url': 'pg://c:1', 'state': 'running'} self.assertIsNone(self.c.update_service({}, {})) self.assertFalse(self.c.update_service({}, d)) self.assertTrue(self.c.update_service(d, d)) self.assertIsNone(self.c.update_service(d, d)) d['state'] = 'stopped' self.assertTrue(self.c.update_service(d, d, force=True)) d['state'] = 'unknown' self.assertIsNone(self.c.update_service({}, d)) d['state'] = 'running' d['role'] = 'bla' self.assertIsNone(self.c.update_service({}, d)) d['role'] = 'primary' self.assertTrue(self.c.update_service({}, d)) @patch.object(KV, 'put', Mock(side_effect=ConsulException)) def test_reload_config(self): self.assertEqual([], self.c._service_tags) self.c.reload_config({'consul': {'token': 'foo', 'register_service': True, 'service_tags': ['foo']}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) self.assertEqual(["foo"], self.c._service_tags) self.c.refresh_session = Mock(return_value=False) d = {'role': 'replica', 'api_url': 'http://a/t', 'conn_url': 'pg://c:1', 'state': 'running'} # Changing register_service from True to False calls deregister() self.c.reload_config({'consul': {'register_service': False}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) with patch.object(Agent.Service, 'deregister') as mock_deregister: self.c.touch_member(d) mock_deregister.assert_called_once() self.assertEqual([], self.c._service_tags) # register_service staying False between reloads does not call deregister() self.c.reload_config({'consul': {'register_service': False}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) with patch.object(Agent.Service, 'deregister') as mock_deregister: self.c.touch_member(d) self.assertFalse(mock_deregister.called) # Changing register_service from False to True calls register() self.c.reload_config({'consul': {'register_service': True}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) with patch.object(Agent.Service, 'register') as mock_register: self.c.touch_member(d) mock_register.assert_called_once() # register_service staying True between reloads does not call register() self.c.reload_config({'consul': {'register_service': True}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) with patch.object(Agent.Service, 'register') as mock_register: self.c.touch_member(d) self.assertFalse(mock_deregister.called) # register_service staying True between reloads does calls register() if other service data has changed self.c.reload_config({'consul': {'register_service': True}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) with patch.object(Agent.Service, 'register') as mock_register: self.c.touch_member(d) mock_register.assert_called_once() # register_service staying True between reloads does calls register() if service_tags have changed self.c.reload_config({'consul': {'register_service': True, 'service_tags': ['foo']}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) with patch.object(Agent.Service, 'register') as mock_register: self.c.touch_member(d) mock_register.assert_called_once() patroni-4.0.4/tests/test_ctl.py000066400000000000000000001202551472010352700165360ustar00rootroot00000000000000import os import unittest from datetime import datetime, timedelta from unittest import mock from unittest.mock import Mock, patch, PropertyMock import click import etcd from click.testing import CliRunner from prettytable import PrettyTable try: from prettytable import HRuleStyle hrule_all = HRuleStyle.ALL except ImportError: from prettytable import ALL as hrule_all from urllib3 import PoolManager from patroni import global_config from patroni.ctl import apply_config_changes, CONFIG_FILE_PATH, ctl, format_config_for_editing, \ format_pg_version, get_all_members, get_any_member, get_cursor, get_dcs, invoke_editor, load_config, \ output_members, parse_dcs, PatroniCtlException, PatronictlPrettyTable, query_member, show_diff from patroni.dcs import Cluster, Failover from patroni.postgresql.config import get_param_diff from patroni.postgresql.mpp import get_mpp from patroni.psycopg import OperationalError from patroni.utils import tzutc from . import MockConnect, MockCursor, MockResponse, psycopg_connect from .test_etcd import etcd_read, socket_getaddrinfo from .test_ha import get_cluster, get_cluster_initialized_with_leader, get_cluster_initialized_with_only_leader, \ get_cluster_initialized_without_leader, get_cluster_not_initialized_without_leader, Member def get_default_config(*args): return { 'scope': 'alpha', 'restapi': {'listen': '::', 'certfile': 'a'}, 'ctl': {'certfile': 'a'}, 'etcd': {'host': 'localhost:2379', 'retry_timeout': 10, 'ttl': 30}, 'citus': {'database': 'citus', 'group': 0}, 'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5} } @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) @patch('patroni.ctl.load_config', get_default_config) @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) class TestCtl(unittest.TestCase): TEST_ROLES = ('primary', 'leader') @patch('socket.getaddrinfo', socket_getaddrinfo) def setUp(self): self.runner = CliRunner() @patch('patroni.ctl.logging.debug') def test_load_config(self, mock_logger_debug): runner = CliRunner() with runner.isolated_filesystem(): self.assertRaises(PatroniCtlException, load_config, './non-existing-config-file', None) with patch('os.path.exists', Mock(return_value=True)), \ patch('patroni.config.Config._load_config_path', Mock(return_value={})): load_config(CONFIG_FILE_PATH, None) mock_logger_debug.assert_called_once() self.assertEqual(('Ignoring configuration file "%s". It does not exists or is not readable.', CONFIG_FILE_PATH), mock_logger_debug.call_args[0]) mock_logger_debug.reset_mock() with patch('os.access', Mock(return_value=True)): load_config(CONFIG_FILE_PATH, '') mock_logger_debug.assert_called_once() self.assertEqual(('Loading configuration from file %s', CONFIG_FILE_PATH), mock_logger_debug.call_args[0]) mock_logger_debug.reset_mock() @patch('patroni.psycopg.connect', psycopg_connect) def test_get_cursor(self): with click.Context(click.Command('query')) as ctx: ctx.obj = {'__config': {}, '__mpp': get_mpp({})} for role in self.TEST_ROLES: self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), None, {}, role=role)) self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {}, role=role)) # MockCursor returns pg_is_in_recovery as false self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), None, {}, role='replica')) self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any')) # Mutually exclusive options with self.assertRaises(PatroniCtlException) as e: get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other', role='replica') self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options') # Invalid member provided self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='invalid')) # Valid member provided self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other')) def test_parse_dcs(self): assert parse_dcs(None) is None assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}} assert parse_dcs('') == {'etcd': {'host': 'localhost:2379'}} assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}} assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}} assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}} assert parse_dcs('consul://localhost') == {'consul': {'host': 'localhost:8500'}} assert parse_dcs('etcd3://random.com:2399') == {'etcd3': {'host': 'random.com:2399'}} self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test') def test_output_members(self): with click.Context(click.Command('list')) as ctx: ctx.obj = {'__config': {}, '__mpp': get_mpp({})} scheduled_at = datetime.now(tzutc) + timedelta(seconds=600) cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at)) del cluster.members[1].data['conn_url'] for fmt in ('pretty', 'json', 'yaml', 'topology'): self.assertIsNone(output_members(cluster, name='abc', fmt=fmt)) with patch('click.echo') as mock_echo: self.assertIsNone(output_members(cluster, name='abc', fmt='tsv')) self.assertEqual(mock_echo.call_args[0][0], 'abc\tother\t\tReplica\trunning\t\tunknown') @patch('patroni.dcs.AbstractDCS.set_failover_value', Mock()) def test_switchover(self): # Confirm result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertEqual(result.exit_code, 0) # Abort result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN') self.assertEqual(result.exit_code, 1) # Without a candidate with --force option result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) self.assertEqual(result.exit_code, 0) # Scheduled (confirm) result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n2300-01-01T12:23:00\ny') self.assertEqual(result.exit_code, 0) # Scheduled (abort) result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN') self.assertEqual(result.exit_code, 1) # Scheduled with --force option result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', '2015-01-01T12:00:00+01:00']) self.assertEqual(result.exit_code, 0) # Scheduled in pause mode with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', '2015-01-01T12:00:00']) self.assertEqual(result.exit_code, 1) self.assertIn("Can't schedule switchover in the paused state", result.output) # Target and source are equal result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny') self.assertEqual(result.exit_code, 1) self.assertIn("Candidate ['other']", result.output) self.assertIn('Member leader is already the leader of cluster dummy', result.output) # Candidate is not a member of the cluster result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny') self.assertEqual(result.exit_code, 1) self.assertIn('Member Reality does not exist in cluster dummy or is tagged as nofailover', result.output) # Invalid timestamp result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', 'invalid']) self.assertEqual(result.exit_code, 1) self.assertIn('Unable to parse scheduled timestamp', result.output) # Invalid timestamp result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', '2115-02-30T12:00:00+01:00']) self.assertEqual(result.exit_code, 1) self.assertIn('Unable to parse scheduled timestamp', result.output) # Specifying wrong leader result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy') self.assertEqual(result.exit_code, 1) self.assertIn('Member dummy is not the leader of cluster dummy', result.output) # Errors while sending Patroni REST API request with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n2300-01-01T12:23:00\ny') self.assertIn('falling back to DCS', result.output) with patch('patroni.ctl.request_patroni') as mock_api_request: mock_api_request.return_value.status = 500 result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertIn('Switchover failed', result.output) mock_api_request.return_value.status = 501 mock_api_request.return_value.data = b'Server does not support this operation' result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertIn('Switchover failed', result.output) # No members available with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertEqual(result.exit_code, 1) self.assertIn('No candidates found to switchover to', result.output) # No leader available with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertEqual(result.exit_code, 1) self.assertIn('This cluster has no leader', result.output) # Citus cluster, no group number specified result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n') self.assertEqual(result.exit_code, 1) self.assertIn('For Citus clusters the --group must me specified', result.output) @patch('patroni.dcs.AbstractDCS.set_failover_value', Mock()) def test_failover(self): # No candidate specified result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n') self.assertIn('Failover could be performed only to a specific candidate', result.output) # Candidate is the same as the leader result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0'], input='leader\n') self.assertIn("Candidate ['other']", result.output) self.assertIn('Member leader is already the leader of cluster dummy', result.output) cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'})) cluster.config.data['synchronous_mode'] = True with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)): # Failover to an async member in sync mode (confirm) result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny') self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output) self.assertEqual(result.exit_code, 0) # Failover to an async member in sync mode (abort) result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N') self.assertEqual(result.exit_code, 1) self.assertIn('Aborting failover', result.output) @patch('patroni.dynamic_loader.iter_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd'])) def test_get_dcs(self): with click.Context(click.Command('list')) as ctx: ctx.obj = {'__config': {'dummy2': {}}, '__mpp': get_mpp({})} self.assertRaises(PatroniCtlException, get_dcs, 'dummy2', 0) @patch('patroni.psycopg.connect', psycopg_connect) @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None))) @patch.object(etcd.Client, 'read', etcd_read) def test_query(self): # Mutually exclusive for role in self.TEST_ROLES: result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', role]) assert result.exit_code == 1 with self.runner.isolated_filesystem(): with open('dummy', 'w') as dummy_file: dummy_file.write('SELECT 1') # Mutually exclusive result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy']) assert result.exit_code == 1 result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--file', 'dummy']) assert result.exit_code == 0 os.remove('dummy') result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1']) assert 'mock column' in result.output # --command or --file is mandatory result = self.runner.invoke(ctl, ['query', 'alpha']) assert result.exit_code == 1 result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root', '--password', '--dbname', 'postgres'], input='ab\nab') assert 'mock column' in result.output def test_query_member(self): with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())): for role in self.TEST_ROLES: rows = query_member(None, None, None, None, role, 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('False' in str(rows)) with patch.object(MockCursor, 'execute', Mock(side_effect=OperationalError('bla'))): rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) with patch('patroni.ctl.get_cursor', Mock(return_value=None)): # No role nor member given -- generic message rows = query_member(None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('No connection is available' in str(rows)) # Member given -- message pointing to member rows = query_member(None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('No connection to member foo' in str(rows)) # Role given -- message pointing to role rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('No connection to role replica' in str(rows)) with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))): rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) def test_dsn(self): result = self.runner.invoke(ctl, ['dsn', 'alpha']) assert 'host=127.0.0.1 port=5435' in result.output # Mutually exclusive options for role in self.TEST_ROLES: result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', role, '--member', 'dummy']) assert result.exit_code == 1 # Non-existing member result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy']) assert result.exit_code == 1 @patch('patroni.ctl.request_patroni') def test_reload(self, mock_post): result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y') assert 'Failed: reload for member' in result.output mock_post.return_value.status = 200 result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y') assert 'No changes to apply on member' in result.output mock_post.return_value.status = 202 result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y') assert 'Reload request received for member' in result.output @patch('patroni.ctl.request_patroni') def test_restart_reinit(self, mock_post): mock_post.return_value.status = 503 result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n') assert 'Failed: restart for' in result.output assert result.exit_code == 0 result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y') assert result.exit_code == 1 # successful reinit result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny') assert result.exit_code == 0 # Aborted restart result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\nN') assert result.exit_code == 1 result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force']) assert result.exit_code == 0 # Aborted scheduled restart result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', '2019-10-01T14:30'], input='N') assert result.exit_code == 1 # Not a member result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='now\ny') assert result.exit_code == 1 # Wrong pg version result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='now\ny') assert 'Error: Invalid PostgreSQL version format' in result.output assert result.exit_code == 1 result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min']) assert result.exit_code == 0 # normal restart, the schedule is actually parsed, but not validated in patronictl result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30']) assert 'Failed: flush scheduled restart' in result.output with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30']) assert result.exit_code == 1 # force restart with restart already present result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30']) assert result.exit_code == 0 ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', '2300-10-01T14:30'] # normal restart, the schedule is actually parsed, but not validated in patronictl mock_post.return_value.status = 200 result = self.runner.invoke(ctl, ctl_args, input='y') assert result.exit_code == 0 # get restart with the non-200 return code # normal restart, the schedule is actually parsed, but not validated in patronictl mock_post.return_value.status = 204 result = self.runner.invoke(ctl, ctl_args, input='y') assert result.exit_code == 0 # get restart with the non-200 return code # normal restart, the schedule is actually parsed, but not validated in patronictl mock_post.return_value.status = 202 result = self.runner.invoke(ctl, ctl_args, input='y') assert 'Success: restart scheduled' in result.output assert result.exit_code == 0 # get restart with the non-200 return code # normal restart, the schedule is actually parsed, but not validated in patronictl mock_post.return_value.status = 409 result = self.runner.invoke(ctl, ctl_args, input='y') assert 'Failed: another restart is already' in result.output assert result.exit_code == 0 def test_remove(self): result = self.runner.invoke(ctl, ['remove', 'dummy'], input='\n') assert 'For Citus clusters the --group must me specified' in result.output result = self.runner.invoke(ctl, ['remove', 'alpha', '--group', '0'], input='alpha\nstandby') assert 'Please confirm' in result.output assert 'You are about to remove all' in result.output # Not typing an exact confirmation assert result.exit_code == 1 # leader specified does not match leader of cluster result = self.runner.invoke(ctl, ['remove', 'alpha', '--group', '0'], input='alpha\nYes I am aware\nstandby') assert result.exit_code == 1 # cluster specified on cmdline does not match verification prompt result = self.runner.invoke(ctl, ['remove', 'alpha', '--group', '0'], input='beta\nleader') assert result.exit_code == 1 result = self.runner.invoke(ctl, ['remove', 'alpha', '--group', '0'], input='alpha\nYes I am aware\nleader') assert result.exit_code == 0 def test_ctl(self): result = self.runner.invoke(ctl, ['--help']) assert 'Usage:' in result.output def test_get_any_member(self): with click.Context(click.Command('list')) as ctx: ctx.obj = {'__config': {}, '__mpp': get_mpp({})} for role in self.TEST_ROLES: self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), None, role=role)) m = get_any_member(get_cluster_initialized_with_leader(), None, role=role) self.assertEqual(m.name, 'leader') def test_get_all_members(self): with click.Context(click.Command('list')) as ctx: ctx.obj = {'__config': {}, '__mpp': get_mpp({})} for role in self.TEST_ROLES: self.assertEqual(list(get_all_members(get_cluster_initialized_without_leader(), None, role=role)), []) r = list(get_all_members(get_cluster_initialized_with_leader(), None, role=role)) self.assertEqual(len(r), 1) self.assertEqual(r[0].name, 'leader') r = list(get_all_members(get_cluster_initialized_with_leader(), None, role='replica')) self.assertEqual(len(r), 1) self.assertEqual(r[0].name, 'other') self.assertEqual(len(list(get_all_members(get_cluster_initialized_without_leader(), None, role='replica'))), 2) def test_members(self): result = self.runner.invoke(ctl, ['list']) assert '127.0.0.1' in result.output assert result.exit_code == 0 assert 'Citus cluster: alpha -' in result.output result = self.runner.invoke(ctl, ['list', '--group', '0']) assert 'Citus cluster: alpha (group: 0, 12345678901) -' in result.output config = get_default_config() del config['citus'] with patch('patroni.ctl.load_config', Mock(return_value=config)): result = self.runner.invoke(ctl, ['list']) assert 'Cluster: alpha (12345678901) -' in result.output with patch('patroni.ctl.load_config', Mock(return_value={})): self.runner.invoke(ctl, ['list']) cluster = get_cluster_initialized_with_leader() cluster.members[1].data['pending_restart'] = True cluster.members[1].data['pending_restart_reason'] = {'param': get_param_diff('', 'very l' + 'o' * 34 + 'ng')} with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)): for cmd in ('list', 'topology'): result = self.runner.invoke(ctl, [cmd, 'dummy']) self.assertIn('param: [hidden - too long]', result.output) result = self.runner.invoke(ctl, ['list', 'dummy', '-f', 'tsv']) self.assertIn('param: ->very l' + 'o' * 34 + 'ng', result.output) cluster.members[1].data['pending_restart_reason'] = {'param': get_param_diff('', 'new')} result = self.runner.invoke(ctl, ['list', 'dummy']) self.assertIn('param: ->new', result.output) def test_list_extended(self): result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp']) assert '2100' in result.output assert 'Scheduled restart' in result.output def test_list_standby_cluster(self): cluster = get_cluster_initialized_without_leader(leader=True, sync=('leader', 'other')) cluster.config.data.update(synchronous_mode=True, standby_cluster={'port': 5433}) with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)): result = self.runner.invoke(ctl, ['list']) self.assertEqual(result.exit_code, 0) self.assertNotIn('Sync Standby', result.output) def test_topology(self): cluster = get_cluster_initialized_with_leader() cluster.members.append(Member(0, 'cascade', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5437/postgres', 'api_url': 'http://127.0.0.1:8012/patroni', 'state': 'running', 'tags': {'replicatefrom': 'other'}})) cluster.members.append(Member(0, 'wrong_cascade', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5438/postgres', 'api_url': 'http://127.0.0.1:8013/patroni', 'state': 'running', 'tags': {'replicatefrom': 'nonexistinghost'}})) with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)): result = self.runner.invoke(ctl, ['topology', 'dummy']) assert '+\n| 0 | leader | 127.0.0.1:5435 | Leader |' in result.output assert '|\n| 0 | + other | 127.0.0.1:5436 | Replica |' in result.output assert '|\n| 0 | + cascade | 127.0.0.1:5437 | Replica |' in result.output assert '|\n| 0 | + wrong_cascade | 127.0.0.1:5438 | Replica |' in result.output with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())): result = self.runner.invoke(ctl, ['topology', 'dummy']) assert '+\n| 0 | + leader | 127.0.0.1:5435 | Replica |' in result.output assert '|\n| 0 | + other | 127.0.0.1:5436 | Replica |' in result.output @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) def test_flush_restart(self): for role in self.TEST_ROLES: result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', role], input='y') assert 'No scheduled restart' in result.output result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force']) assert 'Success: flush scheduled restart' in result.output with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(404))): result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force']) assert 'Failed: flush scheduled restart' in result.output def test_flush_switchover(self): with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())): result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) assert 'No pending scheduled switchover' in result.output scheduled_at = datetime.now(tzutc) + timedelta(seconds=600) with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader(Failover(1, 'a', 'b', scheduled_at)))): result = self.runner.invoke(ctl, ['-k', 'flush', 'dummy', 'switchover']) assert result.output.startswith('Success: ') with patch('patroni.ctl.request_patroni', side_effect=[MockResponse(409), Exception]), \ patch('patroni.dcs.AbstractDCS.manual_failover', Mock()): result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) assert 'Could not find any accessible member of cluster' in result.output @patch('patroni.ctl.polling_loop', Mock(return_value=[1])) def test_pause_cluster(self): with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(500))): result = self.runner.invoke(ctl, ['pause', 'dummy']) assert 'Failed' in result.output with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['pause', 'dummy']) assert 'Cluster is already paused' in result.output result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) assert "'pause' request sent" in result.output with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[get_cluster_initialized_with_leader(), get_cluster(None, None, [], None, None)])): self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[get_cluster_initialized_with_leader(), get_cluster(None, None, [Member(1, 'other', 28, {})], None, None)])): self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) @patch('patroni.ctl.request_patroni') @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) def test_resume_cluster(self, mock_post): mock_post.return_value.status = 200 with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=False)): result = self.runner.invoke(ctl, ['resume', 'dummy']) assert 'Cluster is not paused' in result.output with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['resume', 'dummy']) assert 'Success' in result.output mock_post.return_value.status = 500 result = self.runner.invoke(ctl, ['resume', 'dummy']) assert 'Failed' in result.output mock_post.side_effect = Exception result = self.runner.invoke(ctl, ['resume', 'dummy']) assert 'Can not find accessible cluster member' in result.output def test_apply_config_changes(self): config = {"postgresql": {"parameters": {"work_mem": "4MB"}, "use_pg_rewind": True}, "ttl": 30} before_editing = format_config_for_editing(config) # Spaces are allowed and stripped, numbers and booleans are interpreted after_editing, changed_config = apply_config_changes(before_editing, config, ["postgresql.parameters.work_mem = 5MB", "ttl=15", "postgresql.use_pg_rewind=off", 'a.b=c']) self.assertEqual(changed_config, {"a": {"b": "c"}, "postgresql": {"parameters": {"work_mem": "5MB"}, "use_pg_rewind": False}, "ttl": 15}) # postgresql.parameters namespace is flattened after_editing, changed_config = apply_config_changes(before_editing, config, ["postgresql.parameters.work_mem.sub = x"]) self.assertEqual(changed_config, {"postgresql": {"parameters": {"work_mem": "4MB", "work_mem.sub": "x"}, "use_pg_rewind": True}, "ttl": 30}) # Setting to null deletes after_editing, changed_config = apply_config_changes(before_editing, config, ["postgresql.parameters.work_mem=null"]) self.assertEqual(changed_config, {"postgresql": {"use_pg_rewind": True}, "ttl": 30}) after_editing, changed_config = apply_config_changes(before_editing, config, ["postgresql.use_pg_rewind=null", "postgresql.parameters.work_mem=null"]) self.assertEqual(changed_config, {"ttl": 30}) self.assertRaises(PatroniCtlException, apply_config_changes, before_editing, config, ['a']) @patch('sys.stdout.isatty', return_value=False) @patch('patroni.ctl.markup_to_pager') @patch('os.environ.get', return_value=None) @patch('shutil.which', return_value=None) def test_show_diff(self, mock_which, mock_env_get, mock_markup_to_pager, mock_isatty): # no TTY show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") mock_markup_to_pager.assert_not_called() # TTY but no PAGER nor executable mock_isatty.return_value = True with self.assertRaises(PatroniCtlException) as e: show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") self.assertEqual( str(e.exception), 'No pager could be found. Either set PAGER environment variable with ' 'your pager or install either "less" or "more" in the host.' ) mock_env_get.assert_called_once_with('PAGER') mock_which.assert_has_calls([ mock.call('less'), mock.call('more'), ]) mock_markup_to_pager.assert_not_called() # TTY with PAGER set but invalid mock_env_get.reset_mock() mock_env_get.return_value = 'random' mock_which.reset_mock() with self.assertRaises(PatroniCtlException) as e: show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") self.assertEqual( str(e.exception), 'No pager could be found. Either set PAGER environment variable with ' 'your pager or install either "less" or "more" in the host.' ) mock_env_get.assert_called_once_with('PAGER') mock_which.assert_has_calls([ mock.call('random'), mock.call('less'), mock.call('more'), ]) mock_markup_to_pager.assert_not_called() # TTY with valid executable mock_which.side_effect = [None, '/usr/bin/less', None] show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") mock_markup_to_pager.assert_called_once() # Test that unicode handling doesn't fail with an exception mock_which.side_effect = [None, '/usr/bin/less', None] show_diff(b"foo:\n bar: \xc3\xb6\xc3\xb6\n".decode('utf-8'), b"foo:\n bar: \xc3\xbc\xc3\xbc\n".decode('utf-8')) @patch('subprocess.Popen') @patch('os.environ.get', Mock(return_value='cat')) @patch('sys.stdout.isatty', Mock(return_value=True)) @patch('shutil.which', Mock(return_value='cat')) def test_show_diff_pager(self, mock_popen): show_diff("foo:\n bar: 1\n", "foo:\n bar: 2\n") self.assertEqual(mock_popen.return_value.stdin.write.call_count, 6) self.assertIn(b' bar: ', mock_popen.return_value.stdin.write.call_args_list[5][0][0]) self.assertIn(b' bar: ', mock_popen.return_value.stdin.write.call_args_list[4][0][0]) self.assertIn(b' foo:', mock_popen.return_value.stdin.write.call_args_list[3][0][0]) @patch('subprocess.call', return_value=1) def test_invoke_editor(self, mock_subprocess_call): os.environ.pop('EDITOR', None) for e in ('', '/bin/vi'): with patch('shutil.which', Mock(return_value=e)): self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test') def test_show_config(self): self.runner.invoke(ctl, ['show-config', 'dummy']) @patch('subprocess.call', Mock(return_value=0)) def test_edit_config(self): os.environ['EDITOR'] = 'true' self.runner.invoke(ctl, ['edit-config', 'dummy']) self.runner.invoke(ctl, ['edit-config', 'dummy', '-s', 'foo=bar']) self.runner.invoke(ctl, ['edit-config', 'dummy', '--replace', 'postgres0.yml']) self.runner.invoke(ctl, ['edit-config', 'dummy', '--apply', '-'], input='foo: bar') self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar') with patch('patroni.dcs.etcd.Etcd.set_config_value', Mock(return_value=True)): self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar') with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=Cluster.empty())): result = self.runner.invoke(ctl, ['edit-config', 'dummy']) assert result.exit_code == 1 assert 'The config key does not exist in the cluster dummy' in result.output @patch('patroni.ctl.request_patroni') def test_version(self, mock_request): result = self.runner.invoke(ctl, ['version']) assert 'patronictl version' in result.output mock_request.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}' result = self.runner.invoke(ctl, ['version', 'dummy']) assert '1.2.3' in result.output mock_request.side_effect = Exception result = self.runner.invoke(ctl, ['version', 'dummy']) assert 'failed to get version' in result.output def test_history(self): with patch('patroni.dcs.AbstractDCS.get_cluster') as mock_get_cluster: mock_get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']] result = self.runner.invoke(ctl, ['history']) assert 'Reason' in result.output def test_format_pg_version(self): self.assertEqual(format_pg_version(100001), '10.1') self.assertEqual(format_pg_version(90605), '9.6.5') def test_get_members(self): with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_not_initialized_without_leader())): result = self.runner.invoke(ctl, ['reinit', 'dummy']) assert "cluster doesn\'t have any members" in result.output @patch('time.sleep', Mock()) def test_reinit_wait(self): with patch.object(PoolManager, 'request') as mocked: mocked.side_effect = [Mock(data=s, status=200) for s in [b"reinitialize", b'{"state":"creating replica"}', b'{"state":"running"}']] result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other', '--wait'], input='y\ny') self.assertIn("Waiting for reinitialize to complete on: other", result.output) self.assertIn("Reinitialize is completed on: other", result.output) class TestPatronictlPrettyTable(unittest.TestCase): def setUp(self): self.pt = PatronictlPrettyTable(' header', ['foo', 'bar'], hrules=hrule_all) def test__get_hline(self): expected = '+-----+-----+' self.pt._hrule = expected self.assertEqual(self.pt._hrule, '+ header----+') self.assertFalse(self.pt._is_first_hline()) self.assertEqual(self.pt._hrule, expected) @patch.object(PrettyTable, '_stringify_hrule', Mock(return_value='+-----+-----+')) def test__stringify_hrule(self): self.assertEqual(self.pt._stringify_hrule((), 'top_'), '+ header----+') self.assertFalse(self.pt._is_first_hline()) def test_output(self): self.assertEqual(str(self.pt), '+ header----+\n| foo | bar |\n+-----+-----+') patroni-4.0.4/tests/test_etcd.py000066400000000000000000000413561472010352700166770ustar00rootroot00000000000000import socket import unittest from unittest.mock import Mock, patch, PropertyMock import etcd import urllib3.util.connection from dns.exception import DNSException from urllib3.exceptions import ReadTimeoutError from patroni.dcs import get_dcs from patroni.dcs.etcd import AbstractDCS, Cluster, DnsCachingResolver, Etcd, EtcdClient, EtcdError from patroni.exceptions import DCSError from patroni.postgresql.mpp import get_mpp from patroni.utils import Retry from . import MockResponse, requests_get, SleepException def etcd_watch(self, key, index=None, timeout=None, recursive=None): if timeout == 2.0: raise etcd.EtcdWatchTimedOut elif timeout == 5.0: return etcd.EtcdResult('compareAndSwap', {}) elif 5 < timeout <= 10.0: raise etcd.EtcdException elif timeout == 20.0: raise etcd.EtcdEventIndexCleared def etcd_write(self, key, value, **kwargs): if key == '/service/exists/leader': raise etcd.EtcdAlreadyExist if key in ['/service/test/leader', '/patroni/test/leader'] and \ (kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)): return True raise etcd.EtcdException def etcd_read(self, key, **kwargs): if key == '/service/noleader/': raise DCSError('noleader') elif key == '/service/nocluster/': raise etcd.EtcdKeyNotFound response = {"action": "get", "node": {"key": "/service/batman5", "dir": True, "nodes": [ {"key": "/service/batman5/1", "dir": True, "nodes": [ {"key": "/service/batman5/1/initialize", "value": "2164261704", "modifiedIndex": 20729, "createdIndex": 20729}], "modifiedIndex": 20437, "createdIndex": 20437}, {"key": "/service/batman5/config", "value": '{"synchronous_mode": 0, "failsafe_mode": true}', "modifiedIndex": 1582, "createdIndex": 1582}, {"key": "/service/batman5/failover", "value": "", "modifiedIndex": 1582, "createdIndex": 1582}, {"key": "/service/batman5/initialize", "value": "postgresql0", "modifiedIndex": 1582, "createdIndex": 1582}, {"key": "/service/batman5/leader", "value": "postgresql1", "expiration": "2015-05-15T09:11:00.037397538Z", "ttl": 21, "modifiedIndex": 20728, "createdIndex": 20434}, {"key": "/service/batman5/optime", "dir": True, "nodes": [ {"key": "/service/batman5/optime/leader", "value": "2164261704", "modifiedIndex": 20729, "createdIndex": 20729}], "modifiedIndex": 20437, "createdIndex": 20437}, {"key": "/service/batman5/sync", "value": '{"leader": "leader"}', "modifiedIndex": 1582, "createdIndex": 1582}, {"key": "/service/batman5/members", "dir": True, "nodes": [ {"key": "/service/batman5/members/postgresql1", "value": "postgres://replicator:rep-pass@127.0.0.1:5434/postgres" + "?application_name=http://127.0.0.1:8009/patroni", "expiration": "2015-05-15T09:10:59.949384522Z", "ttl": 21, "modifiedIndex": 20727, "createdIndex": 20727}, {"key": "/service/batman5/members/postgresql0", "value": "postgres://replicator:rep-pass@127.0.0.1:5433/postgres" + "?application_name=http://127.0.0.1:8008/patroni", "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30, "modifiedIndex": 20730, "createdIndex": 20730}], "modifiedIndex": 1581, "createdIndex": 1581}, {"key": "/service/batman5/failsafe", "value": '{', "modifiedIndex": 1582, "createdIndex": 1582}, {"key": "/service/batman5/status", "value": '{"optime":2164261704,"slots":{"ls":12345},"retain_slots":["postgresql0","postgresql1"]}', "modifiedIndex": 1582, "createdIndex": 1582}], "modifiedIndex": 1581, "createdIndex": 1581}} if key == '/service/legacy/': response['node']['nodes'].pop() if key == '/service/broken/': response['node']['nodes'][-1]['value'] = '{' result = etcd.EtcdResult(**response) result.etcd_index = 0 return result def dns_query(name, _): if '-server' not in name or '-ssl' in name: return [] if name == '_etcd-server._tcp.blabla': return [] elif name == '_etcd-server._tcp.exception': raise DNSException() srv = Mock() srv.port = 2380 srv.target.to_text.return_value = \ 'localhost' if name in ['_etcd-server._tcp.foobar', '_etcd-server-baz._tcp.foobar'] else '127.0.0.1' return [srv] def socket_getaddrinfo(*args): if args[0] in ('ok', 'localhost', '127.0.0.1'): return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))] raise socket.gaierror def http_request(method, url, **kwargs): if url == 'http://localhost:2379/timeout': raise ReadTimeoutError(None, None, None) ret = MockResponse() if url == 'http://localhost:2379/v2/machines': ret.content = 'http://localhost:2379,http://localhost:4001' elif url == 'http://localhost:4001/v2/machines': ret.content = '' elif url != 'http://localhost:2379/': raise socket.error return ret class TestDnsCachingResolver(unittest.TestCase): @patch('time.sleep', Mock(side_effect=SleepException)) @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror)) def test_run(self): r = DnsCachingResolver() r._invoke_excepthook = Mock() self.assertIsNone(r.resolve_async('', 0)) r.join() @patch('dns.resolver.query', dns_query) @patch('socket.getaddrinfo', socket_getaddrinfo) @patch('patroni.dcs.etcd.requests_get', requests_get) class TestClient(unittest.TestCase): @patch('dns.resolver.query', dns_query) @patch('socket.getaddrinfo', socket_getaddrinfo) @patch('patroni.dcs.etcd.requests_get', requests_get) @patch.object(EtcdClient, '_get_machines_list', Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])) def setUp(self): self.etcd = get_dcs({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 3, 'etcd': {'srv': 'test'}, 'scope': 'test', 'name': 'foo'}) self.assertIsInstance(self.etcd, Etcd) self.client = self.etcd._client self.client.http.request = http_request self.client.http.request_encode_body = http_request def test_machines(self): self.client._base_uri = 'http://localhost:4002' self.client._machines_cache = ['http://localhost:4002', 'http://localhost:2379'] self.assertIsNotNone(self.client.machines) self.client._base_uri = 'http://localhost:4001' self.client._machines_cache = ['http://localhost:4001'] self.client._update_machines_cache = True machines = None try: machines = self.client.machines self.assertFail() except Exception: self.assertIsNone(machines) @patch.object(EtcdClient, '_get_machines_list', Mock(return_value=['http://localhost:4001', 'http://localhost:2379'])) def test_api_execute(self): self.client._base_uri = 'http://localhost:4001' self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'POST', timeout=0) self.client._base_uri = 'http://localhost:4001' rtry = Retry(deadline=10, max_delay=1, max_tries=-1, retry_exceptions=(etcd.EtcdLeaderElectionInProgress,)) rtry(self.client.api_execute, '/', 'POST', timeout=0, params={'retry': rtry}) self.client._machines_cache_updated = 0 self.client.api_execute('/', 'POST', timeout=0) self.client._machines_cache = [self.client._base_uri] self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'}) self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'}) with patch.object(EtcdClient, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])), \ patch.object(EtcdClient, '_load_machines_cache', Mock(side_effect=Exception)): self.client.http.request = Mock(side_effect=socket.error) self.assertRaises(etcd.EtcdException, rtry, self.client.api_execute, '/', 'GET', params={'retry': rtry}) with patch.object(EtcdClient, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])), \ patch.object(EtcdClient, '_load_machines_cache', Mock(return_value=True)): self.assertRaises(etcd.EtcdException, rtry, self.client.api_execute, '/', 'GET', params={'retry': rtry}) with patch.object(EtcdClient, '_do_http_request', Mock(side_effect=etcd.EtcdException)): self.client._read_timeout = 0.01 self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET') def test_get_srv_record(self): self.assertEqual(self.client.get_srv_record('_etcd-server._tcp.blabla'), []) self.assertEqual(self.client.get_srv_record('_etcd-server._tcp.exception'), []) def test__get_machines_cache_from_srv(self): self.client._get_machines_cache_from_srv('foobar') self.client._get_machines_cache_from_srv('foobar', 'baz') self.client.get_srv_record = Mock(return_value=[('localhost', 2380)]) self.client._get_machines_cache_from_srv('blabla') def test__get_machines_cache_from_dns(self): self.client._get_machines_cache_from_dns('error', 2379) @patch.object(EtcdClient, '_get_machines_list', Mock(side_effect=etcd.EtcdConnectionFailed)) def test__refresh_machines_cache(self): self.assertFalse(self.client._refresh_machines_cache()) self.assertRaises(etcd.EtcdException, self.client._refresh_machines_cache, ['http://localhost:2379']) def test__load_machines_cache(self): self.client._config = {} self.assertRaises(Exception, self.client._load_machines_cache) self.client._config = {'srv': 'blabla'} self.assertRaises(etcd.EtcdException, self.client._load_machines_cache) @patch.object(socket.socket, 'connect') def test_create_connection_patched(self, mock_connect): self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('fail', 2379)) urllib3.util.connection.create_connection(('[localhost]', 2379)) mock_connect.side_effect = socket.error self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('[localhost]', 2379), timeout=1, source_address=('localhost', 53333), socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)]) def test___del__(self): self.client.http.clear = Mock(side_effect=TypeError) del self.client @patch('patroni.dcs.etcd.requests_get', requests_get) @patch('socket.getaddrinfo', socket_getaddrinfo) @patch.object(etcd.Client, 'write', etcd_write) @patch.object(etcd.Client, 'read', etcd_read) @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) class TestEtcd(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) @patch.object(EtcdClient, '_get_machines_list', Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])) def setUp(self): self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10, 'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'}, get_mpp({})) def test_base_path(self): self.assertEqual(self.etcd._base_path, '/patroni/test') @patch('dns.resolver.query', dns_query) @patch('time.sleep', Mock(side_effect=SleepException)) @patch.object(EtcdClient, '_get_machines_list', Mock(side_effect=etcd.EtcdConnectionFailed)) def test_get_etcd_client(self): self.assertRaises(SleepException, self.etcd.get_etcd_client, {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1}, EtcdClient) self.assertRaises(SleepException, self.etcd.get_etcd_client, {'url': 'https://test:2379', 'retry_timeout': 10}, EtcdClient) self.assertRaises(SleepException, self.etcd.get_etcd_client, {'hosts': 'foo:4001,bar', 'retry_timeout': 10}, EtcdClient) with patch.object(EtcdClient, '_get_machines_list', Mock(return_value=[])): self.assertRaises(SleepException, self.etcd.get_etcd_client, {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10}, EtcdClient) def test_get_cluster(self): cluster = self.etcd.get_cluster() self.assertIsInstance(cluster, Cluster) self.etcd._base_path = '/service/legacy' self.assertIsInstance(self.etcd.get_cluster(), Cluster) self.etcd._base_path = '/service/broken' self.assertIsInstance(self.etcd.get_cluster(), Cluster) self.etcd._base_path = '/service/nocluster' cluster = self.etcd.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsNone(cluster.leader) self.etcd._base_path = '/service/noleader' self.assertRaises(EtcdError, self.etcd.get_cluster) def test__get_citus_cluster(self): self.etcd._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) cluster = self.etcd.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsInstance(cluster.workers[1], Cluster) self.etcd._base_path = '/service/nocluster' self.assertTrue(self.etcd.get_cluster().is_empty()) def test_touch_member(self): self.assertFalse(self.etcd.touch_member('')) def test_take_leader(self): self.assertFalse(self.etcd.take_leader()) def test_attempt_to_acquire_leader(self): self.etcd._base_path = '/service/exists' self.assertFalse(self.etcd.attempt_to_acquire_leader()) self.etcd._base_path = '/service/failed' self.assertFalse(self.etcd.attempt_to_acquire_leader()) with patch.object(EtcdClient, 'write', Mock(side_effect=[etcd.EtcdConnectionFailed, Exception])): self.assertRaises(EtcdError, self.etcd.attempt_to_acquire_leader) self.assertRaises(EtcdError, self.etcd.attempt_to_acquire_leader) @patch.object(Cluster, 'min_version', PropertyMock(return_value=(2, 0))) def test_write_leader_optime(self): self.etcd.get_cluster() self.etcd.write_leader_optime('0') def test_update_leader(self): cluster = self.etcd.get_cluster() self.assertTrue(self.etcd.update_leader(cluster, None, failsafe={'foo': 'bar'})) with patch.object(etcd.Client, 'write', Mock(side_effect=[etcd.EtcdConnectionFailed, etcd.EtcdClusterIdChanged, Exception])): self.assertRaises(EtcdError, self.etcd.update_leader, cluster, None) self.assertFalse(self.etcd.update_leader(cluster, None)) self.assertRaises(EtcdError, self.etcd.update_leader, cluster, None) with patch.object(etcd.Client, 'write', Mock(side_effect=etcd.EtcdKeyNotFound)): self.assertFalse(self.etcd.update_leader(cluster, None)) def test_initialize(self): self.assertFalse(self.etcd.initialize()) def test_cancel_initializion(self): self.assertFalse(self.etcd.cancel_initialization()) def test_delete_leader(self): self.assertFalse(self.etcd.delete_leader(self.etcd.get_cluster().leader)) def test_delete_cluster(self): self.assertFalse(self.etcd.delete_cluster()) @patch('time.sleep', Mock(side_effect=SleepException)) @patch.object(etcd.Client, 'watch', etcd_watch) def test_watch(self): self.etcd.watch(None, 0) self.etcd.get_cluster() self.etcd.watch(20729, 1.5) with patch('time.sleep', Mock()): self.etcd.watch(20729, 4.5) with patch.object(AbstractDCS, 'watch', Mock()): self.assertTrue(self.etcd.watch(20729, 19.5)) self.assertRaises(SleepException, self.etcd.watch, 20729, 9.5) def test_other_exceptions(self): self.etcd.retry = Mock(side_effect=AttributeError('foo')) self.assertRaises(EtcdError, self.etcd.cancel_initialization) def test_set_ttl(self): self.etcd.set_ttl(20) self.assertTrue(self.etcd.watch(None, 1)) def test_sync_state(self): self.assertIsNone(self.etcd.write_sync_state('leader', None, 0)) self.assertFalse(self.etcd.delete_sync_state()) def test_set_history_value(self): self.assertFalse(self.etcd.set_history_value('{}')) def test_last_seen(self): self.assertIsNotNone(self.etcd.last_seen) patroni-4.0.4/tests/test_etcd3.py000066400000000000000000000374021472010352700167570ustar00rootroot00000000000000import json import unittest from threading import Thread from unittest.mock import Mock, patch, PropertyMock import etcd import urllib3 from patroni.dcs import get_dcs from patroni.dcs.etcd import DnsCachingResolver from patroni.dcs.etcd3 import AuthFailed, AuthOldRevision, base64_encode, Cluster, Etcd3, \ Etcd3Client, Etcd3ClientError, Etcd3Error, InvalidAuthToken, PatroniEtcd3Client, \ RetryFailedError, Unavailable, Unknown, UnsupportedEtcdVersion, UserEmpty from patroni.postgresql.mpp import get_mpp from . import MockResponse, SleepException def mock_urlopen(self, method, url, **kwargs): ret = MockResponse() if method == 'GET' and url.endswith('/version'): ret.content = '{"etcdserver": "3.3.13", "etcdcluster": "3.3.0"}' elif method != 'POST': raise Exception('Unexpected request method: {0} {1} {2}'.format(method, url, kwargs)) elif url.endswith('/cluster/member/list'): ret.content = '{"members":[{"clientURLs":["http://localhost:2379", "http://localhost:4001"]}]}' elif url.endswith('/auth/authenticate'): ret.content = '{"token":"authtoken"}' elif url.endswith('/lease/grant'): ret.content = '{"ID": "123"}' elif url.endswith('/lease/keepalive'): ret.content = '{"result":{"TTL":30}}' elif url.endswith('/kv/range'): ret.content = json.dumps({ "header": {"revision": "1"}, "kvs": [ {"key": base64_encode('/patroni/test/1/initialize'), "value": base64_encode('12345'), "mod_revision": '1'}, {"key": base64_encode('/patroni/test/leader'), "value": base64_encode('foo'), "lease": "bla", "mod_revision": '1'}, {"key": base64_encode('/patroni/test/members/foo'), "value": base64_encode('{}'), "lease": "123", "mod_revision": '1'}, {"key": base64_encode('/patroni/test/members/bar'), "value": base64_encode('{"version":"1.6.5"}'), "lease": "123", "mod_revision": '1'}, {"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'}, {"key": base64_encode('/patroni/test/failsafe'), "value": base64_encode('{'), "mod_revision": '1'} ] }) elif url.endswith('/watch'): key = base64_encode('/patroni/test/config') ret.read_chunked = Mock(return_value=[json.dumps({ 'result': {'events': [ {'kv': {'key': key, 'value': base64_encode('bar'), 'mod_revision': '2'}}, {'kv': {'key': key, 'value': base64_encode('buzz'), 'mod_revision': '3'}}, {'type': 'DELETE', 'kv': {'key': key, 'mod_revision': '4'}}, {'kv': {'key': base64_encode('/patroni/test/optime/leader'), 'value': base64_encode('1234567'), 'mod_revision': '5'}}, ]} })[:-1].encode('utf-8'), b'}{"error":{"grpc_code":14,"message":"","http_code":503}}']) elif url.endswith('/kv/put') or url.endswith('/kv/txn'): if base64_encode('/patroni/test/sync') in kwargs['body']: ret.content = '{"header":{"revision":"1"},"succeeded":true}' else: ret.status_code = 400 ret.content = '{"code":5,"error":"etcdserver: requested lease not found"}' elif not url.endswith('/kv/deleterange'): raise Exception('Unexpected url: {0} {1} {2}'.format(method, url, kwargs)) return ret class TestEtcd3Client(unittest.TestCase): @patch.object(Thread, 'start', Mock()) @patch.object(urllib3.PoolManager, 'urlopen', mock_urlopen) def test_authenticate(self): etcd3 = Etcd3Client({'host': '127.0.0.1', 'port': 2379, 'use_proxies': True, 'retry_timeout': 10}, DnsCachingResolver()) self.assertIsNotNone(etcd3._cluster_version) class BaseTestEtcd3(unittest.TestCase): @patch.object(Thread, 'start', Mock()) @patch.object(urllib3.PoolManager, 'urlopen', mock_urlopen) def setUp(self): self.etcd3 = get_dcs({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10, 'name': 'foo', 'scope': 'test', 'etcd3': {'host': 'localhost:2378', 'username': 'etcduser', 'password': 'etcdpassword'}}) self.assertIsInstance(self.etcd3, Etcd3) self.client = self.etcd3._client self.kv_cache = self.client._kv_cache class TestKVCache(BaseTestEtcd3): @patch.object(urllib3.PoolManager, 'urlopen', mock_urlopen) @patch.object(Etcd3Client, 'watchprefix', Mock(return_value=urllib3.response.HTTPResponse())) def test__build_cache(self): self.kv_cache._build_cache() def test__do_watch(self): self.client.watchprefix = Mock(return_value=False) self.assertRaises(AttributeError, self.kv_cache._do_watch, '1') @patch('time.sleep', Mock(side_effect=SleepException)) @patch('patroni.dcs.etcd3.KVCache._build_cache', Mock(side_effect=Exception)) def test_run(self): self.assertRaises(SleepException, self.kv_cache.run) @patch.object(urllib3.response.HTTPResponse, 'read_chunked', Mock(return_value=[b'{"error":{"grpc_code":14,"message":"","http_code":503}}'])) @patch.object(Etcd3Client, 'watchprefix', Mock(return_value=urllib3.response.HTTPResponse())) def test_kill_stream(self): self.assertRaises(Unavailable, self.kv_cache._do_watch, '1') with patch.object(urllib3.response.HTTPResponse, 'connection') as mock_conn: self.kv_cache.kill_stream() mock_conn.sock.close.side_effect = Exception self.kv_cache.kill_stream() type(mock_conn).sock = PropertyMock(side_effect=Exception) self.kv_cache.kill_stream() class TestPatroniEtcd3Client(BaseTestEtcd3): @patch('patroni.dcs.etcd3.Etcd3Client.authenticate', Mock(side_effect=AuthFailed)) def test__init__(self): self.assertRaises(SystemExit, self.setUp) @patch.object(urllib3.PoolManager, 'urlopen') def test_call_rpc(self, mock_urlopen): request = {'key': base64_encode('/patroni/test/leader')} mock_urlopen.return_value = MockResponse() mock_urlopen.return_value.content = '{"succeeded":true,"header":{"revision":"1"}}' self.client.call_rpc('/kv/put', request) self.client.call_rpc('/kv/deleterange', request) @patch.object(urllib3.PoolManager, 'urlopen') def test_txn(self, mock_urlopen): mock_urlopen.return_value = MockResponse() mock_urlopen.return_value.content = '{"header":{"revision":"1"}}' self.client.txn({'target': 'MOD', 'mod_revision': '1'}, {'request_delete_range': {'key': base64_encode('/patroni/test/leader')}}) @patch('time.time', Mock(side_effect=[1, 10.9, 100])) def test__wait_cache(self): with self.kv_cache.condition: self.assertRaises(RetryFailedError, self.client._wait_cache, 10) @patch.object(urllib3.PoolManager, 'urlopen') def test__restart_watcher(self, mock_urlopen): mock_urlopen.return_value = MockResponse() mock_urlopen.return_value.status_code = 400 mock_urlopen.return_value.content = '{"code":9,"error":"etcdserver: authentication is not enabled"}' self.client.authenticate() @patch.object(urllib3.PoolManager, 'urlopen') def test__handle_auth_errors(self, mock_urlopen): mock_urlopen.return_value = MockResponse() mock_urlopen.return_value.content = '{"code":3,"error":"etcdserver: user name is empty"}' mock_urlopen.return_value.status_code = 403 self.client._cluster_version = (3, 1, 5) self.assertRaises(UnsupportedEtcdVersion, self.client.deleteprefix, 'foo') self.client._cluster_version = (3, 3, 13) self.assertRaises(UserEmpty, self.client.deleteprefix, 'foo') mock_urlopen.return_value.content = '{"code":16,"error":"etcdserver: invalid auth token"}' self.assertRaises(InvalidAuthToken, self.client.deleteprefix, 'foo') with patch.object(PatroniEtcd3Client, 'authenticate', Mock(return_value=True)): retry = self.etcd3._retry.copy() with patch('time.time', Mock(side_effect=[0, 10, 20, 30, 40])): self.assertRaises(InvalidAuthToken, retry, self.client.deleteprefix, 'foo', retry=retry) with patch('time.time', Mock(side_effect=[0, 10])): self.assertRaises(InvalidAuthToken, self.client.deleteprefix, 'foo') self.client.username = None self.client._reauthenticate = False retry = self.etcd3._retry.copy() self.assertRaises(InvalidAuthToken, retry, self.client.deleteprefix, 'foo', retry=retry) mock_urlopen.return_value.content = '{"code":3,"error":"etcdserver: revision of auth store is old"}' self.client._reauthenticate = False self.assertRaises(AuthOldRevision, retry, self.client.deleteprefix, 'foo', retry=retry) def test__handle_server_response(self): response = MockResponse() response.content = '{"code":0,"error":"' self.assertRaises(etcd.EtcdException, self.client._handle_server_response, response) response.status_code = 400 self.assertRaises(Unknown, self.client._handle_server_response, response) response.content = '{"error":{"grpc_code":0,"message":"","http_code":400}}' try: self.client._handle_server_response(response) except Unknown as e: self.assertEqual(e.as_dict(), {'code': 2, 'codeText': 'OK', 'error': u'', 'status': 400}) @patch.object(urllib3.PoolManager, 'urlopen') def test__ensure_version_prefix(self, mock_urlopen): self.client.version_prefix = None mock_urlopen.return_value = MockResponse() mock_urlopen.return_value.content = '{"etcdserver": "3.0.3", "etcdcluster": "3.0.0"}' self.assertRaises(UnsupportedEtcdVersion, self.client._ensure_version_prefix, '') mock_urlopen.return_value.content = '{"etcdserver": "3.0.4", "etcdcluster": "3.0.0"}' self.client._ensure_version_prefix('') self.assertEqual(self.client.version_prefix, '/v3alpha') mock_urlopen.return_value.content = '{"etcdserver": "3.4.4", "etcdcluster": "3.4.0"}' self.client._ensure_version_prefix('') self.assertEqual(self.client.version_prefix, '/v3') @patch.object(urllib3.PoolManager, 'urlopen', mock_urlopen) class TestEtcd3(BaseTestEtcd3): @patch.object(Thread, 'start', Mock()) @patch.object(urllib3.PoolManager, 'urlopen', mock_urlopen) def setUp(self): super(TestEtcd3, self).setUp() # self.assertRaises(AttributeError, self.kv_cache._build_cache) self.kv_cache._build_cache() self.kv_cache._is_ready = True self.etcd3.get_cluster() def test_get_cluster(self): self.assertIsInstance(self.etcd3.get_cluster(), Cluster) self.client._kv_cache = None with patch.object(urllib3.PoolManager, 'urlopen') as mock_urlopen: mock_urlopen.return_value = MockResponse() mock_urlopen.return_value.content = json.dumps({ "header": {"revision": "1"}, "kvs": [ {"key": base64_encode('/patroni/test/status'), "value": base64_encode('{"optime":1234567,"slots":{"ls":12345},"retain_slots": ["foo"]}'), "mod_revision": '1'} ] }) self.assertIsInstance(self.etcd3.get_cluster(), Cluster) mock_urlopen.return_value.content = json.dumps({ "header": {"revision": "1"}, "kvs": [ {"key": base64_encode('/patroni/test/status'), "value": base64_encode('{'), "mod_revision": '1'} ] }) self.assertIsInstance(self.etcd3.get_cluster(), Cluster) mock_urlopen.side_effect = UnsupportedEtcdVersion('') self.assertRaises(UnsupportedEtcdVersion, self.etcd3.get_cluster) mock_urlopen.side_effect = SleepException() self.assertRaises(Etcd3Error, self.etcd3.get_cluster) def test__get_citus_cluster(self): self.etcd3._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) cluster = self.etcd3.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsInstance(cluster.workers[1], Cluster) def test_touch_member(self): self.etcd3.touch_member({}) self.etcd3._lease = 'bla' self.etcd3.touch_member({}) with patch.object(PatroniEtcd3Client, 'lease_grant', Mock(side_effect=Etcd3ClientError)): self.etcd3.touch_member({}) def test__update_leader(self): cluster = self.etcd3.get_cluster() self.etcd3._lease = None with patch.object(Etcd3Client, 'txn', Mock(return_value={'succeeded': True})): self.etcd3.update_leader(cluster, '123', failsafe={'foo': 'bar'}) self.etcd3._last_lease_refresh = 0 self.etcd3.update_leader(cluster, '124') with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)), \ patch('time.time', Mock(side_effect=[0, 100, 200, 300])): self.assertRaises(Etcd3Error, self.etcd3.update_leader, cluster, '126') self.etcd3._lease = cluster.leader.session self.etcd3.update_leader(cluster, '124') self.etcd3._last_lease_refresh = 0 with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(side_effect=Unknown)): self.assertFalse(self.etcd3.update_leader(cluster, '125')) def test_take_leader(self): self.assertFalse(self.etcd3.take_leader()) def test_attempt_to_acquire_leader(self): self.assertFalse(self.etcd3.attempt_to_acquire_leader()) with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200, 300])): self.assertFalse(self.etcd3.attempt_to_acquire_leader()) with patch('time.time', Mock(side_effect=[0, 100, 200, 300, 400])): self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader) with patch.object(PatroniEtcd3Client, 'put', Mock(return_value=False)): self.assertFalse(self.etcd3.attempt_to_acquire_leader()) def test_set_ttl(self): self.etcd3.set_ttl(20) @patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=False)) def test_refresh_lease(self): self.etcd3._last_lease_refresh = 0 self.etcd3.refresh_lease() @patch('time.sleep', Mock(side_effect=SleepException)) @patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=False)) @patch.object(PatroniEtcd3Client, 'lease_grant', Mock(side_effect=Etcd3ClientError)) def test_create_lease(self): self.etcd3._lease = None self.etcd3._last_lease_refresh = 0 self.assertRaises(SleepException, self.etcd3.create_lease) def test_set_failover_value(self): self.etcd3.set_failover_value('', 1) def test_set_config_value(self): self.etcd3.set_config_value('') def test_initialize(self): self.etcd3.initialize() def test_cancel_initialization(self): self.etcd3.cancel_initialization() def test_delete_leader(self): leader = self.etcd3.get_cluster().leader self.etcd3.delete_leader(leader) self.etcd3._name = 'other' self.etcd3.delete_leader(leader) def test_delete_cluster(self): self.etcd3.delete_cluster() def test_set_history_value(self): self.etcd3.set_history_value('') def test_set_sync_state_value(self): self.etcd3.set_sync_state_value('', 1) def test_delete_sync_state(self): self.etcd3.delete_sync_state('1') def test_watch(self): self.etcd3.set_ttl(10) self.etcd3.watch(None, 0) self.etcd3.watch('5', 0) def test_set_socket_options(self): with patch('socket.SIO_KEEPALIVE_VALS', 1, create=True): self.etcd3.set_socket_options(Mock(), None) patroni-4.0.4/tests/test_exhibitor.py000066400000000000000000000027361472010352700177540ustar00rootroot00000000000000import unittest from unittest.mock import Mock, patch import urllib3 from patroni.dcs import get_dcs from patroni.dcs.exhibitor import Exhibitor, ExhibitorEnsembleProvider from patroni.dcs.zookeeper import ZooKeeperError from . import requests_get, SleepException from .test_zookeeper import MockKazooClient @patch('patroni.dcs.exhibitor.requests_get', requests_get) @patch('time.sleep', Mock(side_effect=SleepException)) class TestExhibitorEnsembleProvider(unittest.TestCase): def test_init(self): self.assertRaises(SleepException, ExhibitorEnsembleProvider, ['localhost'], 8181) def test_poll(self): self.assertFalse(ExhibitorEnsembleProvider(['exhibitor'], 8181).poll()) class TestExhibitor(unittest.TestCase): @patch('urllib3.PoolManager.request', Mock(return_value=urllib3.HTTPResponse( status=200, body=b'{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'))) @patch('patroni.dcs.zookeeper.PatroniKazooClient', MockKazooClient) def setUp(self): self.e = get_dcs({'exhibitor': {'hosts': ['localhost', 'exhibitor'], 'port': 8181}, 'scope': 'test', 'name': 'foo', 'ttl': 30, 'retry_timeout': 10}) self.assertIsInstance(self.e, Exhibitor) @patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True)) @patch.object(MockKazooClient, 'get_children', Mock(side_effect=Exception)) def test_get_cluster(self): self.assertRaises(ZooKeeperError, self.e.get_cluster) patroni-4.0.4/tests/test_file_perm.py000066400000000000000000000026471472010352700177220ustar00rootroot00000000000000import stat import unittest from unittest.mock import Mock, patch from patroni.file_perm import pg_perm class TestFilePermissions(unittest.TestCase): @patch('os.stat') @patch('os.umask') @patch('patroni.file_perm.logger.error') def test_set_umask(self, mock_logger, mock_umask, mock_stat): mock_umask.side_effect = Exception mock_stat.return_value.st_mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP pg_perm.set_permissions_from_data_directory('test') # umask is called with PG_MODE_MASK_GROUP self.assertEqual(mock_umask.call_args[0][0], stat.S_IWGRP | stat.S_IRWXO) self.assertEqual(mock_logger.call_args[0][0], 'Can not set umask to %03o: %r') mock_umask.reset_mock() mock_stat.return_value.st_mode = stat.S_IRWXU pg_perm.set_permissions_from_data_directory('test') # umask is called with PG_MODE_MASK_OWNER (permissions changed from group to owner) self.assertEqual(mock_umask.call_args[0][0], stat.S_IRWXG | stat.S_IRWXO) @patch('os.stat', Mock(side_effect=FileNotFoundError)) @patch('patroni.file_perm.logger.error') def test_set_permissions_from_data_directory(self, mock_logger): pg_perm.set_permissions_from_data_directory('test') self.assertEqual(mock_logger.call_args[0][0], 'Can not check permissions on %s: %r') def test_orig_umask(self): self.assertIsNotNone(pg_perm.orig_umask) patroni-4.0.4/tests/test_ha.py000066400000000000000000003077671472010352700163630ustar00rootroot00000000000000import datetime import os import sys from unittest.mock import MagicMock, Mock, mock_open, patch, PropertyMock import etcd from patroni import global_config from patroni.collections import CaseInsensitiveSet from patroni.config import Config from patroni.dcs import Cluster, ClusterConfig, Failover, get_dcs, Leader, Member, Status, SyncState, TimelineHistory from patroni.dcs.etcd import AbstractEtcdClientWithFailover from patroni.exceptions import DCSError, PatroniFatalException, PostgresConnectionException from patroni.ha import _MemberStatus, Ha from patroni.postgresql import Postgresql from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.callback_executor import CallbackAction from patroni.postgresql.cancellable import CancellableSubprocess from patroni.postgresql.config import ConfigHandler from patroni.postgresql.postmaster import PostmasterProcess from patroni.postgresql.rewind import Rewind from patroni.postgresql.slots import SlotsHandler from patroni.postgresql.sync import _SyncState from patroni.utils import tzutc from patroni.watchdog import Watchdog from . import MockPostmaster, PostgresInit, psycopg_connect, requests_get from .test_etcd import etcd_read, etcd_write, socket_getaddrinfo SYSID = '12345678901' def true(*args, **kwargs): return True def false(*args, **kwargs): return False def get_cluster(initialize, leader, members, failover, sync, cluster_config=None, failsafe=None): t = datetime.datetime.now().isoformat() history = TimelineHistory(1, '[[1,67197376,"no recovery target specified","' + t + '","foo"]]', [(1, 67197376, 'no recovery target specified', t, 'foo')]) cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True, 'member_slots_ttl': 0}, 1) return Cluster(initialize, cluster_config, leader, Status(10, None, []), members, failover, sync, history, failsafe) def get_cluster_not_initialized_without_leader(cluster_config=None): return get_cluster(None, None, [], None, SyncState.empty(), cluster_config) def get_cluster_bootstrapping_without_leader(cluster_config=None): return get_cluster("", None, [], None, SyncState.empty(), cluster_config) def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None, cluster_config=None, failsafe=False): m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5435/postgres', 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4, 'role': 'primary', 'state': 'running'}) leader = Leader(0, 0, m1 if leader else Member(0, '', 28, {})) m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5436/postgres', 'api_url': 'http://127.0.0.1:8011/patroni', 'state': 'running', 'pause': True, 'tags': {'clonefrom': True}, 'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00", 'postgres_version': '99.0.0'}}) syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1], 0) failsafe = {m.name: m.api_url for m in (m1, m2)} if failsafe else None return get_cluster(SYSID, leader, [m1, m2], failover, syncstate, cluster_config, failsafe) def get_cluster_initialized_with_leader(failover=None, sync=None): return get_cluster_initialized_without_leader(leader=True, failover=failover, sync=sync) def get_cluster_initialized_with_only_leader(failover=None, cluster_config=None): leader = get_cluster_initialized_without_leader(leader=True, failover=failover).leader return get_cluster(True, leader, [leader.member], failover, SyncState.empty(), cluster_config) def get_standby_cluster_initialized_with_only_leader(failover=None, sync=None): return get_cluster_initialized_with_only_leader( cluster_config=ClusterConfig(1, { "standby_cluster": { "host": "localhost", "port": 5432, "primary_slot_name": "", }}, 1) ) def get_cluster_initialized_with_leader_and_failsafe(): return get_cluster_initialized_without_leader(leader=True, failsafe=True, cluster_config=ClusterConfig(1, {'failsafe_mode': True}, 1)) def get_node_status(reachable=True, in_recovery=True, dcs_last_seen=0, timeline=2, wal_position=10, nofailover=False, watchdog_failed=False, failover_priority=1): def fetch_node_status(e): tags = {} if nofailover: tags['nofailover'] = True tags['failover_priority'] = failover_priority return _MemberStatus(e, reachable, in_recovery, wal_position, {'tags': tags, 'watchdog_failed': watchdog_failed, 'dcs_last_seen': dcs_last_seen, 'timeline': timeline}) return fetch_node_status future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5) postmaster_start_time = datetime.datetime.now(tzutc) class MockPatroni(object): def __init__(self, p, d): os.environ[Config.PATRONI_CONFIG_VARIABLE] = """ restapi: listen: 0.0.0.0:8008 bootstrap: postgresql: name: foo data_dir: data/postgresql0 pg_rewind: username: postgres password: postgres watchdog: mode: off zookeeper: exhibitor: hosts: [localhost] port: 8181 """ # We rely on sys.argv in Config, so it's necessary to reset # all the extra values that are coming from py.test sys.argv = sys.argv[:1] self.config = Config(None) self.version = '1.5.7' self.postgresql = p self.dcs = d self.api = Mock() self.tags = {'foo': 'bar'} self.nofailover = None self.replicatefrom = None self.api.connection_string = 'http://127.0.0.1:8008' self.clonefrom = None self.nosync = False self.nostream = False self.scheduled_restart = {'schedule': future_restart_time, 'postmaster_start_time': str(postmaster_start_time)} self.watchdog = Watchdog(self.config) self.request = lambda *args, **kwargs: requests_get(args[0].api_url, *args[1:], **kwargs) self.failover_priority = 1 def run_async(self, func, args=()): self.reset_scheduled_action() if args: func(*args) else: func() @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster())) @patch.object(Postgresql, 'is_primary', Mock(return_value=True)) @patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10, 1))) @patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=10)) @patch.object(Postgresql, 'slots', Mock(return_value={'l': 100})) @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False)) @patch.object(Postgresql, 'controldata', Mock(return_value={ 'Database system identifier': SYSID, 'Database cluster state': 'shut down', 'Latest checkpoint location': '0/12345678', "Latest checkpoint's TimeLineID": '2'})) @patch.object(SlotsHandler, 'load_replication_slots', Mock(side_effect=Exception)) @patch.object(ConfigHandler, 'append_pg_hba', Mock()) @patch.object(ConfigHandler, 'write_pgpass', Mock(return_value={})) @patch.object(ConfigHandler, 'write_recovery_conf', Mock()) @patch.object(ConfigHandler, 'write_postgresql_conf', Mock()) @patch.object(Postgresql, 'query', Mock()) @patch.object(Postgresql, 'checkpoint', Mock()) @patch.object(CancellableSubprocess, 'call', Mock(return_value=0)) @patch.object(Postgresql, 'get_replica_timeline', Mock(return_value=2)) @patch.object(Postgresql, 'get_primary_timeline', Mock(return_value=2)) @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) @patch.object(Postgresql, 'resume_wal_replay', Mock()) @patch.object(ConfigHandler, 'restore_configuration_files', Mock()) @patch.object(etcd.Client, 'write', etcd_write) @patch.object(etcd.Client, 'read', etcd_read) @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1))) @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False)) @patch('patroni.async_executor.AsyncExecutor.run_async', run_async) @patch('patroni.postgresql.rewind.Thread', Mock()) @patch('patroni.postgresql.mpp.citus.CitusHandler.start', Mock()) @patch('subprocess.call', Mock(return_value=0)) @patch('time.sleep', Mock()) class TestHa(PostgresInit): @patch('socket.getaddrinfo', socket_getaddrinfo) @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.etcd'])) @patch.object(etcd.Client, 'read', etcd_read) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) @patch.object(Config, '_load_cache', Mock()) def setUp(self): super(TestHa, self).setUp() self.p.set_state('running') self.p.set_role('replica') self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time)) self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False) self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test', 'name': 'foo', 'retry_timeout': 10}, 'citus': {'database': 'citus', 'group': None}}) self.ha = Ha(MockPatroni(self.p, self.e)) self.ha.old_cluster = self.e.get_cluster() self.ha.cluster = get_cluster_initialized_without_leader() global_config.update(self.ha.cluster) self.ha.load_cluster_from_dcs = Mock() def test_update_lock(self): self.ha.is_failsafe_mode = true self.p.last_operation = Mock(side_effect=PostgresConnectionException('')) self.ha.dcs.update_leader = Mock(side_effect=[DCSError(''), Exception]) self.assertRaises(DCSError, self.ha.update_lock) self.assertFalse(self.ha.update_lock(True)) @patch.object(Postgresql, 'received_timeline', Mock(return_value=None)) def test_touch_member(self): self.p._major_version = 110000 self.p.is_primary = false self.p.timeline_wal_position = Mock(return_value=(0, 1, 0)) self.p.replica_cached_timeline = Mock(side_effect=Exception) with patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value='streaming')): self.ha.touch_member() self.p.timeline_wal_position = Mock(return_value=(0, 1, 1)) self.p.set_role('standby_leader') self.ha.touch_member() self.p.set_role('primary') self.ha.dcs.touch_member = true self.ha.touch_member() def test_is_leader(self): self.assertFalse(self.ha.is_leader()) def test_start_as_replica(self): self.p.is_healthy = false self.assertEqual(self.ha.run_cycle(), 'starting as a secondary') @patch('patroni.dcs.etcd.Etcd.initialize', return_value=True) def test_bootstrap_as_standby_leader(self, initialize): self.p.data_directory_empty = true self.ha.cluster = get_cluster_not_initialized_without_leader( cluster_config=ClusterConfig(1, {"standby_cluster": {"port": 5432}}, 1)) self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader') def test_bootstrap_waiting_for_standby_leader(self): self.p.data_directory_empty = true self.ha.cluster = get_cluster_initialized_without_leader() self.ha.cluster.config.data.update({'standby_cluster': {'port': 5432}}) self.assertEqual(self.ha.run_cycle(), 'waiting for standby_leader to bootstrap') @patch.object(Cluster, 'get_clone_member', Mock(return_value=Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres'}))) @patch.object(Bootstrap, 'create_replica', Mock(return_value=0)) def test_start_as_cascade_replica_in_standby_cluster(self): self.p.data_directory_empty = true self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.assertEqual(self.ha.run_cycle(), "trying to bootstrap from replica 'test'") def test_recover_replica_failed(self): self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': SYSID} self.p.is_running = false self.p.follow = false self.assertEqual(self.ha.run_cycle(), 'starting as a secondary') self.assertEqual(self.ha.run_cycle(), 'failed to start postgres') def test_recover_raft(self): self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': SYSID} self.p.is_running = false self.p.follow = true self.assertEqual(self.ha.run_cycle(), 'starting as a secondary') self.p.is_running = true ha_dcs_orig_name = self.ha.dcs.__class__.__name__ self.ha.dcs.__class__.__name__ = 'Raft' self.assertEqual(self.ha.run_cycle(), 'started as a secondary') self.ha.dcs.__class__.__name__ = ha_dcs_orig_name def test_recover_former_primary(self): self.p.follow = false self.p.is_running = false self.p.name = 'leader' self.p.set_role('demoted') self.p.controldata = lambda: {'Database cluster state': 'shut down', 'Database system identifier': SYSID} self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'starting as readonly because i had the session lock') def test_start_primary_after_failure(self): self.p.start = false self.p.is_running = false self.p.name = 'leader' self.p.set_role('primary') self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID} self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'starting primary after failure') @patch.object(Rewind, 'ensure_clean_shutdown', Mock()) def test_crash_recovery(self): self.ha.has_lock = true self.p.is_running = false self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID} self.assertEqual(self.ha.run_cycle(), 'doing crash recovery in a single user mode') with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)), \ patch.object(Ha, 'check_timeline', Mock(return_value=False)): self.ha._async_executor.schedule('doing crash recovery in a single user mode') self.ha.state_handler.cancellable._process = Mock() self.ha._crash_recovery_started -= 600 self.ha.cluster.config.data.update({'maximum_lag_on_failover': 10}) self.assertEqual(self.ha.run_cycle(), 'terminated crash recovery because of startup timeout') @patch.object(Rewind, 'ensure_clean_shutdown', Mock()) @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)) @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True)) def test_crash_recovery_before_rewind(self): self.p.is_primary = false self.p.is_running = false self.p.controldata = lambda: {'Database cluster state': 'in archive recovery', 'Database system identifier': SYSID} self.ha._rewind.trigger_check_diverged_lsn() self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'doing crash recovery in a single user mode') @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)) @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True)) @patch('os.listdir', Mock(return_value=[])) @patch('patroni.postgresql.rewind.fsync_dir', Mock()) def test_recover_with_rewind(self): self.p.is_running = false self.ha.cluster = get_cluster_initialized_with_leader() self.ha.cluster.leader.member.data.update(version='2.0.2', role='primary') self.ha._rewind.pg_rewind = true self.ha._rewind.check_leader_is_not_in_recovery = true with patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)): self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader') with patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=False)), \ patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)): self.p.follow = true self.assertEqual(self.ha.run_cycle(), 'starting as a secondary') self.p.is_running = true self.ha.follow = Mock(return_value='fake') self.assertEqual(self.ha.run_cycle(), 'fake') @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)) @patch.object(Rewind, 'should_remove_data_directory_on_diverged_timelines', PropertyMock(return_value=True)) @patch.object(Bootstrap, 'create_replica', Mock(return_value=1)) def test_recover_with_reinitialize(self): self.p.is_running = false self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'reinitializing due to diverged timelines') @patch('sys.exit', return_value=1) @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True)) def test_sysid_no_match(self, exit_mock): self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': '123'} self.ha.run_cycle() exit_mock.assert_called_once_with(1) @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_start_as_readonly(self): self.p.is_primary = false self.p.is_healthy = true self.ha.has_lock = true self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID} self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because I had the session lock') @patch('patroni.psycopg.connect', psycopg_connect) def test_acquire_lock_as_primary(self): self.assertEqual(self.ha.run_cycle(), 'acquired session lock as a leader') def test_leader_race_stale_primary(self): with patch.object(Postgresql, 'get_primary_timeline', Mock(return_value=1)), \ patch('patroni.ha.logger.warning') as mock_logger: self.assertEqual(self.ha.run_cycle(), 'demoting self because i am not the healthiest node') self.assertEqual(mock_logger.call_args[0][0], 'My timeline %s is behind last known cluster timeline %s') def test_promoted_by_acquiring_lock(self): self.ha.is_healthiest_node = true self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') def test_promotion_cancelled_after_pre_promote_failed(self): self.p.is_primary = false self.p._pre_promote = false self.ha._is_healthiest_node = true self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(self.ha.run_cycle(), 'Promotion cancelled because the pre-promote script failed') self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') def test_lost_leader_lock_during_promote(self): with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)): self.ha._async_executor.schedule('promote') self.assertEqual(self.ha.run_cycle(), 'lost leader before promote') @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_long_promote(self): self.ha.has_lock = true self.p.is_primary = false self.p.set_role('primary') self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') def test_demote_after_failing_to_obtain_lock(self): self.ha.acquire_lock = false self.assertEqual(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock') def test_follow_new_leader_after_failing_to_obtain_lock(self): self.ha.is_healthiest_node = true self.ha.acquire_lock = false self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock') def test_demote_because_not_healthiest(self): self.ha.is_healthiest_node = false self.assertEqual(self.ha.run_cycle(), 'demoting self because i am not the healthiest node') def test_follow_new_leader_because_not_healthiest(self): self.ha.is_healthiest_node = false self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_promote_because_have_lock(self): self.ha.has_lock = true self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because I had the session lock') def test_promote_without_watchdog(self): self.ha.has_lock = true self.p.is_primary = true with patch.object(Watchdog, 'activate', Mock(return_value=False)): self.assertEqual(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated') self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated') def test_leader_with_lock(self): self.ha.cluster = get_cluster_initialized_with_leader() self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') def test_coordinator_leader_with_lock(self): self.ha.cluster = get_cluster_initialized_with_leader() self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') @patch.object(Postgresql, '_wait_for_connection_close', Mock()) @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_demote_because_not_having_lock(self): with patch.object(Watchdog, 'is_running', PropertyMock(return_value=True)): self.assertEqual(self.ha.run_cycle(), 'demoting self because I do not have the lock and I was a leader') @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_demote_because_update_lock_failed(self): self.ha.has_lock = true self.ha.update_lock = false self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS') with patch.object(Ha, '_get_node_to_follow', Mock(side_effect=DCSError('foo'))): self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS') self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS') def test_get_node_to_follow_nostream(self): self.ha.patroni.nostream = True self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha._get_node_to_follow(self.ha.cluster), None) @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_follow(self): self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader ()') self.ha.patroni.replicatefrom = "foo" self.p.config.check_recovery_conf = Mock(return_value=(True, False)) self.ha.cluster.config.data.update({'slots': {'l': {'database': 'a', 'plugin': 'b'}}}) self.ha.cluster.members[1].data['tags']['replicatefrom'] = 'postgresql0' self.ha.patroni.nofailover = True self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader ()') del self.ha.cluster.config.data['slots'] self.ha.cluster.config.data.update({'postgresql': {'use_slots': False}}) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader ()') del self.ha.cluster.config.data['postgresql']['use_slots'] @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_follow_in_pause(self): self.ha.is_paused = true self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock') self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action. I am (postgresql0)') @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)) @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True)) def test_follow_triggers_rewind(self): self.p.is_primary = false self.ha._rewind.trigger_check_diverged_lsn() self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader') def test_no_dcs_connection_primary_demote(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader') self.ha._async_executor.schedule('dummy') self.assertEqual(self.ha.run_cycle(), 'demoted self because DCS is not accessible and I was a leader') def test_check_failsafe_topology(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() global_config.update(self.ha.cluster) self.ha.dcs._last_failsafe = self.ha.cluster.failsafe self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader') self.ha.state_handler.name = self.ha.cluster.leader.name self.assertFalse(self.ha.failsafe_is_active()) self.assertEqual(self.ha.run_cycle(), 'continue to run as a leader because failsafe mode is enabled and all members are accessible') self.assertTrue(self.ha.failsafe_is_active()) with patch.object(Postgresql, 'slots', Mock(side_effect=Exception)): self.ha.patroni.request = Mock(side_effect=Exception) self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader') self.assertFalse(self.ha.failsafe_is_active()) self.ha.dcs._last_failsafe.clear() self.ha.dcs._last_failsafe[self.ha.cluster.leader.name] = self.ha.cluster.leader.member.api_url self.assertEqual(self.ha.run_cycle(), 'continue to run as a leader because failsafe mode is enabled and all members are accessible') def test_no_dcs_connection_primary_failsafe(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() for m in self.ha.cluster.members: if m.name != self.ha.cluster.leader.name: m.data['tags']['replicatefrom'] = 'test' global_config.update(self.ha.cluster) self.ha.dcs._last_failsafe = self.ha.cluster.failsafe self.ha.state_handler.name = self.ha.cluster.leader.name self.assertEqual(self.ha.run_cycle(), 'continue to run as a leader because failsafe mode is enabled and all members are accessible') def test_readonly_dcs_primary_failsafe(self): self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() self.ha.dcs.update_leader = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.dcs._last_failsafe = self.ha.cluster.failsafe self.ha.state_handler.name = self.ha.cluster.leader.name self.assertEqual(self.ha.run_cycle(), 'continue to run as a leader because failsafe mode is enabled and all members are accessible') def test_no_dcs_connection_replica_failsafe(self): self.p.last_operation = Mock(side_effect=PostgresConnectionException('')) self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() global_config.update(self.ha.cluster) self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}}) self.p.is_primary = false with patch('patroni.ha.logger.debug') as mock_logger: self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible') self.assertEqual(mock_logger.call_args_list[0][0][0], 'Failed to fetch current wal lsn: %r') def test_no_dcs_connection_replica_failsafe_not_enabled_but_active(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader() self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}}) self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible') def test_update_failsafe(self): self.assertRaises(Exception, self.ha.update_failsafe, {}) self.p.set_role('primary') self.assertEqual(self.ha.update_failsafe({}), 'Running as a leader') def test_call_failsafe_member(self): member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'}) self.ha.patroni.request = Mock() self.ha.patroni.request.return_value.data = b'Accepted' self.ha.patroni.request.return_value.status = 200 with patch('patroni.ha.logger.info') as mock_logger: ret = self.ha.call_failsafe_member({}, member) self.assertEqual(mock_logger.call_args_list[0][0], ('Got response from %s %s: %s', 'test', 'http://localhost:8011/failsafe', 'Accepted')) self.assertTrue(ret.accepted) e = Exception('request failed') self.ha.patroni.request.side_effect = e with patch('patroni.ha.logger.warning') as mock_logger: ret = self.ha.call_failsafe_member({}, member) self.assertEqual(mock_logger.call_args_list[0][0], ('Request failed to %s: POST %s (%s)', 'test', 'http://localhost:8011/failsafe', e)) self.assertFalse(ret.accepted) @patch('time.sleep', Mock()) def test_bootstrap_from_another_member(self): self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'') def test_bootstrap_waiting_for_leader(self): self.ha.cluster = get_cluster_initialized_without_leader() self.assertEqual(self.ha.bootstrap(), 'waiting for leader to bootstrap') def test_bootstrap_without_leader(self): self.ha.cluster = get_cluster_initialized_without_leader() self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True) self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap (without leader)') def test_bootstrap_not_running_concurrently(self): self.ha.cluster = get_cluster_bootstrapping_without_leader() self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True) self.assertEqual(self.ha.bootstrap(), 'waiting for leader to bootstrap') def test_bootstrap_initialize_lock_failed(self): self.ha.cluster = get_cluster_not_initialized_without_leader() self.assertEqual(self.ha.bootstrap(), 'failed to acquire initialize lock') @patch('patroni.psycopg.connect', psycopg_connect) @patch('patroni.postgresql.mpp.citus.connect', psycopg_connect) @patch('patroni.postgresql.mpp.citus.quote_ident', Mock()) @patch.object(Postgresql, 'connection', Mock(return_value=None)) def test_bootstrap_initialized_new_cluster(self): self.ha.cluster = get_cluster_not_initialized_without_leader() self.e.initialize = true self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap a new cluster') self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap') self.p.is_primary = true self.ha.is_synchronous_mode = true self.assertEqual(self.ha.run_cycle(), 'running post_bootstrap') self.assertEqual(self.ha.run_cycle(), 'initialized a new cluster') def test_bootstrap_release_initialize_key_on_failure(self): self.ha.cluster = get_cluster_not_initialized_without_leader() self.e.initialize = true self.ha.bootstrap() self.p.is_running = false self.assertRaises(PatroniFatalException, self.ha.post_bootstrap) @patch('patroni.psycopg.connect', psycopg_connect) @patch('patroni.postgresql.mpp.citus.connect', psycopg_connect) @patch('patroni.postgresql.mpp.citus.quote_ident', Mock()) @patch.object(Postgresql, 'connection', Mock(return_value=None)) def test_bootstrap_release_initialize_key_on_watchdog_failure(self): self.ha.cluster = get_cluster_not_initialized_without_leader() self.e.initialize = true self.ha.bootstrap() self.p.is_primary = true with patch.object(Watchdog, 'activate', Mock(return_value=False)), \ patch('patroni.ha.logger.error') as mock_logger: self.assertEqual(self.ha.post_bootstrap(), 'running post_bootstrap') self.assertRaises(PatroniFatalException, self.ha.post_bootstrap) self.assertTrue(mock_logger.call_args[0][0].startswith('Cancelling bootstrap because' ' watchdog activation failed')) @patch('patroni.psycopg.connect', psycopg_connect) def test_reinitialize(self): self.assertIsNotNone(self.ha.reinitialize()) self.ha.cluster = get_cluster_initialized_with_leader() self.assertIsNone(self.ha.reinitialize(True)) self.ha._async_executor.schedule('reinitialize') self.assertIsNotNone(self.ha.reinitialize()) self.ha.state_handler.name = self.ha.cluster.leader.name self.assertIsNotNone(self.ha.reinitialize()) @patch('time.sleep', Mock()) def test_restart(self): self.assertEqual(self.ha.restart({}), (True, 'restarted successfully')) self.p.restart = Mock(return_value=None) self.assertEqual(self.ha.restart({}), (False, 'postgres is still starting')) self.p.restart = false self.assertEqual(self.ha.restart({}), (False, 'restart failed')) self.ha.cluster = get_cluster_initialized_with_leader() self.ha._async_executor.schedule('reinitialize') self.assertEqual(self.ha.restart({}), (False, 'reinitialize already in progress')) with patch.object(self.ha, "restart_matches", return_value=False): self.assertEqual(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied")) @patch('time.sleep', Mock()) @patch.object(ConfigHandler, 'replace_pg_hba', Mock()) @patch.object(ConfigHandler, 'replace_pg_ident', Mock()) @patch.object(PostmasterProcess, 'start', Mock(return_value=MockPostmaster())) @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False)) def test_worker_restart(self): self.ha.has_lock = true self.ha.patroni.request = Mock() self.p.is_running = Mock(side_effect=[Mock(), False]) self.assertEqual(self.ha.restart({}), (True, 'restarted successfully')) self.ha.patroni.request.assert_called() self.assertEqual(self.ha.patroni.request.call_args_list[0][0][3]['type'], 'before_demote') self.assertEqual(self.ha.patroni.request.call_args_list[1][0][3]['type'], 'after_promote') @patch('os.kill', Mock()) def test_restart_in_progress(self): with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)): self.ha._async_executor.schedule('restart') self.assertTrue(self.ha.restart_scheduled()) self.assertEqual(self.ha.run_cycle(), 'restart in progress') self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'restart in progress') self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'updated leader lock during restart') self.ha.update_lock = false self.p.set_role('primary') with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)), \ patch('patroni.async_executor.CriticalTask.result', PropertyMock(return_value=PostmasterProcess(os.getpid())), create=True), \ patch('patroni.postgresql.Postgresql.terminate_starting_postmaster') as mock_terminate: self.assertEqual(self.ha.run_cycle(), 'lost leader lock during restart') mock_terminate.assert_called() self.ha.is_paused = true self.assertEqual(self.ha.run_cycle(), 'PAUSE: restart in progress') @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False)) def test_manual_failover_from_leader(self): self.ha.has_lock = true # I am the leader # to me with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None)) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') mock_warning.assert_called_with('%s: I am already the leader, no need to %s', 'manual failover', 'failover') # to a non-existent candidate with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None)) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') mock_warning.assert_called_with( '%s: no healthy members found, %s is not possible', 'manual failover', 'failover') # to an existent candidate self.ha.fetch_node_status = get_node_status() self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'b', None)) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself') # to a candidate on an older timeline with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(timeline=1) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Timeline %s of member %s is behind the cluster timeline %s', 1, 'b', 2)) # to a lagging candidate with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(wal_position=1) self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5}) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'b')) self.ha.cluster.members.pop() @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False)) def test_manual_switchover_from_leader(self): self.ha.has_lock = true # I am the leader self.ha.fetch_node_status = get_node_status() # different leader specified in failover key, no candidate with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None)) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') mock_warning.assert_called_with( '%s: leader name does not match: %s != %s', 'switchover', 'blabla', 'postgresql0') # no candidate self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None)) self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself') self.ha._rewind.rewind_or_reinitialize_needed_and_possible = true self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself') # other members with failover_limitation_s with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(nofailover=True) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not allowed to promote')) with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(watchdog_failed=True) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not watchdog capable')) with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(timeline=1) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Timeline %s of member %s is behind the cluster timeline %s', 1, 'leader', 2)) with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(wal_position=1) self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5}) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader')) @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False)) def test_scheduled_switchover_from_leader(self): self.ha.has_lock = true # I am the leader self.ha.fetch_node_status = get_node_status() # switchover scheduled time must include timezone with patch('patroni.ha.logger.warning') as mock_warning: scheduled = datetime.datetime.now() self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled)) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertIn('Incorrect value of scheduled_at: %s', mock_warning.call_args_list[0][0]) # scheduled now scheduled = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=tzutc) self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled)) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual('switchover: demoting myself', self.ha.run_cycle()) # scheduled in the future with patch('patroni.ha.logger.info') as mock_info: scheduled = scheduled + datetime.timedelta(seconds=30) self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled)) self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle()) self.assertIn('Awaiting %s at %s (in %.0f seconds)', mock_info.call_args_list[0][0]) # stale value with patch('patroni.ha.logger.warning') as mock_warning: scheduled = scheduled + datetime.timedelta(seconds=-600) self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled)) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle()) self.assertIn('Found a stale %s value, cleaning up: %s', mock_warning.call_args_list[0][0]) def test_manual_switchover_from_leader_in_pause(self): self.ha.has_lock = true # I am the leader self.ha.is_paused = true # no candidate self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None)) with patch('patroni.ha.logger.warning') as mock_warning: self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle()) mock_warning.assert_called_with( '%s is possible only to a specific candidate in a paused state', 'Switchover') def test_manual_failover_from_leader_in_pause(self): self.ha.has_lock = true self.ha.fetch_node_status = get_node_status() self.ha.is_paused = true # failover from me, candidate is healthy self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None)) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual('PAUSE: manual failover: demoting myself', self.ha.run_cycle()) self.ha.cluster.members.pop() def test_manual_failover_from_leader_in_synchronous_mode(self): self.ha.is_synchronous_mode = true self.ha.process_sync_replication = Mock() self.ha.fetch_node_status = get_node_status() # I am the leader self.p.is_primary = true self.ha.has_lock = true # the candidate is not in sync members but we allow failover to an async candidate self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None), sync=(self.p.name, 'a')) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual('manual failover: demoting myself', self.ha.run_cycle()) self.ha.cluster.members.pop() def test_manual_switchover_from_leader_in_synchronous_mode(self): self.ha.is_synchronous_mode = true self.ha.process_sync_replication = Mock() # I am the leader self.p.is_primary = true self.ha.has_lock = true # candidate specified is not in sync members with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), sync=(self.p.name, 'blabla')) self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle()) self.assertEqual(mock_warning.call_args_list[0][0], ('%s candidate=%s does not match with sync_standbys=%s', 'Switchover', 'a', 'blabla')) # the candidate is in sync members and is healthy self.ha.fetch_node_status = get_node_status(wal_position=305419896) self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), sync=(self.p.name, 'a')) self.ha.cluster.members.append(Member(0, 'a', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual('switchover: demoting myself', self.ha.run_cycle()) # the candidate is in sync members but is not healthy with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(nofailover=true) self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle()) self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'a', 'not allowed to promote')) def test_manual_failover_process_no_leader(self): self.p.is_primary = false self.p.set_role('replica') # failover to another member, fetch_node_status for candidate fails with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None)) self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(mock_warning.call_args_list[1][0], ('%s: member %s is %s', 'manual failover', 'leader', 'not reachable')) # failover to another member, candidate is accessible, in_recovery self.p.set_role('replica') self.ha.fetch_node_status = get_node_status() self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') # set nofailover flag to True for all members of the cluster # this should elect the current member, as we are not going to call the API for it. self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None)) self.ha.fetch_node_status = get_node_status(nofailover=True) self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # failover to me but I am set to nofailover. In no case I should be elected as a leader self.p.set_role('replica') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None)) self.ha.patroni.nofailover = True self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote') self.ha.patroni.nofailover = False # failover to another member that is on an older timeline (only failover_limitation() is checked) with patch('patroni.ha.logger.info') as mock_info: self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'b', None)) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.ha.fetch_node_status = get_node_status(timeline=1) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0') # failover to another member lagging behind the cluster_lsn (only failover_limitation() is checked) with patch('patroni.ha.logger.info') as mock_info: self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5}) self.ha.fetch_node_status = get_node_status(wal_position=1) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0') def test_manual_switchover_process_no_leader(self): self.p.is_primary = false self.p.set_role('replica') # I was the leader, other members are healthy self.ha.fetch_node_status = get_node_status() self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None)) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') # I was the leader, I am the only healthy member with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not reachable')) self.assertEqual(mock_info.call_args_list[1][0], ('Member %s is %s', 'other', 'not reachable')) def test_manual_failover_process_no_leader_in_synchronous_mode(self): self.ha.is_synchronous_mode = true self.p.is_primary = false self.ha.fetch_node_status = get_node_status(nofailover=True) # other nodes are not healthy # manual failover when our name (postgresql0) isn't in the /sync key and the candidate node is not available self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None), sync=('leader1', 'blabla')) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') # manual failover when the candidate node isn't available but our name is in the /sync key # while other sync node is nofailover with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None), sync=('leader1', 'postgresql0')) self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet())) self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(mock_warning.call_args_list[0][0], ('%s: member %s is %s', 'manual failover', 'other', 'not allowed to promote')) # manual failover to our node (postgresql0), # which name is not in sync nodes list (some sync nodes are available) self.p.set_role('replica') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None), sync=('leader1', 'other')) self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']), CaseInsensitiveSet(['leader1']))) self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') def test_manual_switchover_process_no_leader_in_synchronous_mode(self): self.ha.is_synchronous_mode = true self.p.is_primary = false # to a specific node, which name doesn't match our name (postgresql0) self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'other', None)) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') # to our node (postgresql0), which name is not in sync nodes list self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'postgresql0', None), sync=('leader1', 'blabla')) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') # without candidate, our name (postgresql0) is not in the sync nodes list self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None), sync=('leader', 'blabla')) self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') # switchover from a specific leader, but the only sync node (us, postgresql0) has nofailover tag self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None), sync=('postgresql0')) self.ha.patroni.nofailover = True self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote') def test_manual_failover_process_no_leader_in_pause(self): self.ha.is_paused = true # I am running as primary, cluster is unlocked, the candidate is allowed to promote # but we are in pause self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None)) self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock') def test_manual_switchover_process_no_leader_in_pause(self): self.ha.is_paused = true # I am running as primary, cluster is unlocked, no candidate specified self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None)) self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock') # the candidate is not running with patch('patroni.ha.logger.warning') as mock_warning: self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None)) self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle()) self.assertEqual( mock_warning.call_args_list[0][0], ('%s: removing failover key because failover candidate is not running', 'switchover')) # switchover to me, I am not leader self.p.is_primary = false self.p.set_role('replica') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None)) self.assertEqual(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock') def test_is_healthiest_node(self): self.ha.is_failsafe_mode = true self.ha.state_handler.is_primary = false self.ha.patroni.nofailover = False self.ha.fetch_node_status = get_node_status() self.ha.dcs._last_failsafe = {'foo': ''} self.assertFalse(self.ha.is_healthiest_node()) self.ha.dcs._last_failsafe = {'postgresql0': ''} self.assertTrue(self.ha.is_healthiest_node()) self.ha.dcs._last_failsafe = None with patch.object(Watchdog, 'is_healthy', PropertyMock(return_value=False)): self.assertFalse(self.ha.is_healthiest_node()) self.ha.is_paused = true self.assertFalse(self.ha.is_healthiest_node()) def test__is_healthiest_node(self): self.p.is_primary = false self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name)) global_config.update(self.ha.cluster) self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.fetch_node_status = get_node_status() # accessible, in_recovery self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.fetch_node_status = get_node_status(in_recovery=False) # accessible, not in_recovery self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.fetch_node_status = get_node_status(failover_priority=2) # accessible, in_recovery, higher priority self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) # if there is a higher-priority node but it has a lower WAL position then this node should race self.ha.fetch_node_status = get_node_status(failover_priority=6, wal_position=9) self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.fetch_node_status = get_node_status(wal_position=11) # accessible, in_recovery, wal position ahead self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) # in synchronous_mode consider itself healthy if the former leader is accessible in read-only and ahead of us with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)): self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5}) global_config.update(self.ha.cluster) with patch('patroni.postgresql.Postgresql.last_operation', return_value=1): self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=None): self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=1): self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.patroni.nofailover = True self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.patroni.nofailover = None self.ha.patroni.failover_priority = 0 self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) def test_fetch_node_status(self): member = Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni'}) self.ha.fetch_node_status(member) member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'}) self.ha.patroni.request = Mock() self.ha.patroni.request.return_value.data = b'{"wal":{"location":1},"role":"primary"}' ret = self.ha.fetch_node_status(member) self.assertFalse(ret.in_recovery) @patch.object(Rewind, 'pg_rewind', true) @patch.object(Rewind, 'check_leader_is_not_in_recovery', true) @patch('os.listdir', Mock(return_value=[])) @patch('patroni.postgresql.rewind.fsync_dir', Mock()) @patch.object(Postgresql, 'call_nowait') def test_post_recover(self, mock_call_nowait): self.p.is_running = false self.ha.has_lock = true self.p.set_role('primary') self.assertEqual(self.ha.post_recover(), 'removed leader key after trying and failing to start postgres') self.assertEqual(self.p.role, 'demoted') mock_call_nowait.assert_called_once_with(CallbackAction.ON_ROLE_CHANGE) self.ha.has_lock = false self.assertEqual(self.ha.post_recover(), 'failed to start postgres') leader = Leader(0, 0, Member(0, 'l', 2, {"version": "1.6", "conn_url": "postgres://a", "role": "primary"})) self.ha._rewind.execute(leader) self.p.is_running = true self.assertIsNone(self.ha.post_recover()) def test_schedule_future_restart(self): self.ha.patroni.scheduled_restart = {} # do the restart 2 times. The first one should succeed, the second one should fail self.assertTrue(self.ha.schedule_future_restart({'schedule': future_restart_time})) self.assertFalse(self.ha.schedule_future_restart({'schedule': future_restart_time})) def test_delete_future_restarts(self): self.ha.delete_future_restart() def test_evaluate_scheduled_restart(self): self.p.postmaster_start_time = Mock(return_value=str(postmaster_start_time)) # restart already in progress with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)): self.assertIsNone(self.ha.evaluate_scheduled_restart()) # restart while the postmaster has been already restarted, fails with patch.object(self.ha, 'future_restart_scheduled', Mock(return_value={'postmaster_start_time': str(postmaster_start_time - datetime.timedelta(days=1)), 'schedule': str(future_restart_time)})): self.assertIsNone(self.ha.evaluate_scheduled_restart()) with patch.object(self.ha, 'future_restart_scheduled', Mock(return_value={'postmaster_start_time': str(postmaster_start_time), 'schedule': str(future_restart_time)})): with patch.object(self.ha, 'should_run_scheduled_action', Mock(return_value=True)): # restart in the future, ok self.assertIsNotNone(self.ha.evaluate_scheduled_restart()) with patch.object(self.ha, 'restart', Mock(return_value=(False, "Test"))): # restart in the future, bit the actual restart failed self.assertIsNone(self.ha.evaluate_scheduled_restart()) def test_scheduled_restart(self): self.ha.cluster = get_cluster_initialized_with_leader() with patch.object(self.ha, "evaluate_scheduled_restart", Mock(return_value="restart scheduled")): self.assertEqual(self.ha.run_cycle(), "restart scheduled") def test_restart_matches(self): self.p._role = 'replica' self.p._connection.server_version = 90500 self.p._pending_restart = True self.assertFalse(self.ha.restart_matches("primary", "9.5.0", True)) self.assertFalse(self.ha.restart_matches("replica", "9.4.3", True)) self.p._pending_restart = False self.assertFalse(self.ha.restart_matches("replica", "9.5.2", True)) self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False)) def test_process_healthy_cluster_in_pause(self): self.p.is_primary = false self.ha.is_paused = true self.p.name = 'leader' self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running as primary') self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None)) self.assertEqual(self.ha.run_cycle(), 'PAUSE: waiting to become primary after promote...') @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) @patch('builtins.open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n')) def test_process_healthy_standby_cluster_as_standby_leader(self): self.p.is_primary = false self.p.name = 'leader' self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.p.config.check_recovery_conf = Mock(return_value=(False, False)) self.ha._leader_timeline = 1 self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock') self.assertEqual(self.ha.run_cycle(), 'no action. I am (leader), the standby leader with the lock') self.p.set_role('replica') self.p.config.check_recovery_conf = Mock(return_value=(True, False)) self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock') def test_process_healthy_standby_cluster_as_cascade_replica(self): self.p.is_primary = false self.p.name = 'replica' self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.assertEqual(self.ha.run_cycle(), 'no action. I am (replica), a secondary, and following a standby leader (leader)') with patch.object(Leader, 'conn_url', PropertyMock(return_value='')): self.assertEqual(self.ha.run_cycle(), 'continue following the old known standby leader') @patch.object(Cluster, 'is_unlocked', Mock(return_value=True)) def test_process_unhealthy_standby_cluster_as_standby_leader(self): self.p.is_primary = false self.p.name = 'leader' self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.ha.sysid_valid = true self.p._sysid = True self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader by acquiring session lock') @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)) @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True)) def test_process_unhealthy_standby_cluster_as_cascade_replica(self): self.p.is_primary = false self.p.name = 'replica' self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.assertTrue(self.ha.run_cycle().startswith('running pg_rewind from remote_member:')) def test_recover_unhealthy_leader_in_standby_cluster(self): self.p.is_primary = false self.p.name = 'leader' self.p.is_running = false self.p.follow = false self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.assertEqual(self.ha.run_cycle(), 'starting as a standby leader because i had the session lock') @patch.object(Cluster, 'is_unlocked', Mock(return_value=True)) def test_recover_unhealthy_unlocked_standby_cluster(self): self.p.is_primary = false self.p.name = 'leader' self.p.is_running = false self.p.follow = false self.ha.cluster = get_standby_cluster_initialized_with_only_leader() self.ha.has_lock = false self.assertEqual(self.ha.run_cycle(), 'trying to follow a remote member because standby cluster is unhealthy') def test_failed_to_update_lock_in_pause(self): self.ha.update_lock = false self.ha.is_paused = true self.p.name = 'leader' self.ha.cluster = get_cluster_initialized_with_leader() self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary after failing to update leader lock in DCS') def test_postgres_unhealthy_in_pause(self): self.ha.is_paused = true self.p.is_healthy = false self.assertEqual(self.ha.run_cycle(), 'PAUSE: postgres is not running') self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running') def test_no_etcd_connection_in_pause(self): self.ha.is_paused = true self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.assertEqual(self.ha.run_cycle(), 'PAUSE: DCS is not accessible') @patch('patroni.ha.Ha.update_lock', return_value=True) @patch('patroni.ha.Ha.demote') def test_starting_timeout(self, demote, update_lock): def check_calls(seq): for mock, called in seq: if called: mock.assert_called_once() else: mock.assert_not_called() mock.reset_mock() self.ha.has_lock = true self.ha.cluster = get_cluster_initialized_with_leader() self.p.check_for_startup = true self.p.time_in_state = lambda: 30 self.assertEqual(self.ha.run_cycle(), 'PostgreSQL is still starting up, 270 seconds until timeout') check_calls([(update_lock, True), (demote, False)]) self.p.time_in_state = lambda: 350 self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery self.assertEqual(self.ha.run_cycle(), 'primary start has timed out, but continuing to wait because failover is not possible') check_calls([(update_lock, True), (demote, False)]) self.ha.fetch_node_status = get_node_status() # accessible, in_recovery self.assertEqual(self.ha.run_cycle(), 'stopped PostgreSQL because of startup timeout') check_calls([(update_lock, True), (demote, True)]) update_lock.return_value = False self.assertEqual(self.ha.run_cycle(), 'stopped PostgreSQL while starting up because leader key was lost') check_calls([(update_lock, True), (demote, True)]) self.ha.has_lock = false self.p.is_primary = false self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader (leader)') check_calls([(update_lock, False), (demote, False)]) def test_manual_failover_while_starting(self): self.ha.has_lock = true self.p.check_for_startup = true f = Failover(0, self.p.name, '', None) self.ha.cluster = get_cluster_initialized_with_leader(f) self.ha.fetch_node_status = get_node_status() # accessible, in_recovery self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself') @patch('patroni.ha.Ha.demote') def test_failover_immediately_on_zero_primary_start_timeout(self, demote): self.p.is_running = false self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other')) self.ha.cluster.config.data.update({'synchronous_mode': True, 'primary_start_timeout': 0}) self.ha.has_lock = true self.ha.update_lock = true self.ha.fetch_node_status = get_node_status() # accessible, in_recovery self.assertEqual(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash') demote.assert_called_once() def test_primary_stop_timeout(self): self.assertEqual(self.ha.primary_stop_timeout(), None) self.ha.cluster.config.data.update({'primary_stop_timeout': 30}) global_config.update(self.ha.cluster) with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)): self.assertEqual(self.ha.primary_stop_timeout(), 30) with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=False)): self.assertEqual(self.ha.primary_stop_timeout(), None) self.ha.cluster.config.data['primary_stop_timeout'] = None global_config.update(self.ha.cluster) self.assertEqual(self.ha.primary_stop_timeout(), None) @patch('patroni.postgresql.Postgresql.follow') def test_demote_immediate(self, follow): self.ha.has_lock = true self.e.get_cluster = Mock(return_value=get_cluster_initialized_without_leader()) self.ha.demote('immediate') follow.assert_called_once_with(None) def test__process_multisync_replication(self): self.ha.has_lock = true mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock() mock_cfg_set_sync = self.p.config.set_synchronous_standby_names = Mock() self.p.name = 'leader' # Test sync key removed when sync mode disabled self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync: self.ha.run_cycle() mock_delete_sync.assert_called_once() mock_set_sync.assert_called_once_with(CaseInsensitiveSet()) mock_cfg_set_sync.assert_called_once() mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() # Test sync key not touched when not there self.ha.cluster = get_cluster_initialized_with_leader() with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync: self.ha.run_cycle() mock_delete_sync.assert_not_called() mock_set_sync.assert_not_called() mock_cfg_set_sync.assert_called_once() mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() self.ha.is_synchronous_mode = true # Test sync standby not touched when picking the same node self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1, CaseInsensitiveSet(['other']), CaseInsensitiveSet(['other']))) self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) self.ha.run_cycle() mock_set_sync.assert_not_called() mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() # Test sync standby is replaced when switching standbys self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(), CaseInsensitiveSet(['other2']))) self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.ha.run_cycle() mock_set_sync.assert_called_once_with(CaseInsensitiveSet(['other2'])) mock_cfg_set_sync.assert_not_called() # Test sync standby is replaced when new standby is joined self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1, CaseInsensitiveSet(['other2']), CaseInsensitiveSet(['other2', 'other3']))) self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.ha.run_cycle() self.assertEqual(mock_set_sync.call_args_list[0][0], (CaseInsensitiveSet(['other2']),)) self.assertEqual(mock_set_sync.call_args_list[1][0], (CaseInsensitiveSet(['other2', 'other3']),)) mock_cfg_set_sync.assert_not_called() mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() # Test sync standby is not disabled when updating dcs fails self.ha.dcs.write_sync_state = Mock(return_value=None) self.ha.run_cycle() mock_set_sync.assert_not_called() mock_cfg_set_sync.assert_not_called() mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() # Test changing sync standby self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader', 'other'))) # self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1, CaseInsensitiveSet(['other2']), CaseInsensitiveSet(['other2']))) self.ha.run_cycle() self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2) # Test updating sync standby key failed due to race self.ha.dcs.write_sync_state = Mock(side_effect=[SyncState.empty(), None]) self.ha.run_cycle() self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2) # Test updating sync standby key failed due to DCS being not accessible self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.ha.dcs.get_cluster = Mock(side_effect=DCSError('foo')) self.ha.run_cycle() # Test changing sync standby failed due to race self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('somebodyelse', None))) self.ha.run_cycle() self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2) # Test sync set to '*' when synchronous_mode_strict is enabled mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(), CaseInsensitiveSet())) with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)): self.ha.run_cycle() mock_set_sync.assert_called_once_with(CaseInsensitiveSet('*')) mock_cfg_set_sync.assert_not_called() # Test the value configured by the user for synchronous_standby_names is used when synchronous mode is disabled self.ha.is_synchronous_mode = false mock_set_sync.reset_mock() mock_cfg_set_sync.reset_mock() ssn_mock = PropertyMock(return_value="SOME_SSN") with patch('patroni.postgresql.config.ConfigHandler.synchronous_standby_names', ssn_mock): self.ha.run_cycle() mock_set_sync.assert_not_called() mock_cfg_set_sync.assert_called_once_with("SOME_SSN") def test_sync_replication_become_primary(self): self.ha.is_synchronous_mode = true mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock() self.p.is_primary = false self.p.set_role('replica') self.ha.has_lock = true mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) self.p.name = 'leader' self.ha.cluster = get_cluster_initialized_with_leader(sync=('other', None)) # When we just became primary nobody is sync self.assertEqual(self.ha.enforce_primary_role('msg', 'promote msg'), 'promote msg') mock_set_sync.assert_called_once_with(CaseInsensitiveSet(), 0) mock_write_sync.assert_called_once_with('leader', None, 0, version=0) mock_set_sync.reset_mock() # When we just became primary nobody is sync self.p.set_role('replica') mock_write_sync.return_value = False self.assertTrue(self.ha.enforce_primary_role('msg', 'promote msg') != 'promote msg') mock_set_sync.assert_not_called() def test_unhealthy_sync_mode(self): self.ha.is_synchronous_mode = true self.p.is_primary = false self.p.set_role('replica') self.p.name = 'other' self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2')) mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) mock_acquire = self.ha.acquire_lock = Mock(return_value=True) mock_follow = self.p.follow = Mock() mock_promote = self.p.promote = Mock() # If we don't match the sync replica we are not allowed to acquire lock self.ha.run_cycle() mock_acquire.assert_not_called() mock_follow.assert_called_once() self.assertEqual(mock_follow.call_args[0][0], None) mock_write_sync.assert_not_called() mock_follow.reset_mock() # If we do match we will try to promote self.ha._is_healthiest_node = true self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other')) self.ha.run_cycle() mock_acquire.assert_called_once() mock_follow.assert_not_called() mock_promote.assert_called_once() mock_write_sync.assert_called_once_with('other', None, 0, version=0) def test_disable_sync_when_restarting(self): self.ha.is_synchronous_mode = true self.p.name = 'other' self.p.is_primary = false self.p.set_role('replica') mock_restart = self.p.restart = Mock(return_value=True) self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) self.ha.touch_member = Mock(return_value=True) self.ha.dcs.get_cluster = Mock(side_effect=[ get_cluster_initialized_with_leader(sync=('leader', syncstandby)) for syncstandby in ['other', None]]) with patch('time.sleep') as mock_sleep: self.ha.restart({}) mock_restart.assert_called_once() mock_sleep.assert_called() # Restart is still called when DCS connection fails mock_restart.reset_mock() self.ha.dcs.get_cluster = Mock(side_effect=DCSError("foo")) self.ha.restart({}) mock_restart.assert_called_once() # We don't try to fetch the cluster state when touch_member fails mock_restart.reset_mock() self.ha.dcs.get_cluster.reset_mock() self.ha.touch_member = Mock(return_value=False) self.ha.restart({}) mock_restart.assert_called_once() self.ha.dcs.get_cluster.assert_not_called() @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_enable_synchronous_mode(self): self.ha.is_synchronous_mode = true self.ha.has_lock = true self.p.name = 'leader' self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(), CaseInsensitiveSet())) self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) with patch('patroni.ha.logger.info') as mock_logger: self.ha.run_cycle() self.assertEqual(mock_logger.call_args_list[0][0][0], 'Enabled synchronous replication') self.ha.dcs.write_sync_state = Mock(return_value=None) with patch('patroni.ha.logger.warning') as mock_logger: self.ha.run_cycle() self.assertEqual(mock_logger.call_args[0][0], 'Updating sync state failed') @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_inconsistent_synchronous_state(self): self.ha.is_synchronous_mode = true self.ha.has_lock = true self.p.name = 'leader' self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'a')) self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(), CaseInsensitiveSet('a'))) self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty()) mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock() with patch('patroni.ha.logger.warning') as mock_logger: self.ha.run_cycle() mock_set_sync.assert_called_once() self.assertTrue(mock_logger.call_args_list[0][0][0].startswith('Inconsistent state between ')) self.ha.dcs.write_sync_state = Mock(return_value=None) with patch('patroni.ha.logger.warning') as mock_logger: self.ha.run_cycle() self.assertEqual(mock_logger.call_args[0][0], 'Updating sync state failed') def test_effective_tags(self): self.ha._disable_sync = True self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True}) self.ha._disable_sync = False self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar'}) @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) @patch('builtins.open', Mock(side_effect=Exception)) def test_restore_cluster_config(self): self.ha.cluster.config.data.clear() self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') def test_watch(self): self.ha.cluster = get_cluster_initialized_with_leader() self.ha.watch(0) def test_wakeup(self): self.ha.wakeup() def test_shutdown(self): self.p.is_running = false self.ha.is_leader = true def stop(*args, **kwargs): kwargs['on_shutdown'](123, 120) self.p.stop = stop self.ha.shutdown() self.ha.is_failover_possible = true self.ha.shutdown() @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False)) def test_shutdown_citus_worker(self): self.ha.is_leader = true self.p.is_running = Mock(side_effect=[Mock(), False]) self.ha.patroni.request = Mock() self.ha.shutdown() self.ha.patroni.request.assert_called() self.assertEqual(self.ha.patroni.request.call_args[0][2], 'citus') self.assertEqual(self.ha.patroni.request.call_args[0][3]['type'], 'before_demote') @patch('time.sleep', Mock()) def test_leader_with_not_accessible_data_directory(self): self.ha.cluster = get_cluster_initialized_with_leader() self.ha.has_lock = true self.p.data_directory_empty = Mock(side_effect=OSError(5, "Input/output error: '{}'".format(self.p.data_dir))) self.assertEqual(self.ha.run_cycle(), 'released leader key voluntarily as data dir not accessible and currently leader') self.assertEqual(self.p.role, 'uninitialized') # as has_lock is mocked out, we need to fake the leader key release self.ha.has_lock = false # will not say bootstrap because data directory is not accessible self.assertEqual(self.ha.run_cycle(), "data directory is not accessible: [Errno 5] Input/output error: '{}'".format(self.p.data_dir)) @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) @patch('builtins.open', mock_open(read_data=('1\t0/40159C0\tno recovery target specified\n\n' '2\t1/40159C0\tno recovery target specified\n'))) @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_update_cluster_history(self): self.ha.has_lock = true for tl in (1, 3): self.p.get_primary_timeline = Mock(return_value=tl) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') @patch('sys.exit', return_value=1) def test_abort_join(self, exit_mock): self.ha.cluster = get_cluster_not_initialized_without_leader() self.p.is_primary = false self.ha.run_cycle() exit_mock.assert_called_once_with(1) self.p.set_role('replica') self.ha.dcs.initialize = Mock() with patch.object(Postgresql, 'cb_called', PropertyMock(return_value=True)): self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.dcs.initialize.assert_not_called() @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_after_pause(self): self.ha.has_lock = true self.ha.is_paused = true self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action. I am (postgresql0), the leader with the lock') self.ha.is_paused = false self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') @patch('patroni.psycopg.connect', psycopg_connect) def test_permanent_logical_slots_after_promote(self): self.p._major_version = 110000 config = ClusterConfig(1, {'slots': {'l': {'database': 'postgres', 'plugin': 'test_decoding'}}}, 1) self.p.name = 'other' self.ha.cluster = get_cluster_initialized_without_leader(cluster_config=config) self.assertEqual(self.ha.run_cycle(), 'acquired session lock as a leader') self.ha.cluster = get_cluster_initialized_without_leader(leader=True, cluster_config=config) self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'no action. I am (other), the leader with the lock') @patch.object(Cluster, 'has_member', true) def test_run_cycle(self): self.ha.dcs.touch_member = Mock(side_effect=DCSError('foo')) self.assertEqual(self.ha.run_cycle(), 'Unexpected exception raised, please report it as a BUG') self.ha.dcs.touch_member = Mock(side_effect=PatroniFatalException('foo')) self.assertRaises(PatroniFatalException, self.ha.run_cycle) def test_empty_directory_in_pause(self): self.ha.is_paused = true self.p.data_directory_empty = true self.assertEqual(self.ha.run_cycle(), 'PAUSE: running with empty data directory') self.assertEqual(self.p.role, 'uninitialized') @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True)) def test_sysid_no_match_in_pause(self): self.ha.is_paused = true self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': '123'} self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock') self.ha.has_lock = true self.assertEqual(self.ha.run_cycle(), 'PAUSE: released leader key voluntarily due to the system ID mismatch') @patch('patroni.psycopg.connect', psycopg_connect) @patch('os.path.exists', Mock(return_value=True)) @patch('shutil.rmtree', Mock()) @patch('os.makedirs', Mock()) @patch('os.open', Mock()) @patch('os.fsync', Mock()) @patch('os.close', Mock()) @patch('os.chmod', Mock()) @patch('os.rename', Mock()) @patch('patroni.postgresql.Postgresql.is_starting', Mock(return_value=False)) @patch('builtins.open', mock_open()) @patch.object(ConfigHandler, 'check_recovery_conf', Mock(return_value=(False, False))) @patch.object(Postgresql, 'major_version', PropertyMock(return_value=130000)) @patch.object(SlotsHandler, 'sync_replication_slots', Mock(return_value=['ls'])) def test_follow_copy(self): self.ha.cluster.config.data['slots'] = {'ls': {'database': 'a', 'plugin': 'b'}} self.p.is_primary = false self.assertTrue(self.ha.run_cycle().startswith('Copying logical slots')) def test_acquire_lock(self): self.ha.dcs.attempt_to_acquire_leader = Mock(side_effect=[DCSError('foo'), Exception]) self.assertRaises(DCSError, self.ha.acquire_lock) self.assertFalse(self.ha.acquire_lock()) @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False)) def test_notify_citus_coordinator(self): self.ha.patroni.request = Mock() self.ha.notify_mpp_coordinator('before_demote') self.ha.patroni.request.assert_called_once() self.assertEqual(self.ha.patroni.request.call_args[1]['timeout'], 30) self.ha.patroni.request = Mock(side_effect=Exception) with patch('patroni.ha.logger.warning') as mock_logger: self.ha.notify_mpp_coordinator('before_promote') self.assertEqual(self.ha.patroni.request.call_args[1]['timeout'], 2) mock_logger.assert_called() self.assertTrue(mock_logger.call_args[0][0].startswith('Request to %s coordinator leader')) self.assertEqual(mock_logger.call_args[0][1], 'Citus') @patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)) @patch.object(global_config.__class__, 'is_quorum_commit_mode', PropertyMock(return_value=True)) def test_process_sync_replication_prepromote(self): self.p._major_version = 90500 self.ha.cluster = get_cluster_initialized_without_leader(sync=('other', self.p.name + ',foo')) self.p.is_primary = false self.p.set_role('replica') mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=None) # Postgres 9.5, write_sync_state to DCS failed self.assertEqual(self.ha.run_cycle(), 'Postponing promotion because synchronous replication state was updated by somebody else') self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1) self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0)) self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0}) mock_set_sync = self.p.config.set_synchronous_standby_names = Mock() mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True) # Postgres 9.5, our name is written to leader of the /sync key, while voters list and ssn is empty self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1) self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0)) self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0}) self.assertEqual(mock_set_sync.call_count, 1) self.assertEqual(mock_set_sync.call_args_list[0][0], (None,)) self.p._major_version = 90600 mock_set_sync.reset_mock() mock_write_sync.reset_mock() self.p.set_role('replica') # Postgres 9.6, with quorum commit we avoid updating /sync key and put some nodes to ssn self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(mock_write_sync.call_count, 0) self.assertEqual(mock_set_sync.call_count, 1) self.assertEqual(mock_set_sync.call_args_list[0][0], ('2 (foo,other)',)) self.p._major_version = 150000 mock_set_sync.reset_mock() self.p.set_role('replica') self.p.name = 'nonsync' self.ha.fetch_node_status = get_node_status() # Postgres 15, with quorum commit. Non-sync node promoted we avoid updating /sync key and put some nodes to ssn self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.assertEqual(mock_write_sync.call_count, 0) self.assertEqual(mock_set_sync.call_count, 1) self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 3 (foo,other,postgresql0)',)) @patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)) @patch.object(global_config.__class__, 'is_quorum_commit_mode', PropertyMock(return_value=True)) def test__process_quorum_replication(self): self.p._major_version = 150000 self.ha.has_lock = true mock_set_sync = self.p.config.set_synchronous_standby_names = Mock() self.p.name = 'leader' mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=None) # Test /sync key is attempted to set and failed when missing or invalid self.p.sync_handler.current_state = Mock(return_value=_SyncState('quorum', 1, 1, CaseInsensitiveSet(['other']), CaseInsensitiveSet(['other']))) self.ha.run_cycle() self.assertEqual(mock_write_sync.call_count, 1) self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0)) self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None}) self.assertEqual(mock_set_sync.call_count, 0) self.ha._promote_timestamp = 1 mock_write_sync = self.ha.dcs.write_sync_state = Mock(side_effect=[SyncState(None, self.p.name, None, 0), None]) # Test /sync key is attempted to set and succeed when missing or invalid with patch.object(SyncState, 'is_empty', Mock(side_effect=[True, False])): self.ha.run_cycle() self.assertEqual(mock_write_sync.call_count, 2) self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0)) self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None}) self.assertEqual(mock_write_sync.call_args_list[1][0], (self.p.name, CaseInsensitiveSet(['other']), 0)) self.assertEqual(mock_write_sync.call_args_list[1][1], {'version': None}) self.assertEqual(mock_set_sync.call_count, 0) self.p.sync_handler.current_state = Mock(side_effect=[_SyncState('quorum', 1, 0, CaseInsensitiveSet(['foo']), CaseInsensitiveSet(['other'])), _SyncState('quorum', 1, 1, CaseInsensitiveSet(['foo']), CaseInsensitiveSet(['foo']))]) mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState(1, 'leader', 'foo', 0)) self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'foo')) # Test the sync node is removed from voters, added to ssn with patch.object(Postgresql, 'synchronous_standby_names', Mock(return_value='other')), \ patch('time.sleep', Mock()): self.ha.run_cycle() self.assertEqual(mock_write_sync.call_count, 1) self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, CaseInsensitiveSet(), 0)) self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0}) self.assertEqual(mock_set_sync.call_count, 1) self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 1 (other)',)) # Test ANY 1 (*) when synchronous_mode_strict and no nodes available self.p.sync_handler.current_state = Mock(return_value=_SyncState('quorum', 1, 0, CaseInsensitiveSet(['other', 'foo']), CaseInsensitiveSet())) mock_write_sync.reset_mock() mock_set_sync.reset_mock() with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)): self.ha.run_cycle() self.assertEqual(mock_write_sync.call_count, 1) self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, CaseInsensitiveSet(), 0)) self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0}) self.assertEqual(mock_set_sync.call_count, 1) self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 1 (*)',)) # Test that _process_quorum_replication doesn't take longer than loop_wait with patch('time.time', Mock(side_effect=[30, 60, 90, 120])): self.ha.process_sync_replication() patroni-4.0.4/tests/test_kubernetes.py000066400000000000000000000706221472010352700201250ustar00rootroot00000000000000import base64 import datetime import json import socket import time import unittest from threading import Thread from unittest import mock from unittest.mock import Mock, mock_open, patch, PropertyMock import urllib3 from patroni.dcs import get_dcs from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed, \ K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException, Retry, \ RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME from patroni.postgresql.mpp import get_mpp from . import MockResponse, SleepException def mock_list_namespaced_config_map(*args, **kwargs): k8s_group_label = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}).k8s_group_label metadata = {'resource_version': '1', 'labels': {'f': 'b'}, 'name': 'test-config', 'annotations': {'initialize': '123', 'config': '{}'}} items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))] metadata.update({'name': 'test-leader', 'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'retain_slots': '{', 'failsafe': '{'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-0-leader', 'labels': {k8s_group_label: '0'}, 'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'retain_slots': '{', 'failsafe': '{'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-0-config', 'labels': {k8s_group_label: '0'}, 'annotations': {'initialize': '123', 'config': '{}'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-1-leader', 'labels': {k8s_group_label: '1'}, 'annotations': {'leader': 'p-3', 'ttl': '30s'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-2-config', 'labels': {k8s_group_label: '2'}, 'annotations': {}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata = k8s_client.V1ObjectMeta(resource_version='1') return k8s_client.V1ConfigMapList(metadata=metadata, items=items, kind='ConfigMapList') def mock_read_namespaced_endpoints(*args, **kwargs): target_ref = k8s_client.V1ObjectReference(kind='Pod', resource_version='10', name='p-0', namespace='default', uid='964dfeae-e79b-4476-8a5a-1920b5c2a69d') address0 = k8s_client.V1EndpointAddress(ip='10.0.0.0', target_ref=target_ref) address1 = k8s_client.V1EndpointAddress(ip='10.0.0.1') port = k8s_client.V1EndpointPort(port=5432, name='postgresql', protocol='TCP') subset = k8s_client.V1EndpointSubset(addresses=[address1, address0], ports=[port]) metadata = k8s_client.V1ObjectMeta(resource_version='1', labels={'f': 'b'}, name='test', annotations={'optime': '1234', 'leader': 'p-0', 'ttl': '30s'}) return k8s_client.V1Endpoints(subsets=[subset], metadata=metadata) def mock_list_namespaced_endpoints(*args, **kwargs): return k8s_client.V1EndpointsList(metadata=k8s_client.V1ObjectMeta(resource_version='1'), items=[mock_read_namespaced_endpoints()], kind='V1EndpointsList') def mock_list_namespaced_pod(*args, **kwargs): k8s_group_label = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}).k8s_group_label metadata = k8s_client.V1ObjectMeta(resource_version='1', labels={'f': 'b', k8s_group_label: '1'}, name='p-0', annotations={'status': '{}'}, uid='964dfeae-e79b-4476-8a5a-1920b5c2a69d') status = k8s_client.V1PodStatus(pod_ip='10.0.0.1') spec = k8s_client.V1PodSpec(hostname='p-0', node_name='kind-control-plane', containers=[]) items = [k8s_client.V1Pod(metadata=metadata, status=status, spec=spec)] return k8s_client.V1PodList(items=items, kind='PodList') def mock_namespaced_kind(*args, **kwargs): mock = Mock() mock.metadata.resource_version = '2' return mock def mock_load_k8s_config(self, *args, **kwargs): self._server = 'http://localhost' class TestK8sConfig(unittest.TestCase): def test_load_incluster_config(self): for env in ({}, {SERVICE_HOST_ENV_NAME: '', SERVICE_PORT_ENV_NAME: ''}): with patch('os.environ', env): self.assertRaises(k8s_config.ConfigException, k8s_config.load_incluster_config) with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}), \ patch('os.path.isfile', Mock(side_effect=[False, True, True, False, True, True, True, True])), \ patch('builtins.open', Mock(side_effect=[ mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')(), mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')()])): for _ in range(0, 4): self.assertRaises(k8s_config.ConfigException, k8s_config.load_incluster_config) k8s_config.load_incluster_config() self.assertEqual(k8s_config.server, 'https://a:1') self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer a') def test_refresh_token(self): with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}), \ patch('os.path.isfile', Mock(side_effect=[True, True, False, True, True, True])), \ patch('builtins.open', Mock(side_effect=[ mock_open(read_data='cert')(), mock_open(read_data='a')(), mock_open()(), mock_open(read_data='b')(), mock_open(read_data='c')()])): k8s_config.load_incluster_config(token_refresh_interval=datetime.timedelta(milliseconds=100)) self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer a') time.sleep(0.1) # token file doesn't exist self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer a') # token file is empty self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer a') # token refreshed self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer b') time.sleep(0.1) # token refreshed self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer c') # no need to refresh token self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer c') def test_load_kube_config(self): config = { "current-context": "local", "contexts": [{"name": "local", "context": {"user": "local", "cluster": "local"}}], "clusters": [{"name": "local", "cluster": {"server": "https://a:1/", "certificate-authority": "a"}}], "users": [{"name": "local", "user": {"username": "a", "password": "b", "client-certificate": "c"}}] } with patch('builtins.open', mock_open(read_data=json.dumps(config))): k8s_config.load_kube_config() self.assertEqual(k8s_config.server, 'https://a:1') self.assertEqual(k8s_config.pool_config, {'ca_certs': 'a', 'cert_file': 'c', 'cert_reqs': 'CERT_REQUIRED', 'maxsize': 10, 'num_pools': 10}) config["users"][0]["user"]["token"] = "token" with patch('builtins.open', mock_open(read_data=json.dumps(config))): k8s_config.load_kube_config() self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token') config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8') config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8') with patch('builtins.open', mock_open(read_data=json.dumps(config))), \ patch('os.write', Mock()), patch('os.close', Mock()), \ patch('os.remove') as mock_remove, \ patch('atexit.register') as mock_atexit, \ patch('tempfile.mkstemp') as mock_mkstemp: mock_mkstemp.side_effect = [(3, '1.tmp'), (4, '2.tmp')] k8s_config.load_kube_config() mock_atexit.assert_called_once() mock_remove.side_effect = OSError mock_atexit.call_args[0][0]() # call _cleanup_temp_files mock_remove.assert_has_calls([mock.call('1.tmp'), mock.call('2.tmp')]) @patch('urllib3.PoolManager.request') class TestApiClient(unittest.TestCase): @patch.object(K8sConfig, '_server', '', create=True) @patch('urllib3.PoolManager.request', Mock()) def setUp(self): self.a = k8s_client.ApiClient(True) self.mock_get_ep = MockResponse() self.mock_get_ep.content = '{"subsets":[{"ports":[{"name":"https","protocol":"TCP","port":443}],' +\ '"addresses":[{"ip":"127.0.0.1"},{"ip":"127.0.0.2"}]}]}' def test__do_http_request(self, mock_request): mock_request.side_effect = [self.mock_get_ep] + [socket.timeout] self.assertRaises(K8sException, self.a.call_api, 'GET', 'f') @patch('time.sleep', Mock()) def test_request(self, mock_request): retry = Retry(deadline=10, max_delay=1, max_tries=1, retry_exceptions=KubernetesRetriableException) mock_request.side_effect = [self.mock_get_ep] + 3 * [socket.timeout] + [k8s_client.rest.ApiException(500, '')] self.assertRaises(k8s_client.rest.ApiException, retry, self.a.call_api, 'GET', 'f', _retry=retry) mock_request.side_effect = [self.mock_get_ep, socket.timeout, Mock(), self.mock_get_ep] self.assertRaises(k8s_client.rest.ApiException, retry, self.a.call_api, 'GET', 'f', _retry=retry) retry.deadline = 0.0001 mock_request.side_effect = [socket.timeout, socket.timeout, self.mock_get_ep] self.assertRaises(K8sConnectionFailed, retry, self.a.call_api, 'GET', 'f', _retry=retry) def test__refresh_api_servers_cache(self, mock_request): mock_request.side_effect = k8s_client.rest.ApiException(403, '') self.a.refresh_api_servers_cache() class TestCoreV1Api(unittest.TestCase): @patch('urllib3.PoolManager.request', Mock()) @patch.object(K8sConfig, '_server', '', create=True) def setUp(self): self.a = k8s_client.CoreV1Api() self.a._api_client.pool_manager.request = Mock(return_value=MockResponse()) def test_create_namespaced_service(self): self.assertEqual(str(self.a.create_namespaced_service('default', {}, _request_timeout=2)), '{}') def test_list_namespaced_endpoints(self): self.a._api_client.pool_manager.request.return_value.content = '{"items": [1,2,3]}' self.assertIsInstance(self.a.list_namespaced_endpoints('default'), K8sObject) def test_patch_namespaced_config_map(self): self.assertEqual(str(self.a.patch_namespaced_config_map('foo', 'default', {}, _request_timeout=(1, 2))), '{}') def test_list_namespaced_pod(self): self.a._api_client.pool_manager.request.return_value.status_code = 409 self.a._api_client.pool_manager.request.return_value.content = 'foo' try: self.a.list_namespaced_pod('default', label_selector='foo=bar') self.assertFail() except k8s_client.rest.ApiException as e: self.assertTrue('Reason: ' in str(e)) def test_delete_namespaced_pod(self): self.assertEqual(str(self.a.delete_namespaced_pod('foo', 'default', _request_timeout=(1, 2), body={})), '{}') class BaseTestKubernetes(unittest.TestCase): @patch('urllib3.PoolManager.request', Mock()) @patch('socket.TCP_KEEPIDLE', 4, create=True) @patch('socket.TCP_KEEPINTVL', 5, create=True) @patch('socket.TCP_KEEPCNT', 6, create=True) @patch.object(Thread, 'start', Mock()) @patch.object(K8sConfig, 'load_kube_config', mock_load_k8s_config) @patch.object(K8sConfig, 'load_incluster_config', Mock(side_effect=k8s_config.ConfigException)) @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod, create=True) @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map, create=True) def setUp(self, config=None): config = {'ttl': 30, 'scope': 'test', 'name': 'p-0', 'loop_wait': 10, 'retry_timeout': 10, 'kubernetes': {'labels': {'f': 'b'}, 'bypass_api_service': True, **(config or {})}, 'citus': {'group': 0, 'database': 'postgres'}} self.k = get_dcs(config) self.assertIsInstance(self.k, Kubernetes) self.k._mpp = get_mpp({}) self.assertRaises(AttributeError, self.k._pods._build_cache) self.k._pods._is_ready = True self.assertRaises(TypeError, self.k._kinds._build_cache) self.k._kinds._is_ready = True self.k.get_cluster() @patch('urllib3.PoolManager.request', Mock()) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', mock_namespaced_kind, create=True) class TestKubernetesConfigMaps(BaseTestKubernetes): @patch('time.time', Mock(side_effect=[1, 10.9, 100])) def test__wait_caches(self): self.k._pods._is_ready = False with self.k._condition: self.assertRaises(RetryFailedError, self.k._wait_caches, time.time() + 10) @patch('time.time', Mock(return_value=time.time() + 100)) def test_get_cluster(self): self.k.get_cluster() with patch.object(Kubernetes, '_wait_caches', Mock(side_effect=Exception)): self.assertRaises(KubernetesError, self.k.get_cluster) def test__get_citus_cluster(self): self.k._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) cluster = self.k.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsInstance(cluster.workers[1], Cluster) @patch('patroni.dcs.kubernetes.logger.error') def test_get_mpp_coordinator(self, mock_logger): self.assertIsInstance(self.k.get_mpp_coordinator(), Cluster) with patch.object(Kubernetes, '_postgresql_cluster_loader', Mock(side_effect=Exception)): self.assertIsNone(self.k.get_mpp_coordinator()) mock_logger.assert_called() self.assertEqual(mock_logger.call_args[0][0], 'Failed to load %s coordinator cluster from Kubernetes: %r') self.assertEqual(mock_logger.call_args[0][1], 'Null') self.assertIsInstance(mock_logger.call_args[0][2], KubernetesError) @patch('patroni.dcs.kubernetes.logger.error') def test_get_citus_coordinator(self, mock_logger): self.k._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) self.assertIsInstance(self.k.get_mpp_coordinator(), Cluster) with patch.object(Kubernetes, '_postgresql_cluster_loader', Mock(side_effect=Exception)): self.assertIsNone(self.k.get_mpp_coordinator()) mock_logger.assert_called() self.assertEqual(mock_logger.call_args[0][0], 'Failed to load %s coordinator cluster from Kubernetes: %r') self.assertEqual(mock_logger.call_args[0][1], 'Citus') self.assertIsInstance(mock_logger.call_args[0][2], KubernetesError) def test_attempt_to_acquire_leader(self): with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', create=True) as mock_patch: mock_patch.side_effect = K8sException self.assertRaises(KubernetesError, self.k.attempt_to_acquire_leader) mock_patch.side_effect = k8s_client.rest.ApiException(409, '') self.assertFalse(self.k.attempt_to_acquire_leader()) def test_take_leader(self): self.k.take_leader() self.k._leader_observed_record['leader'] = 'test' self.k.patch_or_create = Mock(return_value=False) self.k.take_leader() def test_manual_failover(self): with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock(side_effect=RetryFailedError('')), create=True): self.k.manual_failover('foo', 'bar') def test_set_config_value(self): with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock(side_effect=k8s_client.rest.ApiException(409, '')), create=True): self.k.set_config_value('{}', 1) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', create=True) def test_touch_member(self, mock_patch_namespaced_pod): mock_patch_namespaced_pod.return_value.metadata.resource_version = '10' self.k.touch_member({'role': 'replica'}) self.k._name = 'p-1' self.k.touch_member({'state': 'running', 'role': 'replica'}) self.k.touch_member({'state': 'stopped', 'role': 'primary'}) self.k._role_label = 'isMaster' self.k._leader_label_value = 'true' self.k._follower_label_value = 'false' self.k._standby_leader_label_value = 'false' self.k._tmp_role_label = 'tmp_role' self.k.touch_member({'state': 'running', 'role': 'replica'}) mock_patch_namespaced_pod.assert_called() self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false') self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'replica') mock_patch_namespaced_pod.rest_mock() self.k._name = 'p-0' self.k.touch_member({'role': 'standby_leader'}) mock_patch_namespaced_pod.assert_called() self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false') self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'primary') mock_patch_namespaced_pod.rest_mock() self.k.touch_member({'role': 'primary'}) mock_patch_namespaced_pod.assert_called() self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'true') self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'primary') def test_initialize(self): self.k.initialize() def test_delete_leader(self): self.k.delete_leader(self.k.get_cluster().leader, 1) def test_cancel_initialization(self): self.k.cancel_initialization() @patch.object(k8s_client.CoreV1Api, 'delete_collection_namespaced_config_map', Mock(side_effect=k8s_client.rest.ApiException(403, '')), create=True) def test_delete_cluster(self): self.k.delete_cluster() def test_watch(self): self.k.set_ttl(10) self.k.watch(None, 0) self.k.watch('5', 0) def test_set_history_value(self): self.k.set_history_value('{}') @patch('patroni.dcs.kubernetes.logger.warning') def test_reload_config(self, mock_warning): self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': '401, 403 '}) self.assertEqual(self.k._api._retriable_http_codes, self.k._api._DEFAULT_RETRIABLE_HTTP_CODES | set([401, 403])) self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': 402}) self.assertEqual(self.k._api._retriable_http_codes, self.k._api._DEFAULT_RETRIABLE_HTTP_CODES | set([402])) self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': [405, 406]}) self.assertEqual(self.k._api._retriable_http_codes, self.k._api._DEFAULT_RETRIABLE_HTTP_CODES | set([405, 406])) self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': True}) mock_warning.assert_called_once() @patch('urllib3.PoolManager.request', Mock()) class TestKubernetesEndpointsNoPodIP(BaseTestKubernetes): @patch.object(k8s_client.CoreV1Api, 'list_namespaced_endpoints', mock_list_namespaced_endpoints, create=True) def setUp(self, config=None): super(TestKubernetesEndpointsNoPodIP, self).setUp({'use_endpoints': True}) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True) def test_update_leader(self, mock_patch_namespaced_endpoints): self.assertIsNotNone(self.k.update_leader(self.k.get_cluster(), '123', failsafe={'foo': 'bar'})) args = mock_patch_namespaced_endpoints.call_args[0] self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '1') self.assertEqual(args[2].subsets[0].addresses[0].ip, '10.0.0.1') @patch('urllib3.PoolManager.request', Mock()) class TestKubernetesEndpoints(BaseTestKubernetes): @patch.object(k8s_client.CoreV1Api, 'list_namespaced_endpoints', mock_list_namespaced_endpoints, create=True) def setUp(self, config=None): super(TestKubernetesEndpoints, self).setUp({'use_endpoints': True, 'pod_ip': '10.0.0.0'}) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True) def test_update_leader(self, mock_patch_namespaced_endpoints): cluster = self.k.get_cluster() self.assertIsNotNone(self.k.update_leader(cluster, '123', failsafe={'foo': 'bar'})) args = mock_patch_namespaced_endpoints.call_args[0] self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '10') self.assertEqual(args[2].subsets[0].addresses[0].ip, '10.0.0.0') self.k._kinds._object_cache['test'].subsets[:] = [] self.assertIsNotNone(self.k.update_leader(cluster, '123')) self.k._kinds._object_cache['test'].metadata.annotations['leader'] = 'p-1' self.assertFalse(self.k.update_leader(cluster, '123')) @patch.object(k8s_client.CoreV1Api, 'read_namespaced_endpoints', create=True) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True) def test__update_leader_with_retry(self, mock_patch, mock_read): cluster = self.k.get_cluster() mock_read.return_value = mock_read_namespaced_endpoints() mock_patch.side_effect = k8s_client.rest.ApiException(502, '') self.assertFalse(self.k.update_leader(cluster, '123')) mock_patch.side_effect = RetryFailedError('') self.assertRaises(KubernetesError, self.k.update_leader, cluster, '123') mock_patch.side_effect = [k8s_client.rest.ApiException(409, ''), k8s_client.rest.ApiException(409, ''), mock_namespaced_kind()] mock_read.return_value.metadata.resource_version = '2' with patch('time.time', Mock(side_effect=[0, 0, 100, 200, 0, 0, 0, 0, 0, 100, 200])): self.assertFalse(self.k.update_leader(cluster, '123')) self.assertFalse(self.k.update_leader(cluster, '123')) mock_patch.side_effect = k8s_client.rest.ApiException(409, '') self.assertFalse(self.k.update_leader(cluster, '123')) mock_patch.side_effect = [k8s_client.rest.ApiException(409, ''), mock_namespaced_kind()] self.assertTrue(self.k._update_leader_with_retry({}, '1', [])) mock_patch.side_effect = [k8s_client.rest.ApiException(409, ''), mock_namespaced_kind()] self.assertIsNotNone(self.k._update_leader_with_retry({'foo': 'bar'}, '1', [])) mock_patch.side_effect = k8s_client.rest.ApiException(409, '') mock_read.side_effect = RetryFailedError('') self.assertRaises(KubernetesError, self.k.update_leader, cluster, '123') mock_read.side_effect = Exception self.assertFalse(self.k.update_leader(cluster, '123')) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', Mock(side_effect=[k8s_client.rest.ApiException(500, ''), k8s_client.rest.ApiException(502, '')]), create=True) def test_delete_sync_state(self): self.assertFalse(self.k.delete_sync_state(1)) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', mock_namespaced_kind, create=True) def test_write_sync_state(self): self.assertIsNotNone(self.k.write_sync_state('a', ['b'], 0, 1)) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', mock_namespaced_kind, create=True) @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', mock_namespaced_kind, create=True) @patch.object(k8s_client.CoreV1Api, 'create_namespaced_service', Mock(side_effect=[True, False, k8s_client.rest.ApiException(409, ''), k8s_client.rest.ApiException(403, ''), k8s_client.rest.ApiException(500, ''), Exception("Unexpected") ]), create=True) @patch('patroni.dcs.kubernetes.logger.exception') def test__create_config_service(self, mock_logger_exception): self.assertIsNotNone(self.k.patch_or_create_config({'foo': 'bar'})) self.assertIsNotNone(self.k.patch_or_create_config({'foo': 'bar'})) self.k.patch_or_create_config({'foo': 'bar'}) mock_logger_exception.assert_not_called() self.k.patch_or_create_config({'foo': 'bar'}) mock_logger_exception.assert_not_called() self.k.patch_or_create_config({'foo': 'bar'}) mock_logger_exception.assert_called_once() self.assertEqual(('create_config_service failed',), mock_logger_exception.call_args[0]) mock_logger_exception.reset_mock() self.k.touch_member({'state': 'running', 'role': 'replica'}) mock_logger_exception.assert_called_once() self.assertEqual(('create_config_service failed',), mock_logger_exception.call_args[0]) @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', mock_namespaced_kind, create=True) def test_write_leader_optime(self): self.k.write_leader_optime(12345) def mock_watch(*args): return urllib3.HTTPResponse() @patch('urllib3.PoolManager.request', Mock()) class TestCacheBuilder(BaseTestKubernetes): @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map, create=True) @patch('patroni.dcs.kubernetes.ObjectCache._watch', mock_watch) @patch.object(urllib3.HTTPResponse, 'read_chunked') def test__build_cache(self, mock_read_chunked): self.k._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) mock_read_chunked.return_value = [json.dumps( {'type': 'MODIFIED', 'object': {'metadata': { 'name': self.k.config_path, 'resourceVersion': '2', 'annotations': {self.k._CONFIG: 'foo'}}}} ).encode('utf-8'), ('\n' + json.dumps( {'type': 'DELETED', 'object': {'metadata': { 'name': self.k.config_path, 'resourceVersion': '3'}}} ) + '\n' + json.dumps( {'type': 'MDIFIED', 'object': {'metadata': {'name': self.k.config_path}}} ) + '\n').encode('utf-8'), b'{"object":{', b'"code":410}}\n'] self.k._kinds._build_cache() @patch('patroni.dcs.kubernetes.logger.error', Mock(side_effect=SleepException)) @patch('patroni.dcs.kubernetes.ObjectCache._build_cache', Mock(side_effect=Exception)) def test_run(self): self.assertRaises(SleepException, self.k._pods.run) @patch('time.sleep', Mock()) def test__list(self): self.k._pods._func = Mock(side_effect=Exception) self.assertRaises(Exception, self.k._pods._list) @patch('patroni.dcs.kubernetes.ObjectCache._watch', Mock(return_value=None)) def test__do_watch(self): self.assertRaises(AttributeError, self.k._kinds._do_watch, '1') @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map, create=True) @patch('patroni.dcs.kubernetes.ObjectCache._watch', mock_watch) @patch.object(urllib3.HTTPResponse, 'read_chunked', Mock(return_value=[])) def test_kill_stream(self): self.k._kinds.kill_stream() with patch.object(urllib3.HTTPResponse, 'connection') as mock_connection: mock_connection.sock.close.side_effect = Exception self.k._kinds._do_watch('1') self.k._kinds.kill_stream() with patch.object(urllib3.HTTPResponse, 'connection', PropertyMock(side_effect=Exception)): self.k._kinds.kill_stream() patroni-4.0.4/tests/test_log.py000066400000000000000000000240751472010352700165400ustar00rootroot00000000000000import logging import os import sys import unittest from io import StringIO from queue import Full, Queue from unittest.mock import Mock, patch import yaml from patroni.config import Config from patroni.log import PatroniLogger try: from pythonjsonlogger import jsonlogger jsonlogger.JsonFormatter(None, None, rename_fields={}, static_fields={}) json_formatter_is_available = True import json # we need json.loads() function except Exception: json_formatter_is_available = False _LOG = logging.getLogger(__name__) class TestPatroniLogger(unittest.TestCase): def setUp(self): self._handlers = logging.getLogger().handlers[:] def tearDown(self): logging.getLogger().handlers[:] = self._handlers @patch('logging.FileHandler._open', Mock()) def test_patroni_logger(self): config = { 'log': { 'traceback_level': 'DEBUG', 'max_queue_size': 5, 'dir': 'foo', 'mode': 0o600, 'file_size': 4096, 'file_num': 5, 'loggers': { 'foo.bar': 'INFO' } }, 'restapi': {}, 'postgresql': {'data_dir': 'foo'} } sys.argv = ['patroni.py'] os.environ[Config.PATRONI_CONFIG_VARIABLE] = yaml.dump(config, default_flow_style=False) logger = PatroniLogger() patroni_config = Config(None) with patch('os.chmod') as mock_chmod: logger.reload_config(patroni_config['log']) self.assertEqual(mock_chmod.call_args[0][1], 0o600) _LOG.exception('test') logger.start() with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)), \ patch('_pytest.logging.LogCaptureHandler.emit', Mock()): logging.error('test') self.assertEqual(logger.log_handler.maxBytes, config['log']['file_size']) self.assertEqual(logger.log_handler.backupCount, config['log']['file_num']) config['log']['level'] = 'DEBUG' config['log'].pop('dir') with patch('logging.Handler.close', Mock(side_effect=Exception)): logger.reload_config(config['log']) with patch.object(logging.Logger, 'makeRecord', Mock(side_effect=[logging.LogRecord('', logging.INFO, '', 0, '', (), None), Exception])): logging.exception('test') logging.error('test') with patch.object(Queue, 'put_nowait', Mock(side_effect=Full)): self.assertRaises(SystemExit, logger.shutdown) self.assertRaises(Exception, logger.shutdown) self.assertLessEqual(logger.queue_size, 2) # "Failed to close the old log handler" could be still in the queue self.assertEqual(logger.records_lost, 0) def test_interceptor(self): logger = PatroniLogger() logger.reload_config({'level': 'INFO'}) logger.start() _LOG.info('Lock owner: ') _LOG.info('blabla') logger.shutdown() self.assertEqual(logger.records_lost, 0) def test_json_list_format(self): config = { 'type': 'json', 'format': [ {'asctime': '@timestamp'}, {'levelname': 'level'}, 'message' ], 'static_fields': { 'app': 'patroni' } } test_message = 'test json logging in case of list format' with patch('sys.stderr', StringIO()) as stderr_output: logger = PatroniLogger() logger.reload_config(config) _LOG.info(test_message) if json_formatter_is_available: target_log = json.loads(stderr_output.getvalue().split('\n')[-2]) self.assertIn('@timestamp', target_log) self.assertEqual(target_log['message'], test_message) self.assertEqual(target_log['level'], 'INFO') self.assertEqual(target_log['app'], 'patroni') self.assertEqual(len(target_log), len(config['format']) + len(config['static_fields'])) def test_json_str_format(self): config = { 'type': 'json', 'format': '%(asctime)s %(levelname)s %(message)s', 'static_fields': { 'app': 'patroni' } } test_message = 'test json logging in case of string format' with patch('sys.stderr', StringIO()) as stderr_output: logger = PatroniLogger() logger.reload_config(config) _LOG.info(test_message) if json_formatter_is_available: target_log = json.loads(stderr_output.getvalue().split('\n')[-2]) self.assertIn('asctime', target_log) self.assertEqual(target_log['message'], test_message) self.assertEqual(target_log['levelname'], 'INFO') self.assertEqual(target_log['app'], 'patroni') def test_plain_format(self): config = { 'type': 'plain', 'format': '[%(asctime)s] %(levelname)s %(message)s', } test_message = 'test plain logging' with patch('sys.stderr', StringIO()) as stderr_output: logger = PatroniLogger() logger.reload_config(config) _LOG.info(test_message) target_log = stderr_output.getvalue() self.assertRegex(target_log, fr'^\[.*\] INFO {test_message}$') def test_dateformat(self): config = { 'format': '[%(asctime)s] %(message)s', 'dateformat': '%Y-%m-%dT%H:%M:%S' } test_message = 'test date format' with patch('sys.stderr', StringIO()) as stderr_output: logger = PatroniLogger() logger.reload_config(config) _LOG.info(test_message) target_log = stderr_output.getvalue() self.assertRegex(target_log, r'\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\]') def test_invalid_dateformat(self): config = { 'format': '[%(asctime)s] %(message)s', 'dateformat': 5 } with self.assertLogs() as captured_log: logger = PatroniLogger() logger.reload_config(config) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'WARNING') self.assertRegex( captured_log_message, r'Expected log dateformat to be a string, but got "int"' ) def test_invalid_plain_format(self): config = { 'type': 'plain', 'format': ['message'] } with self.assertLogs() as captured_log: logger = PatroniLogger() logger.reload_config(config) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'WARNING') self.assertRegex( captured_log_message, r'Expected log format to be a string when log type is plain, but got ".*"' ) def test_invalid_json_format(self): config = { 'type': 'json', 'format': { 'asctime': 'timestamp', 'message': 'message' } } with self.assertLogs() as captured_log: logger = PatroniLogger() logger.reload_config(config) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'WARNING') self.assertRegex(captured_log_message, r'Expected log format to be a string or a list, but got ".*"') with self.assertLogs() as captured_log: config['format'] = [['levelname']] logger.reload_config(config) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'WARNING') self.assertRegex( captured_log_message, r'Expected each item of log format to be a string or dictionary, but got ".*"' ) with self.assertLogs() as captured_log: config['format'] = ['message', {'asctime': ['timestamp']}] logger.reload_config(config) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'WARNING') self.assertRegex(captured_log_message, r'Expected renamed log field to be a string, but got ".*"') def test_fail_to_use_python_json_logger(self): with self.assertLogs() as captured_log: logger = PatroniLogger() with patch('builtins.__import__', Mock(side_effect=ImportError)): logger.reload_config({'type': 'json'}) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'ERROR') self.assertRegex( captured_log_message, r'Failed to import "python-json-logger" library: .*. Falling back to the plain logger' ) with self.assertLogs() as captured_log: logger = PatroniLogger() pythonjsonlogger = Mock() pythonjsonlogger.jsonlogger.JsonFormatter = Mock(side_effect=Exception) with patch('builtins.__import__', Mock(return_value=pythonjsonlogger)): logger.reload_config({'type': 'json'}) captured_log_level = captured_log.records[0].levelname captured_log_message = captured_log.records[0].message self.assertEqual(captured_log_level, 'ERROR') self.assertRegex( captured_log_message, r'Failed to initialize JsonFormatter: .*. Falling back to the plain logger' ) patroni-4.0.4/tests/test_mpp.py000066400000000000000000000033051472010352700165440ustar00rootroot00000000000000from typing import Any from patroni.exceptions import PatroniException from patroni.postgresql.mpp import AbstractMPP, get_mpp, Null from . import BaseTestPostgresql from .test_ha import get_cluster_initialized_with_leader class TestMPP(BaseTestPostgresql): def setUp(self): super(TestMPP, self).setUp() self.cluster = get_cluster_initialized_with_leader() def test_get_handler_impl_exception(self): class DummyMPP(AbstractMPP): def __init__(self) -> None: super().__init__({}) @staticmethod def validate_config(config: Any) -> bool: return True @property def group(self) -> None: return None @property def coordinator_group_id(self) -> None: return None @property def type(self) -> str: return "dummy" mpp = DummyMPP() self.assertRaises(PatroniException, mpp.get_handler_impl, self.p) def test_null_handler(self): config = {} mpp = get_mpp(config) self.assertIsInstance(mpp, Null) self.assertIsNone(mpp.group) self.assertTrue(mpp.validate_config(config)) nullHandler = mpp.get_handler_impl(self.p) self.assertIsNone(nullHandler.handle_event(self.cluster, {})) self.assertIsNone(nullHandler.sync_meta_data(self.cluster)) self.assertIsNone(nullHandler.on_demote()) self.assertIsNone(nullHandler.schedule_cache_rebuild()) self.assertIsNone(nullHandler.bootstrap()) self.assertIsNone(nullHandler.adjust_postgres_gucs({})) self.assertFalse(nullHandler.ignore_replication_slot({})) patroni-4.0.4/tests/test_patroni.py000066400000000000000000000324031472010352700174250ustar00rootroot00000000000000import logging import os import signal import time import unittest from http.server import HTTPServer from threading import Thread from unittest.mock import Mock, patch, PropertyMock import etcd import patroni.config as config from patroni.__main__ import check_psycopg, main as _main, Patroni from patroni.api import RestApiServer from patroni.async_executor import AsyncExecutor from patroni.dcs import Cluster, ClusterConfig, Member from patroni.dcs.etcd import AbstractEtcdClientWithFailover from patroni.exceptions import DCSError from patroni.postgresql import Postgresql from patroni.postgresql.config import ConfigHandler from . import psycopg_connect, SleepException from .test_etcd import etcd_read, etcd_write from .test_postgresql import MockPostmaster def mock_import(*args, **kwargs): ret = Mock() ret.__version__ = '2.5.3.dev1 a b c' if args[0] == 'psycopg2' else '3.1.0' return ret def mock_import2(*args, **kwargs): if args[0] == 'psycopg2': raise ImportError ret = Mock() ret.__version__ = '0.1.2' return ret class MockFrozenImporter(object): toc = set(['patroni.dcs.etcd']) @patch('time.sleep', Mock()) @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) @patch('urllib3.PoolManager.request', Mock(side_effect=Exception)) @patch.object(ConfigHandler, 'append_pg_hba', Mock()) @patch.object(ConfigHandler, 'write_postgresql_conf', Mock()) @patch.object(ConfigHandler, 'write_recovery_conf', Mock()) @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster())) @patch.object(Postgresql, 'call_nowait', Mock()) @patch.object(HTTPServer, '__init__', Mock()) @patch.object(AsyncExecutor, 'run', Mock()) @patch.object(etcd.Client, 'write', etcd_write) @patch.object(etcd.Client, 'read', etcd_read) class TestPatroni(unittest.TestCase): @patch('sys.argv', ['patroni.py']) def test_no_config(self): self.assertRaises(SystemExit, _main) @patch('sys.argv', ['patroni.py', '--validate-config', 'postgres0.yml']) @patch('socket.socket.connect_ex', Mock(return_value=1)) def test_validate_config(self): self.assertRaises(SystemExit, _main) with patch.object(config.Config, '__init__', Mock(return_value=None)): self.assertRaises(SystemExit, _main) @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()])) @patch('urllib3.PoolManager.request', Mock(side_effect=Exception)) @patch('sys.frozen', Mock(return_value=True), create=True) @patch.object(HTTPServer, '__init__', Mock()) @patch.object(etcd.Client, 'read', etcd_read) @patch.object(Thread, 'start', Mock()) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) @patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True})) def setUp(self): self._handlers = logging.getLogger().handlers[:] RestApiServer._BaseServer__is_shut_down = Mock() RestApiServer._BaseServer__shutdown_request = True RestApiServer.socket = 0 os.environ['PATRONI_POSTGRESQL_DATA_DIR'] = 'data/test0' conf = config.Config('postgres0.yml') self.p = Patroni(conf) def tearDown(self): logging.getLogger().handlers[:] = self._handlers def test_apply_dynamic_configuration(self): empty_cluster = Cluster.empty() self.p.config._dynamic_configuration = {} self.p.apply_dynamic_configuration(empty_cluster) self.assertEqual(self.p.config._dynamic_configuration['ttl'], 30) without_config = empty_cluster._asdict() del without_config['config'] cluster = Cluster( config=ClusterConfig(version=1, modify_version=1, data={"ttl": 40}), **without_config ) self.p.config._dynamic_configuration = {} self.p.apply_dynamic_configuration(cluster) self.assertEqual(self.p.config._dynamic_configuration['ttl'], 40) @patch('sys.argv', ['patroni.py', 'postgres0.yml']) @patch('time.sleep', Mock(side_effect=SleepException)) @patch.object(etcd.Client, 'delete', Mock()) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) @patch.object(Thread, 'join', Mock()) @patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True})) def test_patroni_patroni_main(self): with patch('subprocess.call', Mock(return_value=1)): with patch.object(Patroni, 'run', Mock(side_effect=SleepException)): os.environ['PATRONI_POSTGRESQL_DATA_DIR'] = 'data/test0' self.assertRaises(SleepException, _main) with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())): with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)): os.environ['PATRONI_POSTGRESQL_DATA_DIR'] = 'data/test0' _main() @patch('os.getpid') @patch('multiprocessing.Process') @patch('patroni.__main__.patroni_main', Mock()) @patch('sys.argv', ['patroni.py', 'postgres0.yml']) def test_patroni_main(self, mock_process, mock_getpid): mock_getpid.return_value = 2 _main() mock_getpid.return_value = 1 def mock_signal(signo, handler): handler(signo, None) with patch('signal.signal', mock_signal): with patch('os.waitpid', Mock(side_effect=[(1, 0), (0, 0)])): _main() with patch('os.waitpid', Mock(side_effect=OSError)): _main() ref = {'passtochild': lambda signo, stack_frame: 0} def mock_sighup(signo, handler): if hasattr(signal, 'SIGHUP') and signo == signal.SIGHUP: ref['passtochild'] = handler def mock_join(): ref['passtochild'](0, None) mock_process.return_value.join = mock_join with patch('signal.signal', mock_sighup), patch('os.kill', Mock()): self.assertIsNone(_main()) @patch('patroni.config.Config.save_cache', Mock()) @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True)) @patch('patroni.ha.Ha.is_leader', Mock(return_value=True)) @patch.object(Postgresql, 'state', PropertyMock(return_value='running')) @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False)) def test_run(self): self.p.postgresql.set_role('replica') self.p.sighup_handler() self.p.ha.dcs.watch = Mock(side_effect=SleepException) self.p.api.start = Mock() self.p.logger.start = Mock() self.p.config._dynamic_configuration = {} self.assertRaises(SleepException, self.p.run) with patch('patroni.dcs.Cluster.is_unlocked', Mock(return_value=True)): self.assertRaises(SleepException, self.p.run) with patch('patroni.config.Config.reload_local_configuration', Mock(return_value=False)): self.p.sighup_handler() self.assertRaises(SleepException, self.p.run) with patch('patroni.config.Config.set_dynamic_configuration', Mock(return_value=True)): self.assertRaises(SleepException, self.p.run) with patch('patroni.postgresql.Postgresql.data_directory_empty', Mock(return_value=False)): self.assertRaises(SleepException, self.p.run) def test_sigterm_handler(self): self.assertRaises(SystemExit, self.p.sigterm_handler) def test_schedule_next_run(self): self.p.ha.cluster = Mock() self.p.ha.dcs.watch = Mock(return_value=True) self.p.schedule_next_run() self.p.next_run = time.time() - self.p.dcs.loop_wait - 1 self.p.schedule_next_run() def test__filter_tags(self): tags = {'noloadbalance': False, 'clonefrom': False, 'nosync': False, 'smth': 'random'} self.assertEqual(self.p._filter_tags(tags), {'smth': 'random'}) tags['clonefrom'] = True tags['smth'] = False self.assertEqual(self.p._filter_tags(tags), {'clonefrom': True, 'smth': False}) tags = {'nofailover': False, 'failover_priority': 0} self.assertEqual(self.p._filter_tags(tags), tags) tags = {'nofailover': True, 'failover_priority': 1} self.assertEqual(self.p._filter_tags(tags), tags) def test_noloadbalance(self): self.p.tags['noloadbalance'] = True self.assertTrue(self.p.noloadbalance) def test_nofailover(self): for (nofailover, failover_priority, expected) in [ # Without any tags, default is False (None, None, False), # Setting `nofailover: True` has precedence (True, 0, True), (True, 1, True), ('False', 1, True), # because we use bool() for the value # Similarly, setting `nofailover: False` has precedence (False, 0, False), (False, 1, False), ('', 0, False), # Only when we have `nofailover: None` should we got based on priority (None, 0, True), (None, 1, False), ]: with self.subTest(nofailover=nofailover, failover_priority=failover_priority, expected=expected): self.p.tags['nofailover'] = nofailover self.p.tags['failover_priority'] = failover_priority self.assertEqual(self.p.nofailover, expected) def test_failover_priority(self): for (nofailover, failover_priority, expected) in [ # Without any tags, default is 1 (None, None, 1), # Setting `nofailover: True` has precedence (value 0) (True, 0, 0), (True, 1, 0), # Setting `nofailover: False` and `failover_priority: None` gives 1 (False, None, 1), # Normal function of failover_priority (None, 0, 0), (None, 1, 1), (None, 2, 2), ]: with self.subTest(nofailover=nofailover, failover_priority=failover_priority, expected=expected): self.p.tags['nofailover'] = nofailover self.p.tags['failover_priority'] = failover_priority self.assertEqual(self.p.failover_priority, expected) def test_replicatefrom(self): self.assertIsNone(self.p.replicatefrom) self.p.tags['replicatefrom'] = 'foo' self.assertEqual(self.p.replicatefrom, 'foo') def test_reload_config(self): self.p.reload_config() self.p._get_tags = Mock(side_effect=Exception) self.p.reload_config(local=True) def test_nosync(self): self.p.tags['nosync'] = True self.assertTrue(self.p.nosync) self.p.tags['nosync'] = None self.assertFalse(self.p.nosync) def test_nostream(self): self.p.tags['nostream'] = 'True' self.assertTrue(self.p.nostream) self.p.tags['nostream'] = 'None' self.assertFalse(self.p.nostream) self.p.tags['nostream'] = 'foo' self.assertFalse(self.p.nostream) self.p.tags['nostream'] = '' self.assertFalse(self.p.nostream) @patch.object(Thread, 'join', Mock()) def test_shutdown(self): self.p.api.shutdown = Mock(side_effect=Exception) self.p.ha.shutdown = Mock(side_effect=Exception) self.p.shutdown() def test_check_psycopg(self): with patch('builtins.__import__', Mock(side_effect=ImportError)): self.assertRaises(SystemExit, check_psycopg) with patch('builtins.__import__', mock_import): self.assertIsNone(check_psycopg()) with patch('builtins.__import__', mock_import2): self.assertRaises(SystemExit, check_psycopg) def test_ensure_unique_name(self): # None/empty cluster implies unique name self.assertIsNone(self.p.ensure_unique_name(None)) empty_cluster = Cluster.empty() self.assertIsNone(self.p.ensure_unique_name(empty_cluster)) without_members = empty_cluster._asdict() del without_members['members'] # Cluster with members with different names implies unique name okay_cluster = Cluster( members=[Member(version=1, name="distinct", session=1, data={})], **without_members ) self.assertIsNone(self.p.ensure_unique_name(okay_cluster)) # Cluster with a member with the same name that is running bad_cluster = Cluster( members=[Member(version=1, name="postgresql0", session=1, data={ "api_url": "https://127.0.0.1:8008", })], **without_members ) # If the api of the running node cannot be reached, this implies unique name with patch('urllib3.PoolManager.request', Mock(side_effect=ConnectionError)): self.assertIsNone(self.p.ensure_unique_name(bad_cluster)) # Only if the api of the running node is reachable do we throw an error with patch('urllib3.PoolManager.request', Mock()): self.assertRaises(SystemExit, self.p.ensure_unique_name, bad_cluster) @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[DCSError('foo'), DCSError('foo'), None])) def test_ensure_dcs_access(self): with patch('patroni.__main__.logger.warning') as mock_logger: result = self.p.ensure_dcs_access() self.assertEqual(result, None) self.assertEqual(mock_logger.call_count, 2) patroni-4.0.4/tests/test_postgresql.py000066400000000000000000001701631472010352700201620ustar00rootroot00000000000000import datetime import os import re import subprocess import time from copy import deepcopy from pathlib import Path from threading import current_thread, Thread from unittest.mock import MagicMock, Mock, mock_open, patch, PropertyMock import psutil import patroni.psycopg as psycopg from patroni import global_config from patroni.async_executor import CriticalTask from patroni.collections import CaseInsensitiveDict, CaseInsensitiveSet from patroni.dcs import RemoteMember from patroni.exceptions import PatroniException, PostgresConnectionException from patroni.postgresql import Postgresql, STATE_NO_RESPONSE, STATE_REJECT from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.callback_executor import CallbackAction from patroni.postgresql.config import _false_validator, get_param_diff from patroni.postgresql.postmaster import PostmasterProcess from patroni.postgresql.validator import _get_postgres_guc_validators, _load_postgres_gucs_validators, \ _read_postgres_gucs_validators_file, Bool, Enum, EnumBool, Integer, InvalidGucValidatorsFile, \ Real, String, transform_postgresql_parameter_value, ValidatorFactory, ValidatorFactoryInvalidSpec, \ ValidatorFactoryInvalidType, ValidatorFactoryNoType from patroni.utils import RetryFailedError from . import BaseTestPostgresql, GET_PG_SETTINGS_RESULT, \ mock_available_gucs, MockCursor, MockPostmaster, psycopg_connect mtime_ret = {} def mock_mtime(filename): if filename not in mtime_ret: mtime_ret[filename] = time.time() else: mtime_ret[filename] += 1 return mtime_ret[filename] def pg_controldata_string(*args, **kwargs): return b""" pg_control version number: 942 Catalog version number: 201509161 Database system identifier: 6200971513092291716 Database cluster state: shut down in recovery pg_control last modified: Fri Oct 2 10:57:06 2015 Latest checkpoint location: 0/30000C8 Prior checkpoint location: 0/2000060 Latest checkpoint's REDO location: 0/3000090 Latest checkpoint's REDO WAL file: 000000020000000000000003 Latest checkpoint's TimeLineID: 2 Latest checkpoint's PrevTimeLineID: 2 Latest checkpoint's full_page_writes: on Latest checkpoint's NextXID: 0/943 Latest checkpoint's NextOID: 24576 Latest checkpoint's NextMultiXactId: 1 Latest checkpoint's NextMultiOffset: 0 Latest checkpoint's oldestXID: 931 Latest checkpoint's oldestXID's DB: 1 Latest checkpoint's oldestActiveXID: 943 Latest checkpoint's oldestMultiXid: 1 Latest checkpoint's oldestMulti's DB: 1 Latest checkpoint's oldestCommitTs: 0 Latest checkpoint's newestCommitTs: 0 Time of latest checkpoint: Fri Oct 2 10:56:54 2015 Fake LSN counter for unlogged rels: 0/1 Minimum recovery ending location: 0/30241F8 Min recovery ending loc's timeline: 2 Backup start location: 0/0 Backup end location: 0/0 End-of-backup record required: no wal_level setting: hot_standby Current wal_log_hints setting: on Current max_connections setting: 100 Current max_worker_processes setting: 8 Current max_prepared_xacts setting: 0 Current max_locks_per_xact setting: 64 Current track_commit_timestamp setting: off Maximum data alignment: 8 Database block size: 8192 Blocks per segment of large relation: 131072 WAL block size: 8192 Bytes per WAL segment: 16777216 Maximum length of identifiers: 64 Maximum columns in an index: 32 Maximum size of a TOAST chunk: 1996 Size of a large-object chunk: 2048 Date/time type storage: 64-bit integers Float4 argument passing: by value Float8 argument passing: by value Data page checksum version: 0 """ @patch('subprocess.call', Mock(return_value=0)) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1")) @patch('patroni.psycopg.connect', psycopg_connect) @patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestPostgresql(BaseTestPostgresql): @patch('subprocess.call', Mock(return_value=0)) @patch('os.rename', Mock()) @patch('patroni.postgresql.CallbackExecutor', Mock()) @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) @patch.object(Postgresql, 'available_gucs', mock_available_gucs) def setUp(self): super(TestPostgresql, self).setUp() self.p.config.write_postgresql_conf() @patch('subprocess.Popen') @patch.object(Postgresql, 'wait_for_startup') @patch.object(Postgresql, 'wait_for_port_open') @patch.object(Postgresql, 'is_running') @patch.object(Postgresql, 'controldata', Mock()) def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen): mock_is_running.return_value = MockPostmaster() mock_wait_for_port_open.return_value = True mock_wait_for_startup.return_value = False mock_popen.return_value.stdout.readline.return_value = '123' self.assertTrue(self.p.start()) mock_is_running.return_value = None with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)): self.assertIsNone(self.p.start()) mock_postmaster = MockPostmaster() with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster): pg_conf = os.path.join(self.p.data_dir, 'postgresql.conf') open(pg_conf, 'w').close() self.assertFalse(self.p.start(task=CriticalTask())) with open(pg_conf) as f: lines = f.readlines() self.assertTrue("f.oo = 'bar'\n" in lines) mock_wait_for_startup.return_value = None self.assertFalse(self.p.start(10)) self.assertIsNone(self.p.start()) mock_wait_for_port_open.return_value = False self.assertFalse(self.p.start()) task = CriticalTask() task.cancel() self.assertFalse(self.p.start(task=task)) self.p.cancellable.cancel() self.assertFalse(self.p.start()) with patch('patroni.postgresql.config.ConfigHandler.effective_configuration', PropertyMock(side_effect=Exception)): self.assertIsNone(self.p.start()) @patch.object(Postgresql, 'pg_isready') @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1))) def test_wait_for_port_open(self, mock_pg_isready): mock_pg_isready.return_value = STATE_NO_RESPONSE mock_postmaster = MockPostmaster() mock_postmaster.is_running.return_value = None # No pid file and postmaster death self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1)) mock_postmaster.is_running.return_value = True # timeout self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1)) # pg_isready failure mock_pg_isready.return_value = 'garbage' self.assertTrue(self.p.wait_for_port_open(mock_postmaster, 1)) # cancelled self.p.cancellable.cancel() self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1)) @patch('time.sleep', Mock()) @patch.object(Postgresql, 'is_running') @patch.object(Postgresql, '_wait_for_connection_close', Mock()) @patch('patroni.postgresql.cancellable.CancellableSubprocess.call') def test_stop(self, mock_cancellable_call, mock_is_running): # Postmaster is not running mock_callback = Mock() mock_is_running.return_value = None self.assertTrue(self.p.stop(on_safepoint=mock_callback)) mock_callback.assert_called() # Is running, stopped successfully mock_is_running.return_value = mock_postmaster = MockPostmaster() mock_callback.reset_mock() self.assertTrue(self.p.stop(on_safepoint=mock_callback)) mock_callback.assert_called() mock_postmaster.signal_stop.assert_called() # Timed out waiting for fast shutdown triggers immediate shutdown mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), psutil.TimeoutExpired(30), Mock()] mock_callback.reset_mock() self.assertTrue(self.p.stop(on_safepoint=mock_callback, stop_timeout=30)) mock_callback.assert_called() mock_postmaster.signal_stop.assert_called() # Immediate shutdown succeeded mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), Mock()] self.assertTrue(self.p.stop(on_safepoint=mock_callback, stop_timeout=30)) # Ensure before_stop script is called when configured to self.p.config._config['before_stop'] = ':' mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), Mock()] mock_cancellable_call.return_value = 0 with patch('patroni.postgresql.logger.info') as mock_logger: self.p.stop(on_safepoint=mock_callback, stop_timeout=30) self.assertEqual(mock_logger.call_args[0], ('before_stop script `%s` exited with %s', ':', 0)) mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), Mock()] mock_cancellable_call.side_effect = Exception with patch('patroni.postgresql.logger.error') as mock_logger: self.p.stop(on_safepoint=mock_callback, stop_timeout=30) self.assertEqual(mock_logger.call_args_list[1][0][0], 'Exception when calling `%s`: %r') # Stop signal failed mock_postmaster.signal_stop.return_value = False self.assertFalse(self.p.stop()) # Stop signal failed to find process mock_postmaster.signal_stop.return_value = True mock_callback.reset_mock() self.assertTrue(self.p.stop(on_safepoint=mock_callback)) mock_callback.assert_called() # Fast shutdown is timed out but when immediate postmaster is already gone mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), Mock()] mock_postmaster.signal_stop.side_effect = [None, True] self.assertTrue(self.p.stop(on_safepoint=mock_callback, stop_timeout=30)) @patch('time.sleep', Mock()) @patch.object(Postgresql, 'is_running', MockPostmaster) @patch.object(Postgresql, '_wait_for_connection_close', Mock()) @patch.object(Postgresql, 'latest_checkpoint_location', Mock(return_value='7')) def test__do_stop(self): mock_callback = Mock() with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down', "Latest checkpoint's TimeLineID": '1', 'Latest checkpoint location': '1/1'})): self.assertTrue(self.p.stop(on_shutdown=mock_callback, stop_timeout=3)) mock_callback.assert_called() with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down in recovery'})): self.assertTrue(self.p.stop(on_shutdown=mock_callback, stop_timeout=3)) with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shutting down'})): self.assertTrue(self.p.stop(on_shutdown=mock_callback, stop_timeout=3)) def test_restart(self): self.p.start = Mock(return_value=False) self.assertFalse(self.p.restart()) self.assertEqual(self.p.state, 'restart failed (restarting)') @patch('os.chmod', Mock()) @patch('builtins.open', MagicMock()) def test_write_pgpass(self): self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'}) self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'}) def test__pgpass_content(self): pgpass = self.p.config._pgpass_content({'host': '/tmp', 'port': '5432', 'user': 'foo', 'password': 'bar'}) self.assertEqual(pgpass, "/tmp:5432:*:foo:bar\nlocalhost:5432:*:foo:bar\n") def test_checkpoint(self): with patch.object(MockCursor, 'fetchone', Mock(return_value=(True, ))): self.assertEqual(self.p.checkpoint({'user': 'postgres'}), 'is_in_recovery=true') with patch.object(MockCursor, 'execute', Mock(return_value=None)): self.assertIsNone(self.p.checkpoint()) self.assertEqual(self.p.checkpoint(timeout=10), 'not accessible or not healthy') @patch('patroni.postgresql.config.mtime', mock_mtime) @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings') def test_check_recovery_conf(self, mock_get_pg_settings): self.p.call_nowait(CallbackAction.ON_START) mock_get_pg_settings.return_value = { 'primary_conninfo': ['primary_conninfo', 'foo=', None, 'string', 'postmaster', self.p.config._auto_conf], 'recovery_min_apply_delay': ['recovery_min_apply_delay', '0', 'ms', 'integer', 'sighup', 'foo'] } self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) self.p.config.write_recovery_conf({'standby_mode': 'on'}) self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) mock_get_pg_settings.return_value['primary_conninfo'][1] = '' mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '1' self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) mock_get_pg_settings.return_value['recovery_min_apply_delay'][5] = self.p.config._auto_conf self.assertEqual(self.p.config.check_recovery_conf(None), (True, False)) mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '0' self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) conninfo = {'host': '1', 'password': 'bar'} with patch('patroni.postgresql.config.ConfigHandler.primary_conninfo_params', Mock(return_value=conninfo)): mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '1' self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) mock_get_pg_settings.return_value['primary_conninfo'][1] = 'host=1 target_session_attrs=read-write'\ + ' dbname=postgres passfile=' + re.sub(r'([\'\\ ])', r'\\\1', self.p.config._pgpass) mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '0' self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': conninfo.copy()}) self.p.config.write_postgresql_conf() self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) with patch.object(Postgresql, 'primary_conninfo', Mock(return_value='host=1')): mock_get_pg_settings.return_value['primary_slot_name'] = [ 'primary_slot_name', '', '', 'string', 'postmaster', self.p.config._postgresql_conf] self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) @patch.object(Postgresql, 'major_version', PropertyMock(return_value=120000)) @patch.object(Postgresql, 'is_running', MockPostmaster) @patch.object(MockPostmaster, 'create_time', Mock(return_value=1234567), create=True) @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings') def test__read_recovery_params(self, mock_get_pg_settings): self.p.call_nowait(CallbackAction.ON_START) mock_get_pg_settings.return_value = {'primary_conninfo': ['primary_conninfo', '', None, 'string', 'postmaster', self.p.config._postgresql_conf]} self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': {'password': 'foo'}}) self.p.config.write_postgresql_conf() self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) # Config files changed, but can't connect to postgres mock_get_pg_settings.side_effect = PostgresConnectionException('') with patch('patroni.postgresql.config.mtime', mock_mtime): self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) # Config files didn't change, but postgres crashed or in crash recovery with patch.object(MockPostmaster, 'create_time', Mock(return_value=1234568), create=True): self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) # Any other exception raised when executing the query mock_get_pg_settings.side_effect = Exception with patch('patroni.postgresql.config.mtime', mock_mtime): self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) with patch.object(Postgresql, 'is_starting', Mock(return_value=True)): self.assertEqual(self.p.config.check_recovery_conf(None), (False, False)) @patch.object(Postgresql, 'major_version', PropertyMock(return_value=100000)) @patch.object(Postgresql, 'primary_conninfo', Mock(return_value='host=1')) def test__read_recovery_params_pre_v12(self): self.p.config.write_recovery_conf({'standby_mode': 'off', 'primary_conninfo': {'password': 'foo'}}) self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) self.p.config.write_recovery_conf({'restore_command': '\n'}) with patch('patroni.postgresql.config.mtime', mock_mtime): self.assertEqual(self.p.config.check_recovery_conf(None), (True, True)) def test_write_postgresql_and_sanitize_auto_conf(self): read_data = 'primary_conninfo = foo\nfoo = bar\n' with open(os.path.join(self.p.data_dir, 'postgresql.auto.conf'), 'w') as f: f.write(read_data) mock_read_auto = mock_open(read_data=read_data) mock_read_auto.return_value.__iter__ = lambda o: iter(o.readline, '') with patch('builtins.open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])), \ patch('os.chmod', Mock()): self.p.config.write_postgresql_conf() with patch('builtins.open', Mock(side_effect=[mock_open()(), IOError])), patch('os.chmod', Mock()): self.p.config.write_postgresql_conf() self.p.config.write_recovery_conf({'foo': 'bar'}) self.p.config.write_postgresql_conf() @patch.object(Postgresql, 'is_running', Mock(return_value=False)) @patch.object(Postgresql, 'start', Mock()) @patch.object(Postgresql, 'major_version', PropertyMock(return_value=170000)) def test_follow(self): self.p.call_nowait(CallbackAction.ON_START) m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'foo,bar'}}) self.p.follow(m) with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)): self.assertIsNone(self.p.follow(m)) @patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)) def test__query(self): self.assertRaises(PostgresConnectionException, self.p._query, 'blabla') self.p._state = 'restarting' self.assertRaises(RetryFailedError, self.p._query, 'blabla') def test_query(self): self.p.query('select 1') self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError') self.assertRaises(psycopg.ProgrammingError, self.p.query, 'blabla') @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT)) def test_is_primary(self): self.assertTrue(self.p.is_primary()) self.p.reset_cluster_info_state(None) with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))): self.assertFalse(self.p.is_primary()) @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down', 'Latest checkpoint location': '0/1ADBC18', "Latest checkpoint's TimeLineID": '1'})) @patch('subprocess.Popen') def test_latest_checkpoint_location(self, mock_popen): mock_popen.return_value.communicate.return_value = (None, None) self.assertEqual(self.p.latest_checkpoint_location(), 28163096) with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down', 'Latest checkpoint location': 'k/1ADBC18', "Latest checkpoint's TimeLineID": '1'})): self.assertIsNone(self.p.latest_checkpoint_location()) # 9.3 and 9.4 format mock_popen.return_value.communicate.side_effect = [ (b'rmgr: XLOG len (rec/tot): 72/ 104, tx: 0, lsn: 0/01ADBC18, prev 0/01ADBBB8, ' + b'bkp: 0000, desc: checkpoint: redo 0/1ADBC18; tli 1; prev tli 1; fpw true; xid 0/727; oid 16386; multi' + b' 1; offset 0; oldest xid 715 in DB 1; oldest multi 1 in DB 1; oldest running xid 0; shutdown', None), (b'rmgr: Transaction len (rec/tot): 64/ 96, tx: 726, lsn: 0/01ADBBB8, prev 0/01ADBB70, ' + b'bkp: 0000, desc: commit: 2021-02-26 11:19:37.900918 CET; inval msgs: catcache 11 catcache 10', None)] self.assertEqual(self.p.latest_checkpoint_location(), 28163096) mock_popen.return_value.communicate.side_effect = [ (b'rmgr: XLOG len (rec/tot): 72/ 104, tx: 0, lsn: 0/01ADBC18, prev 0/01ADBBB8, ' + b'bkp: 0000, desc: checkpoint: redo 0/1ADBC18; tli 1; prev tli 1; fpw true; xid 0/727; oid 16386; multi' + b' 1; offset 0; oldest xid 715 in DB 1; oldest multi 1 in DB 1; oldest running xid 0; shutdown', None), (b'rmgr: XLOG len (rec/tot): 0/ 32, tx: 0, lsn: 0/01ADBBB8, prev 0/01ADBBA0, ' + b'bkp: 0000, desc: xlog switch ', None)] self.assertEqual(self.p.latest_checkpoint_location(), 28163000) # 9.5+ format mock_popen.return_value.communicate.side_effect = [ (b'rmgr: XLOG len (rec/tot): 114/ 114, tx: 0, lsn: 0/01ADBC18, prev 0/018260F8, ' + b'desc: CHECKPOINT_SHUTDOWN redo 0/1825ED8; tli 1; prev tli 1; fpw true; xid 0:494; oid 16387; multi 1' + b'; offset 0; oldest xid 479 in DB 1; oldest multi 1 in DB 1; oldest/newest commit timestamp xid: 0/0;' + b' oldest running xid 0; shutdown', None), (b'rmgr: XLOG len (rec/tot): 24/ 24, tx: 0, lsn: 0/018260F8, prev 0/01826080, ' + b'desc: SWITCH ', None)] self.assertEqual(self.p.latest_checkpoint_location(), 25321720) def test_reload(self): self.assertTrue(self.p.reload()) @patch.object(Postgresql, 'is_running') def test_is_healthy(self, mock_is_running): mock_is_running.return_value = True self.assertTrue(self.p.is_healthy()) mock_is_running.return_value = False self.assertFalse(self.p.is_healthy()) @patch('psutil.Popen') def test_promote(self, mock_popen): mock_popen.return_value.wait.return_value = 0 task = CriticalTask() self.assertTrue(self.p.promote(0, task)) self.p.set_role('replica') self.p.config._config['pre_promote'] = 'test' with patch('patroni.postgresql.cancellable.CancellableSubprocess.is_cancelled', PropertyMock(return_value=1)): self.assertFalse(self.p.promote(0, task)) mock_popen.side_effect = Exception self.assertFalse(self.p.promote(0, task)) task.reset() task.cancel() self.assertFalse(self.p.promote(0, task)) def test_timeline_wal_position(self): self.assertEqual(self.p.timeline_wal_position(), (1, 2, 1)) Thread(target=self.p.timeline_wal_position).start() @patch.object(PostmasterProcess, 'from_pidfile') def test_is_running(self, mock_frompidfile): # Cached postmaster running mock_postmaster = self.p._postmaster_proc = MockPostmaster() self.assertEqual(self.p.is_running(), mock_postmaster) # Cached postmaster not running, no postmaster running mock_postmaster.is_running.return_value = False mock_frompidfile.return_value = None self.assertEqual(self.p.is_running(), None) self.assertEqual(self.p._postmaster_proc, None) # No cached postmaster, postmaster running mock_frompidfile.return_value = mock_postmaster2 = MockPostmaster() self.assertEqual(self.p.is_running(), mock_postmaster2) self.assertEqual(self.p._postmaster_proc, mock_postmaster2) @patch('shlex.split', Mock(side_effect=OSError)) def test_call_nowait(self): self.p.set_role('replica') self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START)) self.p.bootstrapping = True self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START)) @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster())) def test_is_primary_exception(self): self.p.start() self.p.query = Mock(side_effect=psycopg.OperationalError("not supported")) self.assertTrue(self.p.stop()) @patch('os.rename', Mock()) @patch('os.path.exists', Mock(return_value=True)) @patch('shutil.rmtree', Mock()) @patch('os.path.isdir', Mock(return_value=True)) @patch('os.unlink', Mock()) @patch('os.symlink', Mock()) @patch('patroni.postgresql.Postgresql.pg_wal_realpath', Mock(return_value={'pg_wal': '/mnt/pg_wal'})) @patch('patroni.postgresql.Postgresql.pg_tblspc_realpaths', Mock(return_value={'42': '/mnt/tablespaces/archive'})) def test_move_data_directory(self): self.p.move_data_directory() with patch('os.rename', Mock(side_effect=OSError)): self.p.move_data_directory() @patch('os.listdir', Mock(return_value=['recovery.conf'])) @patch('os.path.exists', Mock(return_value=True)) @patch.object(Postgresql, 'controldata', Mock()) def test_get_postgres_role_from_data_directory(self): self.assertEqual(self.p.get_postgres_role_from_data_directory(), 'replica') @patch('os.remove', Mock()) @patch('shutil.rmtree', Mock()) @patch('os.unlink', Mock(side_effect=OSError)) @patch('os.path.isdir', Mock(return_value=True)) @patch('os.path.exists', Mock(return_value=True)) def test_remove_data_directory(self): with patch('os.path.islink', Mock(return_value=True)): self.p.remove_data_directory() with patch('os.path.isfile', Mock(return_value=True)): self.p.remove_data_directory() with patch('os.path.islink', Mock(side_effect=[False, False, True, True])), \ patch('os.listdir', Mock(return_value=['12345'])), \ patch('os.path.realpath', Mock(side_effect=['../foo', '../foo_tsp'])): self.p.remove_data_directory() @patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True)) def test_controldata(self): with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)): data = self.p.controldata() self.assertEqual(len(data), 50) self.assertEqual(data['Database cluster state'], 'shut down in recovery') self.assertEqual(data['wal_log_hints setting'], 'on') self.assertEqual(int(data['Database block size']), 8192) with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))): self.assertEqual(self.p.controldata(), {}) @patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True)) def test_sysid(self): with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)): self.assertEqual(self.p.sysid, "6200971513092291716") def test_pg_version(self): self.assertEqual(self.p.config.pg_version, 99999) # server_version with patch.object(Postgresql, 'server_version', PropertyMock(side_effect=AttributeError)): self.assertEqual(self.p.config.pg_version, 140000) # PG_VERSION==14, postgres --version == 12.1 with patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 14.1")): self.assertEqual(self.p.config.pg_version, 140001) @patch('os.path.isfile', Mock(return_value=True)) @patch('shutil.copy', Mock(side_effect=IOError)) def test_save_configuration_files(self): self.p.config.save_configuration_files() @patch('os.path.isfile', Mock(side_effect=[False, True, False, True])) @patch('shutil.copy', Mock(side_effect=[None, IOError])) @patch('os.chmod', Mock()) def test_restore_configuration_files(self): self.p.config.restore_configuration_files() def test_can_create_replica_without_replication_connection(self): self.p.config._config['create_replica_method'] = [] self.assertFalse(self.p.can_create_replica_without_replication_connection(None)) self.p.config._config['create_replica_method'] = ['wale', 'basebackup'] self.p.config._config['wale'] = {'command': 'foo', 'no_leader': 1} self.assertTrue(self.p.can_create_replica_without_replication_connection(None)) def test_replica_method_can_work_without_replication_connection(self): self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup')) self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar')) self.p.config._config['foo'] = {'command': 'bar', 'no_leader': 1} self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo')) self.p.config._config['foo'] = {'command': 'bar'} self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo')) @patch('time.sleep', Mock()) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) @patch('patroni.postgresql.config.logger.info') @patch('patroni.postgresql.config.logger.warning') def test_reload_config(self, mock_warning, mock_info): config = deepcopy(self.p.config._config) # Nothing changed self.p.reload_config(config) mock_info.assert_called_once_with('No PostgreSQL configuration items changed, nothing to reload.') mock_warning.assert_not_called() self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) mock_info.reset_mock() # Ignored params changed config['parameters']['archive_cleanup_command'] = 'blabla' self.p.reload_config(config) mock_info.assert_called_once_with('No PostgreSQL configuration items changed, nothing to reload.') self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) mock_info.reset_mock() # Handle wal_buffers self.p.config._config['parameters']['wal_buffers'] = '512' self.p.reload_config(config) mock_info.assert_called_once_with('No PostgreSQL configuration items changed, nothing to reload.') self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) mock_info.reset_mock() config = deepcopy(self.p.config._config) # hba/ident_changed config['pg_hba'] = [''] config['pg_ident'] = [''] self.p.reload_config(config) mock_info.assert_called_once_with('Reloading PostgreSQL configuration.') self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) mock_info.reset_mock() # Postmaster parameter change (pending_restart) init_max_worker_processes = config['parameters']['max_worker_processes'] config['parameters']['max_worker_processes'] *= 2 new_max_worker_processes = config['parameters']['max_worker_processes'] # stale reason to be removed self.p._pending_restart_reason = CaseInsensitiveDict({'max_connections': get_param_diff('200', '100')}) with patch.object(Postgresql, 'get_guc_value', Mock(return_value=str(new_max_worker_processes))), \ patch('patroni.postgresql.Postgresql._query', Mock(side_effect=[ GET_PG_SETTINGS_RESULT, [('max_worker_processes', str(init_max_worker_processes), None, 'integer')]])): self.p.reload_config(config) self.assertEqual(mock_info.call_args_list[0][0], ("Changed %s from '%s' to '%s' (restart might be required)", 'max_worker_processes', str(init_max_worker_processes), config['parameters']['max_worker_processes'])) self.assertEqual(mock_info.call_args_list[1][0], ('Reloading PostgreSQL configuration.',)) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict({'max_worker_processes': get_param_diff(init_max_worker_processes, new_max_worker_processes)})) mock_info.reset_mock() # Reset to the initial value without restart config['parameters']['max_worker_processes'] = init_max_worker_processes self.p.reload_config(config) self.assertEqual(mock_info.call_args_list[0][0], ("Changed %s from '%s' to '%s'", 'max_worker_processes', init_max_worker_processes * 2, config['parameters']['max_worker_processes'])) self.assertEqual(mock_info.call_args_list[1][0], ('Reloading PostgreSQL configuration.',)) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) mock_info.reset_mock() # User-defined parameter changed (removed) config['parameters'].pop('f.oo') self.p.reload_config(config) self.assertEqual(mock_info.call_args_list[0][0], ("Changed %s from '%s' to '%s'", 'f.oo', 'bar', None)) self.assertEqual(mock_info.call_args_list[1][0], ('Reloading PostgreSQL configuration.',)) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) mock_info.reset_mock() # Non-postmaster parameter change config['parameters']['vacuum_cost_delay'] = 2.5 self.p.reload_config(config) self.assertEqual(mock_info.call_args_list[0][0], ("Changed %s from '%s' to '%s'", 'vacuum_cost_delay', '200ms', 2.5)) self.assertEqual(mock_info.call_args_list[1][0], ('Reloading PostgreSQL configuration.',)) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict()) config['parameters']['vacuum_cost_delay'] = 200 mock_info.reset_mock() # Remove invalid parameter config['parameters']['invalid'] = 'value' self.p.reload_config(config) self.assertEqual(mock_warning.call_args_list[0][0], ('Removing invalid parameter `%s` from postgresql.parameters', 'invalid')) config['parameters'].pop('invalid') mock_warning.reset_mock() mock_info.reset_mock() # Non-empty result (outside changes) with patch.object(Postgresql, 'get_guc_value', Mock(side_effect=['73', None, ''])), \ patch('patroni.postgresql.Postgresql._query', Mock(side_effect=[GET_PG_SETTINGS_RESULT, [('shared_buffers', '128MB', '8kB', 'integer')]] * 3)): # pg_settings shared_buffers (current value) == 128MB (16384) # Patroni config shared_buffers == 42MB (should not end up in the restart reason diff) # get_guc_value (will be used after restart) == 73 (584kB) config['parameters']['shared_buffers'] = '42MB' self.p.reload_config(config, True) self.assertEqual(mock_info.call_args_list[0][0], ("Changed %s from '%s' to '%s' (restart might be required)", 'shared_buffers', '128MB', '42MB')) self.assertEqual(mock_info.call_args_list[1][0], ('Reloading PostgreSQL configuration.',)) self.assertEqual(mock_info.call_args_list[2][0], ("PostgreSQL configuration parameters requiring restart" " (%s) seem to be changed bypassing Patroni config." " Setting 'Pending restart' flag", 'shared_buffers')) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict({'shared_buffers': get_param_diff('128MB', '584kB')})) self.p.reload_config(config, True) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict({'shared_buffers': get_param_diff('128MB', '?')})) self.p.reload_config(config, True) self.assertEqual(self.p.pending_restart_reason, CaseInsensitiveDict({'shared_buffers': get_param_diff('128MB', '')})) # Exception while querying pending_restart parameters with patch('patroni.postgresql.Postgresql._query', Mock(side_effect=[GET_PG_SETTINGS_RESULT, Exception])): # Invalid values, just to increase silly coverage in postgresql.validator. # One day we will have proper tests there. config['parameters']['autovacuum'] = 'of' # Bool.transform() config['parameters']['vacuum_cost_limit'] = 'smth' # Number.transform() self.p.reload_config(config, True) self.assertEqual(mock_warning.call_args_list[-1][0][0], 'Exception %r when running query') def test_resolve_connection_addresses(self): self.p.config._config['use_unix_socket'] = self.p.config._config['use_unix_socket_repl'] = True self.p.config.resolve_connection_addresses() self.assertEqual(self.p.config.local_replication_address, {'host': '/tmp', 'port': '5432'}) self.p.config._server_parameters.pop('unix_socket_directories') self.p.config.resolve_connection_addresses() self.assertEqual(self.p.connection_pool.conn_kwargs, {'connect_timeout': 3, 'dbname': 'postgres', 'fallback_application_name': 'Patroni', 'options': '-c statement_timeout=2000', 'password': 'test', 'port': '5432', 'user': 'foo'}) @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True)) def test_get_major_version(self): with patch('builtins.open', mock_open(read_data='9.4')): self.assertEqual(self.p.get_major_version(), 90400) with patch('builtins.open', Mock(side_effect=Exception)): self.assertEqual(self.p.get_major_version(), 0) def test_postmaster_start_time(self): now = datetime.datetime.now() with patch.object(MockCursor, "fetchall", Mock(return_value=[(now, True, '', '', '', '', False)])): self.assertEqual(self.p.postmaster_start_time(), now.isoformat(sep=' ')) t = Thread(target=self.p.postmaster_start_time) t.start() t.join() with patch.object(MockCursor, "execute", side_effect=psycopg.Error): self.assertIsNone(self.p.postmaster_start_time()) def test_check_for_startup(self): with patch('subprocess.call', return_value=0): self.p._state = 'starting' self.assertFalse(self.p.check_for_startup()) self.assertEqual(self.p.state, 'running') with patch('subprocess.call', return_value=1): self.p._state = 'starting' self.assertTrue(self.p.check_for_startup()) self.assertEqual(self.p.state, 'starting') with patch('subprocess.call', return_value=2): self.p._state = 'starting' self.assertFalse(self.p.check_for_startup()) self.assertEqual(self.p.state, 'start failed') with patch('subprocess.call', return_value=0): self.p._state = 'running' self.assertFalse(self.p.check_for_startup()) self.assertEqual(self.p.state, 'running') with patch('subprocess.call', return_value=127): self.p._state = 'running' self.assertFalse(self.p.check_for_startup()) self.assertEqual(self.p.state, 'running') self.p._state = 'starting' self.assertFalse(self.p.check_for_startup()) self.assertEqual(self.p.state, 'running') def test_wait_for_startup(self): state = {'sleeps': 0, 'num_rejects': 0, 'final_return': 0} self.__thread_ident = current_thread().ident def increment_sleeps(*args): if current_thread().ident == self.__thread_ident: print("Sleep") state['sleeps'] += 1 def isready_return(*args): ret = 1 if state['sleeps'] < state['num_rejects'] else state['final_return'] print("Isready {0} {1}".format(ret, state)) return ret def time_in_state(*args): return state['sleeps'] with patch('subprocess.call', side_effect=isready_return): with patch('time.sleep', side_effect=increment_sleeps): self.p.time_in_state = Mock(side_effect=time_in_state) self.p._state = 'stopped' self.assertTrue(self.p.wait_for_startup()) self.assertEqual(state['sleeps'], 0) self.p._state = 'starting' state['num_rejects'] = 5 self.assertTrue(self.p.wait_for_startup()) self.assertEqual(state['sleeps'], 5) self.p._state = 'starting' state['sleeps'] = 0 state['final_return'] = 2 self.assertFalse(self.p.wait_for_startup()) self.p._state = 'starting' state['sleeps'] = 0 state['final_return'] = 0 self.assertFalse(self.p.wait_for_startup(timeout=2)) self.assertEqual(state['sleeps'], 3) with patch.object(Postgresql, 'check_startup_state_changed', Mock(return_value=False)): self.p.cancellable.cancel() self.p._state = 'starting' self.assertIsNone(self.p.wait_for_startup()) def test_get_server_parameters(self): config = {'parameters': {'wal_level': 'hot_standby', 'max_prepared_transactions': 100}, 'listen': '0'} with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)): self.p.config.get_server_parameters(config) with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)): self.p.config.get_server_parameters(config) self.p.config.set_synchronous_standby_names('foo') self.assertTrue(str(self.p.config.get_server_parameters(config)).startswith(' None: kwargs = { 'leader': leader, 'quorum': quorum, 'voters': voters, 'numsync': numsync, 'sync': sync, 'numsync_confirmed': numsync_confirmed, 'active': active, 'sync_wanted': sync_wanted, 'leader_wanted': leader_wanted } result = list(QuorumStateResolver(**kwargs)) self.assertEqual(result, expected) # also check interrupted transitions if len(result) > 0 and result[0][0] != 'restart' and kwargs['leader'] == result[0][1]: if result[0][0] == 'sync': kwargs.update(numsync=result[0][2], sync=result[0][3]) else: kwargs.update(leader=result[0][1], quorum=result[0][2], voters=result[0][3]) kwargs['expected'] = expected[1:] self.check_state_transitions(**kwargs) def test_1111(self): leader = 'a' # Add node self.check_state_transitions(leader=leader, quorum=0, voters=set(), numsync=0, sync=set(), numsync_confirmed=0, active=set('b'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 1, set('b')), ('restart', leader, 0, set()), ]) self.check_state_transitions(leader=leader, quorum=0, voters=set(), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('b'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('b')) ]) self.check_state_transitions(leader=leader, quorum=0, voters=set(), numsync=0, sync=set(), numsync_confirmed=0, active=set('bcde'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bcde')), ('restart', leader, 0, set()), ]) self.check_state_transitions(leader=leader, quorum=0, voters=set(), numsync=2, sync=set('bcde'), numsync_confirmed=1, active=set('bcde'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 3, set('bcde')), ]) def test_1222(self): """2 node cluster""" leader = 'a' # Active set matches state self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('b'), sync_wanted=2, leader_wanted=leader, expected=[]) # Add node by increasing quorum self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('BC'), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 1, set('bC')), ('sync', leader, 1, set('bC')), ]) # Add node by increasing sync self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bc'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bc')), ('quorum', leader, 1, set('bc')), ]) # Reduce quorum after added node caught up self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'), numsync=2, sync=set('bc'), numsync_confirmed=2, active=set('bc'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('bc')), ]) # Add multiple nodes by increasing both sync and quorum self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('BCdE'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bC')), ('quorum', leader, 3, set('bCdE')), ('sync', leader, 2, set('bCdE')), ]) # Reduce quorum after added nodes caught up self.check_state_transitions(leader=leader, quorum=3, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=3, active=set('bcde'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 2, set('bcde')), ]) # Primary is alone self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=1, sync=set('b'), numsync_confirmed=0, active=set(), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 0, set()), ('sync', leader, 0, set()), ]) # Swap out sync replica self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=1, sync=set('b'), numsync_confirmed=0, active=set('c'), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 0, set()), ('sync', leader, 1, set('c')), ('restart', leader, 0, set()), ]) # Update quorum when added node caught up self.check_state_transitions(leader=leader, quorum=0, voters=set(), numsync=1, sync=set('c'), numsync_confirmed=1, active=set('c'), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('c')), ]) def test_1233(self): """Interrupted transition from 2 node cluster to 3 node fully sync cluster""" leader = 'a' # Node c went away, transition back to 2 node cluster self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('b'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 1, set('b')), ]) # Node c is available transition to larger quorum set, but not yet caught up. self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bc'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 1, set('bc')), ]) # Add in a new node at the same time, but node c didn't caught up yet self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bcd'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 2, set('bcd')), ('sync', leader, 2, set('bcd')), ]) # All sync nodes caught up, reduce quorum self.check_state_transitions(leader=leader, quorum=2, voters=set('bcd'), numsync=2, sync=set('bcd'), numsync_confirmed=3, active=set('bcd'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 1, set('bcd')), ]) # Change replication factor at the same time self.check_state_transitions(leader=leader, quorum=0, voters=set('b'), numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bc'), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 1, set('bc')), ('sync', leader, 1, set('bc')), ]) def test_2322(self): """Interrupted transition from 2 node cluster to 3 node cluster with replication factor 2""" leader = 'a' # Node c went away, transition back to 2 node cluster self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('b'), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('b')), ]) # Node c is available transition to larger quorum set. self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bc'), sync_wanted=1, leader_wanted=leader, expected=[ ('sync', leader, 1, set('bc')), ]) # Add in a new node at the same time self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bcd'), sync_wanted=1, leader_wanted=leader, expected=[ ('sync', leader, 1, set('bc')), ('quorum', leader, 2, set('bcd')), ('sync', leader, 1, set('bcd')), ]) # Convert to a fully synced cluster self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'), numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bc'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bc')), ]) # Reduce quorum after all nodes caught up self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'), numsync=2, sync=set('bc'), numsync_confirmed=2, active=set('bc'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('bc')), ]) def test_3535(self): leader = 'a' # remove nodes self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bc'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bc')), ('quorum', leader, 0, set('bc')), ]) self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=3, active=set('bcd'), sync_wanted=2, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bcd')), ('quorum', leader, 1, set('bcd')), ]) # remove nodes and decrease sync self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bc'), sync_wanted=1, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bc')), ('quorum', leader, 1, set('bc')), ('sync', leader, 1, set('bc')), ]) self.check_state_transitions(leader=leader, quorum=1, voters=set('bcde'), numsync=3, sync=set('bcde'), numsync_confirmed=2, active=set('bc'), sync_wanted=1, leader_wanted=leader, expected=[ ('sync', leader, 3, set('bcd')), ('quorum', leader, 1, set('bc')), ('sync', leader, 1, set('bc')), ]) # Increase replication factor and decrease quorum self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bcde'), sync_wanted=3, leader_wanted=leader, expected=[ ('sync', leader, 3, set('bcde')), ]) # decrease quorum after more nodes caught up self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=3, sync=set('bcde'), numsync_confirmed=3, active=set('bcde'), sync_wanted=3, leader_wanted=leader, expected=[ ('quorum', leader, 1, set('bcde')), ]) # Add node with decreasing sync and increasing quorum self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bcdef'), sync_wanted=1, leader_wanted=leader, expected=[ # increase quorum by 2, 1 for added node and another for reduced sync ('quorum', leader, 4, set('bcdef')), # now reduce replication factor to requested value ('sync', leader, 1, set('bcdef')), ]) # Remove node with increasing sync and decreasing quorum self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'), numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bcd'), sync_wanted=3, leader_wanted=leader, expected=[ # node e removed from sync with replication factor increase ('sync', leader, 3, set('bcd')), # node e removed from voters with quorum decrease ('quorum', leader, 1, set('bcd')), ]) def test_remove_nosync_node(self): leader = 'a' self.check_state_transitions(leader=leader, quorum=0, voters=set('bc'), numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('b'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('b')), ('sync', leader, 1, set('b')) ]) def test_swap_sync_node(self): leader = 'a' self.check_state_transitions(leader=leader, quorum=0, voters=set('bc'), numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bd'), sync_wanted=2, leader_wanted=leader, expected=[ ('quorum', leader, 0, set('b')), ('sync', leader, 2, set('bd')), ('quorum', leader, 1, set('bd')) ]) def test_promotion(self): # Beginning stat: 'a' in the primary, 1 of bcd in sync # a fails, c gets quorum votes and promotes self.check_state_transitions(leader='a', quorum=2, voters=set('bcd'), numsync=0, sync=set(), numsync_confirmed=0, active=set(), sync_wanted=1, leader_wanted='c', expected=[ ('sync', 'a', 1, set('abd')), # set a and b to sync ('quorum', 'c', 2, set('abd')), # set c as a leader and move a to voters # and stop because there are no active nodes ]) # next loop, b managed to reconnect self.check_state_transitions(leader='c', quorum=2, voters=set('abd'), numsync=1, sync=set('abd'), numsync_confirmed=0, active=set('b'), sync_wanted=1, leader_wanted='c', expected=[ ('sync', 'c', 1, set('b')), # remove a from sync as inactive ('quorum', 'c', 0, set('b')), # remove a from voters and reduce quorum ]) # alternative reality: next loop, no one reconnected self.check_state_transitions(leader='c', quorum=2, voters=set('abd'), numsync=1, sync=set('abd'), numsync_confirmed=0, active=set(), sync_wanted=1, leader_wanted='c', expected=[ ('quorum', 'c', 0, set()), ('sync', 'c', 0, set()), ]) def test_nonsync_promotion(self): # Beginning state: 1 of bc in sync. e.g. (a primary, ssn = ANY 1 (b c)) # a fails, d sees b and c, knows that it is in sync and decides to promote. # We include in sync state former primary increasing replication factor # and let situation resolve. Node d ssn=ANY 1 (b c) leader = 'd' self.check_state_transitions(leader='a', quorum=1, voters=set('bc'), numsync=0, sync=set(), numsync_confirmed=0, active=set(), sync_wanted=1, leader_wanted=leader, expected=[ # Set a, b, and c to sync and increase replication factor ('sync', 'a', 2, set('abc')), # Set ourselves as the leader and move the old leader to voters ('quorum', leader, 1, set('abc')), # and stop because there are no active nodes ]) # next loop, b and c managed to reconnect self.check_state_transitions(leader=leader, quorum=1, voters=set('abc'), numsync=2, sync=set('abc'), numsync_confirmed=0, active=set('bc'), sync_wanted=1, leader_wanted=leader, expected=[ ('sync', leader, 2, set('bc')), # Remove a from being synced to. ('quorum', leader, 1, set('bc')), # Remove a from quorum ('sync', leader, 1, set('bc')), # Can now reduce replication factor back ]) # alternative reality: next loop, no one reconnected self.check_state_transitions(leader=leader, quorum=1, voters=set('abc'), numsync=2, sync=set('abc'), numsync_confirmed=0, active=set(), sync_wanted=1, leader_wanted=leader, expected=[ ('quorum', leader, 0, set()), ('sync', leader, 0, set()), ]) def test_invalid_states(self): leader = 'a' # Main invariant is not satisfied, system is in an unsafe state resolver = QuorumStateResolver(leader=leader, quorum=0, voters=set('bc'), numsync=1, sync=set('bc'), numsync_confirmed=1, active=set('bc'), sync_wanted=1, leader_wanted=leader) self.assertRaises(QuorumError, resolver.check_invariants) self.assertEqual(list(resolver), [ ('quorum', leader, 1, set('bc')) ]) # Quorum and sync states mismatched, somebody other than Patroni modified system state resolver = QuorumStateResolver(leader=leader, quorum=1, voters=set('bc'), numsync=2, sync=set('bd'), numsync_confirmed=1, active=set('bd'), sync_wanted=1, leader_wanted=leader) self.assertRaises(QuorumError, resolver.check_invariants) self.assertEqual(list(resolver), [ ('quorum', leader, 1, set('bd')), ('sync', leader, 1, set('bd')), ]) self.assertTrue(repr(resolver.sync).startswith('= times: pass else: scope['times'] += 1 raise PatroniException('Failed!') return inner def test_reset(self): retry = Retry(delay=0, max_tries=2) retry(self._fail()) self.assertEqual(retry._attempts, 1) retry.reset() self.assertEqual(retry._attempts, 0) def test_too_many_tries(self): retry = Retry(delay=0) self.assertRaises(RetryFailedError, retry, self._fail(times=999)) self.assertEqual(retry._attempts, 1) def test_maximum_delay(self): retry = Retry(delay=10, max_tries=100) retry(self._fail(times=10)) self.assertTrue(retry._cur_delay < 4000, retry._cur_delay) # gevent's sleep function is picky about the type self.assertEqual(type(retry._cur_delay), float) def test_deadline(self): retry = Retry(deadline=0.0001) self.assertRaises(RetryFailedError, retry, self._fail(times=100)) def test_copy(self): def _sleep(t): pass retry = Retry(sleep_func=_sleep) rcopy = retry.copy() self.assertTrue(rcopy.sleep_func is _sleep) patroni-4.0.4/tests/test_validator.py000066400000000000000000000414431472010352700177420ustar00rootroot00000000000000import copy import os import socket import tempfile import unittest from io import StringIO from unittest.mock import Mock, mock_open, patch from patroni.dcs import dcs_modules from patroni.validator import Directory, populate_validate_params, schema, Schema available_dcs = [m.split(".")[-1] for m in dcs_modules()] config = { "name": "string", "scope": "string", "log": { "type": "plain", "level": "DEBUG", "traceback_level": "DEBUG", "format": "%(asctime)s %(levelname)s: %(message)s", "dateformat": "%Y-%m-%d %H:%M:%S", "max_queue_size": 100, "dir": "/tmp", "file_num": 10, "file_size": 1000000, "loggers": { "patroni.postmaster": "WARNING", "urllib3": "DEBUG" } }, "restapi": { "listen": "127.0.0.2:800", "connect_address": "127.0.0.2:800", "verify_client": 'none' }, "bootstrap": { "dcs": { "ttl": 1000, "loop_wait": 1000, "retry_timeout": 1000, "maximum_lag_on_failover": 1000 }, "initdb": ["string", {"key": "value"}] }, "consul": { "host": "127.0.0.1:5000" }, "etcd": { "hosts": "127.0.0.1:2379,127.0.0.1:2380" }, "etcd3": { "url": "https://127.0.0.1:2379" }, "exhibitor": { "hosts": ["string"], "port": 4000, "pool_interval": 1000 }, "raft": { "self_addr": "127.0.0.1:2222", "bind_addr": "0.0.0.0:2222", "partner_addrs": ["127.0.0.1:2223", "127.0.0.1:2224"], "data_dir": "/", "password": "12345" }, "zookeeper": { "hosts": "127.0.0.1:3379,127.0.0.1:3380" }, "kubernetes": { "namespace": "string", "labels": {}, "scope_label": "string", "role_label": "string", "use_endpoints": False, "pod_ip": "127.0.0.1", "ports": [{"name": "string", "port": 1000}], "retriable_http_codes": [401], }, "postgresql": { "listen": "127.0.0.2,::1:543", "connect_address": "127.0.0.2:543", "proxy_address": "127.0.0.2:5433", "authentication": { "replication": {"username": "user"}, "superuser": {"username": "user"}, "rewind": {"username": "user"}, }, "data_dir": os.path.join(tempfile.gettempdir(), "data_dir"), "bin_dir": os.path.join(tempfile.gettempdir(), "bin_dir"), "parameters": { "unix_socket_directories": "." }, "pg_hba": [u"string"], "pg_ident": ["string"], "pg_ctl_timeout": 1000, "use_pg_rewind": False }, "watchdog": { "mode": "off", "device": "string" }, "tags": { "nofailover": False, "clonefrom": False, "noloadbalance": False, "nosync": False, "nostream": False } } config_2 = { "some_dir": "very_interesting_dir" } schema2 = Schema({ "some_dir": Directory(contains=["very_interesting_subdir", "another_interesting_subdir"]) }) required_binaries = ["pg_ctl", "initdb", "pg_controldata", "pg_basebackup", "postgres", "pg_isready"] directories = [] files = [] binaries = [] def isfile_side_effect(arg): return arg in files def which_side_effect(arg, path=None): binary = arg if path is None else os.path.join(path, arg) return arg if binary in binaries else None def isdir_side_effect(arg): return arg in directories def exists_side_effect(arg): return isfile_side_effect(arg) or isdir_side_effect(arg) def connect_side_effect(host_port): _, port = host_port if port < 1000: return 1 elif port < 10000: return 0 else: raise socket.gaierror() def mock_getaddrinfo(host, port, *args): if port is None or port == "": port = 0 port = int(port) if port not in range(0, 65536): raise socket.gaierror() if host == "127.0.0.1" or host == "" or host is None: return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, '', ('127.0.0.1', port))] elif host == "127.0.0.2": return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, '', ('127.0.0.2', port))] elif host == "::1": return [(socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_TCP, '', ('::1', port, 0, 0))] else: raise socket.gaierror() def parse_output(output): result = [] for s in output.split("\n"): x = s.split(" ")[0] if x and x not in result: result.append(x) result.sort() return result @patch('socket.socket.connect_ex', Mock(side_effect=connect_side_effect)) @patch('socket.getaddrinfo', Mock(side_effect=mock_getaddrinfo)) @patch('os.path.exists', Mock(side_effect=exists_side_effect)) @patch('os.path.isdir', Mock(side_effect=isdir_side_effect)) @patch('os.path.isfile', Mock(side_effect=isfile_side_effect)) @patch('shutil.which', Mock(side_effect=which_side_effect)) @patch('sys.stderr', new_callable=StringIO) @patch('sys.stdout', new_callable=StringIO) class TestValidator(unittest.TestCase): def setUp(self): del files[:] del directories[:] del binaries[:] def test_empty_config(self, mock_out, mock_err): errors = schema({}) output = "\n".join(errors) expected = list(sorted(['name', 'postgresql', 'restapi', 'scope'] + available_dcs)) self.assertEqual(expected, parse_output(output)) def test_complete_config(self, mock_out, mock_err): errors = schema(config) output = "\n".join(errors) self.assertEqual(['postgresql.bin_dir', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) def test_bin_dir_is_file(self, mock_out, mock_err): files.append(config["postgresql"]["data_dir"]) files.append(config["postgresql"]["bin_dir"]) c = copy.deepcopy(config) c["restapi"]["connect_address"] = 'False:blabla' c["postgresql"]["listen"] = '*:543' c["etcd"]["hosts"] = ["127.0.0.1:2379", "1244.0.0.1:2379", "127.0.0.1:invalidport"] c["kubernetes"]["pod_ip"] = "127.0.0.1111" errors = schema(c) output = "\n".join(errors) self.assertEqual(['etcd.hosts.1', 'etcd.hosts.2', 'kubernetes.pod_ip', 'postgresql.bin_dir', 'postgresql.data_dir', 'raft.bind_addr', 'raft.self_addr', 'restapi.connect_address'], parse_output(output)) @patch('socket.inet_pton', Mock(), create=True) def test_bin_dir_is_empty(self, mock_out, mock_err): directories.append(config["postgresql"]["data_dir"]) directories.append(config["postgresql"]["bin_dir"]) files.append(os.path.join(config["postgresql"]["data_dir"], "global", "pg_control")) c = copy.deepcopy(config) c["restapi"]["connect_address"] = "127.0.0.1:8008" c["kubernetes"]["pod_ip"] = "::1" c["consul"]["host"] = "127.0.0.1:50000" c["etcd"]["host"] = "127.0.0.1:237" c["postgresql"]["listen"] = "127.0.0.1:5432" with patch('patroni.validator.open', mock_open(read_data='9')): errors = schema(c) output = "\n".join(errors) self.assertEqual(['consul.host', 'etcd.host', 'postgresql.bin_dir', 'postgresql.data_dir', 'postgresql.listen', 'raft.bind_addr', 'raft.self_addr', 'restapi.connect_address'], parse_output(output)) def test_bin_dir_is_empty_string_executables_in_path(self, mock_out, mock_err): binaries.extend(required_binaries) c = copy.deepcopy(config) c["postgresql"]["bin_dir"] = "" errors = schema(c) output = "\n".join(errors) self.assertEqual(['raft.bind_addr', 'raft.self_addr'], parse_output(output)) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1")) def test_data_dir_contains_pg_version(self, mock_out, mock_err): directories.append(config["postgresql"]["data_dir"]) directories.append(config["postgresql"]["bin_dir"]) directories.append(os.path.join(config["postgresql"]["data_dir"], "pg_wal")) files.append(os.path.join(config["postgresql"]["data_dir"], "global", "pg_control")) files.append(os.path.join(config["postgresql"]["data_dir"], "PG_VERSION")) binaries.extend(required_binaries) c = copy.deepcopy(config) c["postgresql"]["bin_dir"] = "" # to cover postgres --version call from PATH with patch('patroni.validator.open', mock_open(read_data='12')): errors = schema(c) output = "\n".join(errors) self.assertEqual(['raft.bind_addr', 'raft.self_addr'], parse_output(output)) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1")) def test_pg_version_mismatch(self, mock_out, mock_err): directories.append(config["postgresql"]["data_dir"]) directories.append(config["postgresql"]["bin_dir"]) directories.append(os.path.join(config["postgresql"]["data_dir"], "pg_wal")) files.append(os.path.join(config["postgresql"]["data_dir"], "global", "pg_control")) files.append(os.path.join(config["postgresql"]["data_dir"], "PG_VERSION")) binaries.extend([os.path.join(config["postgresql"]["bin_dir"], i) for i in required_binaries]) c = copy.deepcopy(config) c["etcd"]["hosts"] = [] c["postgresql"]["listen"] = '127.0.0.2,*:543' with patch('patroni.validator.open', mock_open(read_data='11')): errors = schema(c) output = "\n".join(errors) self.assertEqual(['etcd.hosts', 'postgresql.data_dir', 'postgresql.listen', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1")) def test_pg_wal_doesnt_exist(self, mock_out, mock_err): binaries.extend([os.path.join(config["postgresql"]["bin_dir"], i) for i in required_binaries]) directories.append(config["postgresql"]["data_dir"]) directories.append(config["postgresql"]["bin_dir"]) files.append(os.path.join(config["postgresql"]["data_dir"], "global", "pg_control")) files.append(os.path.join(config["postgresql"]["data_dir"], "PG_VERSION")) c = copy.deepcopy(config) with patch('patroni.validator.open', mock_open(read_data='11')): errors = schema(c) output = "\n".join(errors) self.assertEqual(['postgresql.data_dir', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) def test_data_dir_is_empty_string(self, mock_out, mock_err): binaries.extend(required_binaries) directories.append(config["postgresql"]["data_dir"]) directories.append(config["postgresql"]["bin_dir"]) c = copy.deepcopy(config) c["kubernetes"] = False c["postgresql"]["pg_hba"] = "" c["postgresql"]["data_dir"] = "" c["postgresql"]["bin_dir"] = "" errors = schema(c) output = "\n".join(errors) self.assertEqual(['kubernetes', 'postgresql.data_dir', 'postgresql.pg_hba', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) def test_directory_contains(self, mock_out, mock_err): directories.extend([config_2["some_dir"], os.path.join(config_2["some_dir"], "very_interesting_subdir")]) errors = schema2(config_2) output = "\n".join(errors) self.assertEqual(['some_dir'], parse_output(output)) def test_validate_binary_name(self, mock_out, mock_err): r = copy.copy(required_binaries) r.remove('postgres') r.append('fake-postgres') binaries.extend(r) c = copy.deepcopy(config) c["postgresql"]["bin_name"] = {"postgres": "fake-postgres"} del c["postgresql"]["bin_dir"] errors = schema(c) output = "\n".join(errors) self.assertEqual(['raft.bind_addr', 'raft.self_addr'], parse_output(output)) def test_validate_binary_name_missing(self, mock_out, mock_err): r = copy.copy(required_binaries) r.remove('postgres') binaries.extend(r) c = copy.deepcopy(config) c["postgresql"]["bin_name"] = {"postgres": "fake-postgres"} del c["postgresql"]["bin_dir"] errors = schema(c) output = "\n".join(errors) self.assertEqual(['postgresql.bin_dir', 'postgresql.bin_name.postgres', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) def test_validate_binary_name_empty_string(self, mock_out, mock_err): r = copy.copy(required_binaries) binaries.extend(r) c = copy.deepcopy(config) c["postgresql"]["bin_name"] = {"postgres": ""} del c["postgresql"]["bin_dir"] errors = schema(c) output = "\n".join(errors) self.assertEqual(['postgresql.bin_dir', 'postgresql.bin_name.postgres', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) def test_one_of(self, _, __): c = copy.deepcopy(config) # Providing neither is fine del c["tags"]["nofailover"] errors = schema(c) self.assertNotIn("tags Multiple of ('nofailover', 'failover_priority') provided", errors) # Just nofailover is fine c["tags"]["nofailover"] = False errors = schema(c) self.assertNotIn("tags Multiple of ('nofailover', 'failover_priority') provided", errors) # Just failover_priority is fine del c["tags"]["nofailover"] c["tags"]["failover_priority"] = 1 errors = schema(c) self.assertNotIn("tags Multiple of ('nofailover', 'failover_priority') provided", errors) # Providing both is not fine c["tags"]["nofailover"] = False errors = schema(c) self.assertIn("tags Multiple of ('nofailover', 'failover_priority') provided", errors) def test_failover_priority_int(self, *args): c = copy.deepcopy(config) del c["tags"]["nofailover"] c["tags"]["failover_priority"] = 'a string' errors = schema(c) self.assertIn('tags.failover_priority a string is not an integer', errors) c = copy.deepcopy(config) del c["tags"]["nofailover"] c["tags"]["failover_priority"] = -6 errors = schema(c) self.assertIn('tags.failover_priority -6 didn\'t pass validation: Wrong value', errors) def test_json_log_format(self, *args): c = copy.deepcopy(config) c["log"]["type"] = "json" c["log"]["format"] = {"levelname": "level"} errors = schema(c) self.assertIn("log.format {'levelname': 'level'} didn't pass validation: Should be a string or a list", errors) c["log"]["format"] = [] errors = schema(c) self.assertIn("log.format [] didn't pass validation: should contain at least one item", errors) c["log"]["format"] = [{"levelname": []}] errors = schema(c) self.assertIn("log.format [{'levelname': []}] didn't pass validation: " "each item should be a string or a dictionary with string values", errors) c["log"]["format"] = [[]] errors = schema(c) self.assertIn("log.format [[]] didn't pass validation: " "each item should be a string or a dictionary with string values", errors) c["log"]["format"] = ['foo'] errors = schema(c) output = "\n".join(errors) self.assertEqual(['postgresql.bin_dir', 'raft.bind_addr', 'raft.self_addr'], parse_output(output)) @patch('socket.socket.connect_ex', Mock(return_value=0)) def test_bound_port_checks_without_ignore(self, mock_out, mock_err): # When ignore_listen_port is False (default case), an error should be raised if the ports are already bound. c = copy.deepcopy(config) c['restapi']['listen'] = "127.0.0.1:8000" c['postgresql']['listen'] = "127.0.0.1:9000" c['raft']['self_addr'] = "127.0.0.2:9200" populate_validate_params(ignore_listen_port=False) errors = schema(c) output = "\n".join(errors) self.assertEqual(['postgresql.bin_dir', 'postgresql.listen', 'raft.bind_addr', 'restapi.listen'], parse_output(output)) @patch('socket.socket.connect_ex', Mock(return_value=0)) def test_bound_port_checks_with_ignore(self, mock_out, mock_err): c = copy.deepcopy(config) c['restapi']['listen'] = "127.0.0.1:8000" c['postgresql']['listen'] = "127.0.0.1:9000" c['raft']['self_addr'] = "127.0.0.2:9200" c['raft']['bind_addr'] = "127.0.0.1:9300" # Case: When ignore_listen_port is True, error should NOT be raised # even if the ports are already bound. populate_validate_params(ignore_listen_port=True) errors = schema(c) output = "\n".join(errors) self.assertEqual(['postgresql.bin_dir'], parse_output(output)) patroni-4.0.4/tests/test_wale_restore.py000066400000000000000000000152211472010352700204430ustar00rootroot00000000000000import subprocess import unittest from threading import current_thread from unittest.mock import Mock, mock_open, patch, PropertyMock import patroni.psycopg as psycopg from patroni.scripts import wale_restore from patroni.scripts.wale_restore import get_major_version, main as _main, WALERestore from . import MockConnect, psycopg_connect wale_output_header = ( b'name\tlast_modified\t' b'expanded_size_bytes\t' b'wal_segment_backup_start\twal_segment_offset_backup_start\t' b'wal_segment_backup_stop\twal_segment_offset_backup_stop\n' ) wale_output_values = ( b'base_00000001000000000000007F_00000040\t2015-05-18T10:13:25.000Z\t' b'167772160\t' b'00000001000000000000007F\t00000040\t' b'00000001000000000000007F\t00000240\n' ) wale_output = wale_output_header + wale_output_values wale_restore.RETRY_SLEEP_INTERVAL = 0.001 # Speed up retries WALE_TEST_RETRIES = 2 @patch('os.access', Mock(return_value=True)) @patch('os.makedirs', Mock(return_value=True)) @patch('os.path.exists', Mock(return_value=True)) @patch('os.path.isdir', Mock(return_value=True)) @patch('patroni.psycopg.connect', psycopg_connect) @patch('subprocess.check_output', Mock(return_value=wale_output)) class TestWALERestore(unittest.TestCase): def setUp(self): self.wale_restore = WALERestore('batman', '/data', 'host=batman port=5432 user=batman', '/etc', 100, 100, 1, 0, WALE_TEST_RETRIES) def test_should_use_s3_to_create_replica(self): self.__thread_ident = current_thread().ident sleeps = [0] def mock_sleep(*args): if current_thread().ident == self.__thread_ident: sleeps[0] += 1 self.assertTrue(self.wale_restore.should_use_s3_to_create_replica()) with patch.object(MockConnect, 'server_version', PropertyMock(return_value=100000)): self.assertTrue(self.wale_restore.should_use_s3_to_create_replica()) with patch('subprocess.check_output', Mock(return_value=wale_output.replace(b'167772160', b'1'))): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) with patch('patroni.psycopg.connect', Mock(side_effect=psycopg.Error("foo"))): save_no_leader = self.wale_restore.no_leader save_leader_connection = self.wale_restore.leader_connection self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) with patch('time.sleep', mock_sleep): self.wale_restore.no_leader = 1 self.assertTrue(self.wale_restore.should_use_s3_to_create_replica()) # verify retries self.assertEqual(sleeps[0], WALE_TEST_RETRIES) self.wale_restore.leader_connection = '' self.assertTrue(self.wale_restore.should_use_s3_to_create_replica()) self.wale_restore.no_leader = save_no_leader self.wale_restore.leader_connection = save_leader_connection with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) with patch('subprocess.check_output', Mock(return_value=wale_output_header)): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) with patch('subprocess.check_output', Mock(return_value=wale_output + wale_output_values)): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) with patch('subprocess.check_output', Mock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) def test_create_replica_with_s3(self): with patch('subprocess.call', Mock(return_value=0)): self.assertEqual(self.wale_restore.create_replica_with_s3(), 0) with patch.object(self.wale_restore, 'fix_subdirectory_path_if_broken', Mock(return_value=False)): self.assertEqual(self.wale_restore.create_replica_with_s3(), 2) with patch('subprocess.call', Mock(side_effect=Exception("foo"))): self.assertEqual(self.wale_restore.create_replica_with_s3(), 1) def test_run(self): self.wale_restore.init_error = True self.assertEqual(self.wale_restore.run(), 2) # this would do 2 retries 1 sec each self.wale_restore.init_error = False with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=True)): with patch.object(self.wale_restore, 'create_replica_with_s3', Mock(return_value=0)): self.assertEqual(self.wale_restore.run(), 0) with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=False)): self.assertEqual(self.wale_restore.run(), 2) with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=None)): self.assertEqual(self.wale_restore.run(), 1) with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(side_effect=Exception)): self.assertEqual(self.wale_restore.run(), 2) @patch('sys.exit', Mock()) def test_main(self): self.__thread_ident = current_thread().ident sleeps = [0] def mock_sleep(*args): if current_thread().ident == self.__thread_ident: sleeps[0] += 1 with patch.object(WALERestore, 'run', Mock(return_value=0)): self.assertEqual(_main(), 0) with patch.object(WALERestore, 'run', Mock(return_value=1)), \ patch('time.sleep', mock_sleep): self.assertEqual(_main(), 1) self.assertEqual(sleeps[0], WALE_TEST_RETRIES) @patch('os.path.isfile', Mock(return_value=True)) def test_get_major_version(self): with patch('builtins.open', mock_open(read_data='9.4')): self.assertEqual(get_major_version("data"), 9.4) with patch('builtins.open', side_effect=OSError): self.assertEqual(get_major_version("data"), 0.0) @patch('os.path.islink', Mock(return_value=True)) @patch('os.readlink', Mock(return_value="foo")) @patch('os.remove', Mock()) @patch('os.mkdir', Mock()) def test_fix_subdirectory_path_if_broken(self): with patch('os.path.exists', Mock(return_value=False)): # overriding the class-wide mock self.assertTrue(self.wale_restore.fix_subdirectory_path_if_broken("data1")) for fn in ('os.remove', 'os.mkdir'): with patch(fn, side_effect=OSError): self.assertFalse(self.wale_restore.fix_subdirectory_path_if_broken("data3")) patroni-4.0.4/tests/test_watchdog.py000066400000000000000000000216521472010352700175550ustar00rootroot00000000000000import ctypes import os import sys import unittest from unittest.mock import Mock, patch, PropertyMock import patroni.watchdog.linux as linuxwd from patroni.watchdog import Watchdog, WatchdogError from patroni.watchdog.base import NullWatchdog from patroni.watchdog.linux import LinuxWatchdogDevice class MockDevice: def __init__(self, fd, filename, flag): self.fd = fd self.filename = filename self.flag = flag self.timeout = 60 self.open = True self.writes = [] mock_devices = [None] def mock_open(filename, flag): fd = len(mock_devices) mock_devices.append(MockDevice(fd, filename, flag)) return fd def mock_ioctl(fd, op, arg=None, mutate_flag=False): assert 0 < fd < len(mock_devices) dev = mock_devices[fd] sys.stderr.write("Ioctl %d %d %r\n" % (fd, op, arg)) if op == linuxwd.WDIOC_GETSUPPORT: sys.stderr.write("Get support\n") assert (mutate_flag is True) arg.options = sum(map(linuxwd.WDIOF.get, ['SETTIMEOUT', 'KEEPALIVEPING'])) arg.identity = (ctypes.c_ubyte * 32)(*map(ord, 'Mock Watchdog')) elif op == linuxwd.WDIOC_GETTIMEOUT: arg.value = dev.timeout elif op == linuxwd.WDIOC_SETTIMEOUT: sys.stderr.write("Set timeout called with %s\n" % arg.value) assert 0 < arg.value < 65535 dev.timeout = arg.value - 1 else: raise Exception("Unknown op %d", op) return 0 def mock_write(fd, string): assert 0 < fd < len(mock_devices) assert len(string) == 1 assert mock_devices[fd].open mock_devices[fd].writes.append(string) def mock_close(fd): assert 0 < fd < len(mock_devices) assert mock_devices[fd].open mock_devices[fd].open = False @unittest.skipIf(os.name == 'nt', "Windows not supported") @patch('os.open', mock_open) @patch('os.write', mock_write) @patch('os.close', mock_close) @patch('fcntl.ioctl', mock_ioctl) class TestWatchdog(unittest.TestCase): def setUp(self): mock_devices[:] = [None] @patch('platform.system', Mock(return_value='Linux')) @patch.object(LinuxWatchdogDevice, 'can_be_disabled', PropertyMock(return_value=True)) def test_unsafe_timeout_disable_watchdog_and_exit(self): watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'safety_margin': -1}}) self.assertEqual(watchdog.activate(), False) self.assertEqual(watchdog.is_running, False) @patch('platform.system', Mock(return_value='Linux')) @patch.object(LinuxWatchdogDevice, 'get_timeout', Mock(return_value=16)) def test_timeout_does_not_ensure_safe_termination(self): Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'auto', 'safety_margin': -1}}).activate() self.assertEqual(len(mock_devices), 2) @patch('platform.system', Mock(return_value='Linux')) @patch.object(Watchdog, 'is_running', PropertyMock(return_value=False)) def test_watchdog_not_activated(self): self.assertFalse(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}).activate()) @patch('platform.system', Mock(return_value='Linux')) @patch.object(LinuxWatchdogDevice, 'is_running', PropertyMock(return_value=False)) def test_watchdog_activate(self): with patch.object(LinuxWatchdogDevice, 'open', Mock(side_effect=WatchdogError(''))): self.assertTrue(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'auto'}}).activate()) self.assertFalse(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}).activate()) @patch('platform.system', Mock(return_value='Linux')) def test_basic_operation(self): watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}) watchdog.activate() self.assertEqual(len(mock_devices), 2) device = mock_devices[-1] self.assertTrue(device.open) self.assertEqual(device.timeout, 24) watchdog.keepalive() self.assertEqual(len(device.writes), 1) watchdog.impl._fd, fd = None, watchdog.impl._fd watchdog.keepalive() self.assertEqual(len(device.writes), 1) watchdog.impl._fd = fd watchdog.disable() self.assertFalse(device.open) self.assertEqual(device.writes[-1], b'V') def test_invalid_timings(self): watchdog = Watchdog({'ttl': 30, 'loop_wait': 20, 'watchdog': {'mode': 'automatic', 'safety_margin': -1}}) watchdog.activate() self.assertEqual(len(mock_devices), 1) self.assertFalse(watchdog.is_running) def test_parse_mode(self): with patch('patroni.watchdog.base.logger.warning', new_callable=Mock()) as warning_mock: watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}}) self.assertEqual(watchdog.config.mode, 'off') warning_mock.assert_called_once() @patch('platform.system', Mock(return_value='Unknown')) def test_unsupported_platform(self): self.assertRaises(SystemExit, Watchdog, {'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required', 'driver': 'bad'}}) def test_exceptions(self): wd = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}}) wd.impl.close = wd.impl.keepalive = Mock(side_effect=WatchdogError('')) self.assertTrue(wd.activate()) self.assertIsNone(wd.keepalive()) self.assertIsNone(wd.disable()) @patch('platform.system', Mock(return_value='Linux')) def test_config_reload(self): watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}}) self.assertTrue(watchdog.activate()) self.assertTrue(watchdog.is_running) watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'off'}}) self.assertFalse(watchdog.is_running) watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}}) self.assertFalse(watchdog.is_running) watchdog.keepalive() self.assertTrue(watchdog.is_running) watchdog.disable() watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'driver': 'unknown'}}) self.assertFalse(watchdog.is_healthy) self.assertFalse(watchdog.activate()) watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}}) self.assertFalse(watchdog.is_running) watchdog.keepalive() self.assertTrue(watchdog.is_running) watchdog.reload_config({'ttl': 60, 'loop_wait': 15, 'watchdog': {'mode': 'required'}}) watchdog.keepalive() self.assertTrue(watchdog.is_running) self.assertEqual(watchdog.config.timeout, 60 - 5) watchdog.reload_config({'ttl': 60, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'safety_margin': -1}}) watchdog.keepalive() self.assertTrue(watchdog.is_running) self.assertEqual(watchdog.config.timeout, 60 // 2) class TestNullWatchdog(unittest.TestCase): def test_basics(self): watchdog = NullWatchdog() self.assertTrue(watchdog.can_be_disabled) self.assertRaises(WatchdogError, watchdog.set_timeout, 1) self.assertEqual(watchdog.describe(), 'NullWatchdog') self.assertIsInstance(NullWatchdog.from_config({}), NullWatchdog) @unittest.skipIf(os.name == 'nt', "Windows not supported") class TestLinuxWatchdogDevice(unittest.TestCase): def setUp(self): self.impl = LinuxWatchdogDevice.from_config({}) @patch('os.open', Mock(return_value=3)) @patch('os.write', Mock(side_effect=OSError)) @patch('fcntl.ioctl', Mock(return_value=0)) def test_basics(self): self.impl.open() try: if self.impl.get_support().has_foo: self.assertFail() except Exception as e: self.assertTrue(isinstance(e, AttributeError)) self.assertRaises(WatchdogError, self.impl.close) self.assertRaises(WatchdogError, self.impl.keepalive) self.assertRaises(WatchdogError, self.impl.set_timeout, -1) @patch('os.open', Mock(return_value=3)) @patch('fcntl.ioctl', Mock(side_effect=OSError)) def test__ioctl(self): self.assertRaises(WatchdogError, self.impl.get_support) self.impl.open() self.assertRaises(WatchdogError, self.impl.get_support) def test_is_healthy(self): self.assertFalse(self.impl.is_healthy) @patch('os.open', Mock(return_value=3)) @patch('fcntl.ioctl', Mock(side_effect=OSError)) def test_error_handling(self): self.impl.open() self.assertRaises(WatchdogError, self.impl.get_timeout) self.assertRaises(WatchdogError, self.impl.set_timeout, 10) # We still try to output a reasonable string even if getting info errors self.assertEqual(self.impl.describe(), "Linux watchdog device") @patch('os.open', Mock(side_effect=OSError)) def test_open(self): self.assertRaises(WatchdogError, self.impl.open) patroni-4.0.4/tests/test_zookeeper.py000066400000000000000000000324151472010352700177570ustar00rootroot00000000000000import select import unittest from unittest.mock import Mock, patch, PropertyMock from kazoo.client import KazooClient from kazoo.exceptions import NodeExistsError, NoNodeError from kazoo.handlers.threading import SequentialThreadingHandler from kazoo.protocol.states import KeeperState, WatchedEvent, ZnodeStat from kazoo.retry import RetryFailedError from patroni.dcs import get_dcs from patroni.dcs.zookeeper import Cluster, PatroniKazooClient, \ PatroniSequentialThreadingHandler, ZooKeeper, ZooKeeperError from patroni.postgresql.mpp import get_mpp class MockKazooClient(Mock): handler = PatroniSequentialThreadingHandler(10) leader = False exists = True def __init__(self, *args, **kwargs): super(MockKazooClient, self).__init__() self._session_timeout = 30000 @property def client_id(self): return (-1, '') @staticmethod def retry(func, *args, **kwargs): return func(*args, **kwargs) def get(self, path, watch=None): if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") if path == '/broken/status': return (b'{', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) elif path in ('/no_node', '/legacy/status'): raise NoNodeError elif '/members/' in path: return ( b'postgres://repuser:rep-pass@localhost:5434/postgres?application_name=http://127.0.0.1:8009/patroni', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0 if self.exists else -1, 0, 0, 0) ) elif path.endswith('/optime/leader'): return (b'500', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) elif path.endswith('/leader'): if self.leader: return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) elif path.endswith('/initialize'): return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) elif path.endswith('/status'): return (b'{"optime":500,"slots":{"ls":1234567},"retain_slots":["postgresql0"]}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) elif path.endswith('/failsafe'): return (b'{a}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) @staticmethod def get_children(path, watch=None, include_data=False): if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") if path.startswith('/no_node'): raise NoNodeError elif path in ['/service/bla/', '/service/test/']: return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync', 'failsafe', '0', '1'] return ['foo', 'bar', 'buzz'] def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False): if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") if not isinstance(value, bytes): raise TypeError("Invalid type for 'value' (must be a byte string)") if b'Exception' in value: raise Exception if path.endswith('/initialize') or path == '/service/test/optime/leader': raise Exception elif b'retry' in value or (b'exists' in value and self.exists): raise NodeExistsError def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False): return self.create(path, value, acl, ephemeral, sequence, makepath) or Mock() @staticmethod def set(path, value, version=-1): if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") if not isinstance(value, bytes): raise TypeError("Invalid type for 'value' (must be a byte string)") if path == '/service/bla/optime/leader': raise Exception if path == '/service/test/members/bar' and b'retry' in value: return if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'): if b'Exception' in value: raise Exception elif value == b'ok': return raise NoNodeError def set_async(self, path, value, version=-1): return self.set(path, value, version) or Mock() def delete(self, path, version=-1, recursive=False): if not isinstance(path, str): raise TypeError("Invalid type for 'path' (string expected)") self.exists = False if path == '/service/test/leader': self.leader = True raise Exception elif path == '/service/test/members/buzz': raise Exception elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar': raise NoNodeError def delete_async(self, path, version=-1, recursive=False): return self.delete(path, version, recursive) or Mock() class TestPatroniSequentialThreadingHandler(unittest.TestCase): def setUp(self): self.handler = PatroniSequentialThreadingHandler(10) @patch.object(SequentialThreadingHandler, 'create_connection', Mock()) def test_create_connection(self): self.assertIsNotNone(self.handler.create_connection(())) self.assertIsNotNone(self.handler.create_connection((), 40)) self.assertIsNotNone(self.handler.create_connection(timeout=40)) def test_select(self): with patch.object(SequentialThreadingHandler, 'select', Mock(side_effect=ValueError)): self.assertRaises(select.error, self.handler.select) with patch.object(SequentialThreadingHandler, 'select', Mock(side_effect=IOError)): self.assertRaises(Exception, self.handler.select) class TestPatroniKazooClient(unittest.TestCase): def test__call(self): c = PatroniKazooClient() with patch.object(KazooClient, '_call', Mock()): self.assertIsNotNone(c._call(None, Mock())) c._state = KeeperState.CONNECTING self.assertFalse(c._call(None, Mock())) class TestZooKeeper(unittest.TestCase): @patch('patroni.dcs.zookeeper.PatroniKazooClient', MockKazooClient) def setUp(self): self.zk = get_dcs({'scope': 'test', 'name': 'foo', 'ttl': 30, 'retry_timeout': 10, 'loop_wait': 10, 'zookeeper': {'hosts': ['localhost:2181'], 'set_acls': {'CN=principal2': ['ALL']}}}) self.assertIsInstance(self.zk, ZooKeeper) def test_reload_config(self): self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 10}) self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 5}) def test_get_node(self): self.assertIsNone(self.zk.get_node('/no_node')) def test_get_children(self): self.assertListEqual(self.zk.get_children('/no_node'), []) def test__cluster_loader(self): self.zk._base_path = self.zk._base_path.replace('test', 'bla') self.zk._postgresql_cluster_loader(self.zk.client_path('')) self.zk._base_path = self.zk._base_path = '/broken' self.zk._postgresql_cluster_loader(self.zk.client_path('')) self.zk._base_path = self.zk._base_path = '/legacy' self.zk._postgresql_cluster_loader(self.zk.client_path('')) self.zk._base_path = self.zk._base_path = '/no_node' self.zk._postgresql_cluster_loader(self.zk.client_path('')) def test_get_cluster(self): cluster = self.zk.get_cluster() self.assertEqual(cluster.status.last_lsn, 500) def test__get_citus_cluster(self): self.zk._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) for _ in range(0, 2): cluster = self.zk.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsInstance(cluster.workers[1], Cluster) @patch('patroni.dcs.logger.error') def test_get_mpp_coordinator(self, mock_logger): self.assertIsInstance(self.zk.get_mpp_coordinator(), Cluster) with patch.object(ZooKeeper, '_postgresql_cluster_loader', Mock(side_effect=Exception)): self.assertIsNone(self.zk.get_mpp_coordinator()) mock_logger.assert_called_once() self.assertEqual(mock_logger.call_args[0][0], 'Failed to load %s coordinator cluster from %s: %r') self.assertEqual(mock_logger.call_args[0][1], 'Null') self.assertEqual(mock_logger.call_args[0][2], 'ZooKeeper') self.assertIsInstance(mock_logger.call_args[0][3], ZooKeeperError) @patch('patroni.dcs.logger.error') def test_get_citus_coordinator(self, mock_logger): self.zk._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}) self.assertIsInstance(self.zk.get_mpp_coordinator(), Cluster) with patch.object(ZooKeeper, '_postgresql_cluster_loader', Mock(side_effect=Exception)): self.assertIsNone(self.zk.get_mpp_coordinator()) mock_logger.assert_called_once() self.assertEqual(mock_logger.call_args[0][0], 'Failed to load %s coordinator cluster from %s: %r') self.assertEqual(mock_logger.call_args[0][1], 'Citus') self.assertEqual(mock_logger.call_args[0][2], 'ZooKeeper') self.assertIsInstance(mock_logger.call_args[0][3], ZooKeeperError) def test_delete_leader(self): self.assertTrue(self.zk.delete_leader(self.zk.get_cluster().leader)) def test_set_failover_value(self): self.zk.set_failover_value('') self.zk.set_failover_value('ok') self.zk.set_failover_value('Exception') def test_set_config_value(self): self.zk.set_config_value('', 1) self.zk.set_config_value('ok') self.zk.set_config_value('Exception') def test_initialize(self): self.assertFalse(self.zk.initialize()) def test_cancel_initialization(self): self.zk.cancel_initialization() with patch.object(MockKazooClient, 'delete', Mock()): self.zk.cancel_initialization() def test_touch_member(self): self.zk._name = 'buzz' self.zk.get_cluster() self.zk.touch_member({'new': 'new'}) self.zk._name = 'bar' self.zk.touch_member({'new': 'new'}) self.zk._name = 'na' self.zk._client.exists = 1 self.zk.touch_member({'Exception': 'Exception'}) self.zk._name = 'bar' self.zk.touch_member({'retry': 'retry'}) self.zk._fetch_cluster = True self.zk.get_cluster() self.zk.touch_member({'retry': 'retry'}) self.zk.touch_member({'conn_url': 'postgres://repuser:rep-pass@localhost:5434/postgres', 'api_url': 'http://127.0.0.1:8009/patroni'}) @patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception])) def test_attempt_to_acquire_leader(self): self.assertRaises(ZooKeeperError, self.zk.attempt_to_acquire_leader) self.assertFalse(self.zk.attempt_to_acquire_leader()) def test_take_leader(self): self.zk.take_leader() with patch.object(MockKazooClient, 'create', Mock(side_effect=Exception)): self.zk.take_leader() def test_update_leader(self): cluster = self.zk.get_cluster() self.assertFalse(self.zk.update_leader(cluster, 12345)) with patch.object(MockKazooClient, 'delete', Mock(side_effect=RetryFailedError)): self.assertRaises(ZooKeeperError, self.zk.update_leader, cluster, 12345) with patch.object(MockKazooClient, 'delete', Mock(side_effect=NoNodeError)): self.assertTrue(self.zk.update_leader(cluster, 12345, failsafe={'foo': 'bar'})) with patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception])): self.assertRaises(ZooKeeperError, self.zk.update_leader, cluster, 12345) self.assertFalse(self.zk.update_leader(cluster, 12345)) @patch.object(Cluster, 'min_version', PropertyMock(return_value=(2, 0))) def test_write_leader_optime(self): self.zk.last_lsn = '0' self.zk.write_leader_optime('1') with patch.object(MockKazooClient, 'create_async', Mock()): self.zk.write_leader_optime('1') with patch.object(MockKazooClient, 'set_async', Mock()): self.zk.write_leader_optime('2') self.zk._base_path = self.zk._base_path.replace('test', 'bla') self.zk.get_cluster() self.zk.write_leader_optime('3') def test_delete_cluster(self): self.assertTrue(self.zk.delete_cluster()) def test_watch(self): self.zk.event.wait = Mock() self.zk.watch(None, 0) self.zk.event.is_set = Mock(return_value=True) self.zk._fetch_status = False self.zk.watch(None, 0) def test__kazoo_connect(self): self.zk._client._retry.deadline = 1 self.zk._orig_kazoo_connect = Mock(return_value=(0, 0)) self.zk._kazoo_connect(None, None) def test_sync_state(self): self.zk.set_sync_state_value('') self.zk.set_sync_state_value('ok') self.zk.set_sync_state_value('Exception') self.zk.delete_sync_state() def test_set_history_value(self): self.zk.set_history_value('{}') def test_watcher(self): self.zk._watcher(WatchedEvent('', '', '')) self.assertTrue(self.zk.watch(1, 1)) patroni-4.0.4/tox.ini000066400000000000000000000141561472010352700145160ustar00rootroot00000000000000[common] python_matrix = {36,37,38,39,310,311} postgres_matrix = pg11: PG_MAJOR = 11 pg12: PG_MAJOR = 12 pg13: PG_MAJOR = 13 pg14: PG_MAJOR = 14 pg15: PG_MAJOR = 15 pg16: PG_MAJOR = 16 psycopg_deps = py{37,38,39,310,311}-{lin,win}: psycopg[binary] mac: psycopg2-binary py36: psycopg2-binary platforms = lin: linux mac: darwin win: win32 [tox] min_version = 4.0 requires = tox>4 env_list = dep lint py{[common]python_matrix}-test-{lin,mac,win} docs skipsdist = True toxworkdir = {env:TOX_WORK_DIR:.tox} skip_missing_interpreters = True [testenv] setenv = PYTHONDONTWRITEBYTECODE = 1 mac: OPEN_CMD = {env:OPEN_CMD:open} lin: OPEN_CMD = {env:OPEN_CMD:xdg-open} passenv = BROWSER DISPLAY [testenv:lint] description = Lint code with flake8 commands = flake8 {posargs:patroni tests setup.py} deps = flake8 [testenv:py{36,37,38,39,310,311}-test-{lin,win,mac}] description = Run unit tests with pytest labels = test commands_pre = - {tty:rm -f "{toxworkdir}{/}cov_report_{env_name}_html{/}index.html":true} - {tty:rm -f "{toxworkdir}{/}pytest_report_{env_name}.html":true} commands = pytest \ -p no:cacheprovider \ --verbose \ --doctest-modules \ --capture=fd \ --cov=patroni \ --cov-report=term-missing \ --cov-append \ {tty::--cov-report="xml\:{toxworkdir}{/}cov_report.{env_name}.xml"} \ {tty:--cov-report="html\:{toxworkdir}{/}cov_report_{env_name}_html":} \ {tty:--html="{toxworkdir}{/}pytest_report_{env_name}.html":} \ {posargs:tests patroni} commands_post = - {tty:{env:OPEN_CMD} "{toxworkdir}{/}cov_report_{env_name}_html{/}index.html":true} - {tty:{env:OPEN_CMD} "{toxworkdir}{/}pytest_report_{env_name}.html":true} deps = -r requirements.txt pytest pytest-cov pytest-html {[common]psycopg_deps} platform = {[common]platforms} allowlist_externals = rm true {env:OPEN_CMD} [testenv:dep] description = Check package dependency problems commands = pipdeptree -w fail deps = -r requirements.txt pipdeptree {[common]psycopg_deps} [testenv:py{37,38,39,310,311}-type-{lin,mac,win}] description = Run static type checking with pyright labels = type deps = -r requirements.txt pyright psycopg2-binary psycopg[binary] commands = pyright --venv-path {toxworkdir}{/}{envname} {posargs:patroni} platform = {[common]platforms} [testenv:black] description = Reformat code with black deps = black commands = black {posargs:patroni tests} [testenv:pg{12,13,14,15,16}-docker-build] description = Build docker containers needed for testing labels = behave docker-build setenv = {[common]postgres_matrix} DOCKER_BUILDKIT = 1 passenv = BASE_IMAGE commands = docker build . \ --tag patroni-dev:{env:PG_MAJOR} \ --build-arg PG_MAJOR \ --build-arg BASE_IMAGE={env:BASE_IMAGE:postgres} \ --file features/Dockerfile allowlist_externals = docker [testenv:pg{12,13,14,15,16}-docker-behave-{etcd,etcd3}-{lin,mac}] description = Run behaviour tests in patroni-dev docker container setenv = etcd: DCS=etcd etcd3: DCS=etcd3 {[common]postgres_matrix} CONTAINER_NAME = tox-{env_name}-{env:PYTHONHASHSEED} labels = behave depends = pg{11,12,13,14,15,16}-docker-build # There's a bug which affects calling multiple envs on the command line # This should be a valid command: tox -e 'py{36,37,38,39,310,311}-behave-{env:DCS}-lin' # Replaced with workaround, see https://github.com/tox-dev/tox/issues/2850 commands = docker run \ --volume {tox_root}:/src \ --env DCS={env:DCS} \ --hostname {env:CONTAINER_NAME} \ --name {env:CONTAINER_NAME} \ --rm \ --tty \ {env:PATRONI_DEV_IMAGE:patroni-dev:{env:PG_MAJOR}} \ tox run -x 'tox.env_list=py{[common]python_matrix}-behave-{env:DCS}-lin' \ -- {posargs} allowlist_externals = docker find platform = lin: linux ; win: win32 mac: darwin [testenv:py{36,38,39,310,311}-behave-{etcd,etcd3}-{lin,win,mac}] description = Run behaviour tests (locally with tox) deps = -r requirements.txt behave coverage {[common]psycopg_deps} setenv = etcd: DCS = {env:DCS:etcd} etcd3: DCS = {env:DCS:etcd3} passenv = ETCD_UNSUPPORTED_ARCH commands = python3 -m behave --format json --format plain --outfile result.json {posargs} mv result.json features/output allowlist_externals = mv platform = {[common]platforms} [testenv:epub-{lin,mac,win}] description = Build Sphinx documentation in epub format labels: docs deps = -r requirements.docs.txt -r requirements.txt commands = python -m sphinx -T -b epub -d _build/doctrees -D language=en . epub allowlist_externals = true {env:OPEN_CMD} platform = {[common]platforms} change_dir = docs [testenv:docs-{lin,mac,win}] description = Build Sphinx documentation in HTML format labels: docs deps = -r requirements.docs.txt -r requirements.txt commands = sphinx-build \ -d "{envtmpdir}{/}doctree" docs "{toxworkdir}{/}docs_out" \ --color -b html \ -T -E -W --keep-going \ {posargs} commands_post = - {tty:{env:OPEN_CMD} "{toxworkdir}{/}docs_out{/}index.html":true:} allowlist_externals = true {env:OPEN_CMD} platform = {[common]platforms} [testenv:pdf-{lin,mac,win}] description = Build Sphinx documentation in PDF format labels: docs deps = -r requirements.docs.txt -r requirements.txt commands = python -m sphinx -T -E -b latex -d _build/doctrees -D language=en . pdf - latexmk -r pdf/latexmkrc -cd -C pdf/Patroni.tex latexmk -r pdf/latexmkrc -cd -pdf -f -dvi- -ps- -jobname=Patroni -interaction=nonstopmode pdf/Patroni.tex commands_post = - {tty:{env:OPEN_CMD} "pdf{/}Patroni.pdf":true:} allowlist_externals = true latexmk {env:OPEN_CMD} platform = {[common]platforms} change_dir = docs [flake8] max-line-length = 120 ignore = D401,W503 [isort] line_length = 120 multi_line_output = 2 balanced_wrapping = true combine_as_imports = true force_alphabetical_sort_within_sections = true lines_between_types = 1 known_third_party = consul patroni-4.0.4/typings/000077500000000000000000000000001472010352700146715ustar00rootroot00000000000000patroni-4.0.4/typings/botocore/000077500000000000000000000000001472010352700165055ustar00rootroot00000000000000patroni-4.0.4/typings/botocore/__init__.pyi000066400000000000000000000000001472010352700207550ustar00rootroot00000000000000patroni-4.0.4/typings/botocore/exceptions.pyi000066400000000000000000000000421472010352700214050ustar00rootroot00000000000000class ClientError(Exception): ... patroni-4.0.4/typings/botocore/utils.pyi000066400000000000000000000013001472010352700203620ustar00rootroot00000000000000from typing import Any, Callable, Dict, Optional DEFAULT_METADATA_SERVICE_TIMEOUT = 1 METADATA_BASE_URL = 'http://169.254.169.254/' class AWSResponse: status_code: int @property def text(self) -> str: ... class IMDSFetcher: def __init__(self, timeout: float = DEFAULT_METADATA_SERVICE_TIMEOUT, num_attempts: int = 1, base_url: str = METADATA_BASE_URL, env: Optional[Dict[str, str]] = None, user_agent: Optional[str] = None, config: Optional[Dict[str, Any]] = None) -> None: ... def _fetch_metadata_token(self) -> Optional[str]: ...: def _get_request(self, url_path: str, retry_func: Optional[Callable[[AWSResponse], bool]] = None, token: Optional[str] = None) -> AWSResponse: ... patroni-4.0.4/typings/cdiff/000077500000000000000000000000001472010352700157445ustar00rootroot00000000000000patroni-4.0.4/typings/cdiff/__init__.pyi000066400000000000000000000002531472010352700202260ustar00rootroot00000000000000import io from typing import Any class PatchStream: def __init__(self, diff_hdl: io.BytesIOBase) -> None: ... def markup_to_pager(stream: Any, opts: Any) -> None: ... patroni-4.0.4/typings/consul/000077500000000000000000000000001472010352700161745ustar00rootroot00000000000000patroni-4.0.4/typings/consul/__init__.pyi000066400000000000000000000002161472010352700204550ustar00rootroot00000000000000from consul.check import Check from consul.base import ConsulException, NotFound __all__ = ['Check', 'ConsulException', 'Consul', 'NotFound'] patroni-4.0.4/typings/consul/base.pyi000066400000000000000000000032061472010352700176320ustar00rootroot00000000000000from typing import Any, Dict, List, Optional, Tuple class ConsulException(Exception): ... class NotFound(ConsulException): ... class Consul: http: Any agent: 'Consul.Agent' session: 'Consul.Session' kv: 'Consul.KV' class KV: def get(self, key: str, index: Optional[int]=None, recurse: bool = False, wait: Optional[str] = None, token: Optional[str] = None, consistency: Optional[str] = None, keys: bool = False, separator: Optional[str] = '', dc: Optional[str] = None) -> Tuple[int, Dict[str, Any]]: ... def put(self, key: str, value: str, cas: Optional[int] = None, flags: Optional[int] = None, acquire: Optional[str] = None, release: Optional[str] = None, token: Optional[str] = None, dc: Optional[str] = None) -> bool: ... def delete(self, key: str, recurse: Optional[bool] = None, cas: Optional[int] = None, token: Optional[str] = None, dc: Optional[str] = None) -> bool: ... class Agent: service: 'Consul.Agent.Service' def self(self) -> Dict[str, Dict[str, Any]]: ... class Service: def register(self, name: str, service_id=..., address=..., port=..., tags=..., check=..., token=..., script=..., interval=..., ttl=..., http=..., timeout=..., enable_tag_override=...) -> bool: ... def deregister(self, service_id: str) -> bool: ... class Session: def create(self, name: Optional[str] = None, node: Optional[str] = [], checks: Optional[List[str]]=None, lock_delay: float = 15, behavior: str = 'release', ttl: Optional[int] = None, dc: Optional[str] = None) -> str: ... def renew(self, session_id: str, dc: Optional[str] = None) -> Optional[str]: ... patroni-4.0.4/typings/consul/check.pyi000066400000000000000000000003061472010352700177730ustar00rootroot00000000000000from typing import Dict, Optional class Check: @classmethod def http(klass, url: str, interval: str, timeout: Optional[str] = None, deregister: Optional[str] = None) -> Dict[str, str]: ... patroni-4.0.4/typings/dns/000077500000000000000000000000001472010352700154555ustar00rootroot00000000000000patroni-4.0.4/typings/dns/resolver.pyi000066400000000000000000000013571472010352700200470ustar00rootroot00000000000000from typing import Union, Optional, Iterator class Name: def to_text(self, omit_final_dot: bool = ...) -> str: ... class Rdata: target: Name = ... port: int = ... class Answer: def __iter__(self) -> Iterator[Rdata]: ... def resolve(qname : str, rdtype : Union[int,str] = 0, rdclass : Union[int,str] = 0, tcp=False, source=None, raise_on_no_answer=True, source_port=0, lifetime : Optional[float]=None, search : Optional[bool]=None) -> Answer: ... def query(qname : str, rdtype : Union[int,str] = 0, rdclass : Union[int,str] = 0, tcp=False, source: Optional[str] = None, raise_on_no_answer=True, source_port=0, lifetime : Optional[float]=None) -> Answer: ... patroni-4.0.4/typings/etcd/000077500000000000000000000000001472010352700156105ustar00rootroot00000000000000patroni-4.0.4/typings/etcd/__init__.pyi000066400000000000000000000020661472010352700200760ustar00rootroot00000000000000from typing import Dict, Optional, Type, List from .client import Client __all__ = ['Client', 'EtcdError', 'EtcdException', 'EtcdEventIndexCleared', 'EtcdWatcherCleared', 'EtcdKeyNotFound', 'EtcdAlreadyExist', 'EtcdResult', 'EtcdConnectionFailed', 'EtcdWatchTimedOut'] class EtcdResult: action: str = ... modifiedIndex: int = ... key: str = ... value: str = ... ttl: Optional[float] = ... @property def leaves(self) -> List['EtcdResult']: ... class EtcdException(Exception): def __init__(self, message=..., payload=...) -> None: ... class EtcdConnectionFailed(EtcdException): def __init__(self, message=..., payload=..., cause=...) -> None: ... class EtcdKeyError(EtcdException): ... class EtcdKeyNotFound(EtcdKeyError): ... class EtcdAlreadyExist(EtcdKeyError): ... class EtcdEventIndexCleared(EtcdException): ... class EtcdWatchTimedOut(EtcdConnectionFailed): ... class EtcdWatcherCleared(EtcdException): ... class EtcdLeaderElectionInProgress(EtcdException): ... class EtcdError: error_exceptions: Dict[int, Type[EtcdException]] = ... patroni-4.0.4/typings/etcd/client.pyi000066400000000000000000000026171472010352700176170ustar00rootroot00000000000000import urllib3 from typing import Any, Optional, Set from . import EtcdResult class Client: _MGET: str _MPUT: str _MPOST: str _MDELETE: str _comparison_conditions: Set[str] _read_options: Set[str] _del_conditions: Set[str] http: urllib3.poolmanager.PoolManager _use_proxies: bool version_prefix: str username: Optional[str] password: Optional[str] def __init__(self, host=..., port=..., srv_domain=..., version_prefix=..., read_timeout=..., allow_redirect=..., protocol=..., cert=..., ca_cert=..., username=..., password=..., allow_reconnect=..., use_proxies=..., expected_cluster_id=..., per_host_pool_size=..., lock_prefix=...): ... @property def protocol(self) -> str: ... @property def read_timeout(self) -> int: ... @property def allow_redirect(self) -> bool: ... def write(self, key: str, value: str, ttl: int = ..., dir: bool = ..., append: bool = ..., **kwdargs: Any) -> EtcdResult: ... def read(self, key: str, **kwdargs: Any) -> EtcdResult: ... def delete(self, key: str, recursive: bool = ..., dir: bool = ..., **kwdargs: Any) -> EtcdResult: ... def set(self, key: str, value: str, ttl: int = ...) -> EtcdResult: ... def watch(self, key: str, index: int = ..., timeout: float = ..., recursive: bool = ...) -> EtcdResult: ... def _handle_server_response(self, response: urllib3.response.HTTPResponse) -> Any: ... patroni-4.0.4/typings/kazoo/000077500000000000000000000000001472010352700160145ustar00rootroot00000000000000patroni-4.0.4/typings/kazoo/client.pyi000066400000000000000000000043631472010352700200230ustar00rootroot00000000000000__all__ = ['KazooState', 'KazooClient', 'KazooRetry'] from kazoo.protocol.connection import ConnectionHandler from kazoo.protocol.states import KazooState, WatchedEvent, ZnodeStat from kazoo.handlers.threading import AsyncResult, SequentialThreadingHandler from kazoo.retry import KazooRetry from kazoo.security import ACL from typing import Any, Callable, Optional, Tuple, List class KazooClient: handler: SequentialThreadingHandler _state: str _connection: ConnectionHandler _session_timeout: int retry: Callable[..., Any] _retry: KazooRetry def __init__(self, hosts=..., timeout=..., client_id=..., handler=..., default_acl=..., auth_data=..., sasl_options=..., read_only=..., randomize_hosts=..., connection_retry=..., command_retry=..., logger=..., keyfile=..., keyfile_password=..., certfile=..., ca=..., use_ssl=..., verify_certs=..., **kwargs) -> None: ... @property def client_id(self) -> Optional[Tuple[Any]]: ... def add_listener(self, listener: Callable[[str], None]) -> None: ... def start(self, timeout: int = ...) -> None: ... def restart(self) -> None: ... def set_hosts(self, hosts: str, randomize_hosts: Optional[bool] = None) -> None: ... def create(self, path: str, value: bytes = b'', acl: Optional[ACL]=None, ephemeral: bool = False, sequence: bool = False, makepath: bool = False, include_data: bool = False) -> None: ... def create_async(self, path: str, value: bytes = b'', acl: Optional[ACL]=None, ephemeral: bool = False, sequence: bool = False, makepath: bool = False, include_data: bool = False) -> AsyncResult: ... def get(self, path: str, watch: Optional[Callable[[WatchedEvent], None]] = None) -> Tuple[bytes, ZnodeStat]: ... def get_children(self, path: str, watch: Optional[Callable[[WatchedEvent], None]] = None, include_data: bool = False) -> List[str]: ... def set(self, path: str, value: bytes, version: int = -1) -> ZnodeStat: ... def set_async(self, path: str, value: bytes, version: int = -1) -> AsyncResult: ... def delete(self, path: str, version: int = -1, recursive: bool = False) -> None: ... def delete_async(self, path: str, version: int = -1) -> AsyncResult: ... def _call(self, request: Tuple[Any], async_object: AsyncResult) -> Optional[bool]: ... patroni-4.0.4/typings/kazoo/exceptions.pyi000066400000000000000000000004361472010352700207230ustar00rootroot00000000000000class KazooException(Exception): ... class ZookeeperError(KazooException): ... class SessionExpiredError(ZookeeperError): ... class ConnectionClosedError(SessionExpiredError): ... class NoNodeError(ZookeeperError): ... class NodeExistsError(ZookeeperError): ... patroni-4.0.4/typings/kazoo/handlers/000077500000000000000000000000001472010352700176145ustar00rootroot00000000000000patroni-4.0.4/typings/kazoo/handlers/threading.pyi000066400000000000000000000004661472010352700223120ustar00rootroot00000000000000import socket from kazoo.handlers import utils from typing import Any class AsyncResult(utils.AsyncResult): ... class SequentialThreadingHandler: def select(self, *args: Any, **kwargs: Any) -> Any: ... def create_connection(self, *args: Any, **kwargs: Any) -> socket.socket: ... patroni-4.0.4/typings/kazoo/handlers/utils.pyi000066400000000000000000000003271472010352700215010ustar00rootroot00000000000000from typing import Any, Optional class AsyncResult: def set_exception(self, exception: Exception) -> None: ... def get(self, block: bool = False, timeout: Optional[float] = None) -> Any: ... patroni-4.0.4/typings/kazoo/protocol/000077500000000000000000000000001472010352700176555ustar00rootroot00000000000000patroni-4.0.4/typings/kazoo/protocol/connection.pyi000066400000000000000000000003061472010352700225360ustar00rootroot00000000000000import socket from typing import Any, Union, Tuple class ConnectionHandler: _socket: socket.socket def _connect(self, *args: Any) -> Tuple[Union[int, float], Union[int, float]]: ... patroni-4.0.4/typings/kazoo/protocol/states.pyi000066400000000000000000000010451472010352700217030ustar00rootroot00000000000000from typing import Any, NamedTuple class KazooState: SUSPENDED: str CONNECTED: str LOST: str class KeeperState: AUTH_FAILED: str CONNECTED: str CONNECTED_RO: str CONNECTING: str CLOSED: str EXPIRED_SESSION: str class WatchedEvent(NamedTuple): type: str state: str path: str class ZnodeStat(NamedTuple): czxid: int mzxid: int ctime: float mtime: float version: int cversion: int aversion: int ephemeralOwner: Any dataLength: int numChildren: int pzxid: int patroni-4.0.4/typings/kazoo/retry.pyi000066400000000000000000000004641472010352700177100ustar00rootroot00000000000000from kazoo.exceptions import KazooException class RetryFailedError(KazooException): ... class KazooRetry: deadline: float def __init__(self, max_tries=..., delay=..., backoff=..., max_jitter=..., max_delay=..., ignore_expire=..., sleep_func=..., deadline=..., interrupt=...) -> None: ... patroni-4.0.4/typings/kazoo/security.pyi000066400000000000000000000004011472010352700204010ustar00rootroot00000000000000from collections import namedtuple class ACL(namedtuple('ACL', 'perms id')): ... def make_acl(scheme: str, credential: str, read: bool = ..., write: bool = ..., create: bool = ..., delete: bool = ..., admin: bool = ..., all: bool = ...) -> ACL: ... patroni-4.0.4/typings/prettytable/000077500000000000000000000000001472010352700172305ustar00rootroot00000000000000patroni-4.0.4/typings/prettytable/__init__.pyi000066400000000000000000000010641472010352700215130ustar00rootroot00000000000000from enum import IntEnum from typing import Any, Dict, List class HRuleStyle(IntEnum): FRAME = 0 ALL = 1 FRAME = HRuleStyle.FRAME ALL = HRuleStyle.ALL class PrettyTable: def __init__(self, *args: str, **kwargs: Any) -> None: ... def _stringify_hrule(self, options: Dict[str, Any], where: str = '') -> str: ... @property def align(self) -> Dict[str, str]: ... @align.setter def align(self, val: str) -> None: ... def add_row(self, row: List[Any]) -> None: ... def __str__(self) -> str: ... def __repr__(self) -> str: ... patroni-4.0.4/typings/psycopg2/000077500000000000000000000000001472010352700164375ustar00rootroot00000000000000patroni-4.0.4/typings/psycopg2/__init__.pyi000066400000000000000000000030641472010352700207240ustar00rootroot00000000000000from collections.abc import Callable from typing import Any, TypeVar, overload # connection and cursor not available at runtime from psycopg2._psycopg import ( BINARY as BINARY, DATETIME as DATETIME, NUMBER as NUMBER, ROWID as ROWID, STRING as STRING, Binary as Binary, DatabaseError as DatabaseError, DataError as DataError, Date as Date, DateFromTicks as DateFromTicks, Error as Error, IntegrityError as IntegrityError, InterfaceError as InterfaceError, InternalError as InternalError, NotSupportedError as NotSupportedError, OperationalError as OperationalError, ProgrammingError as ProgrammingError, Time as Time, TimeFromTicks as TimeFromTicks, Timestamp as Timestamp, TimestampFromTicks as TimestampFromTicks, Warning as Warning, __libpq_version__ as __libpq_version__, apilevel as apilevel, connection as connection, cursor as cursor, paramstyle as paramstyle, threadsafety as threadsafety, ) __version__: str _T_conn = TypeVar("_T_conn", bound=connection) @overload def connect(dsn: str, connection_factory: Callable[..., _T_conn], cursor_factory: None = None, **kwargs: Any) -> _T_conn: ... @overload def connect( dsn: str | None = None, *, connection_factory: Callable[..., _T_conn], cursor_factory: None = None, **kwargs: Any ) -> _T_conn: ... @overload def connect( dsn: str | None = None, connection_factory: Callable[..., connection] | None = None, cursor_factory: Callable[..., cursor] | None = None, **kwargs: Any, ) -> connection: ... patroni-4.0.4/typings/psycopg2/_ipaddress.pyi000066400000000000000000000004451472010352700213020ustar00rootroot00000000000000from _typeshed import Incomplete from typing import Any ipaddress: Any def register_ipaddress(conn_or_curs: Incomplete | None = None) -> None: ... def cast_interface(s, cur: Incomplete | None = None): ... def cast_network(s, cur: Incomplete | None = None): ... def adapt_ipaddress(obj): ... patroni-4.0.4/typings/psycopg2/_json.pyi000066400000000000000000000015401472010352700202720ustar00rootroot00000000000000from _typeshed import Incomplete from typing import Any JSON_OID: int JSONARRAY_OID: int JSONB_OID: int JSONBARRAY_OID: int class Json: adapted: Any def __init__(self, adapted, dumps: Incomplete | None = None) -> None: ... def __conform__(self, proto): ... def dumps(self, obj): ... def prepare(self, conn) -> None: ... def getquoted(self): ... def register_json( conn_or_curs: Incomplete | None = None, globally: bool = False, loads: Incomplete | None = None, oid: Incomplete | None = None, array_oid: Incomplete | None = None, name: str = "json", ): ... def register_default_json(conn_or_curs: Incomplete | None = None, globally: bool = False, loads: Incomplete | None = None): ... def register_default_jsonb(conn_or_curs: Incomplete | None = None, globally: bool = False, loads: Incomplete | None = None): ... patroni-4.0.4/typings/psycopg2/_psycopg.pyi000066400000000000000000000355771472010352700210260ustar00rootroot00000000000000from collections.abc import Callable, Iterable, Mapping, Sequence from types import TracebackType from typing import Any, TypeVar, overload from typing_extensions import Literal, Self, TypeAlias import psycopg2 import psycopg2.extensions from psycopg2.sql import Composable _Vars: TypeAlias = Sequence[Any] | Mapping[str, Any] | None BINARY: Any BINARYARRAY: Any BOOLEAN: Any BOOLEANARRAY: Any BYTES: Any BYTESARRAY: Any CIDRARRAY: Any DATE: Any DATEARRAY: Any DATETIME: Any DATETIMEARRAY: Any DATETIMETZ: Any DATETIMETZARRAY: Any DECIMAL: Any DECIMALARRAY: Any FLOAT: Any FLOATARRAY: Any INETARRAY: Any INTEGER: Any INTEGERARRAY: Any INTERVAL: Any INTERVALARRAY: Any LONGINTEGER: Any LONGINTEGERARRAY: Any MACADDRARRAY: Any NUMBER: Any PYDATE: Any PYDATEARRAY: Any PYDATETIME: Any PYDATETIMEARRAY: Any PYDATETIMETZ: Any PYDATETIMETZARRAY: Any PYINTERVAL: Any PYINTERVALARRAY: Any PYTIME: Any PYTIMEARRAY: Any REPLICATION_LOGICAL: int REPLICATION_PHYSICAL: int ROWID: Any ROWIDARRAY: Any STRING: Any STRINGARRAY: Any TIME: Any TIMEARRAY: Any UNICODE: Any UNICODEARRAY: Any UNKNOWN: Any adapters: dict[Any, Any] apilevel: str binary_types: dict[Any, Any] encodings: dict[Any, Any] paramstyle: str sqlstate_errors: dict[Any, Any] string_types: dict[Any, Any] threadsafety: int __libpq_version__: int class cursor: arraysize: int binary_types: Any closed: Any connection: Any description: Any itersize: Any lastrowid: Any name: Any pgresult_ptr: Any query: Any row_factory: Any rowcount: int rownumber: int scrollable: bool | None statusmessage: Any string_types: Any typecaster: Any tzinfo_factory: Any withhold: bool def __init__(self, conn: connection, name: str | bytes | None = ...) -> None: ... def callproc(self, procname, parameters=...): ... def cast(self, oid, s): ... def close(self): ... def copy_expert(self, sql: str | bytes | Composable, file, size=...): ... def copy_from(self, file, table, sep=..., null=..., size=..., columns=...): ... def copy_to(self, file, table, sep=..., null=..., columns=...): ... def execute(self, query: str | bytes | Composable, vars: _Vars = ...) -> None: ... def executemany(self, query: str | bytes | Composable, vars_list: Iterable[_Vars]) -> None: ... def fetchall(self) -> list[tuple[Any, ...]]: ... def fetchmany(self, size: int | None = ...) -> list[tuple[Any, ...]]: ... def fetchone(self) -> tuple[Any, ...] | None: ... def mogrify(self, *args, **kwargs): ... def nextset(self): ... def scroll(self, value, mode=...): ... def setinputsizes(self, sizes): ... def setoutputsize(self, size, column=...): ... def __enter__(self) -> Self: ... def __exit__( self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> None: ... def __iter__(self) -> Self: ... def __next__(self) -> tuple[Any, ...]: ... _Cursor: TypeAlias = cursor class AsIs: adapted: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class Binary: adapted: Any buffer: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def prepare(self, conn): ... def __conform__(self, *args, **kwargs): ... class Boolean: adapted: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class Column: display_size: Any internal_size: Any name: Any null_ok: Any precision: Any scale: Any table_column: Any table_oid: Any type_code: Any def __init__(self, *args, **kwargs) -> None: ... def __eq__(self, __other): ... def __ge__(self, __other): ... def __getitem__(self, __index): ... def __getstate__(self): ... def __gt__(self, __other): ... def __le__(self, __other): ... def __len__(self) -> int: ... def __lt__(self, __other): ... def __ne__(self, __other): ... def __setstate__(self, state): ... class ConnectionInfo: # Note: the following properties can be None if their corresponding libpq function # returns NULL. They're not annotated as such, because this is very unlikely in # practice---the psycopg2 docs [1] don't even mention this as a possibility! # # - db_name # - user # - password # - host # - port # - options # # (To prove this, one needs to inspect the psycopg2 source code [2], plus the # documentation [3] and source code [4] of the corresponding libpq calls.) # # [1]: https://www.psycopg.org/docs/extensions.html#psycopg2.extensions.ConnectionInfo # [2]: https://github.com/psycopg/psycopg2/blob/1d3a89a0bba621dc1cc9b32db6d241bd2da85ad1/psycopg/conninfo_type.c#L52 and below # [3]: https://www.postgresql.org/docs/current/libpq-status.html # [4]: https://github.com/postgres/postgres/blob/b39838889e76274b107935fa8e8951baf0e8b31b/src/interfaces/libpq/fe-connect.c#L6754 and below @property def backend_pid(self) -> int: ... @property def dbname(self) -> str: ... @property def dsn_parameters(self) -> dict[str, str]: ... @property def error_message(self) -> str | None: ... @property def host(self) -> str: ... @property def needs_password(self) -> bool: ... @property def options(self) -> str: ... @property def password(self) -> str: ... @property def port(self) -> int: ... @property def protocol_version(self) -> int: ... @property def server_version(self) -> int: ... @property def socket(self) -> int: ... @property def ssl_attribute_names(self) -> list[str]: ... @property def ssl_in_use(self) -> bool: ... @property def status(self) -> int: ... @property def transaction_status(self) -> int: ... @property def used_password(self) -> bool: ... @property def user(self) -> str: ... def __init__(self, *args, **kwargs) -> None: ... def parameter_status(self, name: str) -> str | None: ... def ssl_attribute(self, name: str) -> str | None: ... class DataError(psycopg2.DatabaseError): ... class DatabaseError(psycopg2.Error): ... class Decimal: adapted: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class Diagnostics: column_name: str | None constraint_name: str | None context: str | None datatype_name: str | None internal_position: str | None internal_query: str | None message_detail: str | None message_hint: str | None message_primary: str | None schema_name: str | None severity: str | None severity_nonlocalized: str | None source_file: str | None source_function: str | None source_line: str | None sqlstate: str | None statement_position: str | None table_name: str | None def __init__(self, __err: Error) -> None: ... class Error(Exception): cursor: _Cursor | None diag: Diagnostics pgcode: str | None pgerror: str | None def __init__(self, *args, **kwargs) -> None: ... def __reduce__(self): ... def __setstate__(self, state): ... class Float: adapted: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class ISQLQuote: _wrapped: Any def __init__(self, *args, **kwargs) -> None: ... def getbinary(self, *args, **kwargs): ... def getbuffer(self, *args, **kwargs): ... def getquoted(self, *args, **kwargs) -> bytes: ... class Int: adapted: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class IntegrityError(psycopg2.DatabaseError): ... class InterfaceError(psycopg2.Error): ... class InternalError(psycopg2.DatabaseError): ... class List: adapted: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def prepare(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class NotSupportedError(psycopg2.DatabaseError): ... class Notify: channel: Any payload: Any pid: Any def __init__(self, *args, **kwargs) -> None: ... def __eq__(self, __other): ... def __ge__(self, __other): ... def __getitem__(self, __index): ... def __gt__(self, __other): ... def __hash__(self) -> int: ... def __le__(self, __other): ... def __len__(self) -> int: ... def __lt__(self, __other): ... def __ne__(self, __other): ... class OperationalError(psycopg2.DatabaseError): ... class ProgrammingError(psycopg2.DatabaseError): ... class QueryCanceledError(psycopg2.OperationalError): ... class QuotedString: adapted: Any buffer: Any encoding: Any def __init__(self, *args, **kwargs) -> None: ... def getquoted(self, *args, **kwargs): ... def prepare(self, *args, **kwargs): ... def __conform__(self, *args, **kwargs): ... class ReplicationConnection(psycopg2.extensions.connection): autocommit: Any isolation_level: Any replication_type: Any reset: Any set_isolation_level: Any set_session: Any def __init__(self, *args, **kwargs) -> None: ... class ReplicationCursor(cursor): feedback_timestamp: Any io_timestamp: Any wal_end: Any def __init__(self, *args, **kwargs) -> None: ... def consume_stream(self, consumer, keepalive_interval=...): ... def read_message(self, *args, **kwargs): ... def send_feedback(self, write_lsn=..., flush_lsn=..., apply_lsn=..., reply=..., force=...): ... def start_replication_expert(self, command, decode=..., status_interval=...): ... class ReplicationMessage: cursor: Any data_size: Any data_start: Any payload: Any send_time: Any wal_end: Any def __init__(self, *args, **kwargs) -> None: ... class TransactionRollbackError(psycopg2.OperationalError): ... class Warning(Exception): ... class Xid: bqual: Any database: Any format_id: Any gtrid: Any owner: Any prepared: Any def __init__(self, *args, **kwargs) -> None: ... def from_string(self, *args, **kwargs): ... def __getitem__(self, __index): ... def __len__(self) -> int: ... _T_cur = TypeVar("_T_cur", bound=cursor) class connection: DataError: Any DatabaseError: Any Error: Any IntegrityError: Any InterfaceError: Any InternalError: Any NotSupportedError: Any OperationalError: Any ProgrammingError: Any Warning: Any @property def async_(self) -> int: ... autocommit: bool @property def binary_types(self) -> Any: ... @property def closed(self) -> int: ... cursor_factory: Callable[..., _Cursor] @property def dsn(self) -> str: ... @property def encoding(self) -> str: ... @property def info(self) -> ConnectionInfo: ... @property def isolation_level(self) -> int | None: ... @isolation_level.setter def isolation_level(self, __value: str | bytes | int | None) -> None: ... notices: list[Any] notifies: list[Any] @property def pgconn_ptr(self) -> int | None: ... @property def protocol_version(self) -> int: ... @property def deferrable(self) -> bool | None: ... @deferrable.setter def deferrable(self, __value: Literal["default"] | bool | None) -> None: ... @property def readonly(self) -> bool | None: ... @readonly.setter def readonly(self, __value: Literal["default"] | bool | None) -> None: ... @property def server_version(self) -> int: ... @property def status(self) -> int: ... @property def string_types(self) -> Any: ... # Really it's dsn: str, async: int = ..., async_: int = ..., but # that would be a syntax error. def __init__(self, dsn: str, *, async_: int = ...) -> None: ... def cancel(self) -> None: ... def close(self) -> None: ... def commit(self) -> None: ... @overload def cursor(self, name: str | bytes | None = ..., *, withhold: bool = ..., scrollable: bool | None = ...) -> _Cursor: ... def fileno(self) -> int: ... def get_backend_pid(self) -> int: ... def get_dsn_parameters(self) -> dict[str, str]: ... def get_native_connection(self): ... def get_parameter_status(self, parameter: str) -> str | None: ... def get_transaction_status(self) -> int: ... def isexecuting(self) -> bool: ... def lobject( self, oid: int = ..., mode: str | None = ..., new_oid: int = ..., new_file: str | None = ..., lobject_factory: type[lobject] = ..., ) -> lobject: ... def poll(self) -> int: ... def reset(self) -> None: ... def rollback(self) -> None: ... def set_client_encoding(self, encoding: str) -> None: ... def set_isolation_level(self, level: int | None) -> None: ... def set_session( self, isolation_level: str | bytes | int | None = ..., readonly: bool | Literal["default", b"default"] | None = ..., deferrable: bool | Literal["default", b"default"] | None = ..., autocommit: bool = ..., ) -> None: ... def tpc_begin(self, xid: str | bytes | Xid) -> None: ... def tpc_commit(self, __xid: str | bytes | Xid = ...) -> None: ... def tpc_prepare(self) -> None: ... def tpc_recover(self) -> list[Xid]: ... def tpc_rollback(self, __xid: str | bytes | Xid = ...) -> None: ... def xid(self, format_id, gtrid, bqual) -> Xid: ... def __enter__(self) -> Self: ... def __exit__(self, __type: type[BaseException] | None, __name: BaseException | None, __tb: TracebackType | None) -> None: ... class lobject: closed: Any mode: Any oid: Any def __init__(self, *args, **kwargs) -> None: ... def close(self): ... def export(self, filename): ... def read(self, size=...): ... def seek(self, offset, whence=...): ... def tell(self): ... def truncate(self, len=...): ... def unlink(self): ... def write(self, str): ... def Date(year, month, day): ... def DateFromPy(*args, **kwargs): ... def DateFromTicks(ticks): ... def IntervalFromPy(*args, **kwargs): ... def Time(hour, minutes, seconds, tzinfo=...): ... def TimeFromPy(*args, **kwargs): ... def TimeFromTicks(ticks): ... def Timestamp(year, month, day, hour, minutes, seconds, tzinfo=...): ... def TimestampFromPy(*args, **kwargs): ... def TimestampFromTicks(ticks): ... def _connect(*args, **kwargs): ... def adapt(*args, **kwargs): ... def encrypt_password(*args, **kwargs): ... def get_wait_callback(*args, **kwargs): ... def libpq_version(*args, **kwargs): ... def new_array_type(oids, name, baseobj): ... def new_type(oids, name, castobj): ... def parse_dsn(dsn: str | bytes) -> dict[str, Any]: ... def quote_ident(value: Any, scope: connection | cursor | None) -> str: ... def register_type(*args, **kwargs): ... def set_wait_callback(_none): ... patroni-4.0.4/typings/psycopg2/_range.pyi000066400000000000000000000032311472010352700204140ustar00rootroot00000000000000from _typeshed import Incomplete from typing import Any class Range: def __init__( self, lower: Incomplete | None = None, upper: Incomplete | None = None, bounds: str = "[)", empty: bool = False ) -> None: ... @property def lower(self): ... @property def upper(self): ... @property def isempty(self): ... @property def lower_inf(self): ... @property def upper_inf(self): ... @property def lower_inc(self): ... @property def upper_inc(self): ... def __contains__(self, x): ... def __bool__(self) -> bool: ... def __eq__(self, other): ... def __ne__(self, other): ... def __hash__(self) -> int: ... def __lt__(self, other): ... def __le__(self, other): ... def __gt__(self, other): ... def __ge__(self, other): ... def register_range(pgrange, pyrange, conn_or_curs, globally: bool = False): ... class RangeAdapter: name: Any adapted: Any def __init__(self, adapted) -> None: ... def __conform__(self, proto): ... def prepare(self, conn) -> None: ... def getquoted(self): ... class RangeCaster: subtype_oid: Any typecaster: Any array_typecaster: Any def __init__(self, pgrange, pyrange, oid, subtype_oid, array_oid: Incomplete | None = None) -> None: ... def parse(self, s, cur: Incomplete | None = None): ... class NumericRange(Range): ... class DateRange(Range): ... class DateTimeRange(Range): ... class DateTimeTZRange(Range): ... class NumberRangeAdapter(RangeAdapter): def getquoted(self): ... int4range_caster: Any int8range_caster: Any numrange_caster: Any daterange_caster: Any tsrange_caster: Any tstzrange_caster: Any patroni-4.0.4/typings/psycopg2/errorcodes.pyi000066400000000000000000000221331472010352700213320ustar00rootroot00000000000000def lookup(code, _cache={}): ... CLASS_SUCCESSFUL_COMPLETION: str CLASS_WARNING: str CLASS_NO_DATA: str CLASS_SQL_STATEMENT_NOT_YET_COMPLETE: str CLASS_CONNECTION_EXCEPTION: str CLASS_TRIGGERED_ACTION_EXCEPTION: str CLASS_FEATURE_NOT_SUPPORTED: str CLASS_INVALID_TRANSACTION_INITIATION: str CLASS_LOCATOR_EXCEPTION: str CLASS_INVALID_GRANTOR: str CLASS_INVALID_ROLE_SPECIFICATION: str CLASS_DIAGNOSTICS_EXCEPTION: str CLASS_CASE_NOT_FOUND: str CLASS_CARDINALITY_VIOLATION: str CLASS_DATA_EXCEPTION: str CLASS_INTEGRITY_CONSTRAINT_VIOLATION: str CLASS_INVALID_CURSOR_STATE: str CLASS_INVALID_TRANSACTION_STATE: str CLASS_INVALID_SQL_STATEMENT_NAME: str CLASS_TRIGGERED_DATA_CHANGE_VIOLATION: str CLASS_INVALID_AUTHORIZATION_SPECIFICATION: str CLASS_DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST: str CLASS_INVALID_TRANSACTION_TERMINATION: str CLASS_SQL_ROUTINE_EXCEPTION: str CLASS_INVALID_CURSOR_NAME: str CLASS_EXTERNAL_ROUTINE_EXCEPTION: str CLASS_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION: str CLASS_SAVEPOINT_EXCEPTION: str CLASS_INVALID_CATALOG_NAME: str CLASS_INVALID_SCHEMA_NAME: str CLASS_TRANSACTION_ROLLBACK: str CLASS_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION: str CLASS_WITH_CHECK_OPTION_VIOLATION: str CLASS_INSUFFICIENT_RESOURCES: str CLASS_PROGRAM_LIMIT_EXCEEDED: str CLASS_OBJECT_NOT_IN_PREREQUISITE_STATE: str CLASS_OPERATOR_INTERVENTION: str CLASS_SYSTEM_ERROR: str CLASS_SNAPSHOT_FAILURE: str CLASS_CONFIGURATION_FILE_ERROR: str CLASS_FOREIGN_DATA_WRAPPER_ERROR: str CLASS_PL_PGSQL_ERROR: str CLASS_INTERNAL_ERROR: str SUCCESSFUL_COMPLETION: str WARNING: str NULL_VALUE_ELIMINATED_IN_SET_FUNCTION: str STRING_DATA_RIGHT_TRUNCATION_: str PRIVILEGE_NOT_REVOKED: str PRIVILEGE_NOT_GRANTED: str IMPLICIT_ZERO_BIT_PADDING: str DYNAMIC_RESULT_SETS_RETURNED: str DEPRECATED_FEATURE: str NO_DATA: str NO_ADDITIONAL_DYNAMIC_RESULT_SETS_RETURNED: str SQL_STATEMENT_NOT_YET_COMPLETE: str CONNECTION_EXCEPTION: str SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION: str CONNECTION_DOES_NOT_EXIST: str SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION: str CONNECTION_FAILURE: str TRANSACTION_RESOLUTION_UNKNOWN: str PROTOCOL_VIOLATION: str TRIGGERED_ACTION_EXCEPTION: str FEATURE_NOT_SUPPORTED: str INVALID_TRANSACTION_INITIATION: str LOCATOR_EXCEPTION: str INVALID_LOCATOR_SPECIFICATION: str INVALID_GRANTOR: str INVALID_GRANT_OPERATION: str INVALID_ROLE_SPECIFICATION: str DIAGNOSTICS_EXCEPTION: str STACKED_DIAGNOSTICS_ACCESSED_WITHOUT_ACTIVE_HANDLER: str CASE_NOT_FOUND: str CARDINALITY_VIOLATION: str DATA_EXCEPTION: str STRING_DATA_RIGHT_TRUNCATION: str NULL_VALUE_NO_INDICATOR_PARAMETER: str NUMERIC_VALUE_OUT_OF_RANGE: str NULL_VALUE_NOT_ALLOWED_: str ERROR_IN_ASSIGNMENT: str INVALID_DATETIME_FORMAT: str DATETIME_FIELD_OVERFLOW: str INVALID_TIME_ZONE_DISPLACEMENT_VALUE: str ESCAPE_CHARACTER_CONFLICT: str INVALID_USE_OF_ESCAPE_CHARACTER: str INVALID_ESCAPE_OCTET: str ZERO_LENGTH_CHARACTER_STRING: str MOST_SPECIFIC_TYPE_MISMATCH: str SEQUENCE_GENERATOR_LIMIT_EXCEEDED: str NOT_AN_XML_DOCUMENT: str INVALID_XML_DOCUMENT: str INVALID_XML_CONTENT: str INVALID_XML_COMMENT: str INVALID_XML_PROCESSING_INSTRUCTION: str INVALID_INDICATOR_PARAMETER_VALUE: str SUBSTRING_ERROR: str DIVISION_BY_ZERO: str INVALID_PRECEDING_OR_FOLLOWING_SIZE: str INVALID_ARGUMENT_FOR_NTILE_FUNCTION: str INTERVAL_FIELD_OVERFLOW: str INVALID_ARGUMENT_FOR_NTH_VALUE_FUNCTION: str INVALID_CHARACTER_VALUE_FOR_CAST: str INVALID_ESCAPE_CHARACTER: str INVALID_REGULAR_EXPRESSION: str INVALID_ARGUMENT_FOR_LOGARITHM: str INVALID_ARGUMENT_FOR_POWER_FUNCTION: str INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION: str INVALID_ROW_COUNT_IN_LIMIT_CLAUSE: str INVALID_ROW_COUNT_IN_RESULT_OFFSET_CLAUSE: str INVALID_LIMIT_VALUE: str CHARACTER_NOT_IN_REPERTOIRE: str INDICATOR_OVERFLOW: str INVALID_PARAMETER_VALUE: str UNTERMINATED_C_STRING: str INVALID_ESCAPE_SEQUENCE: str STRING_DATA_LENGTH_MISMATCH: str TRIM_ERROR: str ARRAY_SUBSCRIPT_ERROR: str INVALID_TABLESAMPLE_REPEAT: str INVALID_TABLESAMPLE_ARGUMENT: str DUPLICATE_JSON_OBJECT_KEY_VALUE: str INVALID_ARGUMENT_FOR_SQL_JSON_DATETIME_FUNCTION: str INVALID_JSON_TEXT: str INVALID_SQL_JSON_SUBSCRIPT: str MORE_THAN_ONE_SQL_JSON_ITEM: str NO_SQL_JSON_ITEM: str NON_NUMERIC_SQL_JSON_ITEM: str NON_UNIQUE_KEYS_IN_A_JSON_OBJECT: str SINGLETON_SQL_JSON_ITEM_REQUIRED: str SQL_JSON_ARRAY_NOT_FOUND: str SQL_JSON_MEMBER_NOT_FOUND: str SQL_JSON_NUMBER_NOT_FOUND: str SQL_JSON_OBJECT_NOT_FOUND: str TOO_MANY_JSON_ARRAY_ELEMENTS: str TOO_MANY_JSON_OBJECT_MEMBERS: str SQL_JSON_SCALAR_REQUIRED: str FLOATING_POINT_EXCEPTION: str INVALID_TEXT_REPRESENTATION: str INVALID_BINARY_REPRESENTATION: str BAD_COPY_FILE_FORMAT: str UNTRANSLATABLE_CHARACTER: str NONSTANDARD_USE_OF_ESCAPE_CHARACTER: str INTEGRITY_CONSTRAINT_VIOLATION: str RESTRICT_VIOLATION: str NOT_NULL_VIOLATION: str FOREIGN_KEY_VIOLATION: str UNIQUE_VIOLATION: str CHECK_VIOLATION: str EXCLUSION_VIOLATION: str INVALID_CURSOR_STATE: str INVALID_TRANSACTION_STATE: str ACTIVE_SQL_TRANSACTION: str BRANCH_TRANSACTION_ALREADY_ACTIVE: str INAPPROPRIATE_ACCESS_MODE_FOR_BRANCH_TRANSACTION: str INAPPROPRIATE_ISOLATION_LEVEL_FOR_BRANCH_TRANSACTION: str NO_ACTIVE_SQL_TRANSACTION_FOR_BRANCH_TRANSACTION: str READ_ONLY_SQL_TRANSACTION: str SCHEMA_AND_DATA_STATEMENT_MIXING_NOT_SUPPORTED: str HELD_CURSOR_REQUIRES_SAME_ISOLATION_LEVEL: str NO_ACTIVE_SQL_TRANSACTION: str IN_FAILED_SQL_TRANSACTION: str IDLE_IN_TRANSACTION_SESSION_TIMEOUT: str INVALID_SQL_STATEMENT_NAME: str TRIGGERED_DATA_CHANGE_VIOLATION: str INVALID_AUTHORIZATION_SPECIFICATION: str INVALID_PASSWORD: str DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST: str DEPENDENT_OBJECTS_STILL_EXIST: str INVALID_TRANSACTION_TERMINATION: str SQL_ROUTINE_EXCEPTION: str MODIFYING_SQL_DATA_NOT_PERMITTED_: str PROHIBITED_SQL_STATEMENT_ATTEMPTED_: str READING_SQL_DATA_NOT_PERMITTED_: str FUNCTION_EXECUTED_NO_RETURN_STATEMENT: str INVALID_CURSOR_NAME: str EXTERNAL_ROUTINE_EXCEPTION: str CONTAINING_SQL_NOT_PERMITTED: str MODIFYING_SQL_DATA_NOT_PERMITTED: str PROHIBITED_SQL_STATEMENT_ATTEMPTED: str READING_SQL_DATA_NOT_PERMITTED: str EXTERNAL_ROUTINE_INVOCATION_EXCEPTION: str INVALID_SQLSTATE_RETURNED: str NULL_VALUE_NOT_ALLOWED: str TRIGGER_PROTOCOL_VIOLATED: str SRF_PROTOCOL_VIOLATED: str EVENT_TRIGGER_PROTOCOL_VIOLATED: str SAVEPOINT_EXCEPTION: str INVALID_SAVEPOINT_SPECIFICATION: str INVALID_CATALOG_NAME: str INVALID_SCHEMA_NAME: str TRANSACTION_ROLLBACK: str SERIALIZATION_FAILURE: str TRANSACTION_INTEGRITY_CONSTRAINT_VIOLATION: str STATEMENT_COMPLETION_UNKNOWN: str DEADLOCK_DETECTED: str SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION: str INSUFFICIENT_PRIVILEGE: str SYNTAX_ERROR: str INVALID_NAME: str INVALID_COLUMN_DEFINITION: str NAME_TOO_LONG: str DUPLICATE_COLUMN: str AMBIGUOUS_COLUMN: str UNDEFINED_COLUMN: str UNDEFINED_OBJECT: str DUPLICATE_OBJECT: str DUPLICATE_ALIAS: str DUPLICATE_FUNCTION: str AMBIGUOUS_FUNCTION: str GROUPING_ERROR: str DATATYPE_MISMATCH: str WRONG_OBJECT_TYPE: str INVALID_FOREIGN_KEY: str CANNOT_COERCE: str UNDEFINED_FUNCTION: str GENERATED_ALWAYS: str RESERVED_NAME: str UNDEFINED_TABLE: str UNDEFINED_PARAMETER: str DUPLICATE_CURSOR: str DUPLICATE_DATABASE: str DUPLICATE_PREPARED_STATEMENT: str DUPLICATE_SCHEMA: str DUPLICATE_TABLE: str AMBIGUOUS_PARAMETER: str AMBIGUOUS_ALIAS: str INVALID_COLUMN_REFERENCE: str INVALID_CURSOR_DEFINITION: str INVALID_DATABASE_DEFINITION: str INVALID_FUNCTION_DEFINITION: str INVALID_PREPARED_STATEMENT_DEFINITION: str INVALID_SCHEMA_DEFINITION: str INVALID_TABLE_DEFINITION: str INVALID_OBJECT_DEFINITION: str INDETERMINATE_DATATYPE: str INVALID_RECURSION: str WINDOWING_ERROR: str COLLATION_MISMATCH: str INDETERMINATE_COLLATION: str WITH_CHECK_OPTION_VIOLATION: str INSUFFICIENT_RESOURCES: str DISK_FULL: str OUT_OF_MEMORY: str TOO_MANY_CONNECTIONS: str CONFIGURATION_LIMIT_EXCEEDED: str PROGRAM_LIMIT_EXCEEDED: str STATEMENT_TOO_COMPLEX: str TOO_MANY_COLUMNS: str TOO_MANY_ARGUMENTS: str OBJECT_NOT_IN_PREREQUISITE_STATE: str OBJECT_IN_USE: str CANT_CHANGE_RUNTIME_PARAM: str LOCK_NOT_AVAILABLE: str UNSAFE_NEW_ENUM_VALUE_USAGE: str OPERATOR_INTERVENTION: str QUERY_CANCELED: str ADMIN_SHUTDOWN: str CRASH_SHUTDOWN: str CANNOT_CONNECT_NOW: str DATABASE_DROPPED: str SYSTEM_ERROR: str IO_ERROR: str UNDEFINED_FILE: str DUPLICATE_FILE: str SNAPSHOT_TOO_OLD: str CONFIG_FILE_ERROR: str LOCK_FILE_EXISTS: str FDW_ERROR: str FDW_OUT_OF_MEMORY: str FDW_DYNAMIC_PARAMETER_VALUE_NEEDED: str FDW_INVALID_DATA_TYPE: str FDW_COLUMN_NAME_NOT_FOUND: str FDW_INVALID_DATA_TYPE_DESCRIPTORS: str FDW_INVALID_COLUMN_NAME: str FDW_INVALID_COLUMN_NUMBER: str FDW_INVALID_USE_OF_NULL_POINTER: str FDW_INVALID_STRING_FORMAT: str FDW_INVALID_HANDLE: str FDW_INVALID_OPTION_INDEX: str FDW_INVALID_OPTION_NAME: str FDW_OPTION_NAME_NOT_FOUND: str FDW_REPLY_HANDLE: str FDW_UNABLE_TO_CREATE_EXECUTION: str FDW_UNABLE_TO_CREATE_REPLY: str FDW_UNABLE_TO_ESTABLISH_CONNECTION: str FDW_NO_SCHEMAS: str FDW_SCHEMA_NOT_FOUND: str FDW_TABLE_NOT_FOUND: str FDW_FUNCTION_SEQUENCE_ERROR: str FDW_TOO_MANY_HANDLES: str FDW_INCONSISTENT_DESCRIPTOR_INFORMATION: str FDW_INVALID_ATTRIBUTE_VALUE: str FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH: str FDW_INVALID_DESCRIPTOR_FIELD_IDENTIFIER: str PLPGSQL_ERROR: str RAISE_EXCEPTION: str NO_DATA_FOUND: str TOO_MANY_ROWS: str ASSERT_FAILURE: str INTERNAL_ERROR: str DATA_CORRUPTED: str INDEX_CORRUPTED: str patroni-4.0.4/typings/psycopg2/errors.pyi000066400000000000000000000307321472010352700205030ustar00rootroot00000000000000from psycopg2._psycopg import Error as Error, Warning as Warning class DatabaseError(Error): ... class InterfaceError(Error): ... class DataError(DatabaseError): ... class DiagnosticsException(DatabaseError): ... class IntegrityError(DatabaseError): ... class InternalError(DatabaseError): ... class InvalidGrantOperation(DatabaseError): ... class InvalidGrantor(DatabaseError): ... class InvalidLocatorSpecification(DatabaseError): ... class InvalidRoleSpecification(DatabaseError): ... class InvalidTransactionInitiation(DatabaseError): ... class LocatorException(DatabaseError): ... class NoAdditionalDynamicResultSetsReturned(DatabaseError): ... class NoData(DatabaseError): ... class NotSupportedError(DatabaseError): ... class OperationalError(DatabaseError): ... class ProgrammingError(DatabaseError): ... class SnapshotTooOld(DatabaseError): ... class SqlStatementNotYetComplete(DatabaseError): ... class StackedDiagnosticsAccessedWithoutActiveHandler(DatabaseError): ... class TriggeredActionException(DatabaseError): ... class ActiveSqlTransaction(InternalError): ... class AdminShutdown(OperationalError): ... class AmbiguousAlias(ProgrammingError): ... class AmbiguousColumn(ProgrammingError): ... class AmbiguousFunction(ProgrammingError): ... class AmbiguousParameter(ProgrammingError): ... class ArraySubscriptError(DataError): ... class AssertFailure(InternalError): ... class BadCopyFileFormat(DataError): ... class BranchTransactionAlreadyActive(InternalError): ... class CannotCoerce(ProgrammingError): ... class CannotConnectNow(OperationalError): ... class CantChangeRuntimeParam(OperationalError): ... class CardinalityViolation(ProgrammingError): ... class CaseNotFound(ProgrammingError): ... class CharacterNotInRepertoire(DataError): ... class CheckViolation(IntegrityError): ... class CollationMismatch(ProgrammingError): ... class ConfigFileError(InternalError): ... class ConfigurationLimitExceeded(OperationalError): ... class ConnectionDoesNotExist(OperationalError): ... class ConnectionException(OperationalError): ... class ConnectionFailure(OperationalError): ... class ContainingSqlNotPermitted(InternalError): ... class CrashShutdown(OperationalError): ... class DataCorrupted(InternalError): ... class DataException(DataError): ... class DatabaseDropped(OperationalError): ... class DatatypeMismatch(ProgrammingError): ... class DatetimeFieldOverflow(DataError): ... class DependentObjectsStillExist(InternalError): ... class DependentPrivilegeDescriptorsStillExist(InternalError): ... class DiskFull(OperationalError): ... class DivisionByZero(DataError): ... class DuplicateAlias(ProgrammingError): ... class DuplicateColumn(ProgrammingError): ... class DuplicateCursor(ProgrammingError): ... class DuplicateDatabase(ProgrammingError): ... class DuplicateFile(OperationalError): ... class DuplicateFunction(ProgrammingError): ... class DuplicateJsonObjectKeyValue(DataError): ... class DuplicateObject(ProgrammingError): ... class DuplicatePreparedStatement(ProgrammingError): ... class DuplicateSchema(ProgrammingError): ... class DuplicateTable(ProgrammingError): ... class ErrorInAssignment(DataError): ... class EscapeCharacterConflict(DataError): ... class EventTriggerProtocolViolated(InternalError): ... class ExclusionViolation(IntegrityError): ... class ExternalRoutineException(InternalError): ... class ExternalRoutineInvocationException(InternalError): ... class FdwColumnNameNotFound(OperationalError): ... class FdwDynamicParameterValueNeeded(OperationalError): ... class FdwError(OperationalError): ... class FdwFunctionSequenceError(OperationalError): ... class FdwInconsistentDescriptorInformation(OperationalError): ... class FdwInvalidAttributeValue(OperationalError): ... class FdwInvalidColumnName(OperationalError): ... class FdwInvalidColumnNumber(OperationalError): ... class FdwInvalidDataType(OperationalError): ... class FdwInvalidDataTypeDescriptors(OperationalError): ... class FdwInvalidDescriptorFieldIdentifier(OperationalError): ... class FdwInvalidHandle(OperationalError): ... class FdwInvalidOptionIndex(OperationalError): ... class FdwInvalidOptionName(OperationalError): ... class FdwInvalidStringFormat(OperationalError): ... class FdwInvalidStringLengthOrBufferLength(OperationalError): ... class FdwInvalidUseOfNullPointer(OperationalError): ... class FdwNoSchemas(OperationalError): ... class FdwOptionNameNotFound(OperationalError): ... class FdwOutOfMemory(OperationalError): ... class FdwReplyHandle(OperationalError): ... class FdwSchemaNotFound(OperationalError): ... class FdwTableNotFound(OperationalError): ... class FdwTooManyHandles(OperationalError): ... class FdwUnableToCreateExecution(OperationalError): ... class FdwUnableToCreateReply(OperationalError): ... class FdwUnableToEstablishConnection(OperationalError): ... class FeatureNotSupported(NotSupportedError): ... class FloatingPointException(DataError): ... class ForeignKeyViolation(IntegrityError): ... class FunctionExecutedNoReturnStatement(InternalError): ... class GeneratedAlways(ProgrammingError): ... class GroupingError(ProgrammingError): ... class HeldCursorRequiresSameIsolationLevel(InternalError): ... class IdleInTransactionSessionTimeout(InternalError): ... class InFailedSqlTransaction(InternalError): ... class InappropriateAccessModeForBranchTransaction(InternalError): ... class InappropriateIsolationLevelForBranchTransaction(InternalError): ... class IndeterminateCollation(ProgrammingError): ... class IndeterminateDatatype(ProgrammingError): ... class IndexCorrupted(InternalError): ... class IndicatorOverflow(DataError): ... class InsufficientPrivilege(ProgrammingError): ... class InsufficientResources(OperationalError): ... class IntegrityConstraintViolation(IntegrityError): ... class InternalError_(InternalError): ... class IntervalFieldOverflow(DataError): ... class InvalidArgumentForLogarithm(DataError): ... class InvalidArgumentForNthValueFunction(DataError): ... class InvalidArgumentForNtileFunction(DataError): ... class InvalidArgumentForPowerFunction(DataError): ... class InvalidArgumentForSqlJsonDatetimeFunction(DataError): ... class InvalidArgumentForWidthBucketFunction(DataError): ... class InvalidAuthorizationSpecification(OperationalError): ... class InvalidBinaryRepresentation(DataError): ... class InvalidCatalogName(ProgrammingError): ... class InvalidCharacterValueForCast(DataError): ... class InvalidColumnDefinition(ProgrammingError): ... class InvalidColumnReference(ProgrammingError): ... class InvalidCursorDefinition(ProgrammingError): ... class InvalidCursorName(OperationalError): ... class InvalidCursorState(InternalError): ... class InvalidDatabaseDefinition(ProgrammingError): ... class InvalidDatetimeFormat(DataError): ... class InvalidEscapeCharacter(DataError): ... class InvalidEscapeOctet(DataError): ... class InvalidEscapeSequence(DataError): ... class InvalidForeignKey(ProgrammingError): ... class InvalidFunctionDefinition(ProgrammingError): ... class InvalidIndicatorParameterValue(DataError): ... class InvalidJsonText(DataError): ... class InvalidName(ProgrammingError): ... class InvalidObjectDefinition(ProgrammingError): ... class InvalidParameterValue(DataError): ... class InvalidPassword(OperationalError): ... class InvalidPrecedingOrFollowingSize(DataError): ... class InvalidPreparedStatementDefinition(ProgrammingError): ... class InvalidRecursion(ProgrammingError): ... class InvalidRegularExpression(DataError): ... class InvalidRowCountInLimitClause(DataError): ... class InvalidRowCountInResultOffsetClause(DataError): ... class InvalidSavepointSpecification(InternalError): ... class InvalidSchemaDefinition(ProgrammingError): ... class InvalidSchemaName(ProgrammingError): ... class InvalidSqlJsonSubscript(DataError): ... class InvalidSqlStatementName(OperationalError): ... class InvalidSqlstateReturned(InternalError): ... class InvalidTableDefinition(ProgrammingError): ... class InvalidTablesampleArgument(DataError): ... class InvalidTablesampleRepeat(DataError): ... class InvalidTextRepresentation(DataError): ... class InvalidTimeZoneDisplacementValue(DataError): ... class InvalidTransactionState(InternalError): ... class InvalidTransactionTermination(InternalError): ... class InvalidUseOfEscapeCharacter(DataError): ... class InvalidXmlComment(DataError): ... class InvalidXmlContent(DataError): ... class InvalidXmlDocument(DataError): ... class InvalidXmlProcessingInstruction(DataError): ... class IoError(OperationalError): ... class LockFileExists(InternalError): ... class LockNotAvailable(OperationalError): ... class ModifyingSqlDataNotPermitted(InternalError): ... class ModifyingSqlDataNotPermittedExt(InternalError): ... class MoreThanOneSqlJsonItem(DataError): ... class MostSpecificTypeMismatch(DataError): ... class NameTooLong(ProgrammingError): ... class NoActiveSqlTransaction(InternalError): ... class NoActiveSqlTransactionForBranchTransaction(InternalError): ... class NoDataFound(InternalError): ... class NoSqlJsonItem(DataError): ... class NonNumericSqlJsonItem(DataError): ... class NonUniqueKeysInAJsonObject(DataError): ... class NonstandardUseOfEscapeCharacter(DataError): ... class NotAnXmlDocument(DataError): ... class NotNullViolation(IntegrityError): ... class NullValueNoIndicatorParameter(DataError): ... class NullValueNotAllowed(DataError): ... class NullValueNotAllowedExt(InternalError): ... class NumericValueOutOfRange(DataError): ... class ObjectInUse(OperationalError): ... class ObjectNotInPrerequisiteState(OperationalError): ... class OperatorIntervention(OperationalError): ... class OutOfMemory(OperationalError): ... class PlpgsqlError(InternalError): ... class ProgramLimitExceeded(OperationalError): ... class ProhibitedSqlStatementAttempted(InternalError): ... class ProhibitedSqlStatementAttemptedExt(InternalError): ... class ProtocolViolation(OperationalError): ... class QueryCanceledError(OperationalError): ... class RaiseException(InternalError): ... class ReadOnlySqlTransaction(InternalError): ... class ReadingSqlDataNotPermitted(InternalError): ... class ReadingSqlDataNotPermittedExt(InternalError): ... class ReservedName(ProgrammingError): ... class RestrictViolation(IntegrityError): ... class SavepointException(InternalError): ... class SchemaAndDataStatementMixingNotSupported(InternalError): ... class SequenceGeneratorLimitExceeded(DataError): ... class SingletonSqlJsonItemRequired(DataError): ... class SqlJsonArrayNotFound(DataError): ... class SqlJsonMemberNotFound(DataError): ... class SqlJsonNumberNotFound(DataError): ... class SqlJsonObjectNotFound(DataError): ... class SqlJsonScalarRequired(DataError): ... class SqlRoutineException(InternalError): ... class SqlclientUnableToEstablishSqlconnection(OperationalError): ... class SqlserverRejectedEstablishmentOfSqlconnection(OperationalError): ... class SrfProtocolViolated(InternalError): ... class StatementTooComplex(OperationalError): ... class StringDataLengthMismatch(DataError): ... class StringDataRightTruncation(DataError): ... class SubstringError(DataError): ... class SyntaxError(ProgrammingError): ... class SyntaxErrorOrAccessRuleViolation(ProgrammingError): ... class SystemError(OperationalError): ... class TooManyArguments(OperationalError): ... class TooManyColumns(OperationalError): ... class TooManyConnections(OperationalError): ... class TooManyJsonArrayElements(DataError): ... class TooManyJsonObjectMembers(DataError): ... class TooManyRows(InternalError): ... class TransactionResolutionUnknown(OperationalError): ... class TransactionRollbackError(OperationalError): ... class TriggerProtocolViolated(InternalError): ... class TriggeredDataChangeViolation(OperationalError): ... class TrimError(DataError): ... class UndefinedColumn(ProgrammingError): ... class UndefinedFile(OperationalError): ... class UndefinedFunction(ProgrammingError): ... class UndefinedObject(ProgrammingError): ... class UndefinedParameter(ProgrammingError): ... class UndefinedTable(ProgrammingError): ... class UniqueViolation(IntegrityError): ... class UnsafeNewEnumValueUsage(OperationalError): ... class UnterminatedCString(DataError): ... class UntranslatableCharacter(DataError): ... class WindowingError(ProgrammingError): ... class WithCheckOptionViolation(ProgrammingError): ... class WrongObjectType(ProgrammingError): ... class ZeroLengthCharacterString(DataError): ... class DeadlockDetected(TransactionRollbackError): ... class QueryCanceled(QueryCanceledError): ... class SerializationFailure(TransactionRollbackError): ... class StatementCompletionUnknown(TransactionRollbackError): ... class TransactionIntegrityConstraintViolation(TransactionRollbackError): ... class TransactionRollback(TransactionRollbackError): ... def lookup(code): ... patroni-4.0.4/typings/psycopg2/extensions.pyi000066400000000000000000000061631472010352700213670ustar00rootroot00000000000000from _typeshed import Incomplete from typing import Any from psycopg2._psycopg import ( BINARYARRAY as BINARYARRAY, BOOLEAN as BOOLEAN, BOOLEANARRAY as BOOLEANARRAY, BYTES as BYTES, BYTESARRAY as BYTESARRAY, DATE as DATE, DATEARRAY as DATEARRAY, DATETIMEARRAY as DATETIMEARRAY, DECIMAL as DECIMAL, DECIMALARRAY as DECIMALARRAY, FLOAT as FLOAT, FLOATARRAY as FLOATARRAY, INTEGER as INTEGER, INTEGERARRAY as INTEGERARRAY, INTERVAL as INTERVAL, INTERVALARRAY as INTERVALARRAY, LONGINTEGER as LONGINTEGER, LONGINTEGERARRAY as LONGINTEGERARRAY, PYDATE as PYDATE, PYDATEARRAY as PYDATEARRAY, PYDATETIME as PYDATETIME, PYDATETIMEARRAY as PYDATETIMEARRAY, PYDATETIMETZ as PYDATETIMETZ, PYDATETIMETZARRAY as PYDATETIMETZARRAY, PYINTERVAL as PYINTERVAL, PYINTERVALARRAY as PYINTERVALARRAY, PYTIME as PYTIME, PYTIMEARRAY as PYTIMEARRAY, ROWIDARRAY as ROWIDARRAY, STRINGARRAY as STRINGARRAY, TIME as TIME, TIMEARRAY as TIMEARRAY, UNICODE as UNICODE, UNICODEARRAY as UNICODEARRAY, AsIs as AsIs, Binary as Binary, Boolean as Boolean, Column as Column, ConnectionInfo as ConnectionInfo, DateFromPy as DateFromPy, Diagnostics as Diagnostics, Float as Float, Int as Int, IntervalFromPy as IntervalFromPy, ISQLQuote as ISQLQuote, Notify as Notify, QueryCanceledError as QueryCanceledError, QuotedString as QuotedString, TimeFromPy as TimeFromPy, TimestampFromPy as TimestampFromPy, TransactionRollbackError as TransactionRollbackError, Xid as Xid, adapt as adapt, adapters as adapters, binary_types as binary_types, connection as connection, cursor as cursor, encodings as encodings, encrypt_password as encrypt_password, get_wait_callback as get_wait_callback, libpq_version as libpq_version, lobject as lobject, new_array_type as new_array_type, new_type as new_type, parse_dsn as parse_dsn, quote_ident as quote_ident, register_type as register_type, set_wait_callback as set_wait_callback, string_types as string_types, ) ISOLATION_LEVEL_AUTOCOMMIT: int ISOLATION_LEVEL_READ_UNCOMMITTED: int ISOLATION_LEVEL_READ_COMMITTED: int ISOLATION_LEVEL_REPEATABLE_READ: int ISOLATION_LEVEL_SERIALIZABLE: int ISOLATION_LEVEL_DEFAULT: Any STATUS_SETUP: int STATUS_READY: int STATUS_BEGIN: int STATUS_SYNC: int STATUS_ASYNC: int STATUS_PREPARED: int STATUS_IN_TRANSACTION: int POLL_OK: int POLL_READ: int POLL_WRITE: int POLL_ERROR: int TRANSACTION_STATUS_IDLE: int TRANSACTION_STATUS_ACTIVE: int TRANSACTION_STATUS_INTRANS: int TRANSACTION_STATUS_INERROR: int TRANSACTION_STATUS_UNKNOWN: int def register_adapter(typ, callable) -> None: ... class SQL_IN: def __init__(self, seq) -> None: ... def prepare(self, conn) -> None: ... def getquoted(self): ... class NoneAdapter: def __init__(self, obj) -> None: ... def getquoted(self, _null: bytes = b"NULL"): ... def make_dsn(dsn: Incomplete | None = None, **kwargs): ... JSON: Any JSONARRAY: Any JSONB: Any JSONBARRAY: Any def adapt(obj: Any) -> ISQLQuote: ... patroni-4.0.4/typings/psycopg2/extras.pyi000066400000000000000000000205701472010352700204740ustar00rootroot00000000000000from _typeshed import Incomplete from collections import OrderedDict from collections.abc import Callable from typing import Any, NamedTuple, TypeVar, overload from psycopg2._ipaddress import register_ipaddress as register_ipaddress from psycopg2._json import ( Json as Json, register_default_json as register_default_json, register_default_jsonb as register_default_jsonb, register_json as register_json, ) from psycopg2._psycopg import ( REPLICATION_LOGICAL as REPLICATION_LOGICAL, REPLICATION_PHYSICAL as REPLICATION_PHYSICAL, ReplicationConnection as _replicationConnection, ReplicationCursor as _replicationCursor, ReplicationMessage as ReplicationMessage, ) from psycopg2._range import ( DateRange as DateRange, DateTimeRange as DateTimeRange, DateTimeTZRange as DateTimeTZRange, NumericRange as NumericRange, Range as Range, RangeAdapter as RangeAdapter, RangeCaster as RangeCaster, register_range as register_range, ) from .extensions import connection as _connection, cursor as _cursor, quote_ident as quote_ident _T_cur = TypeVar("_T_cur", bound=_cursor) class DictCursorBase(_cursor): def __init__(self, *args, **kwargs) -> None: ... class DictConnection(_connection): @overload def cursor(self, name: str | bytes | None = ..., *, withhold: bool = ..., scrollable: bool | None = ...) -> DictCursor: ... @overload def cursor( self, name: str | bytes | None = ..., *, cursor_factory: Callable[..., _T_cur], withhold: bool = ..., scrollable: bool | None = ..., ) -> _T_cur: ... @overload def cursor( self, name: str | bytes | None, cursor_factory: Callable[..., _T_cur], withhold: bool = ..., scrollable: bool | None = ... ) -> _T_cur: ... class DictCursor(DictCursorBase): def __init__(self, *args, **kwargs) -> None: ... index: Any def execute(self, query, vars: Incomplete | None = None): ... def callproc(self, procname, vars: Incomplete | None = None): ... def fetchone(self) -> DictRow | None: ... # type: ignore[override] def fetchmany(self, size: int | None = None) -> list[DictRow]: ... # type: ignore[override] def fetchall(self) -> list[DictRow]: ... # type: ignore[override] def __next__(self) -> DictRow: ... # type: ignore[override] class DictRow(list[Any]): def __init__(self, cursor) -> None: ... def __getitem__(self, x): ... def __setitem__(self, x, v) -> None: ... def items(self): ... def keys(self): ... def values(self): ... def get(self, x, default: Incomplete | None = None): ... def copy(self): ... def __contains__(self, x): ... def __reduce__(self): ... class RealDictConnection(_connection): @overload def cursor( self, name: str | bytes | None = ..., *, withhold: bool = ..., scrollable: bool | None = ... ) -> RealDictCursor: ... @overload def cursor( self, name: str | bytes | None = ..., *, cursor_factory: Callable[..., _T_cur], withhold: bool = ..., scrollable: bool | None = ..., ) -> _T_cur: ... @overload def cursor( self, name: str | bytes | None, cursor_factory: Callable[..., _T_cur], withhold: bool = ..., scrollable: bool | None = ... ) -> _T_cur: ... class RealDictCursor(DictCursorBase): def __init__(self, *args, **kwargs) -> None: ... column_mapping: Any def execute(self, query, vars: Incomplete | None = None): ... def callproc(self, procname, vars: Incomplete | None = None): ... def fetchone(self) -> RealDictRow | None: ... # type: ignore[override] def fetchmany(self, size: int | None = None) -> list[RealDictRow]: ... # type: ignore[override] def fetchall(self) -> list[RealDictRow]: ... # type: ignore[override] def __next__(self) -> RealDictRow: ... # type: ignore[override] class RealDictRow(OrderedDict[Any, Any]): def __init__(self, *args, **kwargs) -> None: ... def __setitem__(self, key, value) -> None: ... class NamedTupleConnection(_connection): @overload def cursor( self, name: str | bytes | None = ..., *, withhold: bool = ..., scrollable: bool | None = ... ) -> NamedTupleCursor: ... @overload def cursor( self, name: str | bytes | None = ..., *, cursor_factory: Callable[..., _T_cur], withhold: bool = ..., scrollable: bool | None = ..., ) -> _T_cur: ... @overload def cursor( self, name: str | bytes | None, cursor_factory: Callable[..., _T_cur], withhold: bool = ..., scrollable: bool | None = ... ) -> _T_cur: ... class NamedTupleCursor(_cursor): Record: Any MAX_CACHE: int def execute(self, query, vars: Incomplete | None = None): ... def executemany(self, query, vars): ... def callproc(self, procname, vars: Incomplete | None = None): ... def fetchone(self) -> NamedTuple | None: ... def fetchmany(self, size: int | None = None) -> list[NamedTuple]: ... # type: ignore[override] def fetchall(self) -> list[NamedTuple]: ... # type: ignore[override] def __next__(self) -> NamedTuple: ... class LoggingConnection(_connection): log: Any def initialize(self, logobj) -> None: ... def filter(self, msg, curs): ... def cursor(self, *args, **kwargs): ... class LoggingCursor(_cursor): def execute(self, query, vars: Incomplete | None = None): ... def callproc(self, procname, vars: Incomplete | None = None): ... class MinTimeLoggingConnection(LoggingConnection): def initialize(self, logobj, mintime: int = 0) -> None: ... def filter(self, msg, curs): ... def cursor(self, *args, **kwargs): ... class MinTimeLoggingCursor(LoggingCursor): timestamp: Any def execute(self, query, vars: Incomplete | None = None): ... def callproc(self, procname, vars: Incomplete | None = None): ... class LogicalReplicationConnection(_replicationConnection): def __init__(self, *args, **kwargs) -> None: ... class PhysicalReplicationConnection(_replicationConnection): def __init__(self, *args, **kwargs) -> None: ... class StopReplication(Exception): ... class ReplicationCursor(_replicationCursor): def create_replication_slot( self, slot_name, slot_type: Incomplete | None = None, output_plugin: Incomplete | None = None ) -> None: ... def drop_replication_slot(self, slot_name) -> None: ... def start_replication( self, slot_name: Incomplete | None = None, slot_type: Incomplete | None = None, start_lsn: int = 0, timeline: int = 0, options: Incomplete | None = None, decode: bool = False, status_interval: int = 10, ) -> None: ... def fileno(self): ... class UUID_adapter: def __init__(self, uuid) -> None: ... def __conform__(self, proto): ... def getquoted(self): ... def register_uuid(oids: Incomplete | None = None, conn_or_curs: Incomplete | None = None): ... class Inet: addr: Any def __init__(self, addr) -> None: ... def prepare(self, conn) -> None: ... def getquoted(self): ... def __conform__(self, proto): ... def register_inet(oid: Incomplete | None = None, conn_or_curs: Incomplete | None = None): ... def wait_select(conn) -> None: ... class HstoreAdapter: wrapped: Any def __init__(self, wrapped) -> None: ... conn: Any getquoted: Any def prepare(self, conn) -> None: ... @classmethod def parse(cls, s, cur, _bsdec=...): ... @classmethod def parse_unicode(cls, s, cur): ... @classmethod def get_oids(cls, conn_or_curs): ... def register_hstore( conn_or_curs, globally: bool = False, unicode: bool = False, oid: Incomplete | None = None, array_oid: Incomplete | None = None, ) -> None: ... class CompositeCaster: name: Any schema: Any oid: Any array_oid: Any attnames: Any atttypes: Any typecaster: Any array_typecaster: Any def __init__(self, name, oid, attrs, array_oid: Incomplete | None = None, schema: Incomplete | None = None) -> None: ... def parse(self, s, curs): ... def make(self, values): ... @classmethod def tokenize(cls, s): ... def register_composite(name, conn_or_curs, globally: bool = False, factory: Incomplete | None = None): ... def execute_batch(cur, sql, argslist, page_size: int = 100) -> None: ... def execute_values(cur, sql, argslist, template: Incomplete | None = None, page_size: int = 100, fetch: bool = False): ... patroni-4.0.4/typings/psycopg2/pool.pyi000066400000000000000000000017251472010352700201400ustar00rootroot00000000000000from _typeshed import Incomplete from typing import Any import psycopg2 class PoolError(psycopg2.Error): ... class AbstractConnectionPool: minconn: Any maxconn: Any closed: bool def __init__(self, minconn, maxconn, *args, **kwargs) -> None: ... # getconn, putconn and closeall are officially documented as methods of the # abstract base class, but in reality, they only exist on the children classes def getconn(self, key: Incomplete | None = ...): ... def putconn(self, conn: Any, key: Incomplete | None = ..., close: bool = ...) -> None: ... def closeall(self) -> None: ... class SimpleConnectionPool(AbstractConnectionPool): ... class ThreadedConnectionPool(AbstractConnectionPool): # This subclass has a default value for conn which doesn't exist # in the SimpleConnectionPool class, nor in the documentation def putconn(self, conn: Incomplete | None = None, key: Incomplete | None = None, close: bool = False) -> None: ... patroni-4.0.4/typings/psycopg2/sql.pyi000066400000000000000000000027421472010352700177660ustar00rootroot00000000000000from _typeshed import Incomplete from collections.abc import Iterator from typing import Any class Composable: def __init__(self, wrapped) -> None: ... def as_string(self, context) -> str: ... def __add__(self, other) -> Composed: ... def __mul__(self, n) -> Composed: ... def __eq__(self, other) -> bool: ... def __ne__(self, other) -> bool: ... class Composed(Composable): def __init__(self, seq) -> None: ... @property def seq(self) -> list[Composable]: ... def as_string(self, context) -> str: ... def __iter__(self) -> Iterator[Composable]: ... def __add__(self, other) -> Composed: ... def join(self, joiner) -> Composed: ... class SQL(Composable): def __init__(self, string) -> None: ... @property def string(self) -> str: ... def as_string(self, context) -> str: ... def format(self, *args, **kwargs) -> Composed: ... def join(self, seq) -> Composed: ... class Identifier(Composable): def __init__(self, *strings) -> None: ... @property def strings(self) -> tuple[str, ...]: ... @property def string(self) -> str: ... def as_string(self, context) -> str: ... class Literal(Composable): @property def wrapped(self): ... def as_string(self, context) -> str: ... class Placeholder(Composable): def __init__(self, name: Incomplete | None = None) -> None: ... @property def name(self) -> str | None: ... def as_string(self, context) -> str: ... NULL: Any DEFAULT: Any patroni-4.0.4/typings/psycopg2/tz.pyi000066400000000000000000000012721472010352700176210ustar00rootroot00000000000000import datetime from _typeshed import Incomplete from typing import Any ZERO: Any class FixedOffsetTimezone(datetime.tzinfo): def __init__(self, offset: Incomplete | None = None, name: Incomplete | None = None) -> None: ... def __new__(cls, offset: Incomplete | None = None, name: Incomplete | None = None): ... def __eq__(self, other): ... def __ne__(self, other): ... def __getinitargs__(self): ... def utcoffset(self, dt): ... def tzname(self, dt): ... def dst(self, dt): ... STDOFFSET: Any DSTOFFSET: Any DSTDIFF: Any class LocalTimezone(datetime.tzinfo): def utcoffset(self, dt): ... def dst(self, dt): ... def tzname(self, dt): ... LOCAL: Any patroni-4.0.4/typings/pysyncobj/000077500000000000000000000000001472010352700167115ustar00rootroot00000000000000patroni-4.0.4/typings/pysyncobj/__init__.pyi000066400000000000000000000002051472010352700211700ustar00rootroot00000000000000from .syncobj import FAIL_REASON, SyncObj, SyncObjConf, replicated __all__ = ['SyncObj', 'SyncObjConf', 'replicated', 'FAIL_REASON'] patroni-4.0.4/typings/pysyncobj/config.pyi000066400000000000000000000004771472010352700207110ustar00rootroot00000000000000from typing import Optional class FAIL_REASON: SUCCESS = ... QUEUE_FULL = ... MISSING_LEADER = ... DISCARDED = ... NOT_LEADER = ... LEADER_CHANGED = ... REQUEST_DENIED = ... class SyncObjConf: password: Optional[str] autoTickPeriod: int def __init__(self, **kwargs) -> None: ... patroni-4.0.4/typings/pysyncobj/dns_resolver.pyi000066400000000000000000000003631472010352700221430ustar00rootroot00000000000000from typing import Optional class DnsCachingResolver: def setTimeouts(self, cacheTime: float, failCacheTime: float) -> None: ... def resolve(self, hostname: str) -> Optional[str]: ... def globalDnsResolver() -> DnsCachingResolver: ... patroni-4.0.4/typings/pysyncobj/node.pyi000066400000000000000000000001711472010352700203600ustar00rootroot00000000000000class Node: @property def id(self) -> str: ... class TCPNode(Node): @property def host(self) -> str: ... patroni-4.0.4/typings/pysyncobj/syncobj.pyi000066400000000000000000000021021472010352700210760ustar00rootroot00000000000000from typing import Any, Callable, Collection, List, Optional, Set, Type from .config import FAIL_REASON, SyncObjConf from .node import Node from .transport import Transport __all__ = ['FAIL_REASON', 'SyncObj', 'SyncObjConf', 'replicated'] class SyncObj: def __init__(self, selfNode: Optional[str], otherNodes: Collection[str], conf: SyncObjConf=..., consumers=..., nodeClass=..., transport=..., transportClass: Type[Transport]=...) -> None: ... def destroy(self) -> None: ... def doTick(self, timeToWait: float = 0.0) -> None: ... def isNodeConnected(self, node: Node) -> bool: ... @property def selfNode(self) -> Node: ... @property def otherNodes(self) -> Set[Node]: ... @property def raftLastApplied(self) -> int: ... @property def raftCommitIndex(self) -> int: ... @property def conf(self) -> SyncObjConf: ... def _getLeader(self) -> Optional[Node]: ... def _isLeader(self) -> bool: ... def _onTick(self, timeToWait: float = 0.0) -> None: ... def replicated(*decArgs: Any, **decKwargs: Any) -> Callable[..., Any]: ... patroni-4.0.4/typings/pysyncobj/tcp_connection.pyi000066400000000000000000000001301472010352700224330ustar00rootroot00000000000000class CONNECTION_STATE: DISCONNECTED = ... CONNECTING = ... CONNECTED = ... patroni-4.0.4/typings/pysyncobj/transport.pyi000066400000000000000000000010531472010352700214670ustar00rootroot00000000000000from typing import Any, Callable, Collection, Optional from .node import TCPNode from .syncobj import SyncObj from .tcp_connection import CONNECTION_STATE __all__ = ['CONNECTION_STATE', 'TCPTransport'] class Transport: def setOnUtilityMessageCallback(self, message: str, callback: Callable[[Any, Callable[..., Any]], Any]) -> None: ... class TCPTransport(Transport): def __init__(self, syncObj: SyncObj, selfNode: Optional[TCPNode], otherNodes: Collection[TCPNode]) -> None: ... def _connectIfNecessarySingle(self, node: TCPNode) -> bool: ... patroni-4.0.4/typings/pysyncobj/utility.pyi000066400000000000000000000004251472010352700211400ustar00rootroot00000000000000from typing import Any, List, Optional, Union from .node import TCPNode class TcpUtility(Utility): def __init__(self, password: Optional[str] = None, timeout: float=900.0) -> None: ... def executeCommand(self, node: Union[str, TCPNode], command: List[Any]) -> Any: ... patroni-4.0.4/typings/urllib3/000077500000000000000000000000001472010352700162455ustar00rootroot00000000000000patroni-4.0.4/typings/urllib3/__init__.pyi000066400000000000000000000005101472010352700205230ustar00rootroot00000000000000from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .poolmanager import PoolManager from .response import HTTPResponse from .util.request import make_headers from .util.timeout import Timeout __all__ = ['HTTPResponse', 'HTTPConnectionPool', 'HTTPSConnectionPool', 'PoolManager', 'Timeout', 'make_headers'] patroni-4.0.4/typings/urllib3/_collections.pyi000066400000000000000000000015721472010352700214520ustar00rootroot00000000000000from typing import Any, MutableMapping class HTTPHeaderDict(MutableMapping[str, str]): def __init__(self, headers=None, **kwargs) -> None: ... def __setitem__(self, key, val) -> None: ... def __getitem__(self, key): ... def __delitem__(self, key) -> None: ... def __contains__(self, key): ... def __eq__(self, other): ... def __iter__(self) -> NoReturn: ... def __len__(self) -> int: ... def __ne__(self, other): ... values: Any get: Any update: Any iterkeys: Any itervalues: Any def pop(self, key, default=...): ... def discard(self, key): ... def add(self, key, val): ... def extend(self, *args, **kwargs): ... def getlist(self, key): ... getheaders: Any getallmatchingheaders: Any iget: Any def copy(self): ... def iteritems(self): ... def itermerged(self): ... def items(self): ... patroni-4.0.4/typings/urllib3/connection.pyi000066400000000000000000000001451472010352700211270ustar00rootroot00000000000000from http.client import HTTPConnection as _HTTPConnection class HTTPConnection(_HTTPConnection): ... patroni-4.0.4/typings/urllib3/connectionpool.pyi000066400000000000000000000001211472010352700220130ustar00rootroot00000000000000class HTTPConnectionPool: ... class HTTPSConnectionPool(HTTPConnectionPool): ... patroni-4.0.4/typings/urllib3/poolmanager.pyi000066400000000000000000000012671472010352700213020ustar00rootroot00000000000000from typing import Any, Dict, Optional from .response import HTTPResponse class PoolManager: headers: Dict[str, str] connection_pool_kw: Dict[str, Any] def __init__(self, num_pools: int = 10, headers: Optional[Dict[str, str]] = None, **connection_pool_kw: Any) -> None: ... def urlopen(self, method: str, url: str, body: Optional[Any] = None, headers: Optional[Dict[str,str]] = None, encode_multipart: bool = True, multipart_boundary: Optional[str] = None, **kw: Any) -> HTTPResponse: ... def request(self, method: str, url: str, fields: Optional[Any] = None, headers: Optional[Dict[str, str]] = None, **urlopen_kw: Any) -> HTTPResponse: ... def clear(self) -> None: ... patroni-4.0.4/typings/urllib3/response.pyi000066400000000000000000000010211472010352700206200ustar00rootroot00000000000000import io from typing import Any, Iterator, Optional, Union from ._collections import HTTPHeaderDict from .connection import HTTPConnection class HTTPResponse(io.IOBase): headers: HTTPHeaderDict status: int reason: Optional[str] def release_conn(self) -> None: ... @property def data(self) -> Union[bytes, Any]: ... @property def connection(self) -> Optional[HTTPConnection]: ... def read_chunked(self, amt: Optional[int] = None, decode_content: Optional[bool] = None) -> Iterator[bytes]: ... patroni-4.0.4/typings/urllib3/util/000077500000000000000000000000001472010352700172225ustar00rootroot00000000000000patroni-4.0.4/typings/urllib3/util/request.pyi000066400000000000000000000005421472010352700214360ustar00rootroot00000000000000from typing import Optional, Union, Dict, List def make_headers( keep_alive: Optional[bool] = None, accept_encoding: Union[bool, List[str], str, None] = None, user_agent: Optional[str] = None, basic_auth: Optional[str] = None, proxy_basic_auth: Optional[str] = None, disable_cache: Optional[bool] = None, ) -> Dict[str, str]: ... patroni-4.0.4/typings/urllib3/util/timeout.pyi000066400000000000000000000003131472010352700214300ustar00rootroot00000000000000from typing import Any, Optional class Timeout: DEFAULT_TIMEOUT: Any def __init__(self, total: Optional[float] = None, connect: Optional[float] = None, read: Optional[float] = None) -> None: ... patroni-4.0.4/typings/ydiff/000077500000000000000000000000001472010352700157725ustar00rootroot00000000000000patroni-4.0.4/typings/ydiff/__init__.pyi000066400000000000000000000002531472010352700202540ustar00rootroot00000000000000import io from typing import Any class PatchStream: def __init__(self, diff_hdl: io.BytesIOBase) -> None: ... def markup_to_pager(stream: Any, opts: Any) -> None: ...